diff --git a/.buildkite/common/ci_source_file_dependencies.yml b/.buildkite/common/ci_source_file_dependencies.yml index 5dfbf8da4..99bbffa75 100644 --- a/.buildkite/common/ci_source_file_dependencies.yml +++ b/.buildkite/common/ci_source_file_dependencies.yml @@ -453,6 +453,7 @@ source_file_dependencies: - *lingbot_video - *lingbot_world - tests/e2e/online_serving/test_lingbot_video.py + - tests/e2e/online_serving/test_lingbot_video_expansion.py - tests/e2e/online_serving/test_lingbot_video_moe.py - tests/e2e/offline_inference/test_lingbot_world_v2.py diff --git a/.buildkite/cuda/test-merge.yml b/.buildkite/cuda/test-merge.yml index cf0af0d31..6a0372f05 100644 --- a/.buildkite/cuda/test-merge.yml +++ b/.buildkite/cuda/test-merge.yml @@ -175,6 +175,16 @@ steps: " mirror_hardwares: h100_1 + - label: "Diffusion · LingBot-Video Test" + source_file_dependencies: diffusion_lingbot_function + commands: + - | + timeout 40m bash -c " + export VLLM_IMAGE_FETCH_TIMEOUT=60 + pytest -s -v tests/e2e/online_serving/test_lingbot_video.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model' + " + mirror_hardwares: h100_1 + - label: "Diffusion · MiniMax-H3 L3 DLO DP2 and FastH3" source_file_dependencies: diffusion_minimax_h3_function # Keep four server lifecycles in one Buildkite job, but never in one diff --git a/.buildkite/cuda/test-nightly.yml b/.buildkite/cuda/test-nightly.yml index adc94f9e9..843b8d9a3 100644 --- a/.buildkite/cuda/test-nightly.yml +++ b/.buildkite/cuda/test-nightly.yml @@ -349,7 +349,7 @@ steps: commands: - >- pytest -s -v - tests/e2e/online_serving/test_lingbot_video.py + tests/e2e/online_serving/test_lingbot_video_expansion.py tests/e2e/online_serving/test_lingbot_video_moe.py tests/e2e/offline_inference/test_lingbot_world_v2.py -m "full_model and diffusion and H100 and B200 and cards_1" diff --git a/.buildkite/cuda/test-ready.yml b/.buildkite/cuda/test-ready.yml index 1f07b0721..560399bf5 100644 --- a/.buildkite/cuda/test-ready.yml +++ b/.buildkite/cuda/test-ready.yml @@ -260,3 +260,13 @@ steps: pytest -s -v tests/e2e/online_serving/test_hunyuan_video_15_expansion.py -m 'core_model and diffusion' --run-level 'core_model' " mirror_hardwares: h100_1 + + - label: "Diffusion · LingBot-Video Test" + source_file_dependencies: diffusion_lingbot_function + commands: + - | + timeout 40m bash -c " + export VLLM_IMAGE_FETCH_TIMEOUT=60 + pytest -s -v tests/e2e/online_serving/test_lingbot_video.py -m 'core_model and cuda' --run-level 'core_model' + " + mirror_hardwares: h100_1 diff --git a/.claude/skills/production-add-diffusion-model/references/production-validation.md b/.claude/skills/production-add-diffusion-model/references/production-validation.md index eed568415..3b2f505fa 100644 --- a/.claude/skills/production-add-diffusion-model/references/production-validation.md +++ b/.claude/skills/production-add-diffusion-model/references/production-validation.md @@ -340,7 +340,7 @@ On supported GPU hosts, wrap a test or soak with the repository monitor: ```bash bash tests/dfx/stability/scripts/resource_monitor.sh run --backend gpu -- \ - pytest -s -v tests/dfx/stability/scripts/test_stability_.py -m slow + pytest -s -v tests/dfx/stability/scripts/run_stability_.py -m slow ``` Check the target revision's monitor backends. Do not claim CPU/NPU monitoring diff --git a/apps/ComfyUI-vLLM-Omni/README.md b/apps/ComfyUI-vLLM-Omni/README.md index f33e26b40..c26b2a3de 100644 --- a/apps/ComfyUI-vLLM-Omni/README.md +++ b/apps/ComfyUI-vLLM-Omni/README.md @@ -49,6 +49,7 @@ This extension offers the following nodes based on the output modalities (at **C - **Generate Image** for text-to-image and image-to-image tasks - **Generate Video** for text-to-video, first-frame/image-to-video, and reference-conditioned video +- **Latent Mask Editing** for MiniMax-H3 latent-mask editing (source media plus video/audio noise masks) - **FastH3 Deployment** for routing text-to-video requests to a MiniMax-H3 server with FastH3 fused at startup - **Multimodality Understanding** for multimodality-to-text and multimodality-to-audio tasks - **TTS** and **TTS Voice Clone** for TTS tasks @@ -67,7 +68,7 @@ Every node carries the vLLM-Omni mark in its title bar and is tinted by what it | Amber | AR / Diffusion / Multi-Stage Sampling Params | Sampling parameters that apply to any model | | Purple | Qwen TTS Params, Wan Video Params, MiniMax-H3 Video Params | Parameters that only one model family accepts | | Red | LoRA, FastH3 Deployment | Which weights the server is expected to have loaded | -| Teal | Video References | Reference media | +| Teal | Video References, Latent Mask Editing | Reference media | Recolouring a node by hand (right click -> Colors) overrides its tint, and the choice is kept. @@ -263,6 +264,18 @@ Open the **vLLM-Omni FastH3 Text to Video** template, then: The node records which server the workflow targets; it does not start one, nor switch adapters or attention backends on a running server. +#### Latent-mask editing (MiniMax-H3) + +The [WF-05 template](example_workflows/vLLM-Omni%20MiniMax-H3%20Latent%20Mask%20Editing.json) contains inpainting, object removal, continuation, and extension examples in one graph. See the [workflow guide](docs/wf05-h3-latent-editing.md) for inputs, mask settings, dependencies, and preview limitations. + +Connect a **Latent Mask Editing** node to **Generate Video → latent_edit** to edit a source clip instead of generating from scratch. It uploads the source media and serializes the video/audio noise masks the MiniMax H3 API accepts: + +- `source_video` / `source_audio` — the media to edit. +- `video_mask` — a ComfyUI mask image; `0` preserves a region, `1` regenerates it, fractional values blend. A 2D mask `[H, W]` is applied to every frame; a 3D mask `[T, H, W]` is treated as a temporal mask (one slice per frame) for continuation or extension. It is resized to the video latent grid. +- `audio_mask` — a scalar in `[0, 1]`; `0` keeps the source audio, `1` regenerates it, fractional values blend. + +A non-trivial mask requires its matching source, and a source without a mask is rejected. This node only forwards inputs to the server; the served model must declare latent-mask editing support (MiniMax-H3). + ### TTS (e.g., Qwen TTS series) (Also available at **ComfyUI sidebar->Template->vLLM-Omni->vLLM-Omni TTS**) diff --git a/apps/ComfyUI-vLLM-Omni/__init__.py b/apps/ComfyUI-vLLM-Omni/__init__.py index 8c1b03d87..999f85ab2 100644 --- a/apps/ComfyUI-vLLM-Omni/__init__.py +++ b/apps/ComfyUI-vLLM-Omni/__init__.py @@ -21,7 +21,9 @@ from .comfyui_vllm_omni.nodes import ( VLLMOmniGenerateImage, VLLMOmniGenerateMusic, VLLMOmniGenerateVideo, + VLLMOmniLatentMaskEditing, VLLMOmniMiniMaxH3Params, + VLLMOmniMiniMaxH3TemporalMask, VLLMOmniQwenTTSParams, VLLMOmniRemoteLoRA, VLLMOmniSamplingParamsList, @@ -34,6 +36,7 @@ from .comfyui_vllm_omni.nodes import ( # A dictionary that contains all nodes you want to export with their names NODE_CLASS_MAPPINGS = { + "VLLMOmniMiniMaxH3TemporalMask": VLLMOmniMiniMaxH3TemporalMask, # === Generation === "VLLMOmniGenerateImage": VLLMOmniGenerateImage, "VLLMOmniGenerateVideo": VLLMOmniGenerateVideo, @@ -42,6 +45,7 @@ NODE_CLASS_MAPPINGS = { "VLLMOmniGenerateMusic": VLLMOmniGenerateMusic, "VLLMOmniVoiceClone": VLLMOmniVoiceClone, "VLLMOmniVideoReferences": VLLMOmniVideoReferences, + "VLLMOmniLatentMaskEditing": VLLMOmniLatentMaskEditing, # === Params === "VLLMOmniARSampling": VLLMOmniARSampling, "VLLMOmniDiffusionSampling": VLLMOmniDiffusionSampling, @@ -55,6 +59,7 @@ NODE_CLASS_MAPPINGS = { # A dictionary that contains the friendly/humanly readable titles for the nodes NODE_DISPLAY_NAME_MAPPINGS = { + "VLLMOmniMiniMaxH3TemporalMask": "MiniMax-H3 Temporal Mask", # === Generation === "VLLMOmniGenerateImage": "Generate Image", "VLLMOmniGenerateVideo": "Generate Video", @@ -63,6 +68,7 @@ NODE_DISPLAY_NAME_MAPPINGS = { "VLLMOmniGenerateMusic": "Generate Music", "VLLMOmniVoiceClone": "TTS Voice Cloning", "VLLMOmniVideoReferences": "Video References", + "VLLMOmniLatentMaskEditing": "Latent Mask Editing", # === Params === "VLLMOmniARSampling": "AR Sampling Params", "VLLMOmniDiffusionSampling": "Diffusion Sampling Params", diff --git a/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/nodes.py b/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/nodes.py index 6d5006a11..5bddb9345 100644 --- a/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/nodes.py +++ b/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/nodes.py @@ -1,12 +1,14 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project +import math from typing import Literal import torch from comfy_api.input import AudioInput, VideoInput from .utils.api_client import VLLMOmniClient +from .utils.latent_mask import _align_frame_count, _video_latent_t from .utils.logger import get_logger from .utils.models import lookup_model_spec from .utils.types import ( @@ -17,6 +19,7 @@ from .utils.types import ( AutoregressionSamplingParams, DiffusionSamplingParams, FastH3Deployment, + LatentMaskEditing, MiniMaxH3ModelSpecificParams, QwenTTSModelSpecificParams, VideoReferences, @@ -210,6 +213,7 @@ class VLLMOmniGenerateVideo(_VLLMOmniGenerateBase): "lora": ("REMOTE_LORA",), "model_params": ("VIDEO_PARAMS",), "fast_h3": ("FASTH3_DEPLOYMENT",), + "latent_edit": ("LATENT_MASK_EDITING",), }, } @@ -262,6 +266,7 @@ class VLLMOmniGenerateVideo(_VLLMOmniGenerateBase): model_params: dict | None = None, lora: dict | None = None, fast_h3: dict | None = None, + latent_edit: dict | None = None, **kwargs, ): if kwargs: @@ -350,6 +355,7 @@ class VLLMOmniGenerateVideo(_VLLMOmniGenerateBase): sampling_params=sampling_params, lora=lora, model_params=model_params, + latent_edit=latent_edit, ) return (output,) @@ -1051,3 +1057,89 @@ class VLLMOmniVideoReferences: if value is not None: refs[f"{kind}_{index}"] = value return (refs,) + + +class VLLMOmniLatentMaskEditing: + @classmethod + def INPUT_TYPES(cls): + return { + "required": {}, + "optional": { + "source_video": ("VIDEO",), + "source_audio": ("AUDIO",), + "video_mask": ("MASK",), + "audio_mask": ("FLOAT", {"default": -1.0, "min": -1.0, "max": 1.0, "step": 0.01}), + }, + } + + RETURN_TYPES = ("LATENT_MASK_EDITING",) + RETURN_NAMES = ("latent_edit",) + FUNCTION = "get_latent_edit" + CATEGORY = "vLLM-Omni" + + def get_latent_edit( + self, + source_video: VideoInput | None = None, + source_audio: AudioInput | None = None, + video_mask: torch.Tensor | None = None, + audio_mask: float = -1.0, + **kwargs, + ): + if kwargs: + logger.info("Uncaught kwargs: %s", kwargs) + edit = LatentMaskEditing() + if source_video is not None: + edit["source_video"] = source_video + if source_audio is not None: + edit["source_audio"] = source_audio + if video_mask is not None: + edit["video_mask"] = video_mask + if audio_mask >= 0.0: + edit["audio_mask"] = audio_mask + return (edit,) + + +class VLLMOmniMiniMaxH3TemporalMask: + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "images": ("IMAGE",), + "source_fps": ("FLOAT", {"default": 24.0, "min": 0.01}), + "duration": ("FLOAT", {"default": 5.0, "min": 0.01, "max": 15.0}), + "mode": (["continuation", "extension"],), + "preserve_fraction": ("FLOAT", {"default": 0.5, "min": 0.0, "max": 1.0}), + } + } + + RETURN_TYPES = ("MASK", "FLOAT", "IMAGE", "MASK") + RETURN_NAMES = ("mask", "preview_fps", "preview_images", "preview_mask") + FUNCTION = "build" + CATEGORY = "vLLM-Omni" + + def build(self, images, source_fps, duration, mode, preserve_fraction=0.5): + if images.shape[0] <= 0 or not math.isfinite(source_fps) or source_fps <= 0: + raise ValueError("Source must contain frames and have a positive finite FPS.") + if not math.isfinite(duration) or duration <= 0 or not 0 <= preserve_fraction <= 1: + raise ValueError("Invalid duration or preserve_fraction.") + frames = _align_frame_count(max(1, round(duration * 24))) + source_seconds = images.shape[0] / source_fps + if mode == "extension": + if frames / 24 <= source_seconds: + raise ValueError("Extension output must be longer than the source; increase duration.") + boundary = source_seconds + elif mode == "continuation": + boundary = min(source_seconds, frames / 24) * preserve_fraction + else: + raise ValueError(f"Unknown temporal mask mode: {mode}") + available = min(frames, math.floor(boundary * 24 + 1e-8)) + prefix = 0 if available < 5 else 5 + 17 * ((available - 5) // 17) + preserved = _video_latent_t(prefix) if prefix else 0 + total = _video_latent_t(frames) + mask = torch.ones(total, 1, 1) + mask[:preserved] = 0 + indices = torch.arange(frames, device=images.device) + source_indices = (indices * (source_fps / 24)).floor().long().clamp(max=images.shape[0] - 1) + preview_images = images.index_select(0, source_indices) + preview_mask = mask.index_select(0, torch.arange(frames) * total // frames) + return mask, 24.0, preview_images, preview_mask diff --git a/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/api_client.py b/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/api_client.py index 21935414b..69e1af899 100644 --- a/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/api_client.py +++ b/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/api_client.py @@ -30,6 +30,7 @@ from .format import ( video_to_base64, video_to_bytes, ) +from .latent_mask import scalar_mask_to_json, video_mask_to_grid_json from .logger import get_logger, pretty_printer from .models import lookup_model_spec from .types import ( @@ -283,6 +284,7 @@ class VLLMOmniClient: sampling_params: dict | None = None, model_params: dict | None = None, lora: dict | None = None, + latent_edit: dict | None = None, spec_model: str | None = None, **extra_params, ) -> VideoInput: @@ -386,6 +388,53 @@ class VLLMOmniClient: content_type="image/png", ) + # === latent-mask editing (MiniMax H3) === + if latent_edit is not None: + source_video = latent_edit.get("source_video") + source_audio = latent_edit.get("source_audio") + video_mask = latent_edit.get("video_mask") + audio_mask = latent_edit.get("audio_mask") + + if video_mask is None and audio_mask is None: + raise ValueError("Latent-mask editing requires at least one mask.") + + video_mask_trivial = video_mask is None or bool((video_mask == 1.0).all().item()) + audio_mask_trivial = audio_mask is None or audio_mask == 1.0 + if not video_mask_trivial and source_video is None: + raise ValueError("A non-trivial video mask requires a source video.") + if not audio_mask_trivial and source_audio is None and source_video is None: + raise ValueError("A non-trivial audio mask requires a source audio or a source video with audio.") + + if source_video is not None: + form.add_field( + "source_video", + video_to_bytes(source_video, "source.mp4"), + filename="source.mp4", + content_type="video/mp4", + ) + if source_audio is not None: + form.add_field( + "source_audio", + audio_to_bytes(source_audio, "source_audio.mp3"), + filename="source_audio.mp3", + content_type="audio/mpeg", + ) + if video_mask is not None: + mask_json = video_mask_to_grid_json(video_mask, width=width, height=height, num_frames=num_frames) + form.add_field( + "video_noise_mask", + mask_json.encode("utf-8"), + filename="video-mask.json", + content_type="application/json", + ) + if audio_mask is not None: + form.add_field( + "audio_noise_mask", + scalar_mask_to_json(audio_mask).encode("utf-8"), + filename="audio-mask.json", + content_type="application/json", + ) + if len(keyframe_images) == 2: for image_filename, image in keyframe_images: form.add_field( diff --git a/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/latent_mask.py b/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/latent_mask.py new file mode 100644 index 000000000..532cf8184 --- /dev/null +++ b/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/latent_mask.py @@ -0,0 +1,51 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + +import json + +import torch +import torch.nn.functional as F + + +def scalar_mask_to_json(value: float) -> str: + if not 0.0 <= value <= 1.0: + raise ValueError(f"mask value must be in [0, 1], got {value}") + return str(value) + + +def _align_frame_count(frame_count: int) -> int: + if frame_count <= 0: + return 1 + current = int(frame_count) + while current % 17 != 5: + current += 1 + return current + + +def _video_latent_t(frame_count: int) -> int: + if frame_count <= 5: + return 2 + return ((int(frame_count) - 5) // 17) * 5 + 2 + + +def video_mask_to_grid(mask: torch.Tensor, *, width: int, height: int, num_frames: int) -> torch.Tensor: + tv = _video_latent_t(_align_frame_count(num_frames)) + height = int(height) // 32 * 32 + width = int(width) // 32 * 32 + gh, gw = height // 16, width // 16 + + if mask.ndim == 2: + mask = mask.unsqueeze(0) + elif mask.ndim != 3: + raise ValueError(f"expected a 2D or 3D mask tensor, got {mask.ndim}D") + + grid = F.interpolate(mask.unsqueeze(1).float(), size=(gh, gw), mode="area").squeeze(1) + if grid.shape[0] == 1: + grid = grid.expand(tv, gh, gw) + elif grid.shape[0] != tv: + grid = F.interpolate(grid.unsqueeze(0).unsqueeze(0), size=(tv, gh, gw), mode="nearest").squeeze(0).squeeze(0) + return grid + + +def video_mask_to_grid_json(mask: torch.Tensor, *, width: int, height: int, num_frames: int) -> str: + return json.dumps(video_mask_to_grid(mask, width=width, height=height, num_frames=num_frames).tolist()) diff --git a/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/types.py b/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/types.py index 6d12a98c2..6194613de 100644 --- a/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/types.py +++ b/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/types.py @@ -40,6 +40,10 @@ class VideoReferences(dict): pass +class LatentMaskEditing(dict): + pass + + class FastH3Deployment(dict): """Descriptor for a server that fused FastH3 at startup.""" diff --git a/apps/ComfyUI-vLLM-Omni/docs/wf05-h3-latent-editing.md b/apps/ComfyUI-vLLM-Omni/docs/wf05-h3-latent-editing.md new file mode 100644 index 000000000..9ed10273f --- /dev/null +++ b/apps/ComfyUI-vLLM-Omni/docs/wf05-h3-latent-editing.md @@ -0,0 +1,17 @@ +# WF-05: MiniMax-H3 latent-mask editing + +Edit a source video with MiniMax-H3 using spatial or temporal masks. The workflow includes object removal, inpainting, continuation, and extension, with a mask preview for each case. + +## Requirements + +- ComfyUI with ComfyUI-vLLM-Omni installed. +- An H3 service using the FL2VA checkpoint partition with latent-mask editing support. Follow the [H3 recipe](../../../recipes/MiniMaxAI/MiniMax-H3.md) for server setup. + +## Use + +Open **vLLM-Omni MiniMax-H3 Latent Mask Editing.json** from the example workflows and set the Generate Video URL/model for the case you want to run. + +1. Upload your source video in Load Video. +2. Adjust the case's mask. For removal and inpainting, set Mask Canvas to the video dimensions, Mask Rectangle to the region size, and Combine Masks x/y to its position. For continuation and extension, set MiniMax-H3 Temporal Mask's mode and duration; continuation also uses `preserve_fraction`. Match the duration in Generate Video, and use a duration longer than the source for extension. Mask values are `0` to preserve and `1` to regenerate. +3. Adjust the prompt to describe the desired content. Set the output dimensions and duration in Generate Video. If editing audio, update the sound description and audio mask (`0` preserves, `1` regenerates the whole clip). +4. Select the case's Mask Preview Save Video node and choose **Execute to selected output nodes** to preview the mask as a red overlay without inference, or select its Result Save Video node to generate the edited video. Running the entire workflow generates all four cases. diff --git a/apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Latent Mask Editing - Image Mask.json b/apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Latent Mask Editing - Image Mask.json new file mode 100644 index 000000000..17be731bf --- /dev/null +++ b/apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Latent Mask Editing - Image Mask.json @@ -0,0 +1,499 @@ +{ + "id": "latent-mask-editing-7380", + "revision": 0, + "last_node_id": 7, + "last_link_id": 5, + "nodes": [ + { + "id": 1, + "type": "MarkdownNote", + "pos": [ + 40, + 20 + ], + "size": [ + 420, + 210 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [], + "title": "Latent-mask editing server prerequisite", + "properties": {}, + "widgets_values": [ + "Requires a vLLM-Omni server that declares latent-mask editing support (MiniMax-H3, EDIT-01). The source clip and video/audio noise masks are uploaded to /v1/videos; 0 preserves, 1 regenerates, fractional blends." + ], + "color": "#432", + "bgcolor": "#000" + }, + { + "id": 2, + "type": "LoadVideo", + "pos": [ + 40, + 280 + ], + "size": [ + 420, + 120 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [ + { + "localized_name": "file", + "name": "file", + "type": "COMBO", + "widget": { + "name": "file" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "links": [ + 1 + ] + } + ], + "properties": { + "Node name for S&R": "LoadVideo" + }, + "widgets_values": [ + "" + ] + }, + { + "id": 3, + "type": "LoadImageMask", + "pos": [ + 40, + 460 + ], + "size": [ + 420, + 160 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "STRING", + "widget": { + "name": "image" + }, + "link": null + }, + { + "name": "channel", + "type": "STRING", + "widget": { + "name": "channel" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "mask", + "name": "mask", + "type": "MASK", + "links": [ + 2 + ] + } + ], + "properties": { + "Node name for S&R": "LoadImageMask" + }, + "widgets_values": [ + "", + "red" + ] + }, + { + "id": 4, + "type": "VLLMOmniLatentMaskEditing", + "pos": [ + 560, + 260 + ], + "size": [ + 420, + 300 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "localized_name": "source_video", + "name": "source_video", + "type": "VIDEO", + "link": 1, + "shape": 7 + }, + { + "localized_name": "source_audio", + "name": "source_audio", + "type": "AUDIO", + "link": null + }, + { + "localized_name": "video_mask", + "name": "video_mask", + "type": "MASK", + "link": 2, + "shape": 7 + }, + { + "localized_name": "audio_mask", + "name": "audio_mask", + "type": "FLOAT", + "widget": { + "name": "audio_mask" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "latent_edit", + "name": "latent_edit", + "type": "LATENT_MASK_EDITING", + "links": [ + 3 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniLatentMaskEditing" + }, + "widgets_values": [ + 0.5 + ] + }, + { + "id": 5, + "type": "VLLMOmniMiniMaxH3Params", + "pos": [ + 40, + 680 + ], + "size": [ + 420, + 140 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "localized_name": "audio_flow_shift", + "name": "audio_flow_shift", + "type": "FLOAT", + "widget": { + "name": "audio_flow_shift" + }, + "link": null + }, + { + "localized_name": "flow_shift", + "name": "flow_shift", + "type": "FLOAT", + "widget": { + "name": "flow_shift" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "video params", + "name": "video params", + "type": "VIDEO_PARAMS", + "links": [ + 4 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniMiniMaxH3Params" + }, + "widgets_values": [ + 3.0, + 12.0 + ] + }, + { + "id": 6, + "type": "VLLMOmniGenerateVideo", + "pos": [ + 1080, + 200 + ], + "size": [ + 540, + 680 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "localized_name": "frame", + "name": "frame", + "type": "IMAGE", + "link": null, + "shape": 7 + }, + { + "localized_name": "references", + "name": "references", + "type": "VIDEO_REFERENCES", + "link": null, + "shape": 7 + }, + { + "localized_name": "sampling_params", + "name": "sampling_params", + "type": "SAMPLING_PARAMS", + "link": null + }, + { + "localized_name": "lora", + "name": "lora", + "type": "REMOTE_LORA", + "link": null + }, + { + "localized_name": "model_params", + "name": "model_params", + "type": "VIDEO_PARAMS", + "link": 4, + "shape": 7 + }, + { + "localized_name": "fast_h3", + "name": "fast_h3", + "type": "FASTH3_DEPLOYMENT", + "link": null + }, + { + "localized_name": "latent_edit", + "name": "latent_edit", + "type": "LATENT_MASK_EDITING", + "link": 3, + "shape": 7 + }, + { + "localized_name": "url", + "name": "url", + "type": "STRING", + "widget": { + "name": "url" + }, + "link": null + }, + { + "localized_name": "model", + "name": "model", + "type": "STRING", + "widget": { + "name": "model" + }, + "link": null + }, + { + "localized_name": "prompt", + "name": "prompt", + "type": "STRING", + "widget": { + "name": "prompt" + }, + "link": null + }, + { + "localized_name": "negative_prompt", + "name": "negative_prompt", + "type": "STRING", + "widget": { + "name": "negative_prompt" + }, + "link": null + }, + { + "localized_name": "width", + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": null + }, + { + "localized_name": "height", + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": null + }, + { + "localized_name": "fps", + "name": "fps", + "type": "INT", + "widget": { + "name": "fps" + }, + "link": null + }, + { + "localized_name": "duration", + "name": "duration", + "type": "FLOAT", + "widget": { + "name": "duration" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "links": [ + 5 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniGenerateVideo" + }, + "widgets_values": [ + "http://127.0.0.1:8000/v1", + "MiniMaxAI/MiniMax-H3", + "Partially restyle the clip and soundtrack", + "", + 160, + 120, + 24, + 4.458 + ] + }, + { + "id": 7, + "type": "SaveVideo", + "pos": [ + 1700, + 300 + ], + "size": [ + 420, + 160 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "link": 5, + "shape": 7 + }, + { + "localized_name": "filename_prefix", + "name": "filename_prefix", + "type": "STRING", + "widget": { + "name": "filename_prefix" + }, + "link": null + }, + { + "localized_name": "format", + "name": "format", + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "format" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "links": [] + } + ], + "properties": { + "Node name for S&R": "SaveVideo" + }, + "widgets_values": [ + "vllm-omni/latent-edit", + "mp4" + ] + } + ], + "links": [ + [ + 1, + 2, + 0, + 4, + 0, + "VIDEO" + ], + [ + 2, + 3, + 0, + 4, + 2, + "MASK" + ], + [ + 3, + 4, + 0, + 6, + 6, + "LATENT_MASK_EDITING" + ], + [ + 4, + 5, + 0, + 6, + 4, + "VIDEO_PARAMS" + ], + [ + 5, + 6, + 0, + 7, + 0, + "VIDEO" + ] + ], + "groups": [], + "config": {}, + "extra": {}, + "version": 0.4 +} diff --git a/apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Latent Mask Editing - Temporal Mask.json b/apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Latent Mask Editing - Temporal Mask.json new file mode 100644 index 000000000..b2b3a6980 --- /dev/null +++ b/apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Latent Mask Editing - Temporal Mask.json @@ -0,0 +1,560 @@ +{ + "last_node_id": 9, + "last_link_id": 7, + "nodes": [ + { + "id": 2, + "type": "LoadVideo", + "pos": [ + 40, + 100 + ], + "size": [ + 420, + 120 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [ + { + "name": "file", + "type": "STRING", + "widget": { + "name": "file" + }, + "link": null + } + ], + "outputs": [ + { + "name": "video", + "type": "VIDEO", + "links": [ + 1 + ] + } + ], + "properties": { + "Node name for S&R": "LoadVideo" + }, + "widgets_values": [ + "" + ] + }, + { + "id": 3, + "type": "SolidMask", + "pos": [ + 40, + 300 + ], + "size": [ + 420, + 160 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "name": "value", + "type": "FLOAT", + "widget": { + "name": "value" + }, + "link": null + }, + { + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": null + }, + { + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": null + } + ], + "outputs": [ + { + "name": "mask", + "type": "MASK", + "links": [ + 2 + ] + } + ], + "properties": { + "Node name for S&R": "SolidMask" + }, + "widgets_values": [ + 0.0, + 160, + 120 + ] + }, + { + "id": 4, + "type": "SolidMask", + "pos": [ + 40, + 500 + ], + "size": [ + 420, + 160 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "name": "value", + "type": "FLOAT", + "widget": { + "name": "value" + }, + "link": null + }, + { + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": null + }, + { + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": null + } + ], + "outputs": [ + { + "name": "mask", + "type": "MASK", + "links": [ + 3 + ] + } + ], + "properties": { + "Node name for S&R": "SolidMask" + }, + "widgets_values": [ + 1.0, + 160, + 120 + ] + }, + { + "id": 5, + "type": "BatchMasksNode", + "pos": [ + 250, + 380 + ], + "size": [ + 200, + 100 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "name": "masks.mask0", + "type": "MASK", + "link": 2, + "slot_index": 0 + }, + { + "name": "masks.mask1", + "type": "MASK", + "link": 3, + "slot_index": 1 + } + ], + "outputs": [ + { + "name": "MASK", + "type": "MASK", + "links": [ + 4 + ] + } + ], + "properties": { + "Node name for S&R": "BatchMasksNode" + }, + "widgets_values": [] + }, + { + "id": 6, + "type": "VLLMOmniLatentMaskEditing", + "pos": [ + 450, + 180 + ], + "size": [ + 400, + 120 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "name": "source_video", + "type": "VIDEO", + "link": 1 + }, + { + "name": "source_audio", + "type": "AUDIO", + "link": null + }, + { + "name": "video_mask", + "type": "MASK", + "link": 4 + }, + { + "name": "audio_mask", + "type": "FLOAT", + "widget": { + "name": "audio_mask" + }, + "link": null + } + ], + "outputs": [ + { + "name": "latent_edit", + "type": "LATENT_MASK_EDITING", + "links": [ + 5 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniLatentMaskEditing" + }, + "widgets_values": [ + -1.0 + ] + }, + { + "id": 7, + "type": "VLLMOmniMiniMaxH3Params", + "pos": [ + 40, + 700 + ], + "size": [ + 420, + 160 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "name": "audio_flow_shift", + "type": "FLOAT", + "widget": { + "name": "audio_flow_shift" + }, + "link": null + }, + { + "name": "flow_shift", + "type": "FLOAT", + "widget": { + "name": "flow_shift" + }, + "link": null + } + ], + "outputs": [ + { + "name": "video params", + "type": "VIDEO_PARAMS", + "links": [ + 6 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniMiniMaxH3Params" + }, + "widgets_values": [ + 3.0, + 12.0 + ] + }, + { + "id": 8, + "type": "VLLMOmniGenerateVideo", + "pos": [ + 700, + 180 + ], + "size": [ + 450, + 380 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "name": "frame", + "type": "IMAGE", + "link": null + }, + { + "name": "references", + "type": "VIDEO_REFERENCES", + "link": null + }, + { + "name": "sampling_params", + "type": "SAMPLING_PARAMS", + "link": null + }, + { + "name": "lora", + "type": "LORA", + "link": null + }, + { + "name": "model_params", + "type": "VIDEO_PARAMS", + "link": 6 + }, + { + "name": "fast_h3", + "type": "FAST_H3", + "link": null + }, + { + "name": "latent_edit", + "type": "LATENT_MASK_EDITING", + "link": 5 + }, + { + "name": "url", + "type": "STRING", + "widget": { + "name": "url" + }, + "link": null + }, + { + "name": "model", + "type": "STRING", + "widget": { + "name": "model" + }, + "link": null + }, + { + "name": "prompt", + "type": "STRING", + "widget": { + "name": "prompt" + }, + "link": null + }, + { + "name": "negative_prompt", + "type": "STRING", + "widget": { + "name": "negative_prompt" + }, + "link": null + }, + { + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": null + }, + { + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": null + }, + { + "name": "fps", + "type": "INT", + "widget": { + "name": "fps" + }, + "link": null + }, + { + "name": "duration", + "type": "FLOAT", + "widget": { + "name": "duration" + }, + "link": null + } + ], + "outputs": [ + { + "name": "video", + "type": "VIDEO", + "links": [ + 7 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniGenerateVideo" + }, + "widgets_values": [ + "http://127.0.0.1:8000/v1", + "MiniMaxAI/MiniMax-H3", + "temporal mask: regenerate the ending", + "", + 160, + 120, + 24, + 4.458 + ] + }, + { + "id": 9, + "type": "SaveVideo", + "pos": [ + 900, + 180 + ], + "size": [ + 420, + 200 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "video", + "type": "VIDEO", + "link": 7 + }, + { + "name": "filename_prefix", + "type": "STRING", + "widget": { + "name": "filename_prefix" + }, + "link": null + }, + { + "name": "format", + "type": "STRING", + "widget": { + "name": "format" + }, + "link": null + } + ], + "outputs": [ + { + "name": "video", + "type": "VIDEO", + "links": [] + } + ], + "properties": { + "Node name for S&R": "SaveVideo" + }, + "widgets_values": [ + "temporal_mask", + "mp4" + ] + } + ], + "links": [ + [ + 1, + 2, + 0, + 6, + 0, + "VIDEO" + ], + [ + 2, + 3, + 0, + 5, + 0, + "MASK" + ], + [ + 3, + 4, + 0, + 5, + 1, + "MASK" + ], + [ + 4, + 5, + 0, + 6, + 2, + "MASK" + ], + [ + 5, + 6, + 0, + 8, + 6, + "LATENT_MASK_EDITING" + ], + [ + 6, + 7, + 0, + 8, + 4, + "VIDEO_PARAMS" + ], + [ + 7, + 8, + 0, + 9, + 0, + "VIDEO" + ] + ], + "groups": [], + "config": {}, + "extra": {}, + "version": 0.4 +} diff --git a/apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Latent Mask Editing.json b/apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Latent Mask Editing.json new file mode 100644 index 000000000..4192ef4c5 --- /dev/null +++ b/apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Latent Mask Editing.json @@ -0,0 +1,511 @@ +{ + "id": "latent-mask-editing-7380", + "revision": 0, + "last_node_id": 7, + "last_link_id": 5, + "nodes": [ + { + "id": 1, + "type": "MarkdownNote", + "pos": [ + 40, + 20 + ], + "size": [ + 420, + 210 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [], + "title": "Latent-mask editing server prerequisite", + "properties": {}, + "widgets_values": [ + "Requires a vLLM-Omni server that declares latent-mask editing support (MiniMax-H3, EDIT-01). The source clip and video/audio noise masks are uploaded to /v1/videos; 0 preserves, 1 regenerates, fractional blends." + ], + "color": "#432", + "bgcolor": "#000" + }, + { + "id": 2, + "type": "LoadVideo", + "pos": [ + 40, + 280 + ], + "size": [ + 420, + 120 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [ + { + "localized_name": "file", + "name": "file", + "type": "COMBO", + "widget": { + "name": "file" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "links": [ + 1 + ] + } + ], + "properties": { + "Node name for S&R": "LoadVideo" + }, + "widgets_values": [ + "" + ] + }, + { + "id": 3, + "type": "SolidMask", + "pos": [ + 40, + 460 + ], + "size": [ + 420, + 160 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "localized_name": "value", + "name": "value", + "type": "FLOAT", + "widget": { + "name": "value" + }, + "link": null + }, + { + "localized_name": "width", + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": null + }, + { + "localized_name": "height", + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "mask", + "name": "mask", + "type": "MASK", + "links": [ + 2 + ] + } + ], + "properties": { + "Node name for S&R": "SolidMask" + }, + "widgets_values": [ + 1.0, + 160, + 120 + ] + }, + { + "id": 4, + "type": "VLLMOmniLatentMaskEditing", + "pos": [ + 560, + 260 + ], + "size": [ + 420, + 300 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "localized_name": "source_video", + "name": "source_video", + "type": "VIDEO", + "link": 1, + "shape": 7 + }, + { + "localized_name": "source_audio", + "name": "source_audio", + "type": "AUDIO", + "link": null + }, + { + "localized_name": "video_mask", + "name": "video_mask", + "type": "MASK", + "link": 2, + "shape": 7 + }, + { + "localized_name": "audio_mask", + "name": "audio_mask", + "type": "FLOAT", + "widget": { + "name": "audio_mask" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "latent_edit", + "name": "latent_edit", + "type": "LATENT_MASK_EDITING", + "links": [ + 3 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniLatentMaskEditing" + }, + "widgets_values": [ + 0.5 + ] + }, + { + "id": 5, + "type": "VLLMOmniMiniMaxH3Params", + "pos": [ + 40, + 680 + ], + "size": [ + 420, + 140 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "localized_name": "audio_flow_shift", + "name": "audio_flow_shift", + "type": "FLOAT", + "widget": { + "name": "audio_flow_shift" + }, + "link": null + }, + { + "localized_name": "flow_shift", + "name": "flow_shift", + "type": "FLOAT", + "widget": { + "name": "flow_shift" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "video params", + "name": "video params", + "type": "VIDEO_PARAMS", + "links": [ + 4 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniMiniMaxH3Params" + }, + "widgets_values": [ + 3.0, + 12.0 + ] + }, + { + "id": 6, + "type": "VLLMOmniGenerateVideo", + "pos": [ + 1080, + 200 + ], + "size": [ + 540, + 680 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "localized_name": "frame", + "name": "frame", + "type": "IMAGE", + "link": null, + "shape": 7 + }, + { + "localized_name": "references", + "name": "references", + "type": "VIDEO_REFERENCES", + "link": null, + "shape": 7 + }, + { + "localized_name": "sampling_params", + "name": "sampling_params", + "type": "SAMPLING_PARAMS", + "link": null + }, + { + "localized_name": "lora", + "name": "lora", + "type": "REMOTE_LORA", + "link": null + }, + { + "localized_name": "model_params", + "name": "model_params", + "type": "VIDEO_PARAMS", + "link": 4, + "shape": 7 + }, + { + "localized_name": "fast_h3", + "name": "fast_h3", + "type": "FASTH3_DEPLOYMENT", + "link": null + }, + { + "localized_name": "latent_edit", + "name": "latent_edit", + "type": "LATENT_MASK_EDITING", + "link": 3, + "shape": 7 + }, + { + "localized_name": "url", + "name": "url", + "type": "STRING", + "widget": { + "name": "url" + }, + "link": null + }, + { + "localized_name": "model", + "name": "model", + "type": "STRING", + "widget": { + "name": "model" + }, + "link": null + }, + { + "localized_name": "prompt", + "name": "prompt", + "type": "STRING", + "widget": { + "name": "prompt" + }, + "link": null + }, + { + "localized_name": "negative_prompt", + "name": "negative_prompt", + "type": "STRING", + "widget": { + "name": "negative_prompt" + }, + "link": null + }, + { + "localized_name": "width", + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": null + }, + { + "localized_name": "height", + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": null + }, + { + "localized_name": "fps", + "name": "fps", + "type": "INT", + "widget": { + "name": "fps" + }, + "link": null + }, + { + "localized_name": "duration", + "name": "duration", + "type": "FLOAT", + "widget": { + "name": "duration" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "links": [ + 5 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniGenerateVideo" + }, + "widgets_values": [ + "http://127.0.0.1:8000/v1", + "MiniMaxAI/MiniMax-H3", + "Partially restyle the clip and soundtrack", + "", + 160, + 120, + 24, + 4.458 + ] + }, + { + "id": 7, + "type": "SaveVideo", + "pos": [ + 1700, + 300 + ], + "size": [ + 420, + 160 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "link": 5, + "shape": 7 + }, + { + "localized_name": "filename_prefix", + "name": "filename_prefix", + "type": "STRING", + "widget": { + "name": "filename_prefix" + }, + "link": null + }, + { + "localized_name": "format", + "name": "format", + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "format" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "links": [] + } + ], + "properties": { + "Node name for S&R": "SaveVideo" + }, + "widgets_values": [ + "vllm-omni/latent-edit", + "mp4" + ] + } + ], + "links": [ + [ + 1, + 2, + 0, + 4, + 0, + "VIDEO" + ], + [ + 2, + 3, + 0, + 4, + 2, + "MASK" + ], + [ + 3, + 4, + 0, + 6, + 6, + "LATENT_MASK_EDITING" + ], + [ + 4, + 5, + 0, + 6, + 4, + "VIDEO_PARAMS" + ], + [ + 5, + 6, + 0, + 7, + 0, + "VIDEO" + ] + ], + "groups": [], + "config": {}, + "extra": {}, + "version": 0.4 +} diff --git a/apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni MiniMax-H3 Latent Mask Editing.json b/apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni MiniMax-H3 Latent Mask Editing.json new file mode 100644 index 000000000..fc4341730 --- /dev/null +++ b/apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni MiniMax-H3 Latent Mask Editing.json @@ -0,0 +1,5241 @@ +{ + "id": "eebf673c-c66a-4f20-9e87-b728a82f6e5f", + "revision": 0, + "last_node_id": 99, + "last_link_id": 162, + "nodes": [ + { + "id": 5, + "type": "VLLMOmniMiniMaxH3Params", + "pos": [ + 1510, + 2160 + ], + "size": [ + 300, + 130 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [ + { + "localized_name": "audio_flow_shift", + "name": "audio_flow_shift", + "type": "FLOAT", + "widget": { + "name": "audio_flow_shift" + }, + "link": null + }, + { + "localized_name": "flow_shift", + "name": "flow_shift", + "type": "FLOAT", + "widget": { + "name": "flow_shift" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "MiniMax-H3 video params", + "name": "MiniMax-H3 video params", + "type": "VIDEO_PARAMS", + "links": [ + 4 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniMiniMaxH3Params" + }, + "widgets_values": [ + 3, + 12 + ], + "widgets_values_named": { + "audio_flow_shift": 3, + "flow_shift": 12 + } + }, + { + "id": 11, + "type": "VLLMOmniDiffusionSampling", + "pos": [ + 580, + 1870 + ], + "size": [ + 410.68458556883115, + 226 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [ + { + "localized_name": "n", + "name": "n", + "type": "INT", + "widget": { + "name": "n" + }, + "link": null + }, + { + "localized_name": "num_inference_steps", + "name": "num_inference_steps", + "type": "INT", + "widget": { + "name": "num_inference_steps" + }, + "link": null + }, + { + "localized_name": "guidance_scale", + "name": "guidance_scale", + "type": "FLOAT", + "widget": { + "name": "guidance_scale" + }, + "link": null + }, + { + "localized_name": "true_cfg_scale", + "name": "true_cfg_scale", + "type": "FLOAT", + "widget": { + "name": "true_cfg_scale" + }, + "link": null + }, + { + "localized_name": "vae_use_slicing", + "name": "vae_use_slicing", + "type": "BOOLEAN", + "widget": { + "name": "vae_use_slicing" + }, + "link": null + }, + { + "localized_name": "vae_use_tiling", + "name": "vae_use_tiling", + "type": "BOOLEAN", + "widget": { + "name": "vae_use_tiling" + }, + "link": null + }, + { + "localized_name": "seed", + "name": "seed", + "type": "INT", + "widget": { + "name": "seed" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "diffusion sampling params", + "name": "diffusion sampling params", + "type": "SAMPLING_PARAMS", + "links": [ + 10 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniDiffusionSampling" + }, + "widgets_values": [ + 1, + 50, + 1, + 1, + false, + false, + 0, + "fixed" + ], + "widgets_values_named": { + "n": 1, + "num_inference_steps": 50, + "guidance_scale": 1, + "true_cfg_scale": 1, + "vae_use_slicing": false, + "vae_use_tiling": false, + "seed": 0, + "control_after_generate": "fixed" + } + }, + { + "id": 25, + "type": "VLLMOmniMiniMaxH3Params", + "pos": [ + 1510, + 3760 + ], + "size": [ + 300, + 130 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "localized_name": "audio_flow_shift", + "name": "audio_flow_shift", + "type": "FLOAT", + "widget": { + "name": "audio_flow_shift" + }, + "link": null + }, + { + "localized_name": "flow_shift", + "name": "flow_shift", + "type": "FLOAT", + "widget": { + "name": "flow_shift" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "MiniMax-H3 video params", + "name": "MiniMax-H3 video params", + "type": "VIDEO_PARAMS", + "links": [ + 30 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniMiniMaxH3Params" + }, + "widgets_values": [ + 3, + 12 + ], + "widgets_values_named": { + "audio_flow_shift": 3, + "flow_shift": 12 + } + }, + { + "id": 29, + "type": "MarkdownNote", + "pos": [ + 1050, + 3470 + ], + "size": [ + 360, + 330 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [], + "title": "Inpainting guide", + "properties": { + "Node name for S&R": "MarkdownNote" + }, + "widgets_values": [ + "Replace the masked region with a curled orange cat gently swaying its tail.\n\nThis case has its own rectangular mask. Leave room for the entire cat and tail motion; check the red overlay before generating.\n\nAudio mask 0 preserves source audio latents. To request new sounds, change the audio prompt and mask together. Fractional audio masks do not selectively retain the school chime while adding meows." + ], + "widgets_values_named": { + "text": "Replace the masked region with a curled orange cat gently swaying its tail.\n\nThis case has its own rectangular mask. Leave room for the entire cat and tail motion; check the red overlay before generating.\n\nAudio mask 0 preserves source audio latents. To request new sounds, change the audio prompt and mask together. Fractional audio masks do not selectively retain the school chime while adding meows." + }, + "color": "#333", + "bgcolor": "#222" + }, + { + "id": 30, + "type": "VLLMOmniDiffusionSampling", + "pos": [ + 580, + 3470 + ], + "size": [ + 410.68458556883115, + 226 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "localized_name": "n", + "name": "n", + "type": "INT", + "widget": { + "name": "n" + }, + "link": null + }, + { + "localized_name": "num_inference_steps", + "name": "num_inference_steps", + "type": "INT", + "widget": { + "name": "num_inference_steps" + }, + "link": null + }, + { + "localized_name": "guidance_scale", + "name": "guidance_scale", + "type": "FLOAT", + "widget": { + "name": "guidance_scale" + }, + "link": null + }, + { + "localized_name": "true_cfg_scale", + "name": "true_cfg_scale", + "type": "FLOAT", + "widget": { + "name": "true_cfg_scale" + }, + "link": null + }, + { + "localized_name": "vae_use_slicing", + "name": "vae_use_slicing", + "type": "BOOLEAN", + "widget": { + "name": "vae_use_slicing" + }, + "link": null + }, + { + "localized_name": "vae_use_tiling", + "name": "vae_use_tiling", + "type": "BOOLEAN", + "widget": { + "name": "vae_use_tiling" + }, + "link": null + }, + { + "localized_name": "seed", + "name": "seed", + "type": "INT", + "widget": { + "name": "seed" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "diffusion sampling params", + "name": "diffusion sampling params", + "type": "SAMPLING_PARAMS", + "links": [ + 33 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniDiffusionSampling" + }, + "widgets_values": [ + 1, + 50, + 1, + 1, + false, + false, + 0, + "fixed" + ], + "widgets_values_named": { + "n": 1, + "num_inference_steps": 50, + "guidance_scale": 1, + "true_cfg_scale": 1, + "vae_use_slicing": false, + "vae_use_tiling": false, + "seed": 0, + "control_after_generate": "fixed" + } + }, + { + "id": 34, + "type": "VLLMOmniMiniMaxH3Params", + "pos": [ + 1510, + 5360 + ], + "size": [ + 300, + 130 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "localized_name": "audio_flow_shift", + "name": "audio_flow_shift", + "type": "FLOAT", + "widget": { + "name": "audio_flow_shift" + }, + "link": null + }, + { + "localized_name": "flow_shift", + "name": "flow_shift", + "type": "FLOAT", + "widget": { + "name": "flow_shift" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "MiniMax-H3 video params", + "name": "MiniMax-H3 video params", + "type": "VIDEO_PARAMS", + "links": [ + 38 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniMiniMaxH3Params" + }, + "widgets_values": [ + 3, + 12 + ], + "widgets_values_named": { + "audio_flow_shift": 3, + "flow_shift": 12 + } + }, + { + "id": 45, + "type": "MarkdownNote", + "pos": [ + 1050, + 5070 + ], + "size": [ + 360, + 330 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [], + "outputs": [], + "title": "Continuation guide", + "properties": { + "Node name for S&R": "MarkdownNote" + }, + "widgets_values": [ + "Regenerate the ending with an overcast sky, heavy rain, and distant lightning.\n\nPreserve fraction controls how much of the available source prefix is retained. Match Temporal Mask duration to Generate Video duration; these values are not linked.\n\nThe boundary rounds down to complete H3 chunks. White allows regeneration, but does not force existing subjects to disappear.\n\nThe red preview is schematic. Audio mask affects the entire clip, including the preserved video prefix." + ], + "widgets_values_named": { + "text": "Regenerate the ending with an overcast sky, heavy rain, and distant lightning.\n\nPreserve fraction controls how much of the available source prefix is retained. Match Temporal Mask duration to Generate Video duration; these values are not linked.\n\nThe boundary rounds down to complete H3 chunks. White allows regeneration, but does not force existing subjects to disappear.\n\nThe red preview is schematic. Audio mask affects the entire clip, including the preserved video prefix." + }, + "color": "#333", + "bgcolor": "#222" + }, + { + "id": 46, + "type": "VLLMOmniDiffusionSampling", + "pos": [ + 580, + 5070 + ], + "size": [ + 410.68458556883115, + 226 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "localized_name": "n", + "name": "n", + "type": "INT", + "widget": { + "name": "n" + }, + "link": null + }, + { + "localized_name": "num_inference_steps", + "name": "num_inference_steps", + "type": "INT", + "widget": { + "name": "num_inference_steps" + }, + "link": null + }, + { + "localized_name": "guidance_scale", + "name": "guidance_scale", + "type": "FLOAT", + "widget": { + "name": "guidance_scale" + }, + "link": null + }, + { + "localized_name": "true_cfg_scale", + "name": "true_cfg_scale", + "type": "FLOAT", + "widget": { + "name": "true_cfg_scale" + }, + "link": null + }, + { + "localized_name": "vae_use_slicing", + "name": "vae_use_slicing", + "type": "BOOLEAN", + "widget": { + "name": "vae_use_slicing" + }, + "link": null + }, + { + "localized_name": "vae_use_tiling", + "name": "vae_use_tiling", + "type": "BOOLEAN", + "widget": { + "name": "vae_use_tiling" + }, + "link": null + }, + { + "localized_name": "seed", + "name": "seed", + "type": "INT", + "widget": { + "name": "seed" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "diffusion sampling params", + "name": "diffusion sampling params", + "type": "SAMPLING_PARAMS", + "links": [ + 48 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniDiffusionSampling" + }, + "widgets_values": [ + 1, + 50, + 1, + 1, + false, + false, + 0, + "fixed" + ], + "widgets_values_named": { + "n": 1, + "num_inference_steps": 50, + "guidance_scale": 1, + "true_cfg_scale": 1, + "vae_use_slicing": false, + "vae_use_tiling": false, + "seed": 0, + "control_after_generate": "fixed" + } + }, + { + "id": 49, + "type": "VLLMOmniMiniMaxH3Params", + "pos": [ + 1510, + 6960 + ], + "size": [ + 300, + 130 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "localized_name": "audio_flow_shift", + "name": "audio_flow_shift", + "type": "FLOAT", + "widget": { + "name": "audio_flow_shift" + }, + "link": null + }, + { + "localized_name": "flow_shift", + "name": "flow_shift", + "type": "FLOAT", + "widget": { + "name": "flow_shift" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "MiniMax-H3 video params", + "name": "MiniMax-H3 video params", + "type": "VIDEO_PARAMS", + "links": [ + 51 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniMiniMaxH3Params" + }, + "widgets_values": [ + 3, + 12 + ], + "widgets_values_named": { + "audio_flow_shift": 3, + "flow_shift": 12 + } + }, + { + "id": 59, + "type": "VLLMOmniMiniMaxH3TemporalMask", + "pos": [ + 580, + 6040 + ], + "size": [ + 420, + 240 + ], + "flags": {}, + "order": 26, + "mode": 0, + "inputs": [ + { + "localized_name": "images", + "name": "images", + "type": "IMAGE", + "link": 123 + }, + { + "localized_name": "source_fps", + "name": "source_fps", + "type": "FLOAT", + "widget": { + "name": "source_fps" + }, + "link": 124 + }, + { + "localized_name": "duration", + "name": "duration", + "type": "FLOAT", + "widget": { + "name": "duration" + }, + "link": null + }, + { + "localized_name": "mode", + "name": "mode", + "type": "COMBO", + "widget": { + "name": "mode" + }, + "link": null + }, + { + "localized_name": "preserve_fraction", + "name": "preserve_fraction", + "type": "FLOAT", + "widget": { + "name": "preserve_fraction" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "mask", + "name": "mask", + "type": "MASK", + "links": [ + 60 + ] + }, + { + "localized_name": "preview_fps", + "name": "preview_fps", + "type": "FLOAT", + "links": [ + 146 + ] + }, + { + "localized_name": "preview_images", + "name": "preview_images", + "type": "IMAGE", + "links": [ + 153, + 156 + ] + }, + { + "localized_name": "preview_mask", + "name": "preview_mask", + "type": "MASK", + "links": [ + 155 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniMiniMaxH3TemporalMask" + }, + "widgets_values": [ + 24, + 10, + "extension", + 0.5 + ], + "widgets_values_named": { + "source_fps": 24, + "duration": 10, + "mode": "extension", + "preserve_fraction": 0.5 + } + }, + { + "id": 60, + "type": "MarkdownNote", + "pos": [ + 1050, + 6670 + ], + "size": [ + 360, + 330 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [], + "outputs": [], + "title": "Extension guide", + "properties": { + "Node name for S&R": "MarkdownNote" + }, + "widgets_values": [ + "Extend into a summer starry night. Follow a firework rising behind the school with a slow upward camera tilt, ending at its burst.\n\nSet the same duration in Temporal Mask and Generate Video, longer than the source. These values are not linked. The original source length determines the preserved prefix.\n\nThe boundary rounds down to H3 chunks. Preview holds the final source frame under the red overlay; it is not the generated ending.\n\nAudio mask affects the whole clip, not only the extension." + ], + "widgets_values_named": { + "text": "Extend into a summer starry night. Follow a firework rising behind the school with a slow upward camera tilt, ending at its burst.\n\nSet the same duration in Temporal Mask and Generate Video, longer than the source. These values are not linked. The original source length determines the preserved prefix.\n\nThe boundary rounds down to H3 chunks. Preview holds the final source frame under the red overlay; it is not the generated ending.\n\nAudio mask affects the whole clip, not only the extension." + }, + "color": "#333", + "bgcolor": "#222" + }, + { + "id": 2, + "type": "LoadVideo", + "pos": [ + -268.4516045720269, + 1190 + ], + "size": [ + 420, + 314 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "localized_name": "file", + "name": "file", + "type": "COMBO", + "widget": { + "name": "file" + }, + "link": null + }, + { + "localized_name": "choose file to upload", + "name": "upload", + "type": "IMAGEUPLOAD", + "widget": { + "name": "upload" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "VIDEO", + "name": "VIDEO", + "type": "VIDEO", + "links": [ + 1, + 17, + 28, + 36, + 49 + ] + } + ], + "title": "Shared Source Video | All Cases", + "properties": { + "Node name for S&R": "LoadVideo" + }, + "widgets_values": [ + "", + "image" + ], + "widgets_values_named": { + "file": "", + "upload": "image" + } + }, + { + "id": 62, + "type": "MarkdownNote", + "pos": [ + 1050, + 1870 + ], + "size": [ + 360, + 330 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [], + "outputs": [], + "title": "Object Removal guide", + "properties": { + "Node name for S&R": "MarkdownNote" + }, + "widgets_values": [ + "Reconstruct the surrounding wall inside the mask.\n\nSet this case's Mask Canvas, Mask Rectangle, and Rectangle Position to cover the object, handle, and shadow with some background margin. Preview coverage before generating.\n\nSymmetric expansion by d pixels: x/y minus d; width/height plus 2d. A 32-pixel margin improved this school example.\n\nAudio mask 0 preserves source audio latents; reconstruction is not lossless." + ], + "widgets_values_named": { + "text": "Reconstruct the surrounding wall inside the mask.\n\nSet this case's Mask Canvas, Mask Rectangle, and Rectangle Position to cover the object, handle, and shadow with some background margin. Preview coverage before generating.\n\nSymmetric expansion by d pixels: x/y minus d; width/height plus 2d. A 32-pixel margin improved this school example.\n\nAudio mask 0 preserves source audio latents; reconstruction is not lossless." + }, + "color": "#333", + "bgcolor": "#222" + }, + { + "id": 61, + "type": "VLLMOmniDiffusionSampling", + "pos": [ + 580, + 6670 + ], + "size": [ + 410.68458556883115, + 226 + ], + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [ + { + "localized_name": "n", + "name": "n", + "type": "INT", + "widget": { + "name": "n" + }, + "link": null + }, + { + "localized_name": "num_inference_steps", + "name": "num_inference_steps", + "type": "INT", + "widget": { + "name": "num_inference_steps" + }, + "link": null + }, + { + "localized_name": "guidance_scale", + "name": "guidance_scale", + "type": "FLOAT", + "widget": { + "name": "guidance_scale" + }, + "link": null + }, + { + "localized_name": "true_cfg_scale", + "name": "true_cfg_scale", + "type": "FLOAT", + "widget": { + "name": "true_cfg_scale" + }, + "link": null + }, + { + "localized_name": "vae_use_slicing", + "name": "vae_use_slicing", + "type": "BOOLEAN", + "widget": { + "name": "vae_use_slicing" + }, + "link": null + }, + { + "localized_name": "vae_use_tiling", + "name": "vae_use_tiling", + "type": "BOOLEAN", + "widget": { + "name": "vae_use_tiling" + }, + "link": null + }, + { + "localized_name": "seed", + "name": "seed", + "type": "INT", + "widget": { + "name": "seed" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "diffusion sampling params", + "name": "diffusion sampling params", + "type": "SAMPLING_PARAMS", + "links": [ + 61 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniDiffusionSampling" + }, + "widgets_values": [ + 1, + 50, + 1, + 1, + false, + false, + 0, + "fixed" + ], + "widgets_values_named": { + "n": 1, + "num_inference_steps": 50, + "guidance_scale": 1, + "true_cfg_scale": 1, + "vae_use_slicing": false, + "vae_use_tiling": false, + "seed": 0, + "control_after_generate": "fixed" + } + }, + { + "id": 65, + "type": "CreateVideo", + "pos": [ + 1990, + 1190 + ], + "size": [ + 260, + 150 + ], + "flags": {}, + "order": 44, + "mode": 0, + "inputs": [ + { + "localized_name": "images", + "name": "images", + "type": "IMAGE", + "link": 130 + }, + { + "localized_name": "audio", + "name": "audio", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "localized_name": "fps", + "name": "fps", + "type": "FLOAT", + "widget": { + "name": "fps" + }, + "link": 131 + }, + { + "localized_name": "bit_depth", + "name": "bit_depth", + "shape": 7, + "type": "COMBO", + "widget": { + "name": "bit_depth" + }, + "link": null + }, + { + "localized_name": "color_space", + "name": "color_space", + "shape": 7, + "type": "COMBO", + "widget": { + "name": "color_space" + }, + "link": null + }, + { + "localized_name": "codec", + "name": "codec", + "shape": 7, + "type": "COMBO", + "widget": { + "name": "codec" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "VIDEO", + "name": "VIDEO", + "type": "VIDEO", + "links": [ + 77 + ] + } + ], + "title": "Object Removal | Preview Video", + "properties": { + "Node name for S&R": "CreateVideo" + }, + "widgets_values": [ + 30, + "auto", + "sRGB", + "none" + ], + "widgets_values_named": { + "fps": 30, + "bit_depth": "auto", + "color_space": "sRGB", + "codec": "none" + } + }, + { + "id": 69, + "type": "CreateVideo", + "pos": [ + 1990, + 2790 + ], + "size": [ + 260, + 150 + ], + "flags": {}, + "order": 46, + "mode": 0, + "inputs": [ + { + "localized_name": "images", + "name": "images", + "type": "IMAGE", + "link": 137 + }, + { + "localized_name": "audio", + "name": "audio", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "localized_name": "fps", + "name": "fps", + "type": "FLOAT", + "widget": { + "name": "fps" + }, + "link": 138 + }, + { + "localized_name": "bit_depth", + "name": "bit_depth", + "shape": 7, + "type": "COMBO", + "widget": { + "name": "bit_depth" + }, + "link": null + }, + { + "localized_name": "color_space", + "name": "color_space", + "shape": 7, + "type": "COMBO", + "widget": { + "name": "color_space" + }, + "link": null + }, + { + "localized_name": "codec", + "name": "codec", + "shape": 7, + "type": "COMBO", + "widget": { + "name": "codec" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "VIDEO", + "name": "VIDEO", + "type": "VIDEO", + "links": [ + 91 + ] + } + ], + "title": "Inpainting | Preview Video", + "properties": { + "Node name for S&R": "CreateVideo" + }, + "widgets_values": [ + 30, + "auto", + "sRGB", + "none" + ], + "widgets_values_named": { + "fps": 30, + "bit_depth": "auto", + "color_space": "sRGB", + "codec": "none" + } + }, + { + "id": 73, + "type": "CreateVideo", + "pos": [ + 1990, + 4390 + ], + "size": [ + 260, + 150 + ], + "flags": {}, + "order": 48, + "mode": 0, + "inputs": [ + { + "localized_name": "images", + "name": "images", + "type": "IMAGE", + "link": 152 + }, + { + "localized_name": "audio", + "name": "audio", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "localized_name": "fps", + "name": "fps", + "type": "FLOAT", + "widget": { + "name": "fps" + }, + "link": 142 + }, + { + "localized_name": "bit_depth", + "name": "bit_depth", + "shape": 7, + "type": "COMBO", + "widget": { + "name": "bit_depth" + }, + "link": null + }, + { + "localized_name": "color_space", + "name": "color_space", + "shape": 7, + "type": "COMBO", + "widget": { + "name": "color_space" + }, + "link": null + }, + { + "localized_name": "codec", + "name": "codec", + "shape": 7, + "type": "COMBO", + "widget": { + "name": "codec" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "VIDEO", + "name": "VIDEO", + "type": "VIDEO", + "links": [ + 105 + ] + } + ], + "title": "Continuation | Preview Video", + "properties": { + "Node name for S&R": "CreateVideo" + }, + "widgets_values": [ + 30, + "auto", + "sRGB", + "none" + ], + "widgets_values_named": { + "fps": 30, + "bit_depth": "auto", + "color_space": "sRGB", + "codec": "none" + } + }, + { + "id": 77, + "type": "CreateVideo", + "pos": [ + 1990, + 5990 + ], + "size": [ + 260, + 150 + ], + "flags": {}, + "order": 50, + "mode": 0, + "inputs": [ + { + "localized_name": "images", + "name": "images", + "type": "IMAGE", + "link": 158 + }, + { + "localized_name": "audio", + "name": "audio", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "localized_name": "fps", + "name": "fps", + "type": "FLOAT", + "widget": { + "name": "fps" + }, + "link": 146 + }, + { + "localized_name": "bit_depth", + "name": "bit_depth", + "shape": 7, + "type": "COMBO", + "widget": { + "name": "bit_depth" + }, + "link": null + }, + { + "localized_name": "color_space", + "name": "color_space", + "shape": 7, + "type": "COMBO", + "widget": { + "name": "color_space" + }, + "link": null + }, + { + "localized_name": "codec", + "name": "codec", + "shape": 7, + "type": "COMBO", + "widget": { + "name": "codec" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "VIDEO", + "name": "VIDEO", + "type": "VIDEO", + "links": [ + 119 + ] + } + ], + "title": "Extension | Preview Video", + "properties": { + "Node name for S&R": "CreateVideo" + }, + "widgets_values": [ + 30, + "auto", + "sRGB", + "none" + ], + "widgets_values_named": { + "fps": 30, + "bit_depth": "auto", + "color_space": "sRGB", + "codec": "none" + } + }, + { + "id": 79, + "type": "EmptyImage", + "pos": [ + 1090, + 1190 + ], + "size": [ + 260, + 150 + ], + "flags": {}, + "order": 13, + "mode": 0, + "inputs": [ + { + "localized_name": "width", + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": null + }, + { + "localized_name": "height", + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": null + }, + { + "localized_name": "batch_size", + "name": "batch_size", + "type": "INT", + "widget": { + "name": "batch_size" + }, + "link": null + }, + { + "localized_name": "color", + "name": "color", + "type": "INT", + "widget": { + "name": "color" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "IMAGE", + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 126 + ] + } + ], + "properties": { + "Node name for S&R": "EmptyImage" + }, + "widgets_values": [ + 32, + 32, + 1, + 16711680 + ], + "widgets_values_named": { + "width": 32, + "height": 32, + "batch_size": 1, + "color": 16711680 + } + }, + { + "id": 80, + "type": "ImageCompositeMasked", + "pos": [ + 1390, + 1190 + ], + "size": [ + 260, + 150 + ], + "flags": {}, + "order": 28, + "mode": 0, + "inputs": [ + { + "localized_name": "destination", + "name": "destination", + "type": "IMAGE", + "link": 125 + }, + { + "localized_name": "source", + "name": "source", + "type": "IMAGE", + "link": 126 + }, + { + "localized_name": "mask", + "name": "mask", + "shape": 7, + "type": "MASK", + "link": 127 + }, + { + "localized_name": "x", + "name": "x", + "type": "INT", + "widget": { + "name": "x" + }, + "link": null + }, + { + "localized_name": "y", + "name": "y", + "type": "INT", + "widget": { + "name": "y" + }, + "link": null + }, + { + "localized_name": "resize_source", + "name": "resize_source", + "type": "BOOLEAN", + "widget": { + "name": "resize_source" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "IMAGE", + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 129 + ] + } + ], + "properties": { + "Node name for S&R": "ImageCompositeMasked" + }, + "widgets_values": [ + 0, + 0, + true + ], + "widgets_values_named": { + "x": 0, + "y": 0, + "resize_source": true + } + }, + { + "id": 81, + "type": "ImageBlend", + "pos": [ + 1690, + 1190 + ], + "size": [ + 260, + 150 + ], + "flags": {}, + "order": 36, + "mode": 0, + "inputs": [ + { + "localized_name": "image1", + "name": "image1", + "type": "IMAGE", + "link": 128 + }, + { + "localized_name": "image2", + "name": "image2", + "type": "IMAGE", + "link": 129 + }, + { + "localized_name": "blend_factor", + "name": "blend_factor", + "type": "FLOAT", + "widget": { + "name": "blend_factor" + }, + "link": null + }, + { + "localized_name": "blend_mode", + "name": "blend_mode", + "type": "COMBO", + "widget": { + "name": "blend_mode" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "IMAGE", + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 130 + ] + } + ], + "properties": { + "Node name for S&R": "ImageBlend" + }, + "widgets_values": [ + 0.48, + "normal" + ], + "widgets_values_named": { + "blend_factor": 0.48, + "blend_mode": "normal" + } + }, + { + "id": 82, + "type": "EmptyImage", + "pos": [ + 1090, + 2790 + ], + "size": [ + 260, + 150 + ], + "flags": {}, + "order": 14, + "mode": 0, + "inputs": [ + { + "localized_name": "width", + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": null + }, + { + "localized_name": "height", + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": null + }, + { + "localized_name": "batch_size", + "name": "batch_size", + "type": "INT", + "widget": { + "name": "batch_size" + }, + "link": null + }, + { + "localized_name": "color", + "name": "color", + "type": "INT", + "widget": { + "name": "color" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "IMAGE", + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 133 + ] + } + ], + "properties": { + "Node name for S&R": "EmptyImage" + }, + "widgets_values": [ + 32, + 32, + 1, + 16711680 + ], + "widgets_values_named": { + "width": 32, + "height": 32, + "batch_size": 1, + "color": 16711680 + } + }, + { + "id": 83, + "type": "ImageCompositeMasked", + "pos": [ + 1390, + 2790 + ], + "size": [ + 260, + 150 + ], + "flags": {}, + "order": 30, + "mode": 0, + "inputs": [ + { + "localized_name": "destination", + "name": "destination", + "type": "IMAGE", + "link": 132 + }, + { + "localized_name": "source", + "name": "source", + "type": "IMAGE", + "link": 133 + }, + { + "localized_name": "mask", + "name": "mask", + "shape": 7, + "type": "MASK", + "link": 134 + }, + { + "localized_name": "x", + "name": "x", + "type": "INT", + "widget": { + "name": "x" + }, + "link": null + }, + { + "localized_name": "y", + "name": "y", + "type": "INT", + "widget": { + "name": "y" + }, + "link": null + }, + { + "localized_name": "resize_source", + "name": "resize_source", + "type": "BOOLEAN", + "widget": { + "name": "resize_source" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "IMAGE", + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 136 + ] + } + ], + "properties": { + "Node name for S&R": "ImageCompositeMasked" + }, + "widgets_values": [ + 0, + 0, + true + ], + "widgets_values_named": { + "x": 0, + "y": 0, + "resize_source": true + } + }, + { + "id": 84, + "type": "ImageBlend", + "pos": [ + 1690, + 2790 + ], + "size": [ + 260, + 150 + ], + "flags": {}, + "order": 38, + "mode": 0, + "inputs": [ + { + "localized_name": "image1", + "name": "image1", + "type": "IMAGE", + "link": 135 + }, + { + "localized_name": "image2", + "name": "image2", + "type": "IMAGE", + "link": 136 + }, + { + "localized_name": "blend_factor", + "name": "blend_factor", + "type": "FLOAT", + "widget": { + "name": "blend_factor" + }, + "link": null + }, + { + "localized_name": "blend_mode", + "name": "blend_mode", + "type": "COMBO", + "widget": { + "name": "blend_mode" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "IMAGE", + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 137 + ] + } + ], + "properties": { + "Node name for S&R": "ImageBlend" + }, + "widgets_values": [ + 0.48, + "normal" + ], + "widgets_values_named": { + "blend_factor": 0.48, + "blend_mode": "normal" + } + }, + { + "id": 89, + "type": "EmptyImage", + "pos": [ + 1090, + 4390 + ], + "size": [ + 260, + 150 + ], + "flags": {}, + "order": 15, + "mode": 0, + "inputs": [ + { + "localized_name": "width", + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": null + }, + { + "localized_name": "height", + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": null + }, + { + "localized_name": "batch_size", + "name": "batch_size", + "type": "INT", + "widget": { + "name": "batch_size" + }, + "link": null + }, + { + "localized_name": "color", + "name": "color", + "type": "INT", + "widget": { + "name": "color" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "IMAGE", + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 148 + ] + } + ], + "properties": { + "Node name for S&R": "EmptyImage" + }, + "widgets_values": [ + 32, + 32, + 1, + 16711680 + ], + "widgets_values_named": { + "width": 32, + "height": 32, + "batch_size": 1, + "color": 16711680 + } + }, + { + "id": 90, + "type": "ImageCompositeMasked", + "pos": [ + 1390, + 4390 + ], + "size": [ + 260, + 150 + ], + "flags": {}, + "order": 32, + "mode": 0, + "inputs": [ + { + "localized_name": "destination", + "name": "destination", + "type": "IMAGE", + "link": 147 + }, + { + "localized_name": "source", + "name": "source", + "type": "IMAGE", + "link": 148 + }, + { + "localized_name": "mask", + "name": "mask", + "shape": 7, + "type": "MASK", + "link": 149 + }, + { + "localized_name": "x", + "name": "x", + "type": "INT", + "widget": { + "name": "x" + }, + "link": null + }, + { + "localized_name": "y", + "name": "y", + "type": "INT", + "widget": { + "name": "y" + }, + "link": null + }, + { + "localized_name": "resize_source", + "name": "resize_source", + "type": "BOOLEAN", + "widget": { + "name": "resize_source" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "IMAGE", + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 151 + ] + } + ], + "properties": { + "Node name for S&R": "ImageCompositeMasked" + }, + "widgets_values": [ + 0, + 0, + true + ], + "widgets_values_named": { + "x": 0, + "y": 0, + "resize_source": true + } + }, + { + "id": 91, + "type": "ImageBlend", + "pos": [ + 1690, + 4390 + ], + "size": [ + 260, + 150 + ], + "flags": {}, + "order": 40, + "mode": 0, + "inputs": [ + { + "localized_name": "image1", + "name": "image1", + "type": "IMAGE", + "link": 150 + }, + { + "localized_name": "image2", + "name": "image2", + "type": "IMAGE", + "link": 151 + }, + { + "localized_name": "blend_factor", + "name": "blend_factor", + "type": "FLOAT", + "widget": { + "name": "blend_factor" + }, + "link": null + }, + { + "localized_name": "blend_mode", + "name": "blend_mode", + "type": "COMBO", + "widget": { + "name": "blend_mode" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "IMAGE", + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 152 + ] + } + ], + "properties": { + "Node name for S&R": "ImageBlend" + }, + "widgets_values": [ + 0.48, + "normal" + ], + "widgets_values_named": { + "blend_factor": 0.48, + "blend_mode": "normal" + } + }, + { + "id": 92, + "type": "EmptyImage", + "pos": [ + 1090, + 5990 + ], + "size": [ + 260, + 150 + ], + "flags": {}, + "order": 16, + "mode": 0, + "inputs": [ + { + "localized_name": "width", + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": null + }, + { + "localized_name": "height", + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": null + }, + { + "localized_name": "batch_size", + "name": "batch_size", + "type": "INT", + "widget": { + "name": "batch_size" + }, + "link": null + }, + { + "localized_name": "color", + "name": "color", + "type": "INT", + "widget": { + "name": "color" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "IMAGE", + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 154 + ] + } + ], + "properties": { + "Node name for S&R": "EmptyImage" + }, + "widgets_values": [ + 32, + 32, + 1, + 16711680 + ], + "widgets_values_named": { + "width": 32, + "height": 32, + "batch_size": 1, + "color": 16711680 + } + }, + { + "id": 93, + "type": "ImageCompositeMasked", + "pos": [ + 1390, + 5990 + ], + "size": [ + 260, + 150 + ], + "flags": {}, + "order": 34, + "mode": 0, + "inputs": [ + { + "localized_name": "destination", + "name": "destination", + "type": "IMAGE", + "link": 153 + }, + { + "localized_name": "source", + "name": "source", + "type": "IMAGE", + "link": 154 + }, + { + "localized_name": "mask", + "name": "mask", + "shape": 7, + "type": "MASK", + "link": 155 + }, + { + "localized_name": "x", + "name": "x", + "type": "INT", + "widget": { + "name": "x" + }, + "link": null + }, + { + "localized_name": "y", + "name": "y", + "type": "INT", + "widget": { + "name": "y" + }, + "link": null + }, + { + "localized_name": "resize_source", + "name": "resize_source", + "type": "BOOLEAN", + "widget": { + "name": "resize_source" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "IMAGE", + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 157 + ] + } + ], + "properties": { + "Node name for S&R": "ImageCompositeMasked" + }, + "widgets_values": [ + 0, + 0, + true + ], + "widgets_values_named": { + "x": 0, + "y": 0, + "resize_source": true + } + }, + { + "id": 94, + "type": "ImageBlend", + "pos": [ + 1690, + 5990 + ], + "size": [ + 260, + 150 + ], + "flags": {}, + "order": 42, + "mode": 0, + "inputs": [ + { + "localized_name": "image1", + "name": "image1", + "type": "IMAGE", + "link": 156 + }, + { + "localized_name": "image2", + "name": "image2", + "type": "IMAGE", + "link": 157 + }, + { + "localized_name": "blend_factor", + "name": "blend_factor", + "type": "FLOAT", + "widget": { + "name": "blend_factor" + }, + "link": null + }, + { + "localized_name": "blend_mode", + "name": "blend_mode", + "type": "COMBO", + "widget": { + "name": "blend_mode" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "IMAGE", + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 158 + ] + } + ], + "properties": { + "Node name for S&R": "ImageBlend" + }, + "widgets_values": [ + 0.48, + "normal" + ], + "widgets_values_named": { + "blend_factor": 0.48, + "blend_mode": "normal" + } + }, + { + "id": 1, + "type": "MarkdownNote", + "pos": [ + -268.4516045720269, + 2138.3724768826028 + ], + "size": [ + 420, + 210 + ], + "flags": {}, + "order": 17, + "mode": 0, + "inputs": [], + "outputs": [], + "title": "Usage Guide", + "properties": {}, + "widgets_values": [ + "Use a compatible MiniMax-H3 latent-edit server. Select a source video and set the server URL/model in each Generate Video node.\n\nSelect one Save Preview or Save Video output, then use Execute to selected output nodes. Running the whole workflow submits all four cases.\n\nMask: 0 preserves source latents; 1 regenerates. Audio mask applies to the whole clip, not individual sounds. Preview overlays are silent and do not run inference." + ], + "widgets_values_named": { + "text": "Use a compatible MiniMax-H3 latent-edit server. Select a source video and set the server URL/model in each Generate Video node.\n\nSelect one Save Preview or Save Video output, then use Execute to selected output nodes. Running the whole workflow submits all four cases.\n\nMask: 0 preserves source latents; 1 regenerates. Audio mask applies to the whole clip, not individual sounds. Preview overlays are silent and do not run inference." + }, + "color": "#432", + "bgcolor": "#000" + }, + { + "id": 15, + "type": "GetVideoComponents", + "pos": [ + -268.4516045720269, + 2438.3724768826028 + ], + "size": [ + 193.530859375, + 106 + ], + "flags": {}, + "order": 22, + "mode": 0, + "inputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "link": 17 + } + ], + "outputs": [ + { + "localized_name": "images", + "name": "images", + "type": "IMAGE", + "links": [ + 120, + 123, + 125, + 128, + 132, + 135 + ] + }, + { + "localized_name": "audio", + "name": "audio", + "type": "AUDIO", + "links": [] + }, + { + "localized_name": "fps", + "name": "fps", + "type": "FLOAT", + "links": [ + 121, + 124, + 131, + 138 + ] + }, + { + "localized_name": "bit_depth", + "name": "bit_depth", + "type": "COMBO", + "links": [] + }, + { + "localized_name": "color_space", + "name": "color_space", + "type": "COMBO", + "links": [] + } + ], + "properties": { + "Node name for S&R": "GetVideoComponents" + } + }, + { + "id": 95, + "type": "SolidMask", + "pos": [ + 580, + 1240 + ], + "size": [ + 300, + 150 + ], + "flags": {}, + "order": 18, + "mode": 0, + "inputs": [ + { + "localized_name": "value", + "name": "value", + "type": "FLOAT", + "widget": { + "name": "value" + }, + "link": null + }, + { + "localized_name": "width", + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": null + }, + { + "localized_name": "height", + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "MASK", + "name": "MASK", + "type": "MASK", + "links": [ + 159 + ] + } + ], + "title": "Mask Canvas", + "properties": { + "Node name for S&R": "SolidMask" + }, + "widgets_values": [ + 0, + 1344, + 768 + ], + "widgets_values_named": { + "value": 0, + "width": 1344, + "height": 768 + } + }, + { + "id": 97, + "type": "SolidMask", + "pos": [ + 580, + 2840 + ], + "size": [ + 300, + 150 + ], + "flags": {}, + "order": 19, + "mode": 0, + "inputs": [ + { + "localized_name": "value", + "name": "value", + "type": "FLOAT", + "widget": { + "name": "value" + }, + "link": null + }, + { + "localized_name": "width", + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": null + }, + { + "localized_name": "height", + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "MASK", + "name": "MASK", + "type": "MASK", + "links": [ + 161 + ] + } + ], + "title": "Mask Canvas", + "properties": { + "Node name for S&R": "SolidMask" + }, + "widgets_values": [ + 0, + 1344, + 768 + ], + "widgets_values_named": { + "value": 0, + "width": 1344, + "height": 768 + } + }, + { + "id": 66, + "type": "SaveVideo", + "pos": [ + 2530, + 1240 + ], + "size": [ + 420, + 106 + ], + "flags": {}, + "order": 51, + "mode": 0, + "inputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "link": 77 + }, + { + "localized_name": "filename_prefix", + "name": "filename_prefix", + "type": "STRING", + "widget": { + "name": "filename_prefix" + }, + "link": null + }, + { + "localized_name": "format", + "name": "format", + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "format" + }, + "link": null + }, + { + "localized_name": "codec", + "name": "format.codec", + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "format.codec" + }, + "link": null + }, + { + "localized_name": "codec", + "name": "codec", + "shape": 7, + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "codec" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "links": [] + } + ], + "title": "Mask Preview (Object Removal)", + "properties": {}, + "widgets_values": [ + "vllm-omni/WF05-object-removal-mask-preview", + "auto", + "auto", + "auto" + ], + "widgets_values_named": { + "filename_prefix": "vllm-omni/WF05-object-removal-mask-preview", + "format": "auto", + "format.codec": "auto", + "codec": "auto" + } + }, + { + "id": 78, + "type": "SaveVideo", + "pos": [ + 2530, + 6040 + ], + "size": [ + 420, + 106 + ], + "flags": {}, + "order": 54, + "mode": 0, + "inputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "link": 119 + }, + { + "localized_name": "filename_prefix", + "name": "filename_prefix", + "type": "STRING", + "widget": { + "name": "filename_prefix" + }, + "link": null + }, + { + "localized_name": "format", + "name": "format", + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "format" + }, + "link": null + }, + { + "localized_name": "codec", + "name": "format.codec", + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "format.codec" + }, + "link": null + }, + { + "localized_name": "codec", + "name": "codec", + "shape": 7, + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "codec" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "links": [] + } + ], + "title": "Mask Preview (Extension)", + "properties": {}, + "widgets_values": [ + "vllm-omni/WF05-extension-mask-preview", + "auto", + "auto", + "auto" + ], + "widgets_values_named": { + "filename_prefix": "vllm-omni/WF05-extension-mask-preview", + "format": "auto", + "format.codec": "auto", + "codec": "auto" + } + }, + { + "id": 6, + "type": "VLLMOmniGenerateVideo", + "pos": [ + 1940, + 1870 + ], + "size": [ + 440, + 680 + ], + "flags": {}, + "order": 35, + "mode": 0, + "inputs": [ + { + "localized_name": "frame", + "name": "frame", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "localized_name": "references", + "name": "references", + "shape": 7, + "type": "VIDEO_REFERENCES", + "link": null + }, + { + "localized_name": "sampling_params", + "name": "sampling_params", + "shape": 7, + "type": "SAMPLING_PARAMS", + "link": 10 + }, + { + "localized_name": "lora", + "name": "lora", + "shape": 7, + "type": "REMOTE_LORA", + "link": null + }, + { + "localized_name": "model_params", + "name": "model_params", + "shape": 7, + "type": "VIDEO_PARAMS", + "link": 4 + }, + { + "localized_name": "fast_h3", + "name": "fast_h3", + "shape": 7, + "type": "FASTH3_DEPLOYMENT", + "link": null + }, + { + "localized_name": "latent_edit", + "name": "latent_edit", + "shape": 7, + "type": "LATENT_MASK_EDITING", + "link": 3 + }, + { + "localized_name": "url", + "name": "url", + "type": "STRING", + "widget": { + "name": "url" + }, + "link": null + }, + { + "localized_name": "model", + "name": "model", + "type": "STRING", + "widget": { + "name": "model" + }, + "link": null + }, + { + "localized_name": "prompt", + "name": "prompt", + "type": "STRING", + "widget": { + "name": "prompt" + }, + "link": null + }, + { + "localized_name": "negative_prompt", + "name": "negative_prompt", + "type": "STRING", + "widget": { + "name": "negative_prompt" + }, + "link": null + }, + { + "localized_name": "width", + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": null + }, + { + "localized_name": "height", + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": null + }, + { + "localized_name": "fps", + "name": "fps", + "type": "INT", + "widget": { + "name": "fps" + }, + "link": null + }, + { + "localized_name": "duration", + "name": "duration", + "type": "FLOAT", + "widget": { + "name": "duration" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "links": [ + 5 + ] + } + ], + "title": "Generate Video(Object Removal)", + "properties": { + "Node name for S&R": "VLLMOmniGenerateVideo" + }, + "widgets_values": [ + "http://127.0.0.1:8001/v1", + "MiniMaxAI/MiniMax-H3", + "integrated_multimodal_description: The masked area is a flat, seamless continuation of the surrounding wall, matching its color, texture, and lighting. The wall surface remains consistent throughout the video.\n\noverall_soundscape: \n\nnon_diegetic_music: ", + "", + 1344, + 768, + 24, + 5 + ], + "widgets_values_named": { + "url": "http://127.0.0.1:8001/v1", + "model": "MiniMaxAI/MiniMax-H3", + "prompt": "integrated_multimodal_description: The masked area is a flat, seamless continuation of the surrounding wall, matching its color, texture, and lighting. The wall surface remains consistent throughout the video.\n\noverall_soundscape: \n\nnon_diegetic_music: ", + "negative_prompt": "", + "width": 1344, + "height": 768, + "fps": 24, + "duration": 5 + } + }, + { + "id": 96, + "type": "SolidMask", + "pos": [ + 580, + 1360 + ], + "size": [ + 300, + 150 + ], + "flags": {}, + "order": 20, + "mode": 0, + "inputs": [ + { + "localized_name": "value", + "name": "value", + "type": "FLOAT", + "widget": { + "name": "value" + }, + "link": null + }, + { + "localized_name": "width", + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": null + }, + { + "localized_name": "height", + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "MASK", + "name": "MASK", + "type": "MASK", + "links": [ + 160 + ] + } + ], + "title": "Mask Rectangle", + "properties": { + "Node name for S&R": "SolidMask" + }, + "widgets_values": [ + 1, + 180, + 168 + ], + "widgets_values_named": { + "value": 1, + "width": 180, + "height": 168 + } + }, + { + "id": 9, + "type": "MaskComposite", + "pos": [ + 580, + 1540 + ], + "size": [ + 300, + 150 + ], + "flags": {}, + "order": 23, + "mode": 0, + "inputs": [ + { + "localized_name": "destination", + "name": "destination", + "type": "MASK", + "link": 159 + }, + { + "localized_name": "source", + "name": "source", + "type": "MASK", + "link": 160 + }, + { + "localized_name": "x", + "name": "x", + "type": "INT", + "widget": { + "name": "x" + }, + "link": null + }, + { + "localized_name": "y", + "name": "y", + "type": "INT", + "widget": { + "name": "y" + }, + "link": null + }, + { + "localized_name": "operation", + "name": "operation", + "type": "COMBO", + "widget": { + "name": "operation" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "MASK", + "name": "MASK", + "type": "MASK", + "links": [ + 63, + 127 + ] + } + ], + "title": "Spatial Mask | Rectangle Position", + "properties": { + "Node name for S&R": "MaskComposite" + }, + "widgets_values": [ + 1074, + 493, + "add" + ], + "widgets_values_named": { + "x": 1074, + "y": 493, + "operation": "add" + } + }, + { + "id": 98, + "type": "SolidMask", + "pos": [ + 580, + 2960 + ], + "size": [ + 300, + 150 + ], + "flags": {}, + "order": 21, + "mode": 0, + "inputs": [ + { + "localized_name": "value", + "name": "value", + "type": "FLOAT", + "widget": { + "name": "value" + }, + "link": null + }, + { + "localized_name": "width", + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": null + }, + { + "localized_name": "height", + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "MASK", + "name": "MASK", + "type": "MASK", + "links": [ + 162 + ] + } + ], + "title": "Mask Rectangle", + "properties": { + "Node name for S&R": "SolidMask" + }, + "widgets_values": [ + 1, + 180, + 168 + ], + "widgets_values_named": { + "value": 1, + "width": 180, + "height": 168 + } + }, + { + "id": 99, + "type": "MaskComposite", + "pos": [ + 580, + 3140 + ], + "size": [ + 300, + 150 + ], + "flags": {}, + "order": 24, + "mode": 0, + "inputs": [ + { + "localized_name": "destination", + "name": "destination", + "type": "MASK", + "link": 161 + }, + { + "localized_name": "source", + "name": "source", + "type": "MASK", + "link": 162 + }, + { + "localized_name": "x", + "name": "x", + "type": "INT", + "widget": { + "name": "x" + }, + "link": null + }, + { + "localized_name": "y", + "name": "y", + "type": "INT", + "widget": { + "name": "y" + }, + "link": null + }, + { + "localized_name": "operation", + "name": "operation", + "type": "COMBO", + "widget": { + "name": "operation" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "MASK", + "name": "MASK", + "type": "MASK", + "links": [ + 35, + 134 + ] + } + ], + "title": "Spatial Mask | Rectangle Position", + "properties": { + "Node name for S&R": "MaskComposite" + }, + "widgets_values": [ + 1074, + 493, + "add" + ], + "widgets_values_named": { + "x": 1074, + "y": 493, + "operation": "add" + } + }, + { + "id": 70, + "type": "SaveVideo", + "pos": [ + 2530, + 2840 + ], + "size": [ + 420, + 106 + ], + "flags": {}, + "order": 52, + "mode": 0, + "inputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "link": 91 + }, + { + "localized_name": "filename_prefix", + "name": "filename_prefix", + "type": "STRING", + "widget": { + "name": "filename_prefix" + }, + "link": null + }, + { + "localized_name": "format", + "name": "format", + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "format" + }, + "link": null + }, + { + "localized_name": "codec", + "name": "format.codec", + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "format.codec" + }, + "link": null + }, + { + "localized_name": "codec", + "name": "codec", + "shape": 7, + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "codec" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "links": [] + } + ], + "title": "Mask Preview (Inpainting)", + "properties": {}, + "widgets_values": [ + "vllm-omni/WF05-inpainting-mask-preview", + "auto", + "auto", + "auto" + ], + "widgets_values_named": { + "filename_prefix": "vllm-omni/WF05-inpainting-mask-preview", + "format": "auto", + "format.codec": "auto", + "codec": "auto" + } + }, + { + "id": 4, + "type": "VLLMOmniLatentMaskEditing", + "pos": [ + 1510, + 1870 + ], + "size": [ + 300, + 200 + ], + "flags": {}, + "order": 27, + "mode": 0, + "inputs": [ + { + "localized_name": "source_video", + "name": "source_video", + "shape": 7, + "type": "VIDEO", + "link": 1 + }, + { + "localized_name": "source_audio", + "name": "source_audio", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "localized_name": "video_mask", + "name": "video_mask", + "shape": 7, + "type": "MASK", + "link": 63 + }, + { + "localized_name": "audio_mask", + "name": "audio_mask", + "shape": 7, + "type": "FLOAT", + "widget": { + "name": "audio_mask" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "latent_edit", + "name": "latent_edit", + "type": "LATENT_MASK_EDITING", + "links": [ + 3 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniLatentMaskEditing" + }, + "widgets_values": [ + 0 + ], + "widgets_values_named": { + "audio_mask": 0 + } + }, + { + "id": 7, + "type": "SaveVideo", + "pos": [ + 2530, + 1870 + ], + "size": [ + 420, + 106 + ], + "flags": {}, + "order": 43, + "mode": 0, + "inputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "link": 5 + }, + { + "localized_name": "filename_prefix", + "name": "filename_prefix", + "type": "STRING", + "widget": { + "name": "filename_prefix" + }, + "link": null + }, + { + "localized_name": "format", + "name": "format", + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "format" + }, + "link": null + }, + { + "localized_name": "codec", + "name": "format.codec", + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "format.codec" + }, + "link": null + }, + { + "localized_name": "codec", + "name": "codec", + "shape": 7, + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "codec" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "links": [] + } + ], + "title": "Save Video (Object Removal)", + "properties": { + "Node name for S&R": "SaveVideo" + }, + "widgets_values": [ + "vllm-omni/WF05-object-removal", + "mp4", + "auto", + "auto" + ], + "widgets_values_named": { + "filename_prefix": "vllm-omni/WF05-object-removal", + "format": "mp4", + "format.codec": "auto", + "codec": "auto" + } + }, + { + "id": 27, + "type": "SaveVideo", + "pos": [ + 2530, + 3470 + ], + "size": [ + 420, + 106 + ], + "flags": {}, + "order": 45, + "mode": 0, + "inputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "link": 31 + }, + { + "localized_name": "filename_prefix", + "name": "filename_prefix", + "type": "STRING", + "widget": { + "name": "filename_prefix" + }, + "link": null + }, + { + "localized_name": "format", + "name": "format", + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "format" + }, + "link": null + }, + { + "localized_name": "codec", + "name": "format.codec", + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "format.codec" + }, + "link": null + }, + { + "localized_name": "codec", + "name": "codec", + "shape": 7, + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "codec" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "links": [] + } + ], + "title": "Save Video (Inpainting)", + "properties": { + "Node name for S&R": "SaveVideo" + }, + "widgets_values": [ + "vllm-omni/WF05-inpainting", + "mp4", + "auto", + "auto" + ], + "widgets_values_named": { + "filename_prefix": "vllm-omni/WF05-inpainting", + "format": "mp4", + "format.codec": "auto", + "codec": "auto" + } + }, + { + "id": 24, + "type": "VLLMOmniLatentMaskEditing", + "pos": [ + 1510, + 3470 + ], + "size": [ + 300, + 200 + ], + "flags": {}, + "order": 29, + "mode": 0, + "inputs": [ + { + "localized_name": "source_video", + "name": "source_video", + "shape": 7, + "type": "VIDEO", + "link": 28 + }, + { + "localized_name": "source_audio", + "name": "source_audio", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "localized_name": "video_mask", + "name": "video_mask", + "shape": 7, + "type": "MASK", + "link": 35 + }, + { + "localized_name": "audio_mask", + "name": "audio_mask", + "shape": 7, + "type": "FLOAT", + "widget": { + "name": "audio_mask" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "latent_edit", + "name": "latent_edit", + "type": "LATENT_MASK_EDITING", + "links": [ + 29 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniLatentMaskEditing" + }, + "widgets_values": [ + 0 + ], + "widgets_values_named": { + "audio_mask": 0 + } + }, + { + "id": 26, + "type": "VLLMOmniGenerateVideo", + "pos": [ + 1940, + 3470 + ], + "size": [ + 440, + 680 + ], + "flags": {}, + "order": 37, + "mode": 0, + "inputs": [ + { + "localized_name": "frame", + "name": "frame", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "localized_name": "references", + "name": "references", + "shape": 7, + "type": "VIDEO_REFERENCES", + "link": null + }, + { + "localized_name": "sampling_params", + "name": "sampling_params", + "shape": 7, + "type": "SAMPLING_PARAMS", + "link": 33 + }, + { + "localized_name": "lora", + "name": "lora", + "shape": 7, + "type": "REMOTE_LORA", + "link": null + }, + { + "localized_name": "model_params", + "name": "model_params", + "shape": 7, + "type": "VIDEO_PARAMS", + "link": 30 + }, + { + "localized_name": "fast_h3", + "name": "fast_h3", + "shape": 7, + "type": "FASTH3_DEPLOYMENT", + "link": null + }, + { + "localized_name": "latent_edit", + "name": "latent_edit", + "shape": 7, + "type": "LATENT_MASK_EDITING", + "link": 29 + }, + { + "localized_name": "url", + "name": "url", + "type": "STRING", + "widget": { + "name": "url" + }, + "link": null + }, + { + "localized_name": "model", + "name": "model", + "type": "STRING", + "widget": { + "name": "model" + }, + "link": null + }, + { + "localized_name": "prompt", + "name": "prompt", + "type": "STRING", + "widget": { + "name": "prompt" + }, + "link": null + }, + { + "localized_name": "negative_prompt", + "name": "negative_prompt", + "type": "STRING", + "widget": { + "name": "negative_prompt" + }, + "link": null + }, + { + "localized_name": "width", + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": null + }, + { + "localized_name": "height", + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": null + }, + { + "localized_name": "fps", + "name": "fps", + "type": "INT", + "widget": { + "name": "fps" + }, + "link": null + }, + { + "localized_name": "duration", + "name": "duration", + "type": "FLOAT", + "widget": { + "name": "duration" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "links": [ + 31 + ] + } + ], + "title": "Generate Video(Inpainting)", + "properties": { + "Node name for S&R": "VLLMOmniGenerateVideo" + }, + "widgets_values": [ + "http://127.0.0.1:8001/v1", + "MiniMaxAI/MiniMax-H3", + "integrated_multimodal_description: A small orange cat rests curled up in the masked area, gently swaying its tail from side to side. Its appearance matches the surrounding anime style, lighting, and shadows.\n\noverall_soundscape:\n\nnon_diegetic_music: ", + "", + 1344, + 768, + 24, + 5 + ], + "widgets_values_named": { + "url": "http://127.0.0.1:8001/v1", + "model": "MiniMaxAI/MiniMax-H3", + "prompt": "integrated_multimodal_description: A small orange cat rests curled up in the masked area, gently swaying its tail from side to side. Its appearance matches the surrounding anime style, lighting, and shadows.\n\noverall_soundscape:\n\nnon_diegetic_music: ", + "negative_prompt": "", + "width": 1344, + "height": 768, + "fps": 24, + "duration": 5 + } + }, + { + "id": 33, + "type": "VLLMOmniLatentMaskEditing", + "pos": [ + 1510, + 5070 + ], + "size": [ + 300, + 200 + ], + "flags": {}, + "order": 31, + "mode": 0, + "inputs": [ + { + "localized_name": "source_video", + "name": "source_video", + "shape": 7, + "type": "VIDEO", + "link": 36 + }, + { + "localized_name": "source_audio", + "name": "source_audio", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "localized_name": "video_mask", + "name": "video_mask", + "shape": 7, + "type": "MASK", + "link": 47 + }, + { + "localized_name": "audio_mask", + "name": "audio_mask", + "shape": 7, + "type": "FLOAT", + "widget": { + "name": "audio_mask" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "latent_edit", + "name": "latent_edit", + "type": "LATENT_MASK_EDITING", + "links": [ + 37 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniLatentMaskEditing" + }, + "widgets_values": [ + 0.8 + ], + "widgets_values_named": { + "audio_mask": 0.8 + } + }, + { + "id": 35, + "type": "VLLMOmniGenerateVideo", + "pos": [ + 1940, + 5070 + ], + "size": [ + 440, + 680 + ], + "flags": {}, + "order": 39, + "mode": 0, + "inputs": [ + { + "localized_name": "frame", + "name": "frame", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "localized_name": "references", + "name": "references", + "shape": 7, + "type": "VIDEO_REFERENCES", + "link": null + }, + { + "localized_name": "sampling_params", + "name": "sampling_params", + "shape": 7, + "type": "SAMPLING_PARAMS", + "link": 48 + }, + { + "localized_name": "lora", + "name": "lora", + "shape": 7, + "type": "REMOTE_LORA", + "link": null + }, + { + "localized_name": "model_params", + "name": "model_params", + "shape": 7, + "type": "VIDEO_PARAMS", + "link": 38 + }, + { + "localized_name": "fast_h3", + "name": "fast_h3", + "shape": 7, + "type": "FASTH3_DEPLOYMENT", + "link": null + }, + { + "localized_name": "latent_edit", + "name": "latent_edit", + "shape": 7, + "type": "LATENT_MASK_EDITING", + "link": 37 + }, + { + "localized_name": "url", + "name": "url", + "type": "STRING", + "widget": { + "name": "url" + }, + "link": null + }, + { + "localized_name": "model", + "name": "model", + "type": "STRING", + "widget": { + "name": "model" + }, + "link": null + }, + { + "localized_name": "prompt", + "name": "prompt", + "type": "STRING", + "widget": { + "name": "prompt" + }, + "link": null + }, + { + "localized_name": "negative_prompt", + "name": "negative_prompt", + "type": "STRING", + "widget": { + "name": "negative_prompt" + }, + "link": null + }, + { + "localized_name": "width", + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": null + }, + { + "localized_name": "height", + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": null + }, + { + "localized_name": "fps", + "name": "fps", + "type": "INT", + "widget": { + "name": "fps" + }, + "link": null + }, + { + "localized_name": "duration", + "name": "duration", + "type": "FLOAT", + "widget": { + "name": "duration" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "links": [ + 39 + ] + } + ], + "title": "Generate Video(Continuation)", + "properties": { + "Node name for S&R": "VLLMOmniGenerateVideo" + }, + "widgets_values": [ + "http://127.0.0.1:8001/v1", + "MiniMaxAI/MiniMax-H3", + "integrated_multimodal_description: The sky turns completely overcast, with the sun hidden and no direct sunlight, as torrential rain pours over the school entrance in dense visible streaks and splashes on the wet pavement. Distant lightning flashes in the sky behind the school buildings.\n\noverall_soundscape: Loud, steady rainfall, splashing water, and gusts of wind.\n\nnon_diegetic_music: None.", + "", + 1344, + 768, + 24, + 5 + ], + "widgets_values_named": { + "url": "http://127.0.0.1:8001/v1", + "model": "MiniMaxAI/MiniMax-H3", + "prompt": "integrated_multimodal_description: The sky turns completely overcast, with the sun hidden and no direct sunlight, as torrential rain pours over the school entrance in dense visible streaks and splashes on the wet pavement. Distant lightning flashes in the sky behind the school buildings.\n\noverall_soundscape: Loud, steady rainfall, splashing water, and gusts of wind.\n\nnon_diegetic_music: None.", + "negative_prompt": "", + "width": 1344, + "height": 768, + "fps": 24, + "duration": 5 + } + }, + { + "id": 44, + "type": "VLLMOmniMiniMaxH3TemporalMask", + "pos": [ + 580, + 4440 + ], + "size": [ + 420, + 240 + ], + "flags": {}, + "order": 25, + "mode": 0, + "inputs": [ + { + "localized_name": "images", + "name": "images", + "type": "IMAGE", + "link": 120 + }, + { + "localized_name": "source_fps", + "name": "source_fps", + "type": "FLOAT", + "widget": { + "name": "source_fps" + }, + "link": 121 + }, + { + "localized_name": "duration", + "name": "duration", + "type": "FLOAT", + "widget": { + "name": "duration" + }, + "link": null + }, + { + "localized_name": "mode", + "name": "mode", + "type": "COMBO", + "widget": { + "name": "mode" + }, + "link": null + }, + { + "localized_name": "preserve_fraction", + "name": "preserve_fraction", + "type": "FLOAT", + "widget": { + "name": "preserve_fraction" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "mask", + "name": "mask", + "type": "MASK", + "links": [ + 47 + ] + }, + { + "localized_name": "preview_fps", + "name": "preview_fps", + "type": "FLOAT", + "links": [ + 142 + ] + }, + { + "localized_name": "preview_images", + "name": "preview_images", + "type": "IMAGE", + "links": [ + 147, + 150 + ] + }, + { + "localized_name": "preview_mask", + "name": "preview_mask", + "type": "MASK", + "links": [ + 149 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniMiniMaxH3TemporalMask" + }, + "widgets_values": [ + 24, + 5, + "continuation", + 0.2 + ], + "widgets_values_named": { + "source_fps": 24, + "duration": 5, + "mode": "continuation", + "preserve_fraction": 0.2 + } + }, + { + "id": 36, + "type": "SaveVideo", + "pos": [ + 2530, + 5070 + ], + "size": [ + 420, + 106 + ], + "flags": {}, + "order": 47, + "mode": 0, + "inputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "link": 39 + }, + { + "localized_name": "filename_prefix", + "name": "filename_prefix", + "type": "STRING", + "widget": { + "name": "filename_prefix" + }, + "link": null + }, + { + "localized_name": "format", + "name": "format", + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "format" + }, + "link": null + }, + { + "localized_name": "codec", + "name": "format.codec", + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "format.codec" + }, + "link": null + }, + { + "localized_name": "codec", + "name": "codec", + "shape": 7, + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "codec" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "links": [] + } + ], + "title": "Save Video (Continuation)", + "properties": { + "Node name for S&R": "SaveVideo" + }, + "widgets_values": [ + "vllm-omni/WF05-continuation", + "mp4", + "auto", + "auto" + ], + "widgets_values_named": { + "filename_prefix": "vllm-omni/WF05-continuation", + "format": "mp4", + "format.codec": "auto", + "codec": "auto" + } + }, + { + "id": 74, + "type": "SaveVideo", + "pos": [ + 2530, + 4440 + ], + "size": [ + 420, + 106 + ], + "flags": {}, + "order": 53, + "mode": 0, + "inputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "link": 105 + }, + { + "localized_name": "filename_prefix", + "name": "filename_prefix", + "type": "STRING", + "widget": { + "name": "filename_prefix" + }, + "link": null + }, + { + "localized_name": "format", + "name": "format", + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "format" + }, + "link": null + }, + { + "localized_name": "codec", + "name": "format.codec", + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "format.codec" + }, + "link": null + }, + { + "localized_name": "codec", + "name": "codec", + "shape": 7, + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "codec" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "links": [] + } + ], + "title": "Mask Preview (Continuation)", + "properties": {}, + "widgets_values": [ + "vllm-omni/WF05-continuation-mask-preview", + "auto", + "auto", + "auto" + ], + "widgets_values_named": { + "filename_prefix": "vllm-omni/WF05-continuation-mask-preview", + "format": "auto", + "format.codec": "auto", + "codec": "auto" + } + }, + { + "id": 50, + "type": "VLLMOmniGenerateVideo", + "pos": [ + 1940, + 6670 + ], + "size": [ + 440, + 680 + ], + "flags": {}, + "order": 41, + "mode": 0, + "inputs": [ + { + "localized_name": "frame", + "name": "frame", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "localized_name": "references", + "name": "references", + "shape": 7, + "type": "VIDEO_REFERENCES", + "link": null + }, + { + "localized_name": "sampling_params", + "name": "sampling_params", + "shape": 7, + "type": "SAMPLING_PARAMS", + "link": 61 + }, + { + "localized_name": "lora", + "name": "lora", + "shape": 7, + "type": "REMOTE_LORA", + "link": null + }, + { + "localized_name": "model_params", + "name": "model_params", + "shape": 7, + "type": "VIDEO_PARAMS", + "link": 51 + }, + { + "localized_name": "fast_h3", + "name": "fast_h3", + "shape": 7, + "type": "FASTH3_DEPLOYMENT", + "link": null + }, + { + "localized_name": "latent_edit", + "name": "latent_edit", + "shape": 7, + "type": "LATENT_MASK_EDITING", + "link": 50 + }, + { + "localized_name": "url", + "name": "url", + "type": "STRING", + "widget": { + "name": "url" + }, + "link": null + }, + { + "localized_name": "model", + "name": "model", + "type": "STRING", + "widget": { + "name": "model" + }, + "link": null + }, + { + "localized_name": "prompt", + "name": "prompt", + "type": "STRING", + "widget": { + "name": "prompt" + }, + "link": null + }, + { + "localized_name": "negative_prompt", + "name": "negative_prompt", + "type": "STRING", + "widget": { + "name": "negative_prompt" + }, + "link": null + }, + { + "localized_name": "width", + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": null + }, + { + "localized_name": "height", + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": null + }, + { + "localized_name": "fps", + "name": "fps", + "type": "INT", + "widget": { + "name": "fps" + }, + "link": null + }, + { + "localized_name": "duration", + "name": "duration", + "type": "FLOAT", + "widget": { + "name": "duration" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "links": [ + 52 + ] + } + ], + "title": "Generate Video(Extension)", + "properties": { + "Node name for S&R": "VLLMOmniGenerateVideo" + }, + "widgets_values": [ + "http://127.0.0.1:8001/v1", + "MiniMaxAI/MiniMax-H3", + "integrated_multimodal_description: Cut to the same school on a summer night beneath a deep blue sky dotted with stars. A firework rises from behind the school buildings, leaving a glowing trail; the camera slowly tilts upward to follow it into the sky, ending as it bursts into a large bloom of colorful sparks.\n\noverall_soundscape: Soft summer insect sounds, followed by the firework rising with a whistle and bursting with a distant boom and crackling sparks.\n\nnon_diegetic_music: N/A", + "", + 1344, + 768, + 24, + 10 + ], + "widgets_values_named": { + "url": "http://127.0.0.1:8001/v1", + "model": "MiniMaxAI/MiniMax-H3", + "prompt": "integrated_multimodal_description: Cut to the same school on a summer night beneath a deep blue sky dotted with stars. A firework rises from behind the school buildings, leaving a glowing trail; the camera slowly tilts upward to follow it into the sky, ending as it bursts into a large bloom of colorful sparks.\n\noverall_soundscape: Soft summer insect sounds, followed by the firework rising with a whistle and bursting with a distant boom and crackling sparks.\n\nnon_diegetic_music: N/A", + "negative_prompt": "", + "width": 1344, + "height": 768, + "fps": 24, + "duration": 10 + } + }, + { + "id": 48, + "type": "VLLMOmniLatentMaskEditing", + "pos": [ + 1510, + 6670 + ], + "size": [ + 300, + 200 + ], + "flags": {}, + "order": 33, + "mode": 0, + "inputs": [ + { + "localized_name": "source_video", + "name": "source_video", + "shape": 7, + "type": "VIDEO", + "link": 49 + }, + { + "localized_name": "source_audio", + "name": "source_audio", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "localized_name": "video_mask", + "name": "video_mask", + "shape": 7, + "type": "MASK", + "link": 60 + }, + { + "localized_name": "audio_mask", + "name": "audio_mask", + "shape": 7, + "type": "FLOAT", + "widget": { + "name": "audio_mask" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "latent_edit", + "name": "latent_edit", + "type": "LATENT_MASK_EDITING", + "links": [ + 50 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniLatentMaskEditing" + }, + "widgets_values": [ + 0.9 + ], + "widgets_values_named": { + "audio_mask": 0.9 + } + }, + { + "id": 51, + "type": "SaveVideo", + "pos": [ + 2530, + 6670 + ], + "size": [ + 420, + 338 + ], + "flags": {}, + "order": 49, + "mode": 0, + "inputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "link": 52 + }, + { + "localized_name": "filename_prefix", + "name": "filename_prefix", + "type": "STRING", + "widget": { + "name": "filename_prefix" + }, + "link": null + }, + { + "localized_name": "format", + "name": "format", + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "format" + }, + "link": null + }, + { + "localized_name": "codec", + "name": "format.codec", + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "format.codec" + }, + "link": null + }, + { + "localized_name": "codec", + "name": "codec", + "shape": 7, + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "codec" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "links": [] + } + ], + "title": "Save Video (Extension)", + "properties": { + "Node name for S&R": "SaveVideo" + }, + "widgets_values": [ + "vllm-omni/WF05-extension", + "mp4", + "auto", + "auto" + ], + "widgets_values_named": { + "filename_prefix": "vllm-omni/WF05-extension", + "format": "mp4", + "format.codec": "auto", + "codec": "auto" + } + } + ], + "links": [ + [ + 1, + 2, + 0, + 4, + 0, + "VIDEO" + ], + [ + 3, + 4, + 0, + 6, + 6, + "LATENT_MASK_EDITING" + ], + [ + 4, + 5, + 0, + 6, + 4, + "VIDEO_PARAMS" + ], + [ + 5, + 6, + 0, + 7, + 0, + "VIDEO" + ], + [ + 10, + 11, + 0, + 6, + 2, + "SAMPLING_PARAMS" + ], + [ + 17, + 2, + 0, + 15, + 0, + "VIDEO" + ], + [ + 28, + 2, + 0, + 24, + 0, + "VIDEO" + ], + [ + 29, + 24, + 0, + 26, + 6, + "LATENT_MASK_EDITING" + ], + [ + 30, + 25, + 0, + 26, + 4, + "VIDEO_PARAMS" + ], + [ + 31, + 26, + 0, + 27, + 0, + "VIDEO" + ], + [ + 33, + 30, + 0, + 26, + 2, + "SAMPLING_PARAMS" + ], + [ + 35, + 99, + 0, + 24, + 2, + "MASK" + ], + [ + 36, + 2, + 0, + 33, + 0, + "VIDEO" + ], + [ + 37, + 33, + 0, + 35, + 6, + "LATENT_MASK_EDITING" + ], + [ + 38, + 34, + 0, + 35, + 4, + "VIDEO_PARAMS" + ], + [ + 39, + 35, + 0, + 36, + 0, + "VIDEO" + ], + [ + 47, + 44, + 0, + 33, + 2, + "MASK" + ], + [ + 48, + 46, + 0, + 35, + 2, + "SAMPLING_PARAMS" + ], + [ + 49, + 2, + 0, + 48, + 0, + "VIDEO" + ], + [ + 50, + 48, + 0, + 50, + 6, + "LATENT_MASK_EDITING" + ], + [ + 51, + 49, + 0, + 50, + 4, + "VIDEO_PARAMS" + ], + [ + 52, + 50, + 0, + 51, + 0, + "VIDEO" + ], + [ + 60, + 59, + 0, + 48, + 2, + "MASK" + ], + [ + 61, + 61, + 0, + 50, + 2, + "SAMPLING_PARAMS" + ], + [ + 63, + 9, + 0, + 4, + 2, + "MASK" + ], + [ + 77, + 65, + 0, + 66, + 0, + "VIDEO" + ], + [ + 91, + 69, + 0, + 70, + 0, + "VIDEO" + ], + [ + 105, + 73, + 0, + 74, + 0, + "VIDEO" + ], + [ + 119, + 77, + 0, + 78, + 0, + "VIDEO" + ], + [ + 120, + 15, + 0, + 44, + 0, + "IMAGE" + ], + [ + 121, + 15, + 2, + 44, + 1, + "FLOAT" + ], + [ + 123, + 15, + 0, + 59, + 0, + "IMAGE" + ], + [ + 124, + 15, + 2, + 59, + 1, + "FLOAT" + ], + [ + 125, + 15, + 0, + 80, + 0, + "IMAGE" + ], + [ + 126, + 79, + 0, + 80, + 1, + "IMAGE" + ], + [ + 127, + 9, + 0, + 80, + 2, + "MASK" + ], + [ + 128, + 15, + 0, + 81, + 0, + "IMAGE" + ], + [ + 129, + 80, + 0, + 81, + 1, + "IMAGE" + ], + [ + 130, + 81, + 0, + 65, + 0, + "IMAGE" + ], + [ + 131, + 15, + 2, + 65, + 2, + "FLOAT" + ], + [ + 132, + 15, + 0, + 83, + 0, + "IMAGE" + ], + [ + 133, + 82, + 0, + 83, + 1, + "IMAGE" + ], + [ + 134, + 99, + 0, + 83, + 2, + "MASK" + ], + [ + 135, + 15, + 0, + 84, + 0, + "IMAGE" + ], + [ + 136, + 83, + 0, + 84, + 1, + "IMAGE" + ], + [ + 137, + 84, + 0, + 69, + 0, + "IMAGE" + ], + [ + 138, + 15, + 2, + 69, + 2, + "FLOAT" + ], + [ + 142, + 44, + 1, + 73, + 2, + "FLOAT" + ], + [ + 146, + 59, + 1, + 77, + 2, + "FLOAT" + ], + [ + 147, + 44, + 2, + 90, + 0, + "IMAGE" + ], + [ + 148, + 89, + 0, + 90, + 1, + "IMAGE" + ], + [ + 149, + 44, + 3, + 90, + 2, + "MASK" + ], + [ + 150, + 44, + 2, + 91, + 0, + "IMAGE" + ], + [ + 151, + 90, + 0, + 91, + 1, + "IMAGE" + ], + [ + 152, + 91, + 0, + 73, + 0, + "IMAGE" + ], + [ + 153, + 59, + 2, + 93, + 0, + "IMAGE" + ], + [ + 154, + 92, + 0, + 93, + 1, + "IMAGE" + ], + [ + 155, + 59, + 3, + 93, + 2, + "MASK" + ], + [ + 156, + 59, + 2, + 94, + 0, + "IMAGE" + ], + [ + 157, + 93, + 0, + 94, + 1, + "IMAGE" + ], + [ + 158, + 94, + 0, + 77, + 0, + "IMAGE" + ], + [ + 159, + 95, + 0, + 9, + 0, + "MASK" + ], + [ + 160, + 96, + 0, + 9, + 1, + "MASK" + ], + [ + 161, + 97, + 0, + 99, + 0, + "MASK" + ], + [ + 162, + 98, + 0, + 99, + 1, + "MASK" + ] + ], + "groups": [ + { + "id": 1, + "title": "Shared Input", + "bounding": [ + -308.451604572027, + 1080, + 473.7665553206702, + 1508.3724768826028 + ], + "color": "#596477", + "flags": {} + }, + { + "id": 2, + "title": "Output Hub", + "bounding": [ + 2470, + 1080, + 540, + 6400 + ], + "color": "#444", + "flags": {} + }, + { + "id": 10, + "title": "Case 01 | Object Removal", + "bounding": [ + 540, + 1080, + 1880, + 1540 + ], + "color": "#77528d", + "flags": {} + }, + { + "id": 20, + "title": "Case 01 | Object Removal Output", + "bounding": [ + 2490, + 1130, + 500, + 1490 + ], + "color": "#77528d", + "flags": {} + }, + { + "id": 11, + "title": "Case 02 | Inpainting", + "bounding": [ + 540, + 2680, + 1880, + 1540 + ], + "color": "#357e80", + "flags": {} + }, + { + "id": 21, + "title": "Case 02 | Inpainting Output", + "bounding": [ + 2490, + 2730, + 500, + 1490 + ], + "color": "#357e80", + "flags": {} + }, + { + "id": 12, + "title": "Case 03 | Continuation", + "bounding": [ + 540, + 4280, + 1880, + 1540 + ], + "color": "#596477", + "flags": {} + }, + { + "id": 22, + "title": "Case 03 | Continuation Output", + "bounding": [ + 2490, + 4330, + 500, + 1490 + ], + "color": "#596477", + "flags": {} + }, + { + "id": 13, + "title": "Case 04 | Extension", + "bounding": [ + 540, + 5880, + 1880, + 1540 + ], + "color": "#b06634", + "flags": {} + }, + { + "id": 23, + "title": "Case 04 | Extension Output", + "bounding": [ + 2490, + 5930, + 500, + 1490 + ], + "color": "#b06634", + "flags": {} + }, + { + "id": 24, + "title": "Object Removal | Mask Generation", + "bounding": [ + 560, + 1130, + 480, + 620 + ], + "color": "#65794e", + "flags": {} + }, + { + "id": 25, + "title": "Inpainting | Mask Generation", + "bounding": [ + 560, + 2730, + 480, + 620 + ], + "color": "#65794e", + "flags": {} + }, + { + "id": 26, + "title": "Continuation | Mask Generation", + "bounding": [ + 560, + 4330, + 480, + 620 + ], + "color": "#65794e", + "flags": {} + }, + { + "id": 27, + "title": "Extension | Mask Generation", + "bounding": [ + 560, + 5930, + 480, + 620 + ], + "color": "#65794e", + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.9837545257996714, + "offset": [ + -1807.8247536090078, + -6349.252392678693 + ] + } + }, + "version": 0.4 +} diff --git a/apps/ComfyUI-vLLM-Omni/web/main.js b/apps/ComfyUI-vLLM-Omni/web/main.js index 9aba6260e..f8c633e11 100644 --- a/apps/ComfyUI-vLLM-Omni/web/main.js +++ b/apps/ComfyUI-vLLM-Omni/web/main.js @@ -52,6 +52,7 @@ const FAMILY_BY_OUTPUT = { REMOTE_LORA: "deployment", FASTH3_DEPLOYMENT: "deployment", VIDEO_REFERENCES: "media", + LATENT_MASK_EDITING: "media", }; function familyOf(nodeData) { diff --git a/artifacts/pr7759-review/e2e_images/i2i_teapot_blue.png b/artifacts/pr7759-review/e2e_images/i2i_teapot_blue.png new file mode 100644 index 000000000..f9ba01de4 Binary files /dev/null and b/artifacts/pr7759-review/e2e_images/i2i_teapot_blue.png differ diff --git a/artifacts/pr7759-review/e2e_images/t2i_teapot.png b/artifacts/pr7759-review/e2e_images/t2i_teapot.png new file mode 100644 index 000000000..0f62d99f4 Binary files /dev/null and b/artifacts/pr7759-review/e2e_images/t2i_teapot.png differ diff --git a/artifacts/pr7759-review/e2e_slurm.sh b/artifacts/pr7759-review/e2e_slurm.sh new file mode 100644 index 000000000..a48597164 --- /dev/null +++ b/artifacts/pr7759-review/e2e_slurm.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +#SBATCH --partition=batch +#SBATCH --nodes=1 +#SBATCH --time=90 +#SBATCH --job-name=qi21-e2e +#SBATCH --output=/home/mozf/vllm-omni/artifacts/pr7759-review/e2e_slurm_%j.out + +set -euo pipefail +cd /home/mozf/vllm-omni + +export HF_HOME=/mnt/lustre/hf-models +export QWEN_IMAGE_21_TEST_MODEL=Qwen/Qwen-Image-2.1 +export VLLM_LOGGING_LEVEL=INFO + +echo "node: $(hostname)" +nvidia-smi -L | head -4 +.venv/bin/python -c 'import vllm; print("vllm", vllm.__version__)' + +.venv/bin/python -m pytest tests/e2e/online_serving/test_qwen_image_21.py \ + -o addopts="" -v -s 2>&1 diff --git a/artifacts/pr7759-review/e2e_slurm_30092.out b/artifacts/pr7759-review/e2e_slurm_30092.out new file mode 100644 index 000000000..a1b0e6453 --- /dev/null +++ b/artifacts/pr7759-review/e2e_slurm_30092.out @@ -0,0 +1,742 @@ +node: gb200-rack1-01 +GPU 0: NVIDIA GB200 (UUID: GPU-56e80c56-4d0c-e81e-db5f-70f33aab4e18) +GPU 1: NVIDIA GB200 (UUID: GPU-2ce1a9bc-dbf1-3bbd-116d-6b4e61022280) +GPU 2: NVIDIA GB200 (UUID: GPU-f7cdefd9-3d67-384f-dbc1-53bc44d93985) +GPU 3: NVIDIA GB200 (UUID: GPU-7c9bf698-f3f3-4634-9309-dc1e32f62fba) +vllm 0.29.0 +INFO 09-21 08:43:18 [patch.py:252] NVFP4 W4A4 weight_scale NaN-clamp: installed. +INFO 09-21 08:43:21 [quack_fp8.py:76] Quack FP8 fused-bias GEMM enabled (CuteDSL). +INFO 09-21 08:43:21 [quack_fp8.py:155] Patched FlashInfer FP8 ScaledMM to use quack fused-bias GEMM. +INFO 09-21 08:43:21 [patch.py:503] inductor factorable-divisibility patch: installed. +INFO 09-21 08:43:21 [patch.py:568] [cumem-cuda] CuMemAllocator._python_free_callback patched: asleep guard extended to all platforms. +============================= test session starts ============================== +platform linux -- Python 3.12.3, pytest-9.1.1, pluggy-1.6.0 -- /home/mozf/vllm-omni/.venv/bin/python +cachedir: .pytest_cache +rootdir: /home/mozf/vllm-omni +configfile: pyproject.toml +plugins: asyncio-1.4.0, mock-3.15.1, anyio-4.14.2 +asyncio: mode=Mode.AUTO, debug=False, asyncio_default_fixture_loop_scope=None, asyncio_default_test_loop_scope=function +collecting ... INFO 09-21 08:43:27 [fa.py:105] Using CuTe FlashAttention-4 on Blackwell +collected 4 items + +tests/e2e/online_serving/test_qwen_image_21.py::test_text_to_image[request] INFO 09-21 08:43:32 [scheduler.py:277] Chunked prefill is enabled with max_num_batched_tokens=2048. +INFO 09-21 08:43:32 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']) +/home/mozf/vllm-omni/vllm_omni/version.py:55: RuntimeWarning: vLLM and vLLM-Omni appear to have mismatched major/minor versions: + --> vLLM-Omni version 0.26.1.dev1+g76de642c5 + --> vLLM version 0.29.0 +This will likely cause compatibility issues. + warn_if_misaligned_vllm_version() +INFO 09-21 08:43:50 [patch.py:252] NVFP4 W4A4 weight_scale NaN-clamp: installed. +INFO 09-21 08:43:50 [quack_fp8.py:76] Quack FP8 fused-bias GEMM enabled (CuteDSL). +INFO 09-21 08:43:50 [quack_fp8.py:155] Patched FlashInfer FP8 ScaledMM to use quack fused-bias GEMM. +INFO 09-21 08:43:50 [patch.py:503] inductor factorable-divisibility patch: installed. +INFO 09-21 08:43:50 [patch.py:568] [cumem-cuda] CuMemAllocator._python_free_callback patched: asleep guard extended to all platforms. +INFO 09-21 08:43:55 [fa.py:105] Using CuTe FlashAttention-4 on Blackwell +INFO 09-21 08:43:57 [serve.py:313] Detected diffusion model: Qwen/Qwen-Image-2.1 +INFO 09-21 08:43:57 [logo.py:52] █ █ █▄ ▄█ ▄▀▀▀▀▄ █▄ ▄█ █▄ █ ▀█▀ +INFO 09-21 08:43:57 [logo.py:52] ▄▄ ▄█ █ █ █ ▀▄▀ █ ▄▄▄ █ █ █ ▀▄▀ █ █ ▀▄ █ █ +INFO 09-21 08:43:57 [logo.py:52] █▄█▀ █ █ █ █ █ █ █ █ █ ▀▄█ █ +INFO 09-21 08:43:57 [logo.py:52] ▀▀ ▀▀▀▀▀ ▀▀▀▀▀ ▀ ▀ ▀▀▀▀ ▀ ▀ ▀ ▀ ▀▀▀ +INFO 09-21 08:43:57 [logo.py:52] +(APIServer pid=2856381) INFO 09-21 08:43:57 [api_utils.py:347] vLLM server version 0.29.0, serving model Qwen/Qwen-Image-2.1 +(APIServer pid=2856381) INFO 09-21 08:43:57 [api_utils.py:286] non-default args: {'model_tag': 'Qwen/Qwen-Image-2.1', 'host': '127.0.0.1', 'port': 45115, 'model': 'Qwen/Qwen-Image-2.1', 'max_num_seqs': 2} +(APIServer pid=2856381) INFO 09-21 08:43:58 [weight_utils.py:58] Using model weights format ['*'] +(APIServer pid=2856381) INFO 09-21 08:45:02 [weight_utils.py:90] Time spent downloading weights for Qwen/Qwen-Image-2.1: 64.568430 seconds +(APIServer pid=2856381) INFO 09-21 08:45:02 [omni_base.py:208] [AsyncOmni] Initializing with model Qwen/Qwen-Image-2.1 +(APIServer pid=2856381) INFO 09-21 08:45:02 [omni_engine_base.py:163] [OmniEngine] Initializing with model Qwen/Qwen-Image-2.1 +(APIServer pid=2856381) INFO 09-21 08:45:05 [config_factory.py:697] Resolved generic diffusion final_output_type='image' for model_class_name='QwenImage21Pipeline'. +(APIServer pid=2856381) INFO 09-21 08:45:05 [omni_engine_base.py:312] [OmniEngine] Launching Orchestrator thread with 1 stages +(APIServer pid=2856381) INFO 09-21 08:45:05 [weight_utils.py:58] Using model weights format ['*'] +(APIServer pid=2856381) INFO 09-21 08:45:05 [weight_utils.py:58] Using model weights format ['*'] +(APIServer pid=2856381) INFO 09-21 08:45:06 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']) +(APIServer pid=2856381) INFO 09-21 08:45:06 [diffusion_worker.py:332] Final IR op priority after setting vLLM-Omni overrides: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native']) +(APIServer pid=2856381) INFO 09-21 08:45:06 [diffusion_worker.py:344] Worker 0: Initialized device and distributed environment. +(APIServer pid=2856381) INFO 09-21 08:45:06 [parallel_state.py:610] Building SP subgroups from explicit sp_group_ranks (sp_size=1, ulysses=1, ring=1, use_ulysses_low=True). +(APIServer pid=2856381) INFO 09-21 08:45:06 [parallel_state.py:652] SP group details for rank 0: sp_group=[0], ulysses_group=[0], ring_group=[0] +(DiffusionWorker pid=2856381) INFO 09-21 08:45:07 [hub_prefetch.py:299] Prefetching Qwen/Qwen-Image-2.1 subfolders: ['scheduler', 'text_encoder', 'vae', 'processor'] +(DiffusionWorker pid=2856381) INFO 09-21 08:45:07 [hub_prefetch.py:224] Acquired flock prefetch lock for Qwen/Qwen-Image-2.1 at /mnt/lustre/hf-models/locks/vllm-omni-prefetch/Qwen__Qwen-Image-2.1.lock +(DiffusionWorker pid=2856381) Fetching 22 files: 0%| | 0/22 [00:00 vLLM-Omni version 0.26.1.dev1+g76de642c5 + --> vLLM version 0.29.0 +This will likely cause compatibility issues. + warn_if_misaligned_vllm_version() +INFO 09-21 08:45:52 [patch.py:252] NVFP4 W4A4 weight_scale NaN-clamp: installed. +INFO 09-21 08:45:53 [quack_fp8.py:76] Quack FP8 fused-bias GEMM enabled (CuteDSL). +INFO 09-21 08:45:53 [quack_fp8.py:155] Patched FlashInfer FP8 ScaledMM to use quack fused-bias GEMM. +INFO 09-21 08:45:53 [patch.py:503] inductor factorable-divisibility patch: installed. +INFO 09-21 08:45:53 [patch.py:568] [cumem-cuda] CuMemAllocator._python_free_callback patched: asleep guard extended to all platforms. +INFO 09-21 08:45:57 [fa.py:105] Using CuTe FlashAttention-4 on Blackwell +INFO 09-21 08:45:58 [serve.py:313] Detected diffusion model: Qwen/Qwen-Image-2.1 +INFO 09-21 08:45:58 [logo.py:52] █ █ █▄ ▄█ ▄▀▀▀▀▄ █▄ ▄█ █▄ █ ▀█▀ +INFO 09-21 08:45:58 [logo.py:52] ▄▄ ▄█ █ █ █ ▀▄▀ █ ▄▄▄ █ █ █ ▀▄▀ █ █ ▀▄ █ █ +INFO 09-21 08:45:58 [logo.py:52] █▄█▀ █ █ █ █ █ █ █ █ █ ▀▄█ █ +INFO 09-21 08:45:58 [logo.py:52] ▀▀ ▀▀▀▀▀ ▀▀▀▀▀ ▀ ▀ ▀▀▀▀ ▀ ▀ ▀ ▀ ▀▀▀ +INFO 09-21 08:45:58 [logo.py:52] +(APIServer pid=2864057) INFO 09-21 08:45:58 [api_utils.py:347] vLLM server version 0.29.0, serving model Qwen/Qwen-Image-2.1 +(APIServer pid=2864057) INFO 09-21 08:45:58 [api_utils.py:286] non-default args: {'model_tag': 'Qwen/Qwen-Image-2.1', 'host': '127.0.0.1', 'port': 44729, 'model': 'Qwen/Qwen-Image-2.1', 'max_num_seqs': 2} +(APIServer pid=2864057) INFO 09-21 08:45:58 [weight_utils.py:58] Using model weights format ['*'] +(APIServer pid=2864057) INFO 09-21 08:45:58 [omni_base.py:208] [AsyncOmni] Initializing with model Qwen/Qwen-Image-2.1 +(APIServer pid=2864057) INFO 09-21 08:45:58 [omni_engine_base.py:163] [OmniEngine] Initializing with model Qwen/Qwen-Image-2.1 +(APIServer pid=2864057) INFO 09-21 08:46:00 [config_factory.py:697] Resolved generic diffusion final_output_type='image' for model_class_name='QwenImage21Pipeline'. +(APIServer pid=2864057) INFO 09-21 08:46:00 [omni_engine_base.py:312] [OmniEngine] Launching Orchestrator thread with 1 stages +(APIServer pid=2864057) INFO 09-21 08:46:00 [weight_utils.py:58] Using model weights format ['*'] +(APIServer pid=2864057) INFO 09-21 08:46:00 [weight_utils.py:58] Using model weights format ['*'] +(APIServer pid=2864057) INFO 09-21 08:46:01 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']) +(APIServer pid=2864057) INFO 09-21 08:46:01 [diffusion_worker.py:332] Final IR op priority after setting vLLM-Omni overrides: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native']) +(APIServer pid=2864057) INFO 09-21 08:46:01 [diffusion_worker.py:344] Worker 0: Initialized device and distributed environment. +(APIServer pid=2864057) INFO 09-21 08:46:01 [parallel_state.py:610] Building SP subgroups from explicit sp_group_ranks (sp_size=1, ulysses=1, ring=1, use_ulysses_low=True). +(APIServer pid=2864057) INFO 09-21 08:46:01 [parallel_state.py:652] SP group details for rank 0: sp_group=[0], ulysses_group=[0], ring_group=[0] +(DiffusionWorker pid=2864057) INFO 09-21 08:46:01 [hub_prefetch.py:299] Prefetching Qwen/Qwen-Image-2.1 subfolders: ['scheduler', 'text_encoder', 'vae', 'processor'] +(DiffusionWorker pid=2864057) INFO 09-21 08:46:01 [hub_prefetch.py:224] Acquired flock prefetch lock for Qwen/Qwen-Image-2.1 at /mnt/lustre/hf-models/locks/vllm-omni-prefetch/Qwen__Qwen-Image-2.1.lock +(DiffusionWorker pid=2864057) Fetching 22 files: 0%| | 0/22 [00:00 vLLM-Omni version 0.26.1.dev1+g76de642c5 + --> vLLM version 0.29.0 + This will likely cause compatibility issues. + warn_if_misaligned_vllm_version() + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +--- Running Summary +================== 4 passed, 15 warnings in 173.25s (0:02:53) ================== diff --git a/artifacts/pr7759-review/gen_images_30094.out b/artifacts/pr7759-review/gen_images_30094.out new file mode 100644 index 000000000..d2c9d150d --- /dev/null +++ b/artifacts/pr7759-review/gen_images_30094.out @@ -0,0 +1,204 @@ +node: gb200-rack1-01 +=== T2I === +/home/mozf/vllm-omni/vllm_omni/version.py:55: RuntimeWarning: vLLM and vLLM-Omni appear to have mismatched major/minor versions: + --> vLLM-Omni version 0.26.1.dev1+g76de642c5 + --> vLLM version 0.29.0 +This will likely cause compatibility issues. + warn_if_misaligned_vllm_version() +INFO 09-21 08:49:45 [patch.py:252] NVFP4 W4A4 weight_scale NaN-clamp: installed. +INFO 09-21 08:49:46 [quack_fp8.py:76] Quack FP8 fused-bias GEMM enabled (CuteDSL). +INFO 09-21 08:49:46 [quack_fp8.py:155] Patched FlashInfer FP8 ScaledMM to use quack fused-bias GEMM. +INFO 09-21 08:49:46 [patch.py:503] inductor factorable-divisibility patch: installed. +INFO 09-21 08:49:46 [patch.py:568] [cumem-cuda] CuMemAllocator._python_free_callback patched: asleep guard extended to all platforms. +INFO 09-21 08:49:47 [fa.py:105] Using CuTe FlashAttention-4 on Blackwell +INFO 09-21 08:49:49 [weight_utils.py:58] Using model weights format ['*'] +INFO 09-21 08:49:49 [omni_base.py:208] [Omni] Initializing with model Qwen/Qwen-Image-2.1 +INFO 09-21 08:49:49 [omni_engine_base.py:163] [OmniEngine] Initializing with model Qwen/Qwen-Image-2.1 +INFO 09-21 08:49:51 [config_factory.py:697] Resolved generic diffusion final_output_type='image' for model_class_name='QwenImage21Pipeline'. +INFO 09-21 08:49:51 [omni_engine_base.py:312] [OmniEngine] Launching Orchestrator thread with 1 stages +INFO 09-21 08:49:51 [stage_init_utils.py:775] [stage_init] Set VLLM_WORKER_MULTIPROC_METHOD=spawn +INFO 09-21 08:49:51 [weight_utils.py:58] Using model weights format ['*'] +INFO 09-21 08:49:51 [weight_utils.py:58] Using model weights format ['*'] +INFO 09-21 08:49:51 [scheduler.py:277] Chunked prefill is enabled with max_num_batched_tokens=2048. +INFO 09-21 08:49:51 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']) +INFO 09-21 08:49:51 [diffusion_worker.py:332] Final IR op priority after setting vLLM-Omni overrides: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native']) +INFO 09-21 08:49:51 [diffusion_worker.py:344] Worker 0: Initialized device and distributed environment. +INFO 09-21 08:49:51 [parallel_state.py:610] Building SP subgroups from explicit sp_group_ranks (sp_size=1, ulysses=1, ring=1, use_ulysses_low=True). +INFO 09-21 08:49:51 [parallel_state.py:652] SP group details for rank 0: sp_group=[0], ulysses_group=[0], ring_group=[0] +(DiffusionWorker pid=2877460) INFO 09-21 08:49:52 [hub_prefetch.py:299] Prefetching Qwen/Qwen-Image-2.1 subfolders: ['scheduler', 'text_encoder', 'vae', 'processor'] +(DiffusionWorker pid=2877460) INFO 09-21 08:49:52 [hub_prefetch.py:224] Acquired flock prefetch lock for Qwen/Qwen-Image-2.1 at /mnt/lustre/hf-models/locks/vllm-omni-prefetch/Qwen__Qwen-Image-2.1.lock +(DiffusionWorker pid=2877460) Fetching 22 files: 0%| | 0/22 [00:00 vLLM-Omni version 0.26.1.dev1+g76de642c5 + --> vLLM version 0.29.0 +This will likely cause compatibility issues. + warn_if_misaligned_vllm_version() +INFO 09-21 08:50:24 [patch.py:252] NVFP4 W4A4 weight_scale NaN-clamp: installed. +INFO 09-21 08:50:26 [quack_fp8.py:76] Quack FP8 fused-bias GEMM enabled (CuteDSL). +INFO 09-21 08:50:26 [quack_fp8.py:155] Patched FlashInfer FP8 ScaledMM to use quack fused-bias GEMM. +INFO 09-21 08:50:26 [patch.py:503] inductor factorable-divisibility patch: installed. +INFO 09-21 08:50:26 [patch.py:568] [cumem-cuda] CuMemAllocator._python_free_callback patched: asleep guard extended to all platforms. +INFO 09-21 08:50:29 [fa.py:105] Using CuTe FlashAttention-4 on Blackwell +INFO 09-21 08:50:31 [weight_utils.py:58] Using model weights format ['*'] +INFO 09-21 08:50:31 [omni_base.py:208] [Omni] Initializing with model Qwen/Qwen-Image-2.1 +INFO 09-21 08:50:31 [omni_engine_base.py:163] [OmniEngine] Initializing with model Qwen/Qwen-Image-2.1 +INFO 09-21 08:50:33 [config_factory.py:697] Resolved generic diffusion final_output_type='image' for model_class_name='QwenImage21Pipeline'. +INFO 09-21 08:50:33 [omni_engine_base.py:312] [OmniEngine] Launching Orchestrator thread with 1 stages +INFO 09-21 08:50:33 [stage_init_utils.py:775] [stage_init] Set VLLM_WORKER_MULTIPROC_METHOD=spawn +INFO 09-21 08:50:33 [weight_utils.py:58] Using model weights format ['*'] +INFO 09-21 08:50:33 [weight_utils.py:58] Using model weights format ['*'] +INFO 09-21 08:50:33 [scheduler.py:277] Chunked prefill is enabled with max_num_batched_tokens=2048. +INFO 09-21 08:50:33 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']) +INFO 09-21 08:50:33 [diffusion_worker.py:332] Final IR op priority after setting vLLM-Omni overrides: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native']) +INFO 09-21 08:50:33 [diffusion_worker.py:344] Worker 0: Initialized device and distributed environment. +INFO 09-21 08:50:33 [parallel_state.py:610] Building SP subgroups from explicit sp_group_ranks (sp_size=1, ulysses=1, ring=1, use_ulysses_low=True). +INFO 09-21 08:50:33 [parallel_state.py:652] SP group details for rank 0: sp_group=[0], ulysses_group=[0], ring_group=[0] +(DiffusionWorker pid=2879936) INFO 09-21 08:50:34 [hub_prefetch.py:299] Prefetching Qwen/Qwen-Image-2.1 subfolders: ['scheduler', 'text_encoder', 'vae', 'processor'] +(DiffusionWorker pid=2879936) INFO 09-21 08:50:34 [hub_prefetch.py:224] Acquired flock prefetch lock for Qwen/Qwen-Image-2.1 at /mnt/lustre/hf-models/locks/vllm-omni-prefetch/Qwen__Qwen-Image-2.1.lock +(DiffusionWorker pid=2879936) Fetching 22 files: 0%| | 0/22 [00:00 str: + """Return an ``http(s)`` or ``data:`` URL for the session's first frame.""" + if value is None: + from vllm.assets.image import ImageAsset + + value = str(ImageAsset(_DEFAULT_IMAGE_ASSET).get_path("jpg")) + if value.startswith(("http://", "https://", "data:")): + return value + path = Path(value).expanduser() + if not path.is_file(): + raise BenchmarkError(f"--image is neither a URL nor a readable file: {value}") + media_type = mimetypes.guess_type(path.name)[0] or "image/png" + return f"data:{media_type};base64,{base64.b64encode(path.read_bytes()).decode()}" + + +def build_workload(args: argparse.Namespace) -> Workload: + if args.workload is not None: + # The file names its own first frame, so resolving happens inside + # load_workload -- never eagerly, which would fetch the default asset a + # --workload run has no use for. + return load_workload( + args.workload, + image_resolver=lambda value: resolve_image_reference(value if args.image is None else args.image), + overrides={ + "width": args.width_override, + "height": args.height_override, + "fps": args.fps_override, + "seed": args.seed_override, + "prompt": args.prompt_override, + "flow_shift": args.flow_shift, + "negative_prompt": args.negative_prompt, + }, + ) + return default_workload( + image_reference=resolve_image_reference(args.image), + num_chunks=args.num_chunks, + camera_pattern=args.camera_pattern, + prompt=args.prompt_override if args.prompt_override is not None else DEFAULT_PROMPT, + width=args.width_override if args.width_override is not None else DEFAULT_WIDTH, + height=args.height_override if args.height_override is not None else DEFAULT_HEIGHT, + fps=args.fps_override if args.fps_override is not None else DEFAULT_FPS, + seed=args.seed_override if args.seed_override is not None else DEFAULT_SEED, + flow_shift=args.flow_shift if args.flow_shift is not None else DEFAULT_FLOW_SHIFT, + negative_prompt=args.negative_prompt, + ) + + +# --------------------------------------------------------------------------- # +# One streaming session +# --------------------------------------------------------------------------- # + + +@dataclass +class SessionResult: + request_id: str | None + records: list[ChunkRecord] + media: bytes + ttfc_s: float | None + video_start_s: float | None + wall_s: float + done_event: dict[str, Any] + + +async def _keepalive(websocket: Any, send_lock: asyncio.Lock, interval: float) -> None: + """Refresh the server's stall clock while a slow first chunk is generating. + + A compiled deploy config records CUDA graphs on the first block, which can + outlast the server's ~60 s stall timeout. ``session.ping`` is the documented + way to say the client is still there. + """ + while True: + await asyncio.sleep(interval) + async with send_lock: + await websocket.send(json.dumps({"type": "session.ping"})) + + +async def run_session( + url: str, + model: str, + workload: Workload, + *, + first_chunk_timeout: float, + chunk_timeout: float, + ping_interval: float, + collect_media: bool, + print_chunks: bool, +) -> SessionResult: + """Drive one rollout and timestamp every chunk as its bytes land.""" + try: + from websockets.asyncio.client import connect + except ImportError: # pragma: no cover - exercised only without websockets + try: + from websockets import connect # type: ignore[attr-defined] + except ImportError as exc: + raise BenchmarkError("This benchmark needs the websockets package: pip install websockets") from exc + + payload = workload.session_start_payload(model) + records: list[ChunkRecord] = [] + media = bytearray() + pending: dict[str, Any] | None = None + request_id: str | None = None + ttfc_s: float | None = None + video_start_s: float | None = None + last_arrival: float | None = None + done_event: dict[str, Any] = {} + send_lock = asyncio.Lock() + keepalive: asyncio.Task[None] | None = None + # Fire remaining prompt updates in chunk order as their boundary passes. + updates = sorted(workload.prompt_updates, key=lambda update: update.after_chunk) + update_index = 0 + + async with connect(url, max_size=None, ping_interval=None) as websocket: + async with send_lock: + await websocket.send(json.dumps(payload, ensure_ascii=False)) + started = time.perf_counter() + if ping_interval > 0: + keepalive = asyncio.create_task(_keepalive(websocket, send_lock, ping_interval)) + + try: + while True: + timeout = first_chunk_timeout if not records else chunk_timeout + try: + deadline = started + (last_arrival if last_arrival is not None else 0.0) + timeout + remaining = deadline - time.perf_counter() + if remaining <= 0: + raise asyncio.TimeoutError + message = await asyncio.wait_for(websocket.recv(), timeout=remaining) + except asyncio.TimeoutError as exc: + raise BenchmarkError( + f"No media chunk for {timeout:g} s after {len(records)} chunk(s); " + "the server may still be loading, compiling, or capturing graphs." + ) from exc + now = time.perf_counter() + + if isinstance(message, (bytes, bytearray)): + if pending is None: + raise BenchmarkError("Received a binary frame with no preceding video.chunk_metadata event.") + if collect_media: + media.extend(message) + if pending.get("kind") == "media": + index = int(pending["generation_chunk_index"]) + arrival = now - started + if ttfc_s is None: + ttfc_s = arrival + records.append( + ChunkRecord( + index=index, + arrival_s=arrival, + inter_arrival_s=arrival - (last_arrival if last_arrival is not None else 0.0), + byte_length=len(message), + num_frames=int(pending["num_frames"]), + ) + ) + last_arrival = arrival + if print_chunks: + print( + f" chunk {index:3d} t={arrival:8.3f}s " + f"interval={records[-1].inter_arrival_s * 1000:8.1f}ms " + f"frames={records[-1].num_frames:3d} bytes={len(message)}", + flush=True, + ) + while update_index < len(updates) and updates[update_index].after_chunk <= index: + async with send_lock: + await websocket.send(json.dumps(updates[update_index].to_payload(), ensure_ascii=False)) + update_index += 1 + pending = None + continue + + event = json.loads(message) + event_type = event.get("type") + if event_type == "video.chunk_metadata": + pending = event + elif event_type == "video.start": + request_id = event.get("request_id") + video_start_s = now - started + elif event_type == "error": + # Say what was asked for alongside what the server refused: + # some builds cap a rollout's num_frames, and the operator + # needs to know which --num-chunks would fit. + raise BenchmarkError( + f"Server rejected the session after {len(records)} chunk(s): " + f"{event.get('message', event)} " + f"(requested {workload.num_chunks} chunks = num_frames {workload.num_frames})" + ) + elif event_type == "session.done": + done_event = event + break + finally: + if keepalive is not None: + keepalive.cancel() + try: + await keepalive + except asyncio.CancelledError: + pass + + wall_s = time.perf_counter() - started + + if not records: + raise BenchmarkError("The session finished without delivering a single media chunk.") + if done_event.get("stopped"): + raise BenchmarkError(f"The session was stopped before it finished: {done_event}") + observed = [record.index for record in records] + if observed != list(range(len(records))): + raise BenchmarkError(f"Chunk indices are not contiguous from zero: {observed}") + if len(records) != workload.num_chunks: + raise BenchmarkError( + f"Expected {workload.num_chunks} chunks for num_frames={workload.num_frames}, received {len(records)}." + ) + + return SessionResult( + request_id=request_id, + records=records, + media=bytes(media), + ttfc_s=ttfc_s, + video_start_s=video_start_s, + wall_s=wall_s, + done_event=done_event, + ) + + +# --------------------------------------------------------------------------- # +# Reporting +# --------------------------------------------------------------------------- # + + +def _git_revision() -> str | None: + try: + return subprocess.check_output( + ["git", "-C", str(REPO_ROOT), "rev-parse", "HEAD"], + text=True, + stderr=subprocess.DEVNULL, + ).strip() + except (OSError, subprocess.CalledProcessError): + return None + + +def _print_row(label: str, value: str) -> None: + print(f"{label:<44}{value:>16}") + + +def print_report( + metrics: dict[str, Any], + *, + workload: Workload, + endpoint: str, + sessions: int, + target_fps: float, +) -> None: + header = " LingBot-World realtime streaming benchmark " + print("\n{s:{c}^{n}}".format(s=header, n=60, c="=")) + _print_row("Endpoint:", endpoint) + _print_row("Geometry:", f"{workload.width}x{workload.height}") + _print_row("Chunks per session:", str(workload.num_chunks)) + _print_row("Sessions:", str(sessions)) + frames = workload.num_frames + _print_row("Real-time basis:", f"{target_fps:g} fps") + _print_row("Video seconds per session:", f"{frames / target_fps:.2f} ({frames} frames)") + if abs(target_fps - workload.fps) > 1e-9: + _print_row("Mux label sent to server:", f"{workload.fps} fps") + + if sessions == 1: + print("{s:{c}^{n}}".format(s=" Cadence ", n=60, c="-")) + if metrics.get("ttfc_ms") is not None: + _print_row("TTFC (session.start -> first chunk):", f"{metrics['ttfc_ms']:.1f} ms") + interval_all = metrics.get("interval_all") or {} + interval_steady = metrics.get("interval_steady") or {} + for title, summary in (("All chunks", interval_all), ("Steady state", interval_steady)): + if not summary: + continue + print(f" {title} (n={int(summary['count'])}):") + _print_row(" mean / median:", f"{summary['mean_ms']:.1f} / {summary['median_ms']:.1f} ms") + _print_row(" std / max:", f"{summary['std_ms']:.1f} / {summary['max_ms']:.1f} ms") + tail = f"{summary['p90_ms']:.0f} / {summary['p95_ms']:.0f} / {summary['p99_ms']:.0f} ms" + _print_row(" p90 / p95 / p99:", tail) + if summary["count"] < 100: + print( + " NOTE: fewer than 100 intervals; tail percentiles are descriptive, not reliable tail estimates." + ) + + print("{s:{c}^{n}}".format(s=" Real time ", n=60, c="-")) + _print_row("VIDEO_RTF (wall s / video s, <1 real time):", f"{metrics['video_rtf']:.3f}") + _print_row("VIDEO_RTFX (video s / wall s):", f"{metrics['video_rtfx']:.3f}") + if metrics.get("steady_video_rtf") is not None: + _print_row("VIDEO_RTF steady state:", f"{metrics['steady_video_rtf']:.3f}") + _print_row("Frames per second:", f"{metrics['frames_per_second']:.2f}") + _print_row("Chunk deadline:", f"{metrics['chunk_deadline_ms']:.1f} ms") + if metrics.get("slo_attainment") is not None: + _print_row( + "Steady chunks within deadline:", + f"{metrics['slo_attainment'] * 100:.1f}% ({metrics['slo_violations']} late)", + ) + if not metrics.get("steady_state_reached", False): + print( + f" NOTE: no chunk reached the steady state; the attention window needs " + f"{metrics['warmup_chunks']} chunks. Raise --num-chunks." + ) + + playback = metrics.get("playback") or {} + print("{s:{c}^{n}}".format(s=" Playback continuity ", n=60, c="-")) + _print_row("Start buffer:", f"{playback.get('buffer_chunks')} chunk(s)") + _print_row("Underruns:", str(playback.get("underrun_count"))) + _print_row("Total stall:", f"{playback.get('total_stall_ms', 0.0):.1f} ms") + _print_row("Stall ratio:", f"{playback.get('stall_ratio', 0.0) * 100:.2f}%") + if playback.get("first_underrun_chunk") is not None: + _print_row("First underrun at chunk:", str(playback["first_underrun_chunk"])) + else: + print("{s:{c}^{n}}".format(s=" Across sessions ", n=60, c="-")) + if metrics.get("ttfc_ms_mean") is not None: + _print_row("TTFC mean / max:", f"{metrics['ttfc_ms_mean']:.1f} / {metrics['ttfc_ms_max']:.1f} ms") + if metrics.get("steady_interval_ms_mean") is not None: + _print_row("Steady interval mean:", f"{metrics['steady_interval_ms_mean']:.1f} ms") + _print_row("Steady interval spread:", f"{metrics['steady_interval_ms_spread']:.1f} ms") + _print_row("VIDEO_RTF mean (lower is better):", f"{metrics['video_rtf_mean']:.3f}") + _print_row( + "VIDEO_RTF best / worst:", + f"{metrics['video_rtf_best']:.3f} / {metrics['video_rtf_worst']:.3f}", + ) + _print_row("Chunk deadline:", f"{metrics['chunk_deadline_ms']:.1f} ms") + _print_row("Underruns (all sessions):", str(metrics["underrun_count"])) + print("=" * 60) + + +def save_video(media: bytes, path: Path, *, fps: int) -> None: + from vllm_omni.diffusion.utils.media_utils import finalize_streaming_video_bytes + + playback_bytes = finalize_streaming_video_bytes(media, input_format="m4s", fps=fps) + path.parent.mkdir(parents=True, exist_ok=True) + tmp_path = path.with_suffix(path.suffix + ".tmp") + tmp_path.write_bytes(playback_bytes) + os.replace(tmp_path, path) + print(f"Saved {len(media)} streamed bytes -> {len(playback_bytes)} playback bytes at {path}") + + +# --------------------------------------------------------------------------- # +# CLI +# --------------------------------------------------------------------------- # + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Benchmark LingBot-World 2.0 realtime streaming over WS /v1/realtime/video.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + endpoint = parser.add_argument_group("endpoint") + endpoint.add_argument("--host", default="127.0.0.1") + endpoint.add_argument("--port", type=int, default=8000) + endpoint.add_argument("--base-url", default=None, help="Full ws:// URL; overrides --host/--port.") + endpoint.add_argument("--model", default=DEFAULT_MODEL) + + workload = parser.add_argument_group("workload") + workload.add_argument("--workload", default=None, help="JSON rollout spec; overrides the built-in workload.") + workload.add_argument("--image", default=None, help="First frame: path or URL. Defaults to the shared vLLM asset.") + workload.add_argument("--prompt", dest="prompt_override", default=None, help=f"Default: {DEFAULT_PROMPT!r}") + workload.add_argument("--negative-prompt", default=None) + workload.add_argument("--num-chunks", type=int, default=16, help="AR blocks per session; ignored with --workload.") + workload.add_argument( + "--camera-pattern", + choices=CAMERA_PATTERNS, + default="forward", + help="Camera pattern for the built-in workload only; ignored with --workload.", + ) + workload.add_argument("--width", dest="width_override", type=int, default=None, help=f"Default: {DEFAULT_WIDTH}") + workload.add_argument("--height", dest="height_override", type=int, default=None, help=f"Default: {DEFAULT_HEIGHT}") + workload.add_argument("--fps", dest="fps_override", type=int, default=None, help=f"Default: {DEFAULT_FPS}") + workload.add_argument("--seed", dest="seed_override", type=int, default=None, help=f"Default: {DEFAULT_SEED}") + workload.add_argument("--flow-shift", type=float, default=None, help=f"Default: {DEFAULT_FLOW_SHIFT}") + + run = parser.add_argument_group("run") + run.add_argument("--sessions", type=int, default=1, help="Sequential measured rollouts.") + run.add_argument("--warmup-sessions", type=int, default=0, help="Unmeasured rollouts first (pays compile cost).") + run.add_argument( + "--warmup-chunks", + type=int, + default=DEFAULT_WARMUP_CHUNKS, + help="Chunks excluded from steady state; the attention window saturates after 6.", + ) + run.add_argument( + "--chunk-slo-ms", + type=float, + default=None, + help="Deadline a steady chunk must meet. Default: one chunk of video at --target-fps.", + ) + run.add_argument( + "--target-fps", + type=float, + default=None, + help=( + "Playback rate used for real-time factor, the chunk deadline, and the playback " + "simulation. The checkpoint declares no frame rate, so this is a consumption " + "choice, not a model property. Default: the --fps sent to the server." + ), + ) + run.add_argument("--playback-buffer-chunks", type=int, default=1, help="Chunks buffered before playback starts.") + run.add_argument("--first-chunk-timeout", type=float, default=900.0) + run.add_argument("--chunk-timeout", type=float, default=300.0) + run.add_argument("--ping-interval", type=float, default=20.0, help="session.ping cadence; 0 disables.") + + output = parser.add_argument_group("output") + output.add_argument("--output-json", default=None) + output.add_argument("--save-video", default=None, help="Write the last measured session's video here.") + output.add_argument("--print-chunks", action="store_true") + + args = parser.parse_args(argv) + if args.sessions < 1: + parser.error("--sessions must be at least 1.") + if args.warmup_sessions < 0: + parser.error("--warmup-sessions must be non-negative.") + if args.warmup_chunks < 0: + parser.error("--warmup-chunks must be non-negative.") + if args.playback_buffer_chunks < 1: + parser.error("--playback-buffer-chunks must be at least 1.") + if args.target_fps is not None and args.target_fps <= 0: + parser.error("--target-fps must be positive.") + if args.workload is None and args.num_chunks < 1: + parser.error("--num-chunks must be at least 1.") + return args + + +def endpoint_url(args: argparse.Namespace) -> str: + if args.base_url is not None: + base = args.base_url.rstrip("/") + for prefix, replacement in (("http://", "ws://"), ("https://", "wss://")): + if base.startswith(prefix): + base = replacement + base[len(prefix) :] + break + return base if base.endswith(_WS_PATH) else base + _WS_PATH + return f"ws://{args.host}:{args.port}{_WS_PATH}" + + +async def run_benchmark(args: argparse.Namespace) -> dict[str, Any]: + workload = build_workload(args) + url = endpoint_url(args) + # The mux label and the real-time basis are different questions; only the + # latter decides whether a run passes. + target_fps = args.target_fps if args.target_fps is not None else float(workload.fps) + if max(1, args.warmup_chunks) >= workload.num_chunks - 1: + print( + f"WARNING: --warmup-chunks={args.warmup_chunks} leaves no nonterminal steady-state chunk in a " + f"{workload.num_chunks}-chunk rollout; steady-state metrics will be empty.", + file=sys.stderr, + ) + + print(f"Connecting to {url}") + print( + f"Rollout: {workload.num_chunks} chunks, num_frames={workload.num_frames}, " + f"{workload.video_seconds:.2f}s of video" + ) + print( + f"Real-time basis: {target_fps:g} fps -> a steady 12-frame chunk is " + f"{steady_chunk_deadline_ms(target_fps):.1f} ms of video" + + ("" if args.target_fps is None else f" (mux label stays {workload.fps} fps)") + ) + + for index in range(args.warmup_sessions): + print(f"Warmup session {index + 1}/{args.warmup_sessions} (not measured)...") + await run_session( + url, + args.model, + workload, + first_chunk_timeout=args.first_chunk_timeout, + chunk_timeout=args.chunk_timeout, + ping_interval=args.ping_interval, + collect_media=False, + print_chunks=False, + ) + + session_metrics: list[dict[str, Any]] = [] + session_payloads: list[dict[str, Any]] = [] + last_result: SessionResult | None = None + for index in range(args.sessions): + if args.sessions > 1: + print(f"Session {index + 1}/{args.sessions}...") + result = await run_session( + url, + args.model, + workload, + first_chunk_timeout=args.first_chunk_timeout, + chunk_timeout=args.chunk_timeout, + ping_interval=args.ping_interval, + collect_media=args.save_video is not None, + print_chunks=args.print_chunks, + ) + last_result = result + metrics = compute_metrics( + result.records, + fps=target_fps, + warmup_chunks=args.warmup_chunks, + slo_ms=args.chunk_slo_ms, + playback_buffer_chunks=args.playback_buffer_chunks, + ttfc_s=result.ttfc_s, + session_wall_s=result.wall_s, + ) + session_metrics.append(metrics) + session_payloads.append( + { + "session_index": index, + "request_id": result.request_id, + "video_start_ms": (result.video_start_s * 1000.0) if result.video_start_s is not None else None, + "chunks": [record.to_dict() for record in result.records], + "metrics": metrics, + } + ) + + aggregate = aggregate_metrics(session_metrics, fps=target_fps) + print_report(aggregate, workload=workload, endpoint=url, sessions=args.sessions, target_fps=target_fps) + + if args.save_video is not None and last_result is not None: + save_video(last_result.media, Path(args.save_video), fps=workload.fps) + + document = { + "benchmark": "lingbot_world_realtime", + "timestamp": datetime.now(timezone.utc).isoformat(), + "endpoint": url, + "model": args.model, + "git_revision": _git_revision(), + "hostname": platform.node(), + "workload": workload.describe(), + "config": { + "sessions": args.sessions, + "warmup_sessions": args.warmup_sessions, + "warmup_chunks": args.warmup_chunks, + "camera_pattern": args.camera_pattern, + "chunk_slo_ms": args.chunk_slo_ms, + "target_fps": target_fps, + "mux_fps": workload.fps, + "playback_buffer_chunks": args.playback_buffer_chunks, + "workload_file": args.workload, + }, + "sessions": session_payloads, + "metrics": aggregate, + } + if args.output_json is not None: + path = Path(args.output_json) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(document, indent=2, sort_keys=True) + "\n") + print(f"Wrote {path}") + return document + + +def main(argv: list[str] | None = None) -> int: + args = parse_args(argv) + try: + asyncio.run(run_benchmark(args)) + except BenchmarkError as error: + print(f"ERROR: {error}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/lingbot_world/configs/single_gpu_eager.yaml b/benchmarks/lingbot_world/configs/single_gpu_eager.yaml new file mode 100644 index 000000000..58f36e4f0 --- /dev/null +++ b/benchmarks/lingbot_world/configs/single_gpu_eager.yaml @@ -0,0 +1,45 @@ +# Benchmark deploy config: LingBot-World 2.0 on one GPU, eager. +# +# This is the portable smoke configuration -- it matches the shipped +# vllm_omni/deploy/lingbot_world_v2_stepwise.yaml so a number measured here is +# comparable to the default deployment. It is NOT a realtime configuration: one +# GPU runs all five forwards of every AR block serially and nothing is compiled, +# so expect a chunk interval several times the real-time deadline. +# +# Use configs/usp4_compiled.yaml for the realtime topology. +# +# vllm serve robbyant/lingbot-world-v2-14b-causal-fast-diffusers \ +# --omni --deploy-config benchmarks/lingbot_world/configs/single_gpu_eager.yaml \ +# --port 8000 + +pipeline: lingbot_world +async_chunk: false +distributed_executor_backend: mp +dtype: bfloat16 + +stages: + - stage_id: 0 + devices: "0" + # AR-Diffusion binds one session per runner invocation; the runner rejects + # anything larger. + max_num_seqs: 1 + enforce_eager: true + model_class_name: LingBotWorldCausalDMDPipeline + engine_backend: vllm_omni.experimental.ar_diffusion.engine.ARDiffusionEngine + step_execution: true + streaming_output: true + default_sampling_params: + # The causal-fast checkpoint pins a four-step DMD schedule; this sizes the + # step budget before the pipeline is reached. + num_inference_steps: 4 + output_type: pil + extra_args: + flow_shift: 5.0 + model_config: + # The AR cache geometry is fixed at load time, so a benchmark request must + # ask for exactly this resolution. + ar_diffusion_height: 480 + ar_diffusion_width: 832 + ar_diffusion_kv_config: + gpu_memory_fraction: 0.6 + warmup_cudagraph: true diff --git a/benchmarks/lingbot_world/configs/usp4_compiled.yaml b/benchmarks/lingbot_world/configs/usp4_compiled.yaml new file mode 100644 index 000000000..ed71bc13e --- /dev/null +++ b/benchmarks/lingbot_world/configs/usp4_compiled.yaml @@ -0,0 +1,60 @@ +# Benchmark deploy config: LingBot-World 2.0 on four GPUs, Ulysses SP-4, compiled. +# +# This is the realtime topology. Two things differ from the single-GPU config, +# and both matter to the number the benchmark reports: +# +# parallel_config Pure Ulysses sequence parallelism over four ranks. The +# pipeline validates this: LingBot requires +# sequence_parallel_size == ulysses_degree, ring_degree and +# allgather_degree of 1, ulysses_mode "strict", and +# ulysses_a2a_permute disabled. Pipeline, CFG, and VAE +# parallel sizes above 1 are rejected. This config uses +# TP=1; the validator does not reject larger TP sizes. +# enforce_eager false puts LingBotDMDBlockRunner on the compiled path +# (regional torch.compile with cudagraph step marking). The +# first rollout therefore pays compile and capture cost -- +# use --warmup-sessions 1 before measured repetitions. +# Chunk warmup only excludes the attention-window ramp. +# +# Serve it under a GPU reservation, since it needs four cards: +# +# gpu run --gpus 4 -- vllm serve robbyant/lingbot-world-v2-14b-causal-fast-diffusers \ +# --omni --deploy-config benchmarks/lingbot_world/configs/usp4_compiled.yaml \ +# --port 8000 +# +# `devices` is the rank->card mapping inside the reservation, so it stays +# "0,1,2,3" no matter which physical cards CUDA_VISIBLE_DEVICES selected. + +pipeline: lingbot_world +async_chunk: false +distributed_executor_backend: mp +dtype: bfloat16 + +stages: + - stage_id: 0 + devices: "0,1,2,3" + max_num_seqs: 1 + enforce_eager: false + model_class_name: LingBotWorldCausalDMDPipeline + engine_backend: vllm_omni.experimental.ar_diffusion.engine.ARDiffusionEngine + step_execution: true + streaming_output: true + parallel_config: + tensor_parallel_size: 1 + sequence_parallel_size: 4 + ulysses_degree: 4 + ring_degree: 1 + default_sampling_params: + num_inference_steps: 4 + output_type: pil + extra_args: + flow_shift: 5.0 + model_config: + ar_diffusion_height: 480 + ar_diffusion_width: 832 + ar_diffusion_kv_config: + # Each rank holds its own shard of the paged KV cache, so the per-card + # fraction can stay at the single-GPU value; lower it if the four-rank + # start OOMs on smaller cards. + gpu_memory_fraction: 0.6 + warmup_cudagraph: true diff --git a/benchmarks/lingbot_world/workload.py b/benchmarks/lingbot_world/workload.py new file mode 100644 index 000000000..4b59c8899 --- /dev/null +++ b/benchmarks/lingbot_world/workload.py @@ -0,0 +1,631 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project +"""Workload construction and metric math for the LingBot-World realtime benchmark. + +Nothing here imports torch, vllm, or websockets, so the arithmetic that decides +what a benchmark number *means* is unit-testable without a GPU or a server. + +Two model properties drive every formula in this module, both read from the +checkpoint's ``transformer/config.json`` rather than assumed: + +``num_frames_per_block = 3`` + One AR block commits three latent frames, so one streamed chunk is three + latent frames and a request's ``camera_action_script`` carries exactly one + three-entry action list per chunk. + +``sliding_window_num_frames = 18`` with ``sink_size = 9`` + Self-attention sees nine sink frames plus a nine-frame recent window, so the + key count a block attends over keeps growing until eighteen latent frames of + history exist -- six chunks. Chunk cost is therefore *not* stationary before + chunk six, and a mean taken over a short rollout reports a number no steady + session will ever produce. :data:`DEFAULT_WARMUP_CHUNKS` exists for that + reason and is not a round-number guess. + +The causal VAE expands the first latent frame to one pixel frame and every later +latent frame to four, so chunk zero carries nine pixel frames and every chunk +after it carries twelve. Video-duration math has to respect that asymmetry or +real-time factor comes out wrong on short rollouts. +""" + +from __future__ import annotations + +import json +from collections.abc import Iterable, Mapping, Sequence +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +# transformer.config.num_frames_per_block: latent frames committed per AR block. +FRAMES_PER_BLOCK = 3 +# vae_scale_factor_temporal: latent frames -> pixel frames, after the first. +TEMPORAL_COMPRESSION = 4 +# (sink_size 9 + recent window 9) / FRAMES_PER_BLOCK: chunks until the attention +# window saturates and per-chunk cost becomes stationary. +DEFAULT_WARMUP_CHUNKS = 6 + +DEFAULT_MODEL = "robbyant/lingbot-world-v2-14b-causal-fast-diffusers" +DEFAULT_PROMPT = "The camera moves slowly forward through the scene." +# The AR KV cache geometry is fixed at load time, so a request must ask for +# exactly the resolution the deploy config declares. +DEFAULT_WIDTH = 832 +DEFAULT_HEIGHT = 480 +DEFAULT_FPS = 16 +DEFAULT_FLOW_SHIFT = 5.0 +DEFAULT_SEED = 42 + +_HOLD = "hold" +_ACTION_KEYS = ("w", "a", "s", "d") + + +def num_frames_for_chunks(num_chunks: int) -> int: + """Pixel-frame count whose rollout is exactly ``num_chunks`` chunks. + + Inverse of :func:`chunks_for_num_frames`, which is the server's own + ``((num_frames - 1) // 4 + 1) // 3``. + """ + if num_chunks < 1: + raise ValueError("num_chunks must be at least 1.") + return (num_chunks * FRAMES_PER_BLOCK - 1) * TEMPORAL_COMPRESSION + 1 + + +def chunks_for_num_frames(num_frames: int) -> int: + """Chunks the server generates for ``num_frames``, mirroring its own formula.""" + if num_frames < 1: + raise ValueError("num_frames must be at least 1.") + return ((num_frames - 1) // TEMPORAL_COMPRESSION + 1) // FRAMES_PER_BLOCK + + +def pixel_frames_in_chunk(chunk_index: int) -> int: + """Pixel frames carried by one chunk. + + The causal decoder expands the session's opening latent frame once and every + later latent frame four times, so chunk zero is short by three frames. + """ + if chunk_index < 0: + raise ValueError("chunk_index must be non-negative.") + if chunk_index == 0: + return (FRAMES_PER_BLOCK - 1) * TEMPORAL_COMPRESSION + 1 + return FRAMES_PER_BLOCK * TEMPORAL_COMPRESSION + + +def camera_actions_for_chunk(pattern: str, chunk_index: int) -> list[list[str]]: + """One three-entry action list for ``chunk_index``. + + Camera actions select camera embeddings; they change what the world does, + not how much arithmetic it costs. The pattern therefore exists to keep a + benchmark rollout representative rather than to sweep a cost dimension. + """ + if chunk_index < 0: + raise ValueError("chunk_index must be non-negative.") + if pattern == "forward": + return [["w"] for _ in range(FRAMES_PER_BLOCK)] + if pattern == _HOLD: + return [[] for _ in range(FRAMES_PER_BLOCK)] + if pattern == "orbit": + key = _ACTION_KEYS[chunk_index % len(_ACTION_KEYS)] + return [[key] for _ in range(FRAMES_PER_BLOCK)] + raise ValueError(f"Unknown camera pattern {pattern!r}; expected one of {sorted(CAMERA_PATTERNS)}.") + + +CAMERA_PATTERNS = ("forward", "orbit", _HOLD) + + +def build_camera_script(num_chunks: int, pattern: str) -> list[list[list[str]]]: + """Full ``camera_action_script``: one action list per generated chunk.""" + if num_chunks < 1: + raise ValueError("num_chunks must be at least 1.") + return [camera_actions_for_chunk(pattern, index) for index in range(num_chunks)] + + +def _validate_camera_script(script: Sequence[Any]) -> list[list[list[str]]]: + validated: list[list[list[str]]] = [] + for chunk_index, actions in enumerate(script): + if not isinstance(actions, Sequence) or isinstance(actions, (str, bytes)): + raise ValueError(f"camera_action_script[{chunk_index}] must be a list of three action lists.") + if len(actions) != FRAMES_PER_BLOCK: + raise ValueError( + f"camera_action_script[{chunk_index}] must hold exactly {FRAMES_PER_BLOCK} action lists " + f"(got {len(actions)}); one AR block is {FRAMES_PER_BLOCK} latent frames." + ) + frames: list[list[str]] = [] + for frame_index, frame in enumerate(actions): + if not isinstance(frame, Sequence) or isinstance(frame, (str, bytes)): + raise ValueError(f"camera_action_script[{chunk_index}][{frame_index}] must be a list of strings.") + if any(not isinstance(action, str) for action in frame): + raise ValueError(f"camera_action_script[{chunk_index}][{frame_index}] must contain only strings.") + frames.append([str(action) for action in frame]) + validated.append(frames) + if not validated: + raise ValueError("camera_action_script must not be empty.") + return validated + + +@dataclass(frozen=True) +class PromptUpdate: + """A mid-rollout ``session.interaction`` scheduled on a chunk boundary.""" + + after_chunk: int + prompt: str + transition_chunks: int = 3 + + def __post_init__(self) -> None: + if self.after_chunk < 0: + raise ValueError("PromptUpdate.after_chunk must be non-negative.") + if not self.prompt.strip(): + raise ValueError("PromptUpdate.prompt must contain non-whitespace text.") + if self.transition_chunks < 0: + raise ValueError("PromptUpdate.transition_chunks must be non-negative.") + + def to_payload(self) -> dict[str, Any]: + return { + "type": "session.interaction", + "interaction": { + "event_id": f"chunk-{self.after_chunk}", + "event": {"prompt": self.prompt}, + "transition_chunks": self.transition_chunks, + }, + } + + +@dataclass(frozen=True) +class Workload: + """One rollout: what the benchmark asks the server to generate.""" + + prompt: str + image_reference: str + camera_script: list[list[list[str]]] + width: int = DEFAULT_WIDTH + height: int = DEFAULT_HEIGHT + fps: int = DEFAULT_FPS + seed: int = DEFAULT_SEED + flow_shift: float = DEFAULT_FLOW_SHIFT + negative_prompt: str | None = None + prompt_updates: tuple[PromptUpdate, ...] = () + extra_params: Mapping[str, Any] = field(default_factory=dict) + + def __post_init__(self) -> None: + if not self.prompt.strip(): + raise ValueError("Workload.prompt must contain non-whitespace text.") + if not self.image_reference: + raise ValueError("Workload.image_reference is required; LingBot-World is image-conditioned.") + if self.width <= 0 or self.height <= 0 or self.width % 16 or self.height % 16: + raise ValueError("Workload width/height must be positive multiples of 16.") + if self.fps <= 0: + raise ValueError("Workload.fps must be positive.") + object.__setattr__(self, "camera_script", _validate_camera_script(self.camera_script)) + seen_update_chunks: set[int] = set() + for update in self.prompt_updates: + if update.after_chunk in seen_update_chunks: + raise ValueError(f"Duplicate prompt update after chunk {update.after_chunk}; event IDs must be unique.") + seen_update_chunks.add(update.after_chunk) + if update.after_chunk >= self.num_chunks: + raise ValueError( + f"Prompt update after chunk {update.after_chunk} never fires in a {self.num_chunks}-chunk rollout." + ) + + @property + def num_chunks(self) -> int: + return len(self.camera_script) + + @property + def num_frames(self) -> int: + return num_frames_for_chunks(self.num_chunks) + + @property + def video_seconds(self) -> float: + return self.num_frames / self.fps + + def session_start_payload(self, model: str) -> dict[str, Any]: + """The ``session.start`` event for ``WS /v1/realtime/video``.""" + extra_params: dict[str, Any] = { + "flow_shift": self.flow_shift, + "camera_action_script": self.camera_script, + } + extra_params.update(self.extra_params) + payload: dict[str, Any] = { + "type": "session.start", + "model": model, + "prompt": self.prompt, + "image_reference": {"image_url": self.image_reference}, + "width": self.width, + "height": self.height, + "num_frames": self.num_frames, + "fps": self.fps, + "seed": self.seed, + "extra_params": extra_params, + } + if self.negative_prompt is not None: + payload["negative_prompt"] = self.negative_prompt + return payload + + def describe(self) -> dict[str, Any]: + return { + "num_chunks": self.num_chunks, + "num_frames": self.num_frames, + "width": self.width, + "height": self.height, + "fps": self.fps, + "seed": self.seed, + "flow_shift": self.flow_shift, + "video_seconds": self.video_seconds, + "prompt_updates": [ + { + "after_chunk": update.after_chunk, + "transition_chunks": update.transition_chunks, + } + for update in self.prompt_updates + ], + } + + +def default_workload( + *, + image_reference: str, + num_chunks: int, + camera_pattern: str = "forward", + prompt: str = DEFAULT_PROMPT, + width: int = DEFAULT_WIDTH, + height: int = DEFAULT_HEIGHT, + fps: int = DEFAULT_FPS, + seed: int = DEFAULT_SEED, + flow_shift: float = DEFAULT_FLOW_SHIFT, + negative_prompt: str | None = None, +) -> Workload: + """Built-in rollout: a fixed prompt and a generated camera script.""" + return Workload( + prompt=prompt, + image_reference=image_reference, + camera_script=build_camera_script(num_chunks, camera_pattern), + width=width, + height=height, + fps=fps, + seed=seed, + flow_shift=flow_shift, + negative_prompt=negative_prompt, + ) + + +def load_workload( + path: str | Path, + *, + image_resolver: Any = None, + overrides: Mapping[str, Any] | None = None, +) -> Workload: + """Read a rollout spec from JSON. + + ``image_resolver`` turns the spec's ``image`` field into an ``http(s)`` or + ``data:`` URL; the caller owns that because inlining a local file is I/O. + Keys in ``overrides`` that are not ``None`` win over the file, so CLI flags + stay meaningful next to ``--workload``. + """ + document = json.loads(Path(path).read_text()) + if not isinstance(document, dict): + raise ValueError("A workload file must contain a JSON object.") + + merged = dict(document) + for key, value in (overrides or {}).items(): + if value is not None: + merged[key] = value + + camera_script = merged.get("camera_action_script") + if camera_script is None: + num_chunks = merged.get("num_chunks") + if num_chunks is None: + raise ValueError("A workload file needs either camera_action_script or num_chunks.") + camera_script = build_camera_script(int(num_chunks), str(merged.get("camera_pattern", "forward"))) + + image = merged.get("image") or merged.get("image_reference") + if image is None: + raise ValueError("A workload file needs an image or image_reference field.") + image_reference = image_resolver(str(image)) if image_resolver is not None else str(image) + + updates_field = merged.get("prompt_updates") or () + prompt_updates = tuple( + PromptUpdate( + after_chunk=int(entry["after_chunk"]), + prompt=str(entry["prompt"]), + transition_chunks=int(entry.get("transition_chunks", 3)), + ) + for entry in updates_field + ) + + extra_params = merged.get("extra_params") or {} + if not isinstance(extra_params, Mapping): + raise ValueError("Workload extra_params must be a JSON object.") + + return Workload( + prompt=str(merged.get("prompt", DEFAULT_PROMPT)), + image_reference=image_reference, + camera_script=camera_script, + width=int(merged.get("width", DEFAULT_WIDTH)), + height=int(merged.get("height", DEFAULT_HEIGHT)), + fps=int(merged.get("fps", DEFAULT_FPS)), + seed=int(merged.get("seed", DEFAULT_SEED)), + flow_shift=float(merged.get("flow_shift", DEFAULT_FLOW_SHIFT)), + negative_prompt=merged.get("negative_prompt"), + prompt_updates=prompt_updates, + extra_params=dict(extra_params), + ) + + +# --------------------------------------------------------------------------- # +# Metrics +# --------------------------------------------------------------------------- # + + +@dataclass +class ChunkRecord: + """One media chunk as the client observed it.""" + + index: int + arrival_s: float + """Seconds from ``session.start`` being sent to this chunk's bytes arriving.""" + inter_arrival_s: float + """Seconds since the previous chunk; for chunk zero this equals ``arrival_s``.""" + byte_length: int + num_frames: int + + def to_dict(self) -> dict[str, Any]: + return { + "index": self.index, + "arrival_s": self.arrival_s, + "inter_arrival_ms": self.inter_arrival_s * 1000.0, + "byte_length": self.byte_length, + "num_frames": self.num_frames, + } + + +def percentile(values: Sequence[float], fraction: float) -> float: + """Linearly interpolated percentile, so a 10-sample run still reports a p95. + + Implemented here rather than pulled from numpy to keep this module importable + in a bare environment; the benchmark's own numbers must not depend on which + optional package happens to be installed. + """ + if not values: + raise ValueError("percentile() needs at least one value.") + if not 0.0 <= fraction <= 1.0: + raise ValueError("percentile fraction must be within [0, 1].") + ordered = sorted(values) + if len(ordered) == 1: + return ordered[0] + position = fraction * (len(ordered) - 1) + lower = int(position) + upper = min(lower + 1, len(ordered) - 1) + weight = position - lower + return ordered[lower] * (1.0 - weight) + ordered[upper] * weight + + +def _stddev(values: Sequence[float]) -> float: + if len(values) < 2: + return 0.0 + mean = sum(values) / len(values) + return (sum((value - mean) ** 2 for value in values) / (len(values) - 1)) ** 0.5 + + +def _latency_summary(values: Sequence[float]) -> dict[str, float]: + """Millisecond summary of a set of inter-arrival intervals.""" + if not values: + return {} + milliseconds = [value * 1000.0 for value in values] + return { + "count": float(len(milliseconds)), + "mean_ms": sum(milliseconds) / len(milliseconds), + "median_ms": percentile(milliseconds, 0.5), + "std_ms": _stddev(milliseconds), + "min_ms": min(milliseconds), + "p90_ms": percentile(milliseconds, 0.90), + "p95_ms": percentile(milliseconds, 0.95), + "p99_ms": percentile(milliseconds, 0.99), + "max_ms": max(milliseconds), + } + + +@dataclass(frozen=True) +class PlaybackResult: + """What a viewer would have seen, given a start buffer and a playback rate.""" + + underrun_count: int + total_stall_s: float + playback_span_s: float + first_underrun_chunk: int | None + + def to_dict(self) -> dict[str, Any]: + return { + "underrun_count": self.underrun_count, + "total_stall_ms": self.total_stall_s * 1000.0, + "playback_span_s": self.playback_span_s, + "stall_ratio": (self.total_stall_s / self.playback_span_s) if self.playback_span_s > 0 else 0.0, + "first_underrun_chunk": self.first_underrun_chunk, + } + + +def simulate_playback( + records: Sequence[ChunkRecord], + *, + fps: float, + buffer_chunks: int = 1, +) -> PlaybackResult: + """Replay arrivals against a wall clock and count stalls. + + Playback starts when ``buffer_chunks`` chunks have arrived and then consumes + video at ``fps``. A chunk that has not arrived by the moment the player needs + it is an underrun, and the player resumes only when the bytes land -- which + is what a viewer experiences, and what a single p99 number cannot tell you: + the same p99 is harmless behind a three-chunk buffer and fatal behind none. + """ + if fps <= 0: + raise ValueError("fps must be positive.") + if buffer_chunks < 1: + raise ValueError("buffer_chunks must be at least 1.") + if not records: + return PlaybackResult(0, 0.0, 0.0, None) + if buffer_chunks > len(records): + raise ValueError(f"buffer_chunks={buffer_chunks} exceeds the {len(records)} chunks received.") + + ordered = sorted(records, key=lambda record: record.index) + start = ordered[buffer_chunks - 1].arrival_s + cursor = start + underruns = 0 + total_stall = 0.0 + first_underrun: int | None = None + + for record in ordered: + if record.arrival_s > cursor: + stall = record.arrival_s - cursor + underruns += 1 + total_stall += stall + if first_underrun is None: + first_underrun = record.index + cursor = record.arrival_s + cursor += record.num_frames / fps + + return PlaybackResult( + underrun_count=underruns, + total_stall_s=total_stall, + playback_span_s=cursor - start, + first_underrun_chunk=first_underrun, + ) + + +def steady_chunk_deadline_ms(fps: float) -> float: + """Wall-clock budget a steady chunk must meet to hold real time. + + ``fps`` here is the rate the frames are *intended to be played at*, which the + checkpoint does not declare: nothing in ``model_index.json``, the scheduler + config, or the transformer config carries a frame rate, and the pipeline + never reads one. It is a property of how the output is consumed, so both + this deadline and real-time factor scale linearly with the choice and the + caller has to state it rather than inherit it from a muxing default. + """ + if fps <= 0: + raise ValueError("fps must be positive.") + return pixel_frames_in_chunk(1) / fps * 1000.0 + + +def compute_metrics( + records: Sequence[ChunkRecord], + *, + fps: float, + warmup_chunks: int = DEFAULT_WARMUP_CHUNKS, + slo_ms: float | None = None, + playback_buffer_chunks: int = 1, + ttfc_s: float | None = None, + session_wall_s: float | None = None, +) -> dict[str, Any]: + """Summarize one session. + + ``warmup_chunks`` are excluded from the steady-state block because the + attention window has not saturated before then (see the module docstring). + The terminal chunk also stays out of steady metrics because it skips + next-chunk preparation. All chunks stay in the playback simulation, which is + what a session really costs end to end. + """ + if not records: + raise ValueError("compute_metrics() needs at least one chunk record.") + if warmup_chunks < 0: + raise ValueError("warmup_chunks must be non-negative.") + + ordered = sorted(records, key=lambda record: record.index) + deadline_ms = slo_ms if slo_ms is not None else steady_chunk_deadline_ms(fps) + + all_intervals = [record.inter_arrival_s for record in ordered[1:]] + # The terminal chunk skips next-chunk preparation and is not representative + # of sustained generation. Keep it in end-to-end and playback metrics. + steady = [record for record in ordered[:-1] if record.index >= warmup_chunks] + steady_intervals = [record.inter_arrival_s for record in steady if record.index > 0] + + total_frames = sum(record.num_frames for record in ordered) + video_seconds = total_frames / fps + wall_seconds = session_wall_s if session_wall_s is not None else ordered[-1].arrival_s + + metrics: dict[str, Any] = { + "chunks_received": len(ordered), + "warmup_chunks": warmup_chunks, + "steady_chunks": len(steady_intervals), + "steady_state_reached": bool(steady_intervals), + "total_frames": total_frames, + "total_bytes": sum(record.byte_length for record in ordered), + "video_seconds": video_seconds, + "wall_seconds": wall_seconds, + # RTF is processing time over content duration, as vLLM-Omni computes it + # (`patch.py`: generation_s / video_duration; `compute_audio_rtf`: latency + # / audio_duration, "SLO red line < 1"). LOWER IS BETTER and below 1.0 is + # real time. RTFX is the reciprocal the repo reports under its own name, + # kept here because "fraction of real time achieved" reads better on a + # dashboard -- but the two must never be confused for one another. + "video_rtf": (wall_seconds / video_seconds) if video_seconds > 0 else 0.0, + "video_rtfx": (video_seconds / wall_seconds) if wall_seconds > 0 else 0.0, + "frames_per_second": (total_frames / wall_seconds) if wall_seconds > 0 else 0.0, + "chunk_deadline_ms": deadline_ms, + "interval_all": _latency_summary(all_intervals), + "interval_steady": _latency_summary(steady_intervals), + } + if ttfc_s is not None: + metrics["ttfc_ms"] = ttfc_s * 1000.0 + + if steady_intervals: + steady_frames = sum(record.num_frames for record in steady if record.index > 0) + steady_wall = sum(steady_intervals) + steady_video = steady_frames / fps + metrics["steady_video_rtf"] = (steady_wall / steady_video) if steady_video > 0 else 0.0 + metrics["steady_video_rtfx"] = (steady_video / steady_wall) if steady_wall > 0 else 0.0 + within = sum(1 for value in steady_intervals if value * 1000.0 <= deadline_ms) + metrics["slo_attainment"] = within / len(steady_intervals) + metrics["slo_violations"] = len(steady_intervals) - within + + metrics["playback"] = simulate_playback( + ordered, + fps=fps, + buffer_chunks=playback_buffer_chunks, + ).to_dict() + metrics["playback"]["buffer_chunks"] = playback_buffer_chunks + return metrics + + +def aggregate_metrics(sessions: Iterable[Mapping[str, Any]], *, fps: float) -> dict[str, Any]: + """Combine per-session metrics across sequential repeats. + + Steady intervals are pooled rather than averaging per-session means, so a + session that happened to produce fewer steady chunks does not carry the same + weight as a full one. + """ + entries = list(sessions) + if not entries: + raise ValueError("aggregate_metrics() needs at least one session.") + if len(entries) == 1: + single = dict(entries[0]) + single["sessions"] = 1 + return single + + deadline_ms = entries[0]["chunk_deadline_ms"] + if any(entry["chunk_deadline_ms"] != deadline_ms for entry in entries): + raise ValueError("Cannot aggregate sessions with different chunk deadlines.") + ttfcs = [float(entry["ttfc_ms"]) for entry in entries if entry.get("ttfc_ms") is not None] + # Lower is better, so "max" is the worst session, not the best. + rtfs = [float(entry["video_rtf"]) for entry in entries] + steady_means = [ + float(entry["interval_steady"]["mean_ms"]) for entry in entries if entry.get("interval_steady", {}).get("count") + ] + steady_count = sum(entry.get("interval_steady", {}).get("count", 0) for entry in entries) + steady_total_ms = sum( + entry["interval_steady"]["mean_ms"] * entry["interval_steady"]["count"] + for entry in entries + if entry.get("interval_steady", {}).get("count") + ) + return { + "sessions": len(entries), + "chunks_received": sum(int(entry["chunks_received"]) for entry in entries), + "total_frames": sum(int(entry["total_frames"]) for entry in entries), + "video_seconds": sum(float(entry["video_seconds"]) for entry in entries), + "wall_seconds": sum(float(entry["wall_seconds"]) for entry in entries), + "video_rtf_mean": sum(rtfs) / len(rtfs), + "video_rtf_best": min(rtfs), + "video_rtf_worst": max(rtfs), + "ttfc_ms_mean": (sum(ttfcs) / len(ttfcs)) if ttfcs else None, + "ttfc_ms_max": max(ttfcs) if ttfcs else None, + "steady_interval_ms_mean": (steady_total_ms / steady_count) if steady_count else None, + "steady_interval_ms_spread": (max(steady_means) - min(steady_means)) if steady_means else None, + "underrun_count": sum(int(entry["playback"]["underrun_count"]) for entry in entries), + "chunk_deadline_ms": deadline_ms, + } diff --git a/docs/assets/WeChat.jpg b/docs/assets/WeChat.jpg index f55fa90f3..925d77a05 100644 Binary files a/docs/assets/WeChat.jpg and b/docs/assets/WeChat.jpg differ diff --git a/docs/configuration/environment_variables.md b/docs/configuration/environment_variables.md index db204bdd5..e113c9d7c 100644 --- a/docs/configuration/environment_variables.md +++ b/docs/configuration/environment_variables.md @@ -85,7 +85,7 @@ depends on the installed kernels and model path. | `SPEAKER_MAX_UPLOADED` | Integer; default `1000` | Speech server; read when speaker storage initializes | Environment-only setting. A non-integer logs a warning and uses `1000`; range is not otherwise validated. | Stable | | `VLLM_OMNI_ABORT_TIMEOUT` | Float seconds; default `2` | Engine abort wait for Videos DELETE and `generate()` cancel/error cleanup; read when `async_omni` imports | Environment-only setting. A non-float raises `ValueError` during import. | Experimental | | `VLLM_OMNI_ASYNC_OUTPUT_TIMEOUT` | Float seconds; default `600` | Diffusion engine async-output wait in `step_streaming`; resolved per call on the request path, not at import | Environment-only setting. A non-float or `<=0` value warns once and uses the default. | Experimental | -| `VLLM_OMNI_EVENT_DRIVEN_ORCH` | `1`, `true`, `yes` or `on` enables; default `0` (off) | Orchestration loop and the serving-side final-output drain; read once when the `Orchestrator` is constructed | Environment-only setting. Values are stripped and case-normalized; any unrecognized value leaves the legacy poll loop selected. | Experimental | +| `VLLM_OMNI_EVENT_DRIVEN_ORCH` | `1`, `true`, `yes` or `on` enables; defaults to on for Qwen3-TTS and off for other pipelines | Pipeline default computed from `pipeline_config.model_type` at engine initialization; env override resolved at `Orchestrator` construction and separately when the serving-side final-output drain starts | An explicit env value wins; otherwise both consumers use the engine's pipeline default. Values are stripped and case-normalized; any unrecognized value selects the legacy poll loop. Set before server startup. | Experimental | | `VLLM_OMNI_INPUT_WAIT_TIMEOUT_S` | Float seconds; default `600`; `<=0` disables | Full-payload input coordinator, not async-chunk transfer; read when the scheduler module imports in each worker | Environment-only setting. A non-float logs a warning and uses `600`. | Stable operational control | | `VLLM_OMNI_ORCH_MONITOR_PATH` | Filesystem path; default `/vllm_omni_orch_monitor_.json` | Orchestrator monitor enabled by `--enable-orch-monitor`; read when the monitor is created | Environment-only path override. Parent directories are created; write errors are logged. | Diagnostic | | `VLLM_OMNI_VIDEO_SYNC_TIMEOUT` | Float seconds; default `600` | Synchronous Videos API; read when the API server module imports | Environment-only setting. A non-float raises `ValueError` during import. | Experimental | diff --git a/docs/configuration/stage_configs.md b/docs/configuration/stage_configs.md index 2540b19d9..9836bcff9 100644 --- a/docs/configuration/stage_configs.md +++ b/docs/configuration/stage_configs.md @@ -293,19 +293,25 @@ vllm serve Qwen/Qwen2.5-Omni-7B --omni --port 8091 --deploy-config /path/to/depl ## Qwen3-TTS with Model Runner V2 -Qwen3-TTS can opt into the native CUDA Model Runner V2 pipeline on vLLM -0.29.0. Select one of these deployment profiles: +Qwen3-TTS runs the native CUDA Model Runner V2 pipeline by default on vLLM +0.29.0. MRV2 is an experimental feature for this model: the bundled default +profile selects it on CUDA only, and its scheduler and delivery paths are +still being qualified. Set `model_runner: v1` in a copy of the deploy config +to opt out; the `platforms:` sections of `qwen3_tts.yaml` keep V1 on NPU, XPU, +ROCm and MUSA. Select one of these deployment profiles: | Profile | Runner | Code2Wav graph batches | Intended use | | --- | --- | --- | --- | -| `qwen3_tts.yaml` | V1 | Existing defaults | Existing deployment / regression control | -| `qwen3_tts_mrv2.yaml` | V2 | B1 | Native runner validation | +| `qwen3_tts.yaml` | V2 (default) | Existing defaults | Shipped default; experimental | +| `qwen3_tts_mrv2.yaml` | V2 | B1 | Explicit MRV2 profile (same runner selection as the default) | | `qwen3_tts_high_concurrency_mrv2.yaml` | V2 | B1, B2 | Opt-in throughput tuning | | `qwen3_tts_high_concurrency_mrv2_b4.yaml` | V2 | B1, B2, B3, B4 | Experimental throughput / buffered playback | +| `qwen3_tts_high_concurrency.yaml` | V1 | Existing defaults | V1 high-concurrency control | ```bash +# MRV2 is the default; pass a copy with `model_runner: v1` to force V1. vllm serve Qwen/Qwen3-TTS-12Hz-1.7B-Base --omni \ - --deploy-config vllm_omni/deploy/qwen3_tts_mrv2.yaml + --deploy-config /path/to/qwen3_tts_v1.yaml ``` The V2 profiles bound Talker prefill to 512 tokens per step and select the diff --git a/docs/contributing/ci/test_execution_guide.md b/docs/contributing/ci/test_execution_guide.md index 7597f72e4..d3b02d27f 100644 --- a/docs/contributing/ci/test_execution_guide.md +++ b/docs/contributing/ci/test_execution_guide.md @@ -177,10 +177,10 @@ Failed jobs: 1/2 cd tests # Stability: Qwen3-Omni - pytest -s -v dfx/stability/scripts/test_stability_qwen3_omni.py -m slow + pytest -s -v dfx/stability/scripts/run_stability_qwen3_omni.py -m slow # Stability: Wan2.2 (v1/videos diffusion benchmark loop) - pytest -s -v dfx/stability/scripts/test_stability_wan22.py -m slow + pytest -s -v dfx/stability/scripts/run_stability_wan22.py -m slow # Reliability: Qwen3-Omni (H100 × 2) pytest -s -v dfx/reliability/test_reliability_qwen3_omni.py -m slow diff --git a/docs/contributing/ci/test_writing_guide.md b/docs/contributing/ci/test_writing_guide.md index 762863bb3..47a629112 100644 --- a/docs/contributing/ci/test_writing_guide.md +++ b/docs/contributing/ci/test_writing_guide.md @@ -555,7 +555,7 @@ L5 level testing focuses on the performance of model services under ***long-runn - ***Trigger Timing***: **`Weekly`** (weekly) or **`Days before Release`** (several days before a major release). Due to long execution times, the frequency is lower. - ***Run Command***: - - ***Stability***: `pytest -s -v tests/dfx/stability/scripts/test_stability_qwen3_omni.py` or `pytest -s -v tests/dfx/stability/scripts/test_stability_wan22.py` (or add `test_stability_.py` alongside a matching JSON config) + - ***Stability***: `pytest -s -v tests/dfx/stability/scripts/run_stability_qwen3_omni.py` or `pytest -s -v tests/dfx/stability/scripts/run_stability_wan22.py` (or add `run_stability_.py` alongside a matching JSON config) - ***Reliability***: `pytest -s -v tests/dfx/reliability/test_reliability_.py -m slow` (current suites: `qwen3_omni`, `wan22`, `hunyuan_image`). - ***Script Example***: @@ -564,7 +564,7 @@ L5 level testing focuses on the performance of model services under ***long-runn ##### Stability -When you want to add L5-level stability test cases, add or extend the appropriate JSON file under `tests/dfx/stability/tests/` (for example `test_qwen3_omni.json` for Omni bench traffic, or `test_wan22.json` for diffusion `/v1/videos` workloads). Pair the JSON with `tests/dfx/stability/scripts/test_stability_.py`. The following illustrates the Qwen3-Omni shape: +When you want to add L5-level stability test cases, add or extend the appropriate JSON file under `tests/dfx/stability/tests/` (for example `test_qwen3_omni.json` for Omni bench traffic, or `test_wan22.json` for diffusion `/v1/videos` workloads). Pair the JSON with `tests/dfx/stability/scripts/run_stability_.py`. The following illustrates the Qwen3-Omni shape: ```json { diff --git a/docs/contributing/profiling.md b/docs/contributing/profiling.md index 62fff29c5..4e61d2531 100644 --- a/docs/contributing/profiling.md +++ b/docs/contributing/profiling.md @@ -22,7 +22,7 @@ profiler_config: Supported fields: | Field | Description | -|---|---| +| --- | --- | | `profiler` | Profiler backend. Supported values: `torch`, `cuda`. Use `torch` for `trace.json`, Excel operator tables, and optional memory snapshots. Use `cuda` for Nsight Systems only. | | `torch_profiler_dir` | Output directory for torch-profiler artifacts. Required when `profiler: torch`. | | `torch_profiler_use_gzip` | Compress `trace_rank*.json` into `trace_rank*.json.gz`. | @@ -287,6 +287,52 @@ Recommended viewers: For upstream background on the underlying vLLM profiling infrastructure, see the [vLLM profiling guide](https://docs.vllm.ai/en/stable/contributing/profiling/). +## 6. Profiling Omni AR/TTS stages + +For Omni AR and TTS stages, put `profiler_config` under each stage that should +be captured. The serving API then controls a narrow window with +`POST /start_profile` and `POST /stop_profile`; model initialization and +warmup stay outside the trace. + +Example stage override for a host-stack torch trace: + +```bash +PROFILE='{"profiler":"torch","torch_profiler_dir":"/tmp/qwen3_tts_profile",'\ +'"torch_profiler_with_stack":true,"torch_profiler_record_shapes":false,'\ +'"torch_profiler_use_gzip":true,"torch_profiler_dump_cuda_time_total":true}' + +vllm serve Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice --omni \ + --deploy-config vllm_omni/deploy/qwen3_tts_high_concurrency.yaml \ + --stage-overrides "{\"0\":{\"profiler_config\":$PROFILE},\"1\":{\"profiler_config\":$PROFILE}}" +``` + +After one or more warmup requests, capture one short measured request: + +```bash +curl -sS -X POST http://127.0.0.1:8000/start_profile \ + -H 'Content-Type: application/json' -d '{"stages":[0,1]}' +# send one fixed /v1/audio/speech or /v1/chat/completions request here +curl -sS -X POST http://127.0.0.1:8000/stop_profile \ + -H 'Content-Type: application/json' -d '{"stages":[0,1]}' +``` + +For an Nsight Systems CUDA timeline, use the same stage override with +`{"profiler":"cuda"}` and wrap the server: + +```bash +nsys profile --trace=cuda,nvtx,osrt --sample=process-tree \ + --cpuctxsw=process-tree --trace-fork-before-exec=true \ + --capture-range=cudaProfilerApi --capture-range-end=stop-shutdown \ + --cuda-graph-trace=node -o /tmp/qwen3_tts_nsys \ + vllm serve Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice --omni ... +``` + +For Omni AR workers, `profiler="cuda"` requires the native CUDA profiler +wrapper and uses `cudaProfilerStart/Stop`; the report is only valid if the +`/start_profile` and `/stop_profile` calls both succeed. Use `nsys stats` for +CUDA API, GPU kernel, OS runtime, and NVTX summaries. Treat profiler latency as +diagnostic only; use a separate no-profiler run for performance numbers. + ## 7. Orchestrator Monitor > **Warning:** Diagnostic only. Adds lightweight counters on the orchestrator poll loop and per-replica queue depth. Disable in production unless you are actively debugging orchestrator saturation. @@ -314,9 +360,11 @@ If unset, the monitor writes `./vllm_omni_orch_monitor_.json` at shutd Each 1-second window records: | Series | Description | -|---|---| +| --- | --- | | `windows.duration_s` | Wall time covered by the window | | `windows.loop_idle` / `windows.loop_active` | Orchestrator poll-loop iterations with no work vs. active forwarding | +| `windows.dispatch_queue_size` | Ready-queue size sampled at each monitor window boundary | +| `windows.dispatch_queue_high_water` | Maximum Event Driven ready-queue size observed in each window | | `replicas..outputs_queue_size` | MP client `outputs_queue` backlog for that replica | | `replicas..inflight` | Requests currently bound/routed to the replica | @@ -331,12 +379,16 @@ orchestrator wakes only once per 0.5 s reconcile timeout, while a busy one still records one iteration per routed output. Window counts and the `loop_active_pct` summary are therefore not comparable across the two modes. +For Event Driven runs, use `dispatch_queue_high_water` rather than only the +window-boundary `dispatch_queue_size`: a serial dispatcher can drain a burst +before the window closes, hiding a transient ready-queue backlog in the latter. + ### Relationship to other diagnostics This monitor is intentionally separate from the existing profiling tools: | Tool | Scope | Output | -|---|---|---| +| --- | --- | --- | | `--enable-diffusion-pipeline-profiler` | Diffusion stage functions (`vae.decode`, `diffuse`, …) | Per-request `stage_durations` / logs | | `--enable-ar-profiler` | AR stage generation time | Per-request `stage_durations` | | `profiler_config` (`torch` / `cuda`) | GPU/CPU kernels inside a stage worker | `trace.json`, Nsight, operator tables | diff --git a/docs/design/module/engine_orchestration.md b/docs/design/module/engine_orchestration.md index df22f3ce1..c4dd4bb36 100644 --- a/docs/design/module/engine_orchestration.md +++ b/docs/design/module/engine_orchestration.md @@ -79,7 +79,7 @@ the boundary affected by the in-flight stage client/process refactor in responsibilities proposed only by that PR are not current contracts. The orchestration loop also has an opt-in event-driven mode -(`VLLM_OMNI_EVENT_DRIVEN_ORCH=1`, default off) proposed in +(`VLLM_OMNI_EVENT_DRIVEN_ORCH=1`, default off except for Qwen3-TTS) proposed in [#5221](https://github.com/vllm-project/vllm-omni/pull/5221). It changes poll cadence only: the routing, ordering, and terminal-state contracts below hold identically on both loops. diff --git a/docs/features/sleep_mode.md b/docs/features/sleep_mode.md index b4eec162d..9a740519f 100644 --- a/docs/features/sleep_mode.md +++ b/docs/features/sleep_mode.md @@ -9,9 +9,11 @@ This feature is inherited from [vLLM’s Sleep Mode](https://blog.vllm.ai/2025/1 ## 1. Feature Documentation ### Overview + Omni Sleep Mode provides a mechanism to "sleep" specific model stages. When a stage enters sleep, its physical VRAM is reclaimed by the system, while the process state is preserved for rapid "wake-up" without full re-initialization. ### Sleep Levels + We support two levels of hibernation to balance recovery speed and memory efficiency: | Level | Name | Mechanism | Recovery Speed | Memory Freed | @@ -29,15 +31,67 @@ Omni Sleep Mode is optimized for high-performance computing backends: * **Huawei NPU**: Supported via Ascend memory scavenging. ### Hardware Requirements + * **Memory Considerations**: System RAM must be sufficient to hold offloaded weights during sleep. * **TP Support**: Tensor Parallel groups synchronize sleep/wake transitions across all workers. ---- +### Quiescing a busy diffusion stage: `pause_generation(mode="keep")` +`sleep()` on a diffusion stage is a worker-level RPC: it offloads memory but does not stop the +stage's scheduler, so a stage that still has queued requests must be made quiet first. Aborting +discards the batch that is running. `pause_generation(mode="keep")` keeps it instead: + +* The batch that is already running finishes on the weights it started with and its outputs are + delivered normally. +* Requests that have not started stay queued and run after `resume_generation()`, on whatever + weights are loaded by then. A request never continues denoising across a weight change. +* The call returns only after every replica has acknowledged: the batch that was running has + finished executing, its asynchronous output copies are done, and the device is synchronized; its + outputs are then delivered on the normal path. On a non-preemptible backend that acknowledgement + arrives at the next batch boundary, so expect it to take as long as the remaining batch. +* Control RPCs (`list_loras`, `sleep`, `wake_up`, `resume_generation`) keep working while paused. + +Recommended order for a weight update on a diffusion stage: + +```python +await engine.pause_generation(mode="keep", clear_cache=False) +await engine.sleep(level=1) # optional: free memory for the trainer +await engine.wake_up() # admission stays closed until resume +# install new weights here +await engine.resume_generation() +``` + +Support matrix and limits: + +| Configuration | `mode="keep"` | +| :--- | :--- | +| Request-level execution (`step_execution=False`) with the in-process (`uni`) executor | Supported | +| Request-level execution with the multi-process (`mp`) executor, including asynchronous output | Supported; the acknowledgement waits for the output copies | +| Request-level execution with asynchronous KV prefetch | Supported; the acknowledgement joins any prefetch still in flight | +| Step-level execution (`step_execution=True` / streaming output) | Not supported: the call raises `NotImplementedError` | +| `mode="abort"` / `mode="wait"` / `wait_for_inflight_requests=True` on a diffusion stage | Unchanged: frontend admission is paused only | + +Behavior notes: + +* Callers that already passed `mode="keep"` for a diffusion stage used to get an immediate return + that only closed frontend admission. They now also stop the backend and wait for the ACK. +* A timeout or error from `pause_generation` means the pause did **not** complete; the scheduler + stays closed and the caller must retry the pause or call `resume_generation()` explicitly. Do not + sleep or mutate weights after a failed pause. +* `resume_generation(stage_ids=...)` that leaves another paused stage closed (an AR stage, or a + keep-paused diffusion stage) keeps frontend admission closed as well, so new requests cannot + queue on a stage that will not schedule them. Admission reopens once every stage paused by + `pause_generation` has been resumed. +* `clear_cache` has no diffusion-specific effect. +* Without a pause call nothing changes: no barrier runs and batching, abort and output ordering are + as before. + +--- ## 2. Usage Examples ### Python API Example + You can programmatically control the lifecycle of stages using the `AsyncOmni` engine. ```python @@ -65,6 +119,7 @@ if __name__ == "__main__": ``` ### server command Example + Start the server with sleep mode enabled: The first method @@ -92,9 +147,6 @@ python3 -m vllm_omni.entrypoints.openai.api_server \ ``` - - - ### Test Scenarios & Commands #### Scenario 1: LLM Engine Sleep @@ -113,9 +165,8 @@ curl -X POST http://localhost:8000/v1/omni/sleep \ Tip: Open a new terminal and run rocm-smi or nvidia-smi or to observe the immediate drop in VRAM usage. - - #### Scenario 2: Diffusion Sleep + Objective: Verify VRAM reclamation for Stage 1 (Diffusion). Trigger sleep (Level 1 or Level 2) via client: @@ -128,9 +179,8 @@ curl -X POST http://localhost:8000/v1/omni/sleep \ ``` - - #### Scenario 3: Multi-Stage Coordinated Stress Test + Objective: Test concurrent sleep and rapid wake-up across multiple stages. Concurrent Sleep (Stage 0 & 1): @@ -143,7 +193,6 @@ curl -X POST http://localhost:8000/v1/omni/sleep \ ``` - Rapid Wake-up: ``` @@ -154,8 +203,8 @@ curl -X POST http://localhost:8000/v1/omni/wakeup \ ``` - #### Scenario 4: Full Lifecycle Memory Audit & Functional Integrity + Objective: Audit the complete flow from Sleep to Wake-up followed by an Inference validation. Check Initial State: Observe baseline VRAM usage. @@ -196,20 +245,16 @@ curl -X POST http://localhost:8000/v1/images/generations \ ``` - - - ## 3. API Reference - ### Methods | Method | Arguments | Return Type | Description | | :--- | :--- | :--- | :--- | | **sleep** | `stage_ids: List[int], level: int` | `List[OmniACK]` | Triggers hibernation for specified stages. | | **wake_up** | `stage_ids: List[int]` | `List[OmniACK]` | Reloads weights and re-maps memory. | - - +| **pause_generation** | `mode: str, stage_ids: List[int]` | `None` | Stops admission; with `mode="keep"` also stops diffusion schedulers and returns after their ACK. | +| **resume_generation** | `stage_ids: List[int]` | `None` | Reopens paused schedulers, then admission. | ### OmniACK Dataclass Fields @@ -233,7 +278,8 @@ The metadata field is a dynamic dictionary containing hardware-specific telemetr } ``` -#### Core Utility: +#### Core Utility + **VRAM Reclamation Audit (total_freed_gib)**: Converts raw freed_bytes into human-readable GiB. It serves as the primary metric to verify that Level 2 sleep has successfully purged model weights from VRAM. **Residual & Fragmentation Monitoring (rank_residual_gib)**: Reports the remaining VRAM footprint after memory de-mapping. A low residual value (e.g., 2.07 GiB) confirms a successful "clean" state, ensuring the device is ready for high-memory co-located tasks like training or diffusion pipelines. diff --git a/docs/serving/speech_api.md b/docs/serving/speech_api.md index c0b8fe5e2..d973b51a5 100644 --- a/docs/serving/speech_api.md +++ b/docs/serving/speech_api.md @@ -957,22 +957,24 @@ Use `/v1/audio/voices` to list available voices for the loaded model. ## Orchestration Loop (experimental) Multi-stage omni deployments route stage outputs through a single orchestrator -loop. By default that loop polls every stage replica on a 1 ms cadence. An -opt-in event-driven mode replaces the poll with one reader task per live stage +loop. The legacy loop polls every stage replica on a 1 ms cadence. The +event-driven mode replaces the poll with one reader task per live stage replica awaiting its client directly, and switches the serving-side -final-output drain to a condition-variable wakeup at the same time. +final-output drain to a condition-variable wakeup at the same time. Qwen3-TTS +uses this mode by default; other pipelines keep the legacy poll unless enabled. **Configuration (environment variables):** | Variable | Default | Description | | --- | --- | --- | -| `VLLM_OMNI_EVENT_DRIVEN_ORCH` | `0` (off) | Switches the orchestration loop and the final-output drain from the legacy 1 ms poll to event-driven wakeups. Enabled by `1`, `true`, `yes`, or `on`, matched case-insensitively after surrounding whitespace is stripped; any other value leaves it off. | +| `VLLM_OMNI_EVENT_DRIVEN_ORCH` | On for Qwen3-TTS; off for other pipelines | Switches the orchestration loop and the final-output drain from the legacy 1 ms poll to event-driven wakeups. An explicit value wins; otherwise the pipeline default computed at engine initialization is used. The override is resolved at orchestrator construction and separately when the final-output drain starts. `1`, `true`, `yes`, or `on` enables it, ignoring case and surrounding whitespace; other values select the legacy poll loop. | -Set it on the process that runs the orchestrator (stage 0 of an omni -deployment) before starting the server: +Set it on the process that runs the orchestrator (stage 0 of an omni deployment) +before starting the server when overriding the pipeline default. For example, +explicitly disable event-driven orchestration for Qwen3-TTS: ```bash -export VLLM_OMNI_EVENT_DRIVEN_ORCH=1 +VLLM_OMNI_EVENT_DRIVEN_ORCH=0 \ vllm serve Qwen/Qwen3-TTS-12Hz-1.7B-Base \ --omni \ --port 8091 @@ -982,8 +984,9 @@ The server logs the selected loop mode and its reader/poller counts once at startup, so you can confirm which loop is live. Routing, output ordering, and terminal-state behavior are identical on both -loops; only the poll cadence changes. Leaving the variable unset keeps the -legacy poll loop, which is the supported default. +loops; only the poll cadence changes. Leaving the variable unset selects the +pipeline default: event-driven for Qwen3-TTS, and legacy polling for other +pipelines, including Qwen3-Omni. **Known limitations:** diff --git a/docs/user_guide/examples/online_serving/aura_omni.md b/docs/user_guide/examples/online_serving/aura_omni.md index 8b7f56ac5..2042d0d37 100644 --- a/docs/user_guide/examples/online_serving/aura_omni.md +++ b/docs/user_guide/examples/online_serving/aura_omni.md @@ -6,29 +6,29 @@ ASR -> AURA -> Qwen3-TTS Talker -> Code2Wav ``` -Qwen3-TTS remains two engine stages so the pipeline reuses the existing native -Talker and Code2Wav implementation. +**Supported online path:** Realtime duplex at `/v1/realtime?duplex=1`. + +The default deploy profile (`vllm_omni/deploy/aura_omni.yaml`) sets +`session_mode: duplex`. The `aura_omni` pipeline declares `duplex_plugin`, and +`DuplexOmniEngine` requires that mode (same pattern as MiniCPM-o duplex +deploys). Turn-based `chat/completions` is not a supported AURA online mode +against this profile. + +`async_chunk` defaults to `false` in that file (safer if a turn-based +`OmniOrchestrator` ever loads it). Duplex Realtime Stage2→3 still needs +async chunks — use the smoke deploy below, or set `async_chunk: true` +yourself. ```bash -vllm serve aurateam/AURA \ - --omni \ - --deploy-config vllm_omni/deploy/aura_omni.yaml \ - --served-model-name aurateam/AURA \ - --trust-remote-code +bash examples/online_serving/aura_omni/run_duplex_smoke_serve.sh +python examples/online_serving/aura_omni/smoke_duplex_realtime_client.py ``` -Configure local checkpoints by editing per-stage `model` values in -`vllm_omni/deploy/aura_omni.yaml`. The deploy file sets -`pipeline: aura_omni`, so the four-stage topology is used even if the -command-line `--model` points at one of the component checkpoints. +Configure local checkpoints by editing per-stage `model` values in the deploy +or smoke YAML. The file sets `pipeline: aura_omni`, so the four-stage topology +is used even if the command-line `--model` points at one component checkpoint. -Send requests with `"model": "aurateam/AURA"`. The ASR, AURA, and Qwen3-TTS -checkpoint paths are internal stage models from the deploy YAML, not the -OpenAI-facing served model name. - -The AURA stage can emit `<|silent|>`. Silent outputs are treated as a gate: -they produce no Qwen3-TTS Talker input, so no audio is synthesized for that -turn. +Silent Stage1 outputs (`<|silent|>` / id `151669`) skip TTS for that turn. ## GPU Utilization Recommendation @@ -40,26 +40,21 @@ VRAM each stage can reserve. Start with this split for a single GPU: - Stage 2 (Qwen3-TTS Talker): `0.20` - Stage 3 (Qwen3-TTS Code2Wav): `0.20` -## TTS Modes +## TTS modes (stage extras) -`aura_omni` can pass AURA text to Qwen3-TTS in two task modes: +When the duplex session supplies TTS extras, AURA text can feed Qwen3-TTS as: -- `Base`: voice clone from `tts_ref_audio` with ICL enabled in the AURA - pipeline. Provide both `tts_ref_audio` and `tts_ref_text`. Set - `tts_x_vector_only_mode=true` to disable ICL and use speaker embedding only. -- `CustomVoice`: predefined speaker mode. Use a Qwen3-TTS CustomVoice - checkpoint for stages 2 and 3 in `aura_omni.yaml`, then pass - `tts_task_type=CustomVoice` and `tts_speaker`. +- `Base`: voice clone from `tts_ref_audio` (optional x-vector-only mode) +- `CustomVoice`: predefined speaker (`tts_speaker`) with a CustomVoice checkpoint + on stages 2 and 3 -By default, AURA responses are passed to Qwen3-TTS as text. Set -`additional_information.tts_pass_token_ids=true` to pass AURA-generated -assistant token ids directly instead. Even when token passthrough is disabled, -the stage processor uses AURA token ids when available to estimate the Talker -prefill length, so scheduling does not rely on raw character length. +Optional `tts_pass_token_ids` passes AURA assistant token ids into Talker +instead of detokenized text. -The example client exposes this as: +## Old chat / Gradio / curl scripts (unsupported) -```bash -python examples/online_serving/aura_omni/openai_chat_completion_client.py \ - --tts-pass-token-ids -``` +`examples/online_serving/aura_omni/` still contains chat-completions, curl, and +Gradio helpers from an older turn-based path. They do **not** work against the +shipped duplex profile. That path was an incomplete stand-in before streaming +I/O and session history were available; it is not a supported AURA mode. Use +the Realtime duplex smoke client for online verification. diff --git a/docs/user_guide/examples/online_serving/bagel.md b/docs/user_guide/examples/online_serving/bagel.md index 6c10c4349..6b1e20391 100644 --- a/docs/user_guide/examples/online_serving/bagel.md +++ b/docs/user_guide/examples/online_serving/bagel.md @@ -247,12 +247,18 @@ curl http://localhost:8091/v1/chat/completions \ -d @payload.json ``` -BAGEL img2img derives its generated image size from the input image. It -preserves the input aspect ratio, aligns dimensions to the latent stride, and -applies the checkpoint size limit; explicit `height` and `width` values do not -override this model-specific resize policy. In step mode, the effective size is -resolved before scheduler admission so requests with different resulting -shapes are not combined. +When no output size is requested (`size=auto` on `/v1/images/edits`, or no +`height`/`width` in a chat request), BAGEL img2img derives the canvas from the +input image: it preserves the input aspect ratio, aligns dimensions to the +latent stride (16 px), and applies the checkpoint size limit. An explicit +`size`, `height`, or `width` is honored instead, in both the single-stage and +the two-stage deployment; the input image is still resized to a stride-aligned +size for the VAE/ViT prefill. The latent grid needs multiples of 16, so a +requested side that is not one is floored to the next multiple (1000x700 +generates 992x688) and the server logs a warning; the response reports the +generated size. A side above the checkpoint limit is rejected with HTTP 400. +In step mode, the effective size is resolved before scheduler admission so +requests with different resulting shapes are not combined. ### Image to Text (img2text) @@ -318,8 +324,8 @@ curl http://localhost:8091/v1/chat/completions \ | `--server` / `-s` | `http://localhost:8091` | Server URL | | `--image-url` / `-i` | `None` | Input image URL or local path (img2img/img2text) | | `--modality` / `-m` | `text2img` | `text2img`, `img2img`, `img2text`, `text2text` | -| `--height` | `512` | Text-to-image output height; BAGEL img2img derives its size from the input image | -| `--width` | `512` | Text-to-image output width; BAGEL img2img derives its size from the input image | +| `--height` | `512` | Output height, a multiple of 16; omit both `--height` and `--width` to let img2img derive the size from the input image | +| `--width` | `512` | Output width, a multiple of 16; omit both `--height` and `--width` to let img2img derive the size from the input image | | `--steps` | `25` | Number of inference steps | | `--seed` | `42` | Random seed | | `--negative` | `None` | Negative prompt for CFG | diff --git a/examples/online_serving/aura_omni/README.md b/examples/online_serving/aura_omni/README.md index 5f246c336..903850469 100644 --- a/examples/online_serving/aura_omni/README.md +++ b/examples/online_serving/aura_omni/README.md @@ -6,139 +6,89 @@ Qwen3-ASR -> AURA/Qwen3-VL -> Qwen3-TTS Talker -> Qwen3-TTS Code2Wav ``` -The pipeline has three semantic modules, but four engine stages because the -existing Qwen3-TTS implementation is natively split into Talker and Code2Wav. +**AURA's supported online path is Realtime duplex** (`/v1/realtime?duplex=1`). +The default deploy profile sets `session_mode: duplex` because the pipeline +declares `duplex_plugin` and `DuplexOmniEngine` requires that mode. -Start the server with the deploy profile: +This is AURA v1 / Qwen3-VL (silent / ChatML ids `151669` / `151645`), not +AURA v2 / Qwen3.5-VL. + +## Duplex Realtime (only supported online path) + +Stage2→3 codec handoff needs `async_chunk: true`. The default +`vllm_omni/deploy/aura_omni.yaml` keeps `async_chunk: false` as a safer +baseline (turn-based `OmniOrchestrator` never sees the duplex async-chunk +gate). For Realtime, use the smoke deploy (already `async_chunk: true`) or +set that flag yourself: ```bash +bash examples/online_serving/aura_omni/run_duplex_smoke_serve.sh +python examples/online_serving/aura_omni/smoke_duplex_realtime_client.py +``` + +Smoke deploy file: `examples/online_serving/aura_omni/aura_omni_duplex_smoke.yaml`. + +Equivalent serve with the default topology and async chunks enabled: + +```bash +# Copy or edit so async_chunk: true, then: vllm serve aurateam/AURA \ --omni \ --port 8091 \ - --deploy-config vllm_omni/deploy/aura_omni.yaml \ + --deploy-config examples/online_serving/aura_omni/aura_omni_duplex_smoke.yaml \ --served-model-name aurateam/AURA \ --trust-remote-code ``` -The deploy file sets per-stage model repos: +Connect clients to `/v1/realtime?duplex=1`. Silent Stage1 outputs gate TTS (no +audio for that turn). Overlapped input and vision-follow are AURA duplex +capabilities. + +### Browser UI + +AURA has no client VAD: control is **push-to-talk** on the shared shell from +[#7585](https://github.com/vllm-project/vllm-omni/pull/7585) +([`examples/online_serving/realtime_web/`](../realtime_web/README.md)), profile +`aura-ptt` (hold = `is_speech=true` PCM + sticky frames; release = `commit`; +silent+frames for vision-follow). Thin wrapper: + +```bash +python -m examples.online_serving.aura_omni.realtime_web \ + --ws-backend ws://127.0.0.1:8099 --model aurateam/AURA --port 7862 +``` + +Do **not** fork a separate `aura_omni/realtime_web` asset tree; MiniCPM / Qwen +profiles are unchanged and hide the PTT control. + +### Per-stage models + +Edit `model` on each stage in `vllm_omni/deploy/aura_omni.yaml` (or the smoke +YAML) for local checkpoints: - Stage 0 ASR: `Qwen/Qwen3-ASR-1.7B` - Stage 1 AURA: `aurateam/AURA` -- Stage 2/3 TTS: `Qwen/Qwen3-TTS-12Hz-1.7B-Base` +- Stage 2/3 TTS: `Qwen/Qwen3-TTS-12Hz-1.7B-Base` (or CustomVoice) -For local weights, edit the `model` value on each stage in -`vllm_omni/deploy/aura_omni.yaml`. The deploy profile includes -`pipeline: aura_omni`, so the server uses this four-stage topology even when -the command-line model path points at one component checkpoint. +### GPU utilization -Expected request shape: - -- Send microphone audio as the Stage 0 multimodal audio input. -- Include video frames in the original request `multi_modal_data`; the - `asr2aura` processor carries them forward to AURA. -- Optional `additional_information` keys: - - `aura_system_prompt` - - `tts_task_type` - - `tts_language` - - `tts_speaker` - - `tts_instruct` - - `tts_ref_audio` - - `tts_ref_text` - - `tts_x_vector_only_mode` - - `tts_pass_token_ids` - -If AURA emits `<|silent|>`, the `aura2tts` processor returns no TTS request, so -the TTS stages are skipped for that turn. - -## GPU Utilization Recommendation - -Tune `gpu_memory_utilization` per stage in `vllm_omni/deploy/aura_omni.yaml`. -Recommended baseline on one GPU for H200 +Tune `gpu_memory_utilization` per stage. Baseline on one large GPU: - Stage 0 (ASR): `0.10` -- Stage 1 (AURA): `0.4` -- Stage 2 (Qwen3-TTS Talker): `0.20` -- Stage 3 (Qwen3-TTS Code2Wav): `0.20` +- Stage 1 (AURA): `0.40` +- Stage 2 (Talker): `0.20` +- Stage 3 (Code2Wav): `0.20` -## Python Client +## Old chat / Gradio / curl scripts (unsupported) -```bash -python examples/online_serving/aura_omni/openai_chat_completion_client.py \ - --host localhost \ - --port 8091 \ - --model aurateam/AURA \ - --modalities text,audio -``` +`openai_chat_completion_client.py`, `run_curl_multimodal_generation.sh`, and +`run_gradio_demo.sh` target the older turn-based Omni path. They do **not** +work against the shipped `session_mode: duplex` profile. -Use local media: - -```bash -python examples/online_serving/aura_omni/openai_chat_completion_client.py \ - --audio-path /path/to/input.wav \ - --video-path /path/to/video.mp4 \ - --output-dir output_aura_omni_online -``` - -Base voice clone mode (default, recommended as x-vector while debugging ICL): - -```bash -python examples/online_serving/aura_omni/openai_chat_completion_client.py \ - --tts-task-type Base \ - --tts-ref-audio vllm-omni/tests/assets/qwen3_tts/clone_2.wav \ - --tts-ref-text "Okay. Yeah. I resent you. I love you. I respect you. But you know what? You blew it! And thanks to you." -``` - -Enable AURA token-id passthrough explicitly: - -```bash -python examples/online_serving/aura_omni/openai_chat_completion_client.py \ - --tts-pass-token-ids -``` - -CustomVoice mode requires stages 2 and 3 in `aura_omni.yaml` to point at a -Qwen3-TTS CustomVoice checkpoint: - -```bash -python examples/online_serving/aura_omni/openai_chat_completion_client.py \ - --tts-task-type CustomVoice \ - --tts-speaker Vivian -``` - -By default, AURA responses are passed to Qwen3-TTS as text. Set -`tts_pass_token_ids=true` to pass AURA-generated assistant token ids directly -to Qwen3-TTS instead. The processor still uses AURA token ids, when available, -to estimate the Talker prompt length in the default text path. - -## Curl - -```bash -cd examples/online_serving/aura_omni -bash run_curl_multimodal_generation.sh -``` - -Set `PORT`, `MODEL`, or `OUTPUT_DIR` to override defaults: - -```bash -PORT=8666 MODEL=aurateam/AURA bash run_curl_multimodal_generation.sh -TTS_PASS_TOKEN_IDS=true PORT=8666 MODEL=aurateam/AURA bash run_curl_multimodal_generation.sh -``` - -## Gradio - -Launch the server and Gradio UI together: - -```bash -cd examples/online_serving/aura_omni -bash run_gradio_demo.sh -``` - -If the server is already running: - -```bash -python examples/online_serving/aura_omni/gradio_demo.py \ - --model aurateam/AURA \ - --api-base http://localhost:8091/v1 -``` +That turn-based path was an incomplete June-era stand-in: vLLM-Omni did not +yet support streaming I/O for this stack, and adding session history then would +have required a much larger orchestrator change. It is not a supported AURA +mode. Do not run those scripts against the default deploy. Prefer the duplex +smoke client above. ## Offline diff --git a/examples/online_serving/aura_omni/aura_omni_duplex_smoke.yaml b/examples/online_serving/aura_omni/aura_omni_duplex_smoke.yaml new file mode 100644 index 000000000..2be7ea0e7 --- /dev/null +++ b/examples/online_serving/aura_omni/aura_omni_duplex_smoke.yaml @@ -0,0 +1,119 @@ +# AURA duplex Realtime smoke (local weights, session_mode: duplex). +# Stage1 = /workspace/models/AURA + AuraQwen3VLForConditionalGeneration (AURA v1 / Qwen3-VL). +pipeline: aura_omni +# Qwen3-TTS Stage2→3 codec handoff requires async_chunk=true (same as native / +# duplex explore). async_chunk=false reaches talker/code_predictor then never +# delivers audio to Stage3/client. +async_chunk: true +session_mode: duplex +duplex_session: + max_sessions: 1 + +connectors: + connector_of_shared_memory: + name: SharedMemoryConnector + extra: + codec_streaming: true + connector_get_sleep_s: 0.01 + connector_get_max_wait_first_chunk: 3000 + connector_get_max_wait: 300 + codec_chunk_frames: 25 + codec_left_context_frames: 72 + # Match native/duplex explore. initial=1 can stall Code2Wav on the first + # undersized chunk (observed hang after "Code2Wav codec: frames=1"). + initial_codec_chunk_frames: 8 + decode_cudagraph_capture_sizes: [25, 73, 97, 169, 325] + +stages: + - stage_id: 0 + max_num_seqs: 1 + gpu_memory_utilization: 0.08 + enforce_eager: true + trust_remote_code: true + enable_prefix_caching: true + async_scheduling: true + max_model_len: 4096 + max_num_batched_tokens: 2048 + devices: "0" + model: /workspace/models/Qwen3-ASR-1.7B + default_sampling_params: + temperature: 0.0 + top_p: 1.0 + top_k: -1 + max_tokens: 256 + + - stage_id: 1 + max_num_seqs: 1 + gpu_memory_utilization: 0.55 + enforce_eager: true + trust_remote_code: true + enable_prefix_caching: true + async_scheduling: true + max_model_len: 32768 + max_num_batched_tokens: 8192 + devices: "0" + model: /workspace/models/AURA + model_arch: AuraQwen3VLForConditionalGeneration + default_sampling_params: + temperature: 0.0 + top_p: 1.0 + top_k: -1 + max_tokens: 256 + repetition_penalty: 1.0 + stop_token_ids: [151669, 151645] + + - stage_id: 2 + # Allow prior TTS to drain while the next Talker starts (not assumed 1). + max_num_seqs: 2 + gpu_memory_utilization: 0.12 + enforce_eager: true + trust_remote_code: true + enable_prefix_caching: false + async_scheduling: true + max_num_batched_tokens: 4096 + max_model_len: 4096 + devices: "0" + # Stage1→2 is orchestrator-fed via aura2tts (custom_process_input_func). + # A from_stage_1 SHM edge parks Talker in WAITING_FOR_CHUNK and clears + # additional_information.text → Missing Qwen3-TTS text conditioning. + # Only Stage2→3 codec handoff needs a connector. + output_connectors: + to_stage_3: connector_of_shared_memory + model: /workspace/models/Qwen3-TTS-12Hz-1.7B-CustomVoice + default_sampling_params: + temperature: 0.5 + top_k: 50 + # 12Hz × 240 ≈ 20s. Stay under the smoke 25s audio gate so a missed + # 2150 (English/quotes humming) cannot overlap the next vision-follow. + max_tokens: 240 + min_tokens: 8 + repetition_penalty: 1.05 + # Qwen3-TTS Talker codec EOS — without this Talker can run to max_tokens + # (~26s garbage audio while Stage1 text was correct). + stop_token_ids: [2150] + subtalker_sampling_params: + do_sample: true + temperature: 0.5 + top_k: 50 + top_p: 1.0 + + - stage_id: 3 + max_num_seqs: 2 + gpu_memory_utilization: 0.12 + enforce_eager: true + trust_remote_code: true + enable_prefix_caching: false + async_scheduling: true + max_num_batched_tokens: 16384 + max_model_len: 16384 + devices: "0" + input_connectors: + from_stage_2: connector_of_shared_memory + model: /workspace/models/Qwen3-TTS-12Hz-1.7B-CustomVoice + default_sampling_params: + temperature: 0.0 + top_p: 1.0 + top_k: -1 + max_tokens: 16384 + seed: 42 + repetition_penalty: 1.0 diff --git a/examples/online_serving/aura_omni/realtime_web/__init__.py b/examples/online_serving/aura_omni/realtime_web/__init__.py new file mode 100644 index 000000000..d305da15b --- /dev/null +++ b/examples/online_serving/aura_omni/realtime_web/__init__.py @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + +"""AURA push-to-talk browser call UI.""" diff --git a/examples/online_serving/aura_omni/realtime_web/__main__.py b/examples/online_serving/aura_omni/realtime_web/__main__.py new file mode 100644 index 000000000..1f89c5d41 --- /dev/null +++ b/examples/online_serving/aura_omni/realtime_web/__main__.py @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + +from examples.online_serving.realtime_web.server import main + +if __name__ == "__main__": + main(default_profile="aura-ptt") diff --git a/examples/online_serving/aura_omni/run_duplex_smoke_serve.sh b/examples/online_serving/aura_omni/run_duplex_smoke_serve.sh new file mode 100755 index 000000000..ed8f3dc40 --- /dev/null +++ b/examples/online_serving/aura_omni/run_duplex_smoke_serve.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# Minimal AURA duplex Realtime server for smoke. +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" +cd "$REPO_ROOT" + +PORT="${PORT:-8099}" +MODEL="${MODEL:-/workspace/models/AURA}" +DEPLOY_CONFIG="${DEPLOY_CONFIG:-$REPO_ROOT/examples/online_serving/aura_omni/aura_omni_duplex_smoke.yaml}" + +# AURA v1 / Qwen3-VL ChatML ids (do not use AURA v2 248070/248046 here). +export VLLM_AURA_SILENT_TOKEN_ID="${VLLM_AURA_SILENT_TOKEN_ID:-151669}" +export VLLM_AURA_IM_END_TOKEN_ID="${VLLM_AURA_IM_END_TOKEN_ID:-151645}" +export VLLM_AURA_IM_START_TOKEN_ID="${VLLM_AURA_IM_START_TOKEN_ID:-151644}" +export VLLM_AURA_ASSISTANT_TOKEN_ID="${VLLM_AURA_ASSISTANT_TOKEN_ID:-77091}" +# Multi-stage colocated init can make earlier stages' Process.sentinel spuriously +# readable; prefer spawn for stage engine subprocesses (override if needed). +export VLLM_WORKER_MULTIPROC_METHOD="${VLLM_WORKER_MULTIPROC_METHOD:-spawn}" +# Do NOT set TORCHDYNAMO_DISABLE here: it breaks Stage0/1 AOT when those +# stages are not enforce_eager. Stage2/3 use enforce_eager; code_predictor +# honors that and skips its own torch.compile. + +exec "$REPO_ROOT/.venv/bin/vllm-omni" serve "$MODEL" \ + --omni \ + --deploy-config "$DEPLOY_CONFIG" \ + --trust-remote-code \ + --host 0.0.0.0 \ + --port "$PORT" diff --git a/examples/online_serving/aura_omni/smoke_duplex_realtime_client.py b/examples/online_serving/aura_omni/smoke_duplex_realtime_client.py new file mode 100755 index 000000000..431e13d90 --- /dev/null +++ b/examples/online_serving/aura_omni/smoke_duplex_realtime_client.py @@ -0,0 +1,153 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +"""Minimal AURA duplex Realtime smoke via DuplexClient.""" + +from __future__ import annotations + +import argparse +import asyncio +import base64 +import json +import sys +import time +from io import BytesIO +from pathlib import Path + +from PIL import Image + +REPO_ROOT = Path(__file__).resolve().parents[3] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from vllm_omni.clients.duplex import ( # noqa: E402 + DuplexClient, + EventCollector, + SessionConfig, + read_pcm16_wav, +) + +PROGRESS_TYPES = { + "response.created", + "response.done", + "response.audio.delta", + "response.output_audio.delta", + "response.audio_transcript.delta", + "response.output_text.delta", + "response.text.delta", + "response.listen", + "response.model_listen", + "error", +} + + +def _tiny_jpeg_b64() -> str: + img = Image.new("RGB", (64, 64), color=(200, 40, 40)) + buf = BytesIO() + img.save(buf, format="JPEG", quality=85) + return base64.b64encode(buf.getvalue()).decode("ascii") + + +async def run(url: str, model: str, wav: Path, timeout_s: float) -> int: + pcm16 = read_pcm16_wav(wav) + if not pcm16: + print("empty wav", file=sys.stderr) + return 2 + + frame = _tiny_jpeg_b64() + collector = EventCollector() + config = SessionConfig( + modalities=("text", "audio"), + instructions="You are AURA. Reply briefly in Chinese.", + extra_body={ + "aura_system_prompt": "You are AURA. Reply briefly in Chinese.", + "tts_task_type": "CustomVoice", + "tts_language": "Chinese", + "tts_speaker": "Vivian", + }, + ) + + started = time.time() + client = DuplexClient( + url, + model=model, + config=config, + reconnect=None, + heartbeat_interval_s=None, + handshake_timeout_s=min(60.0, timeout_s), + ) + + async with client: + consume_task = asyncio.create_task(collector.consume(client)) + collector.add({"type": "session.created", "session": getattr(client, "session_info", {})}) + + await client.append_audio(pcm16, is_speech=True, video_frames=[frame]) + await client.commit(create_response=True) + + progress = False + got_done = False + while time.time() - started < timeout_s: + types = [e.get("type") for e in collector.events] + if any(t in PROGRESS_TYPES for t in types): + progress = True + if "response.done" in types or "response.completed" in types: + got_done = True + progress = True + break + for e in collector.events: + meta = e.get("metadata") if isinstance(e.get("metadata"), dict) else {} + if meta.get("model_listen") or meta.get("listen_source") == "aura_silent": + progress = True + break + if collector.count("error") > 0 or collector.count("session.closed") > 0: + break + # Keep draining until response.done so TTS is not truncated. + await asyncio.sleep(0.2) + + try: + await client.close(timeout_s=30.0) + except Exception as exc: # noqa: BLE001 + print(f"close_warn={exc}", file=sys.stderr) + try: + await asyncio.wait_for(consume_task, timeout=5.0) + except (TimeoutError, asyncio.CancelledError): + consume_task.cancel() + + event_types = [e.get("type") for e in collector.events] + errors = [json.dumps(e, ensure_ascii=False)[:500] for e in collector.errors()] + result = { + "ok": progress and (got_done or collector.count("response.done") > 0), + "got_response_progress": progress, + "got_response_done": got_done or collector.count("response.done") > 0, + "errors": errors, + "event_types": event_types, + "n_events": len(event_types), + "audio_bytes": len(collector.audio_bytes()), + "llm_text": collector.response_text(collector.response_ids[0]) if collector.response_ids else "", + "elapsed_s": round(time.time() - started, 2), + } + print(json.dumps(result, ensure_ascii=False, indent=2)) + log_path = Path("/tmp/aura_duplex_smoke/client_events.jsonl") + log_path.parent.mkdir(parents=True, exist_ok=True) + log_path.write_text( + "".join(json.dumps(e, ensure_ascii=False) + "\n" for e in collector.events), + encoding="utf-8", + ) + print(f"events_log={log_path}", file=sys.stderr) + return 0 if progress else 1 + + +def main() -> int: + p = argparse.ArgumentParser() + p.add_argument("--url", default="ws://127.0.0.1:8099/v1/realtime") + p.add_argument("--model", default="/workspace/models/AURA") + p.add_argument( + "--wav", + default=str(REPO_ROOT / "tests/assets/minicpmo_4_5/response_required_16k.wav"), + ) + p.add_argument("--timeout-s", type=float, default=300.0) + args = p.parse_args() + return asyncio.run(run(args.url, args.model, Path(args.wav), args.timeout_s)) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/online_serving/realtime_web/README.md b/examples/online_serving/realtime_web/README.md index 854aa4718..7aba2ee10 100644 --- a/examples/online_serving/realtime_web/README.md +++ b/examples/online_serving/realtime_web/README.md @@ -9,6 +9,7 @@ This implements [RFC #7222](https://github.com/vllm-project/vllm-omni/issues/722 | `minicpm-native` | Model-controlled listen/speak, continuous audio input | Frames accompany audio | Yes | | `qwen3-turn --stt` (default) | User presses **Send turn** | No | No | | `qwen3-turn --vad` | Server detects trailing silence; speech interrupts replies | Sampled frames with each spoken turn | Yes | +| `aura-ptt` | Hold **Hold to talk**; release commits. Hold also stops local playback | Sticky frames with speech; vision-follow every 2 frames only while unlocked (locked from PTT release until text-final / listen) | Yes | Qwen3 VAD uses an engine-owned duplex plugin: microphone upload continues while replies stream, and speech can interrupt generation and playback. The @@ -29,6 +30,38 @@ The compatibility wrapper serves the shared assets with `minicpm-native`. The current native query, `extra_body`, reference voice, continuous microphone upload, camera frames and playback acknowledgements are retained. +## AURA: push-to-talk duplex + +AURA has no client VAD. Start the duplex serve, then the thin wrapper: + +```bash +bash examples/online_serving/aura_omni/run_duplex_smoke_serve.sh + +python -m examples.online_serving.aura_omni.realtime_web \ + --ws-backend ws://127.0.0.1:8099 --model aurateam/AURA --port 7862 +``` + +Or select the profile on the shared host: + +```bash +python -m examples.online_serving.realtime_web --profile aura-ptt \ + --ws-backend ws://127.0.0.1:8099 --model aurateam/AURA --port 7862 +``` + +Open the UI, start a session, optionally enable **Camera**, then **hold** +**Hold to talk** to stream `is_speech=true` PCM (with sticky camera frames). +**Release** to `input_audio_buffer.commit`. Holding the button stops playback +in the browser only; the in-flight response is not cancelled on the server. +With the camera on, frames are still captured at 2 fps. While the button is +up they are committed once per two frames (`is_speech=false` + silent PCM). +Both frames of that commit are sent and packed as one video (`<|video_pad|>`). +Turns still open from this client commit, not from a server-side auto trigger. +but only after the prior spoken turn unlocks (text-final or `response.listen`). +Release itself locks vision immediately so a follow-up frame cannot abort the +just-committed speech turn on Stage0/1 (`max_num_seqs=1`). Audio still playing +does not hold the next vision turn once text is done. +MiniCPM / Qwen profiles do not set `pushToTalk`, so the PTT control stays hidden. + ## Qwen3: explicit-turn STT adapter Start the backend, then the UI in a second terminal: @@ -49,6 +82,7 @@ accepts only model selection, so system-prompt controls are hidden in this mode. The adapter follows the shipped `qwen3_omni/openai_realtime_client.py` and `vllm_omni/entrypoints/openai/realtime_connection.py`: + 1. Explicit `duplex=0` selects the legacy STT handler on a turn deployment. It does not enable STT on a duplex deployment. 2. Send `{type: "session.update", model: ...}` and `commit(final=false)`. @@ -181,7 +215,7 @@ conversation API. This demo does not add WebRTC, semantic VAD, or tool calling. ## Shared host options -The shared entry point can select either profile explicitly: +The shared entry point can select a profile explicitly: ```bash python -m examples.online_serving.realtime_web --profile qwen3-turn \ @@ -193,7 +227,7 @@ python -m examples.online_serving.realtime_web --profile qwen3-turn \ - `--public-realtime-url`: optional browser-visible WebSocket URL; otherwise the static host proxies `/v1/realtime` on the same origin. - `--host`, `--port`: UI bind address and port (default port 7862). -- `--ref-audio`: required for MiniCPM; rejected for Qwen3. +- `--ref-audio`: required for MiniCPM; rejected for Qwen3 and AURA. Microphone and camera access require `localhost` or HTTPS. For a remote backend, use an SSH tunnel to the UI host or an HTTPS reverse proxy with WebSocket support. diff --git a/examples/online_serving/realtime_web/app/index.html b/examples/online_serving/realtime_web/app/index.html index a38a731e4..d4662d5a1 100644 --- a/examples/online_serving/realtime_web/app/index.html +++ b/examples/online_serving/realtime_web/app/index.html @@ -22,6 +22,7 @@
+ @@ -75,6 +76,7 @@ + diff --git a/examples/online_serving/realtime_web/app/static/app.js b/examples/online_serving/realtime_web/app/static/app.js index 1dd01d4b3..f57a3f064 100644 --- a/examples/online_serving/realtime_web/app/static/app.js +++ b/examples/online_serving/realtime_web/app/static/app.js @@ -5,6 +5,7 @@ const profile = window.OmniRealtimeProfiles[config.profile || 'minicpm-native'](config); const callButton = document.getElementById('callButton'); const sendTurnButton = document.getElementById('sendTurnButton'); + const pttButton = document.getElementById('pttButton'); const muteButton = document.getElementById('muteButton'); const cameraButton = document.getElementById('cameraButton'); const cameraPreview = document.getElementById('cameraPreview'); @@ -28,6 +29,7 @@ const ECHO_GUARD_MS = 300; const INITIAL_PLAYBACK_BUFFER_MS = 400; const SESSION_CLOSE_TIMEOUT_MS = 1000; + const SILENT_PCM_SAMPLES = 160; const PROMPT_PRESETS = profile.presets; document.title = profile.title; @@ -36,7 +38,21 @@ document.getElementById('profileDescription').textContent = profile.description; document.getElementById('policyLabel').textContent = profile.policy; cameraButton.hidden = !profile.camera; + if (profile.cameraPreviewLarge) { + cameraPreview.classList.add('camera-preview-large'); + // Inline, so a cached stylesheet cannot leave the 96×72 HTML default. + cameraPreview.style.width = '100%'; + cameraPreview.style.height = 'auto'; + cameraPreview.style.maxHeight = '480px'; + cameraPreview.style.flex = '0 0 100%'; + const cameraActions = cameraPreview.closest('.call-actions'); + if (cameraActions) { + cameraActions.style.flexWrap = 'wrap'; + cameraActions.style.gridColumn = '1 / -1'; + } + } sendTurnButton.hidden = !profile.clientCommit; + pttButton.hidden = !profile.pushToTalk; promptPreset.replaceChildren(); for (const name of [...Object.keys(PROMPT_PRESETS), 'custom']) { const option = document.createElement('option'); @@ -64,6 +80,12 @@ let cameraStream = null; let cameraTimer = null; let cameraPendingFrame = null; + let cameraLastFrame = null; + // AURA vision clock only. Other profiles leave this empty and never read it. + let visionFollowQueue = []; + let visionTurnLocked = false; + let suppressedPlaybackId = null; + let pttHeld = false; const cameraCanvas = document.createElement('canvas'); let playbackRate = OUTPUT_RATE; let pendingCapture = []; @@ -87,6 +109,8 @@ const interruptedResponses = new Set(); const pendingEvents = new Set(); let assistantTextChannel = null; + const assistantRowByResponse = new Map(); + let lastClosedAssistantText = ''; let connectionReady = false; let turnCounter = 0; let stopping = null; @@ -134,7 +158,7 @@ assistantTextChannel = null; if (profile.halfDuplex) pendingCapture = []; sendTurnButton.disabled = !profile.clientCommit; - setModel(profile.waiting); + if (!pttHeld) setModel(profile.waiting); }, ECHO_GUARD_MS); } @@ -240,6 +264,7 @@ current.value = finalText; current.text.textContent = finalText; } + if (role === 'assistant' && current.value) lastClosedAssistantText = current.value; current.row.classList.remove('turn-live'); if (role === 'user') liveUserTurn = null; else liveAssistantTurn = null; @@ -317,11 +342,67 @@ } function microphoneUploadEnabled() { - return running && connectionReady && !muted && (!profile.halfDuplex || !assistantActive); + if (!running || !connectionReady || muted) return false; + if (profile.pushToTalk) return pttHeld; + return !profile.halfDuplex || !assistantActive; + } + + function queueVisionFrame(frame) { + visionFollowQueue.push(frame); + if (visionFollowQueue.length > 2) visionFollowQueue.shift(); + } + + function commitVisionFollow() { + // Two frames still gate the turn (client commit). Both frames are one clip; + // the server packs them as a single <|video_pad|>, same as Native. + const frames = visionFollowQueue.slice(); + visionFollowQueue = []; + const silent = new Int16Array(SILENT_PCM_SAMPLES); + const event = profile.append(int16ToBase64(silent), frames, { isSpeech: false }); + socket.send(JSON.stringify(event)); + for (const message of profile.commitMessages()) socket.send(JSON.stringify(message)); + } + + function releaseVisionTurn(responseId) { + if (!profile.visionFollowWhileSpeaking || !visionTurnLocked) return; + if (responseId && currentResponseId && responseId !== currentResponseId) return; + visionTurnLocked = false; + } + + function suppressLocalPlayback() { + // Hold-to-talk cuts the speaker only. The in-flight response stays on the server. + if (currentResponseId) suppressedPlaybackId = currentResponseId; + if (playbackNode) playbackNode.port.postMessage({ type: 'clear' }); + playbackComplete = true; + responseHasAudio = false; + setPlayback('Idle'); + } + + function playbackSuppressed(responseId) { + return Boolean(suppressedPlaybackId) && responseId === suppressedPlaybackId; } function flushCapture() { - if (!socket || socket.readyState !== WebSocket.OPEN || pendingCapture.length === 0) return; + if (!socket || socket.readyState !== WebSocket.OPEN) return; + + // AURA: capture stays 2 fps, but a turn opens only every 2 frames, and + // not until the previous response's text is done (or it listened). + // Playback still running is not the lock. PTT speech is not batched here. + if (profile.pushToTalk && profile.visionFollowWhileSpeaking && !pttHeld) { + if (cameraPendingFrame) { + queueVisionFrame(cameraPendingFrame); + cameraPendingFrame = null; + } + if (!visionTurnLocked && visionFollowQueue.length >= 2) commitVisionFollow(); + return; + } + + if (profile.pushToTalk && !pttHeld) { + pendingCapture = []; + return; + } + + if (pendingCapture.length === 0) return; if (!microphoneUploadEnabled()) { pendingCapture = []; return; @@ -335,11 +416,44 @@ } pendingCapture = []; const pcm = resampleInt16(merged, captureRate, profile.inputSampleRate || INPUT_RATE); - const appendEvent = profile.append(int16ToBase64(pcm), cameraPendingFrame); + const frame = cameraPendingFrame || (profile.stickyCamera ? cameraLastFrame : null); cameraPendingFrame = null; + const appendEvent = profile.pushToTalk + ? profile.append(int16ToBase64(pcm), frame, { isSpeech: true }) + : profile.append(int16ToBase64(pcm), frame); socket.send(JSON.stringify(appendEvent)); } + function setPttHeld(held) { + if (!profile.pushToTalk || !running || pttHeld === held) return; + if (held) { + pttHeld = true; + suppressLocalPlayback(); + pttButton.classList.toggle('is-active', true); + pttButton.textContent = 'Release to send'; + setModel('Talking'); + appendLog('PTT down'); + return; + } + // Flush remaining speech while still held, then commit. + flushCapture(); + if (socket && socket.readyState === WebSocket.OPEN) { + for (const event of profile.commitMessages()) socket.send(JSON.stringify(event)); + } + // Lock vision before clearing pttHeld. Stage0/1 are max_num_seqs=1: a + // vision-follow commit in the gap before response.begin aborts the speech + // turn (server sees a new ephemeral and kills the prior ASR request). + if (profile.visionFollowWhileSpeaking) { + visionTurnLocked = true; + visionFollowQueue = []; + } + pttHeld = false; + pttButton.classList.toggle('is-active', false); + pttButton.textContent = 'Hold to talk'; + setModel(profile.waiting); + appendLog('PTT up · commit'); + } + function beginAssistant(responseId) { currentResponseId = responseId || currentResponseId; responseHasAudio = false; @@ -351,6 +465,7 @@ } function feedPlayback(decoded, responseId) { + if (playbackSuppressed(responseId || currentResponseId)) return; if (!decoded || !decoded.pcm || decoded.pcm.length === 0 || !playbackNode) return; const pcm = resampleInt16(decoded.pcm, decoded.sourceRate, playbackRate); responseHasAudio = true; @@ -392,13 +507,18 @@ } async function handleAudioEvent(action) { + const responseId = action.responseId || currentResponseId || `turn-${turnCounter}`; + if (playbackSuppressed(responseId)) return; markBusy(); - currentResponseId = action.responseId || currentResponseId || `turn-${turnCounter}`; + currentResponseId = responseId; setModel('Speaking'); const generation = sessionGeneration; - const responseId = currentResponseId; const decoded = await decodeAudioDelta(action.event); - if (generation === sessionGeneration && !interruptedResponses.has(responseId)) feedPlayback(decoded, responseId); + if ( + generation === sessionGeneration + && !interruptedResponses.has(responseId) + && !playbackSuppressed(responseId) + ) feedPlayback(decoded, responseId); } function handleTranscriptEvent(action) { @@ -408,12 +528,43 @@ if (profile.deduplicateTranscript && assistantTextChannel && action.channel !== assistantTextChannel) return; if (action.text) assistantTextChannel = action.channel || assistantTextChannel; } - if (action.kind === 'text') addTranscript(action.role, action.text); - else if (profile.deduplicateTranscript && action.role === 'assistant') { + const responseId = action.responseId || null; + const existing = responseId && action.role === 'assistant' ? assistantRowByResponse.get(responseId) : null; + if (existing) { + if (action.kind === 'text') { + if (!action.text) return; + existing.value += action.text; + } else if (action.text) { + existing.value = action.text; + } + existing.text.textContent = existing.value; + liveAssistantTurn = existing; + return; + } + if ( + action.role === 'assistant' + && !liveAssistantTurn + && action.text + && action.text === lastClosedAssistantText + ) { + // response.done already closed this sentence. A later full transcript, + // including one stamped with a newer vision-follow response id, must + // not open a second bubble with the same words. + return; + } + if (action.kind === 'text') { + addTranscript(action.role, action.text); + if (action.role === 'assistant' && responseId && liveAssistantTurn) { + assistantRowByResponse.set(responseId, liveAssistantTurn); + } + return; + } + if (profile.deduplicateTranscript && action.role === 'assistant') { if (action.text) { const turn = ensureTurn('assistant'); turn.value = action.text; turn.text.textContent = action.text; + if (responseId) assistantRowByResponse.set(responseId, turn); } } else finishTranscript(action.role, action.text); } @@ -459,12 +610,14 @@ } case 'listen': assistantActive = false; + releaseVisionTurn(responseId); setModel(profile.waiting); break; case 'begin': if (profile.halfDuplex) armTurnTimeout(); if (echoTimer !== null) { clearTimeout(echoTimer); echoTimer = null; } beginAssistant(responseId); + if (profile.visionFollowWhileSpeaking) visionTurnLocked = true; break; case 'audio': await handleAudioEvent(action); @@ -474,11 +627,13 @@ break; case 'text': case 'text-final': handleTranscriptEvent(action); + if (action.kind === 'text-final' && action.role === 'assistant') releaseVisionTurn(responseId); break; case 'done': clearTimeout(turnTimeout); responseComplete = true; finishTranscript('assistant'); + releaseVisionTurn(responseId); requestPlaybackDrain(responseId); finishResponseIfReady(); break; @@ -671,7 +826,11 @@ await openSocket(); running = true; muted = false; + pttHeld = false; assistantActive = false; + visionFollowQueue = []; + visionTurnLocked = false; + suppressedPlaybackId = null; sendTimer = window.setInterval(flushCapture, profile.sendIntervalMs || SEND_INTERVAL_MS); startClock(); callButton.textContent = 'End session'; @@ -679,6 +838,7 @@ muteButton.disabled = false; cameraButton.disabled = !profile.camera; sendTurnButton.disabled = !profile.clientCommit; + pttButton.disabled = !profile.pushToTalk; setConnection('Connected', 'online'); setModel(profile.waiting); appendLog('session started'); @@ -698,8 +858,9 @@ cameraPreview.srcObject = cameraStream; cameraPreview.style.display = ''; await cameraPreview.play().catch(() => {}); - // Official omni-duplex cadence: one JPEG (quality 0.7) per ~1 s chunk, - // no client-side resize (the server normalizes at scale_resolution=448). + // JPEG quality 0.7; no client-side resize (server scale_resolution=448). + // Interval is per-profile: AURA 2 fps, MiniCPM/Qwen stay at 1 s. + const cameraIntervalMs = profile.cameraIntervalMs || 1000; const captureCameraFrame = () => { if (!cameraStream || cameraPreview.videoWidth === 0) return; const scale = profile.cameraMaxDimension @@ -712,14 +873,17 @@ if (connectionReady && socket?.readyState === WebSocket.OPEN) { for (const event of profile.imageMessages(frame)) socket.send(JSON.stringify(event)); } - } else cameraPendingFrame = frame; + } else { + cameraPendingFrame = frame; + if (profile.stickyCamera) cameraLastFrame = frame; + } }; // Do not make the first spoken turn race a one-second timer. captureCameraFrame(); - cameraTimer = window.setInterval(captureCameraFrame, 1000); + cameraTimer = window.setInterval(captureCameraFrame, cameraIntervalMs); cameraButton.textContent = 'Camera off'; cameraButton.classList.add('is-active'); - appendLog('camera on (1 fps omni frames)'); + appendLog(`camera on (${(1000 / cameraIntervalMs).toFixed(0)} fps omni frames)`); } function stopCamera() { @@ -730,6 +894,7 @@ } cameraStream = null; cameraPendingFrame = null; + cameraLastFrame = null; cameraPreview.srcObject = null; cameraPreview.style.display = 'none'; cameraButton.textContent = 'Camera'; @@ -778,9 +943,18 @@ playbackComplete = true; turnSubmitted = false; assistantTextChannel = null; + assistantRowByResponse.clear(); + lastClosedAssistantText = ''; sendTurnButton.disabled = true; + pttHeld = false; + pttButton.disabled = true; + pttButton.classList.remove('is-active'); + pttButton.textContent = 'Hold to talk'; running = false; assistantActive = false; + visionFollowQueue = []; + visionTurnLocked = false; + suppressedPlaybackId = null; pendingCapture = []; if (sendTimer !== null) clearInterval(sendTimer); if (clockTimer !== null) clearInterval(clockTimer); @@ -854,6 +1028,23 @@ setModel('Thinking'); appendLog('turn submitted'); }); + const pttDown = (event) => { + event.preventDefault(); + setPttHeld(true); + }; + const pttUp = (event) => { + event.preventDefault(); + setPttHeld(false); + }; + pttButton.addEventListener('pointerdown', pttDown); + pttButton.addEventListener('pointerup', pttUp); + pttButton.addEventListener('pointercancel', pttUp); + pttButton.addEventListener('pointerleave', () => { + if (pttHeld) setPttHeld(false); + }); + window.addEventListener('pointerup', () => { + if (pttHeld) setPttHeld(false); + }); muteButton.addEventListener('click', toggleMute); clearLogButton.addEventListener('click', () => { eventLog.textContent = ''; diff --git a/examples/online_serving/realtime_web/app/static/profiles/aura_ptt.js b/examples/online_serving/realtime_web/app/static/profiles/aura_ptt.js new file mode 100644 index 000000000..242aa1b6a --- /dev/null +++ b/examples/online_serving/realtime_web/app/static/profiles/aura_ptt.js @@ -0,0 +1,79 @@ +(function (root) { + 'use strict'; + const profiles = root.OmniRealtimeProfiles; + profiles['aura-ptt'] = () => ({ + title: 'AURA Realtime', + eyebrow: 'Push-to-talk duplex', + policy: 'Hold to talk • release to commit', + description: + 'Hold to talk, release to send. Camera stays on whether you speak or not. ' + + 'Holding the button stops local playback.', + waiting: 'Ready', + camera: true, + // Display only. Shared CSS puts .camera-preview-large on its own row. + // Other profiles keep the 96×72 preview in index.html. + cameraPreviewLarge: true, + playbackAck: true, + clientCommit: false, + pushToTalk: true, + stickyCamera: true, + // OmniInteract / Native AURA clock: 2 fps. Other profiles omit this and stay at 1 s. + cameraIntervalMs: 500, + visionFollowWhileSpeaking: true, + deduplicateTranscript: true, + halfDuplex: false, + closeSession: true, + reconnectEachTurn: false, + readyEvent: 'session.updated', + instructions: true, + sendIntervalMs: 200, + presets: { + omni: 'You are receiving a live video stream where the final frame is the present moment. Respond only when a response is needed based on the user\'s message or the visual context. Otherwise, output `<|silent|>` to signify silence.', + }, + url(config, location) { + const url = profiles.url(config, location); + url.searchParams.set('duplex', '1'); + url.searchParams.set('model', config.model || 'aurateam/AURA'); + return url.toString(); + }, + initialMessages(config, instructions) { + const session = { + modalities: ['audio', 'text'], + extra_body: { auto_response: true }, + }; + if (instructions) session.instructions = instructions; + return [{ type: 'session.update', session }]; + }, + append(audio, frame, opts) { + const isSpeech = !opts || opts.isSpeech !== false; + const event = { + type: 'input_audio_buffer.append', + audio, + format: 'pcm16', + sample_rate_hz: 16000, + is_speech: isSpeech, + }; + const frames = Array.isArray(frame) ? frame.filter(Boolean) : (frame ? [frame] : []); + if (frames.length) event.video_frames = frames; + return event; + }, + commitMessages: () => [{ type: 'input_audio_buffer.commit', create_response: true }], + ack(responseId, playedMs) { + return { + type: 'playback.ack', + response_id: responseId, + item_id: `item_${responseId}`, + played_ms: playedMs, + committed_ms: playedMs, + }; + }, + mapEvent(event) { + if (event.type === 'response.listen') return { kind: 'listen' }; + if (event.type === 'playback.acknowledged') { + return { kind: 'ack', committedMs: (event.event || event).committed_ms || 0 }; + } + return profiles.event(event); + }, + connectionHint: 'Check the AURA duplex serve (run_duplex_smoke_serve.sh) and --ws-backend port.', + }); +})(globalThis); diff --git a/examples/online_serving/realtime_web/app/static/styles.css b/examples/online_serving/realtime_web/app/static/styles.css index bf25c3eff..53f98df7c 100644 --- a/examples/online_serving/realtime_web/app/static/styles.css +++ b/examples/online_serving/realtime_web/app/static/styles.css @@ -71,6 +71,18 @@ main { width: min(1040px, calc(100% - 40px)); margin: 28px auto 48px; } } .call-actions { display: flex; gap: 8px; } +/* AURA only: class is added solely by the aura-ptt profile. */ +.call-bar:has(.camera-preview-large) .call-actions { grid-column: 1 / -1; } +.call-actions:has(.camera-preview-large) { flex-wrap: wrap; align-items: center; } +.call-actions:has(.camera-preview-large) .camera-preview-large { + order: -1; + flex: 0 0 100%; + width: 100% !important; + height: auto !important; + max-height: 480px; + aspect-ratio: 4 / 3; +} +.call-actions:has(.camera-preview-large) button { flex: 0 0 auto; } button { border-radius: 5px; border: 1px solid transparent; cursor: pointer; } button:disabled { opacity: .46; cursor: default; } .primary { padding: 10px 15px; color: #ffffff; background: #16834b; font-weight: 700; } diff --git a/examples/online_serving/realtime_web/server.py b/examples/online_serving/realtime_web/server.py index 85e4b63d5..f6a10c4d3 100644 --- a/examples/online_serving/realtime_web/server.py +++ b/examples/online_serving/realtime_web/server.py @@ -76,15 +76,20 @@ def build_app( profile: str = "minicpm-native", adapter: str = "stt", ) -> FastAPI: - if profile not in {"minicpm-native", "qwen3-turn"}: + if profile not in {"minicpm-native", "qwen3-turn", "aura-ptt"}: raise ValueError(f"Unknown profile: {profile}") if adapter not in {"stt", "vad"}: raise ValueError(f"Unknown adapter: {adapter}") if profile == "qwen3-turn" and ref_audio: raise ValueError("Qwen3 profile does not accept --ref-audio") + if profile == "aura-ptt" and ref_audio: + raise ValueError("AURA profile does not accept --ref-audio") native = profile == "minicpm-native" - ws_backend = ws_backend or ("ws://127.0.0.1:8099" if native else "ws://127.0.0.1:8091") - model = model or ("openbmb/MiniCPM-o-4_5" if native else "Qwen/Qwen3-Omni-30B-A3B-Instruct") + aura = profile == "aura-ptt" + ws_backend = ws_backend or ("ws://127.0.0.1:8099" if native or aura else "ws://127.0.0.1:8091") + model = model or ( + "openbmb/MiniCPM-o-4_5" if native else "aurateam/AURA" if aura else "Qwen/Qwen3-Omni-30B-A3B-Instruct" + ) app = FastAPI(title="Omni Realtime Web UI") index_path = APP_DIR / "index.html" app_version_hash = hashlib.sha256() @@ -182,7 +187,7 @@ def main(default_profile: str = "minicpm-native") -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--host", default="0.0.0.0") parser.add_argument("--port", type=int, default=7862) - parser.add_argument("--profile", choices=("minicpm-native", "qwen3-turn"), default=default_profile) + parser.add_argument("--profile", choices=("minicpm-native", "qwen3-turn", "aura-ptt"), default=default_profile) parser.add_argument("--ws-backend", "--backend", dest="ws_backend") mode = parser.add_mutually_exclusive_group() mode.add_argument("--stt", dest="adapter", action="store_const", const="stt") @@ -202,12 +207,15 @@ def main(default_profile: str = "minicpm-native") -> None: ) args = parser.parse_args() native = args.profile == "minicpm-native" + aura = args.profile == "aura-ptt" if native and not args.ref_audio: parser.error("--ref-audio is required for the minicpm-native profile") if not native and args.ref_audio: parser.error("--ref-audio is only supported by minicpm-native") - args.model = args.model or ("openbmb/MiniCPM-o-4_5" if native else "Qwen/Qwen3-Omni-30B-A3B-Instruct") - args.ws_backend = args.ws_backend or ("ws://127.0.0.1:8099" if native else "ws://127.0.0.1:8091") + args.model = args.model or ( + "openbmb/MiniCPM-o-4_5" if native else "aurateam/AURA" if aura else "Qwen/Qwen3-Omni-30B-A3B-Instruct" + ) + args.ws_backend = args.ws_backend or ("ws://127.0.0.1:8099" if native or aura else "ws://127.0.0.1:8091") logging.basicConfig(level=logging.INFO) uvicorn.run( diff --git a/recipes/Bagel/BAGEL-7B-MoT.md b/recipes/Bagel/BAGEL-7B-MoT.md index 19ace3d33..17e10fb69 100644 --- a/recipes/Bagel/BAGEL-7B-MoT.md +++ b/recipes/Bagel/BAGEL-7B-MoT.md @@ -120,12 +120,19 @@ python examples/offline_inference/image_to_image/image_edit.py \ The `--extra-args` JSON forwards BAGEL-specific parameters (e.g. `cfg_text_scale`, `cfg_img_scale`, `cfg_interval`, `cfg_renorm_type`) into `OmniDiffusionSamplingParams.extra_args` via the model-extras registry. -For BAGEL img2img, the pipeline derives the generated image size from the input -image: it preserves the input aspect ratio, aligns dimensions to the latent -stride, and applies the checkpoint size limit. Explicit `--height` and -`--width` values do not override that BAGEL-specific resize policy. Step-mode -admission resolves this effective size before batching so differently shaped -requests are scheduled separately. +For BAGEL img2img, an explicit `--height`/`--width` (or `size` on +`/v1/images/edits`) sets the output canvas, in both the single-stage and the +two-stage deployment; the input image is still resized to a stride-aligned size +for the VAE/ViT prefill. The latent grid needs multiples of 16, so a requested +side that is not one is floored to the next multiple (1000x700 generates +992x688) with a server-side warning, and the response reports the generated +size. A side above the checkpoint limit is rejected with HTTP 400. When no size +is requested +(`size=auto`, or no `height`/`width`), the pipeline derives the canvas from the +input image: it preserves the input aspect ratio, aligns dimensions to the +latent stride, and applies the checkpoint size limit. Step-mode admission +resolves this effective size before batching so differently shaped requests are +scheduled separately. Run text-to-text with the shared understanding example. BAGEL's default `bagel.yaml` deploy config is discovered from the checkpoint, so no model-specific diff --git a/recipes/Qwen/Qwen3-TTS.md b/recipes/Qwen/Qwen3-TTS.md index d0dbe57df..d6a1a850c 100644 --- a/recipes/Qwen/Qwen3-TTS.md +++ b/recipes/Qwen/Qwen3-TTS.md @@ -71,8 +71,10 @@ Alternatively, use the convenience script: ``` The bundled deploy config (`vllm_omni/deploy/qwen3_tts.yaml`) enables async -chunking for low first-audio latency. For advanced deployment tuning, pass a -custom deploy config: +chunking for low first-audio latency, and selects the experimental Model +Runner V2 on CUDA. Copy it and set `model_runner: v1` to run V1; NPU, XPU, +ROCm and MUSA keep V1 through the `platforms:` sections. For advanced +deployment tuning, pass a custom deploy config: ```bash vllm serve Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice \ diff --git a/recipes/Robbyant/LingBot-World-2.0.md b/recipes/Robbyant/LingBot-World-2.0.md index 24a36653d..b20796726 100644 --- a/recipes/Robbyant/LingBot-World-2.0.md +++ b/recipes/Robbyant/LingBot-World-2.0.md @@ -53,6 +53,26 @@ source frames, at least as many as requested and at most 4096. The runtime consumes only the prefix needed by the request. Longer rollouts also require sufficient device memory. +## Experimental KV writeback reuse + +To reuse KV from the final denoising step instead of running the additional +clean-latent writeback, pass this option in the `model_config` supplied to +`Omni` or `AsyncOmni`: + +```python +model_config = {"lingbot_reuse_last_step_kv": True} +``` + +The default is `False`. The value must be a boolean, not a string or integer. +It is validated and fixed when the pipeline instance is constructed; changing +it requires a new instance. It is not a per-request `extra_args` option. +The previous `VLLM_OMNI_LINGBOT_REUSE_LAST_STEP_KV` environment variable is no +longer read. + +This experimental mode stores the final noisy probe's KV rather than clean +`x0` KV, so generated outputs can differ. The four denoising steps, chunk +boundaries, and paged-context finalization remain the same. + ## Realtime in-process generation (deprecated) Prefer [Streaming video serving](#streaming-video-serving) for mid-session @@ -193,6 +213,47 @@ accepts: `s`, `d`, `i`, `j`, `k`, and `l`; - `lingbot.camera_trajectory.v1` for explicit pose/intrinsics trajectories. +## Online FP8 linear layers (experimental) + +Pass `quantization_config={"method": "fp8"}` to the existing `AsyncOmni` +construction to use vLLM's online FP8 linear implementation. On the tested +Hopper-class GPU this dispatches to CUTLASS FP8 GEMM with online per-tensor +activation scaling; no LingBot-specific quantization kernel is introduced. + +Only the transformer's existing vLLM parallel linear layers are eligible: +self-attention, cross-attention, FFN, camera injectors, and C2WS projections. +The ordinary PyTorch linear layers (including the output head and +time/text embeddings), normalization, convolutions, VAE, text encoder, and +AR KV cache are not converted to FP8 by this option. + +Quality-sensitive projections can remain BF16 through fully qualified +`ignored_layers` names. For example, the following configuration tests +retaining the camera/C2WS path in BF16: + +```python +quantization_config = { + "method": "fp8", + "ignored_layers": [ + f"transformer.blocks.{i}.cam_injector_layer{j}" + for i in range(40) + for j in (1, 2) + ] + + [ + "transformer.c2ws_hidden_states_layer1", + "transformer.c2ws_hidden_states_layer2", + ], +} +``` + +Use `transformer.blocks..self_attn.qkv` to exclude the fused Q/K/V +projection as a unit. The example exclusion list is an ablation, not a +universal quality guarantee or a default. Validate the intended scene, seed, +camera trajectory, and session length against BF16 before selecting a policy. +An FP8 kernel speedup alone is not an end-to-end performance claim. +The long-session E8 quality gate in +[#7074](https://github.com/vllm-project/vllm-omni/issues/7074) remains required +for Tier-2 acceptance. + ## Validation Real-checkpoint validation uses 480x832 output, four DMD steps, and seed 42. @@ -226,7 +287,7 @@ tested commit. AR blocks in one request. `max_num_seqs` must be one in both cases. - Stateful streaming VAE decode is not implemented; the realtime example emits latent chunks. -- SP/USP, pipeline/CFG parallelism, HSDP, VAE parallelism, quantization, +- SP/USP, pipeline/CFG parallelism, HSDP, VAE parallelism, quantization methods other than online FP8, Cache-DiT, TeaCache, causal-pretrain, and the 1.3B checkpoint are not claimed. - No AMD GPU, Ascend NPU, or Intel GPU support is claimed. diff --git a/tests/benchmarks/test_aura_omniinteract_clock.py b/tests/benchmarks/test_aura_omniinteract_clock.py new file mode 100644 index 000000000..5e815f5fe --- /dev/null +++ b/tests/benchmarks/test_aura_omniinteract_clock.py @@ -0,0 +1,157 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + +from __future__ import annotations + +import asyncio +from pathlib import Path + +import pytest + +from vllm_omni.benchmarks.omniinteract import _session_capabilities, _wants_annotation_video_clock +from vllm_omni.model_executor.models.aura_omni.benchmarks.omniinteract_clock import ( + stream_annotation_clock, + wants_annotation_video_clock, +) + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +class _Events: + def __init__(self, events): + self.events = events + + +class _Client: + def __init__(self, capabilities): + self.events = _Events([{"type": "session.created", "session": {"capabilities": capabilities}}]) + self._client = None + + +def test_annotation_clock_follows_capabilities_not_model_name(): + aura = {"required_input_modalities": ["video"], "supports_turn_commit_only": True} + minicpm = {"required_input_modalities": ["audio"], "supports_turn_commit_only": False} + audio_commit = {"required_input_modalities": ["audio"], "supports_turn_commit_only": True} + assert wants_annotation_video_clock(aura) + assert not wants_annotation_video_clock(minicpm) + assert not wants_annotation_video_clock(audio_commit) + assert not wants_annotation_video_clock(None) + assert _wants_annotation_video_clock(_Client(minicpm)) is False + assert _wants_annotation_video_clock(_Client(aura)) is True + assert _session_capabilities(_Client(aura))["required_input_modalities"] == ["video"] + + +def test_annotation_clock_commits_only_the_due_wav(monkeypatch, tmp_path: Path): + video = tmp_path / "videos" / "0002.mp4" + video.parent.mkdir() + video.write_bytes(b"") + sent = [] + + class Client: + async def send(self, event): + sent.append(event) + + async def commit(self): + sent.append({"type": "commit"}) + + class Playback: + async def acknowledge(self, client): + del client + + monkeypatch.setattr( + "vllm_omni.model_executor.models.aura_omni.benchmarks.omniinteract_clock.load_annotation_utterances", + lambda path: [{"at_sec": 0.0, "pcm": b"\x01\x00" * 160, "sent": False, "qa_index": 0}], + ) + chunks, frames, _, _ = asyncio.run( + stream_annotation_clock( + Client(), + ["frame-a", "frame-b"], + Playback(), + video_path=video, + fps=2.0, + ) + ) + assert (chunks, frames) == (2, 2) + assert sent[0]["is_speech"] is True + assert sent[0]["video_frames"] == ["frame-a"] + assert sent[1] == {"type": "commit"} + # The trailing silent frame is one vision turn, committed after the clip. + assert sent[2]["is_speech"] is False + assert sent[2]["video_frames"] == ["frame-b"] + assert sent[3] == {"type": "commit"} + assert sent.count({"type": "commit"}) == 2 + + +def test_annotation_clock_commits_silent_frames_in_pairs(monkeypatch, tmp_path: Path): + video = tmp_path / "videos" / "0002.mp4" + video.parent.mkdir() + video.write_bytes(b"") + sent = [] + + class Client: + async def send(self, event): + sent.append(event) + + async def commit(self): + sent.append({"type": "commit"}) + + class Playback: + async def acknowledge(self, client): + del client + + monkeypatch.setattr( + "vllm_omni.model_executor.models.aura_omni.benchmarks.omniinteract_clock.load_annotation_utterances", + lambda path: [{"at_sec": 100.0, "pcm": b"\x01\x00" * 160, "sent": False, "qa_index": 0}], + ) + asyncio.run( + stream_annotation_clock( + Client(), + ["f1", "f2", "f3", "f4"], + Playback(), + video_path=video, + fps=2.0, + ) + ) + vision = [event for event in sent if event.get("is_speech") is False] + assert [event["video_frames"] for event in vision] == [["f1", "f2"], ["f3", "f4"]] + assert sent.count({"type": "commit"}) == 2 + + +def test_annotation_clock_holds_vision_while_a_response_is_open(monkeypatch, tmp_path: Path): + video = tmp_path / "videos" / "0002.mp4" + video.parent.mkdir() + video.write_bytes(b"") + sent = [] + + class Events: + def __init__(self): + self.events = [{"type": "response.created", "response": {"id": "resp-open"}}] + + class Client: + def __init__(self): + self.events = Events() + + async def send(self, event): + sent.append(event) + + async def commit(self): + sent.append({"type": "commit"}) + + class Playback: + async def acknowledge(self, client): + del client + + monkeypatch.setattr( + "vllm_omni.model_executor.models.aura_omni.benchmarks.omniinteract_clock.load_annotation_utterances", + lambda path: [{"at_sec": 100.0, "pcm": b"\x01\x00" * 160, "sent": False, "qa_index": 0}], + ) + asyncio.run( + stream_annotation_clock( + Client(), + ["f1", "f2", "f3", "f4"], + Playback(), + video_path=video, + fps=2.0, + ) + ) + assert sent == [] diff --git a/tests/benchmarks/test_lingbot_world_realtime.py b/tests/benchmarks/test_lingbot_world_realtime.py new file mode 100644 index 000000000..9926513b4 --- /dev/null +++ b/tests/benchmarks/test_lingbot_world_realtime.py @@ -0,0 +1,798 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project +"""Unit tests for the LingBot-World realtime streaming benchmark. + +The arithmetic is tested against the server's own formulas rather than against +hand-copied expectations, and the protocol handling runs against a real +WebSocket server that replays ``/v1/realtime/video`` with scripted delays -- so +a benchmark number can be wrong only if the server itself is, not because the +client parsed incorrectly an event or timed the wrong edge. +""" + +from __future__ import annotations + +# ruff: noqa: E402, I001 +import asyncio +import json +import sys +from pathlib import Path + +import pytest + +pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu] + +REPO_ROOT = Path(__file__).resolve().parents[2] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from benchmarks.lingbot_world.benchmark_lingbot_world_realtime import ( + BenchmarkError, + build_workload, + print_report, + endpoint_url, + parse_args, + resolve_image_reference, + run_session, +) +from benchmarks.lingbot_world.workload import ( + DEFAULT_WARMUP_CHUNKS, + FRAMES_PER_BLOCK, + ChunkRecord, + PromptUpdate, + Workload, + aggregate_metrics, + build_camera_script, + chunks_for_num_frames, + compute_metrics, + default_workload, + load_workload, + num_frames_for_chunks, + percentile, + pixel_frames_in_chunk, + simulate_playback, + steady_chunk_deadline_ms, +) + +_IMAGE = "data:image/png;base64,Zm9v" + + +def _workload(num_chunks: int = 8, fps: int = 16) -> Workload: + return default_workload(image_reference=_IMAGE, num_chunks=num_chunks, fps=fps) + + +# --------------------------------------------------------------------------- # +# Chunk and frame arithmetic +# --------------------------------------------------------------------------- # + + +@pytest.mark.parametrize("num_chunks", list(range(1, 33))) +def test_frame_count_round_trips_through_the_server_formula(num_chunks: int) -> None: + """``num_frames`` must land on a rollout of exactly the requested chunks. + + The server derives chunk count as ``((num_frames - 1) // 4 + 1) // 3``; if + the benchmark's inverse is off by one frame the run silently generates a + different number of blocks than it reports. + """ + assert chunks_for_num_frames(num_frames_for_chunks(num_chunks)) == num_chunks + + +def test_recipe_frame_counts_match_the_documented_chunk_counts() -> None: + # The recipe states three chunks for num_frames 33 and seven for 81. + assert chunks_for_num_frames(33) == 3 + assert chunks_for_num_frames(81) == 7 + assert num_frames_for_chunks(3) == 33 + assert num_frames_for_chunks(7) == 81 + + +def test_first_chunk_is_short_because_the_decoder_expands_it_once() -> None: + assert pixel_frames_in_chunk(0) == 9 + assert pixel_frames_in_chunk(1) == 12 + assert pixel_frames_in_chunk(7) == 12 + # A whole rollout's frames are the sum of its chunks' frames. + assert sum(pixel_frames_in_chunk(index) for index in range(5)) == num_frames_for_chunks(5) + + +def test_camera_script_holds_one_three_entry_action_list_per_chunk() -> None: + script = build_camera_script(4, "orbit") + assert len(script) == 4 + assert all(len(actions) == FRAMES_PER_BLOCK for actions in script) + # orbit advances through the action keys so successive chunks differ. + assert script[0] != script[1] + + +def test_unknown_camera_pattern_is_rejected() -> None: + with pytest.raises(ValueError, match="Unknown camera pattern"): + build_camera_script(2, "sideways") + + +# --------------------------------------------------------------------------- # +# Workload +# --------------------------------------------------------------------------- # + + +def test_session_start_payload_matches_the_documented_protocol() -> None: + workload = _workload(num_chunks=3) + payload = workload.session_start_payload("robbyant/lingbot-world-v2-14b-causal-fast-diffusers") + assert payload["type"] == "session.start" + assert payload["num_frames"] == 33 + assert payload["image_reference"] == {"image_url": _IMAGE} + assert payload["extra_params"]["flow_shift"] == 5.0 + assert len(payload["extra_params"]["camera_action_script"]) == 3 + + +def test_workload_rejects_a_camera_script_with_the_wrong_block_width() -> None: + with pytest.raises(ValueError, match="exactly 3 action lists"): + Workload(prompt="p", image_reference=_IMAGE, camera_script=[[["w"], ["w"]]]) + + +def test_workload_rejects_geometry_the_cache_cannot_serve() -> None: + with pytest.raises(ValueError, match="multiples of 16"): + Workload(prompt="p", image_reference=_IMAGE, camera_script=build_camera_script(1, "forward"), width=831) + + +def test_prompt_update_beyond_the_rollout_is_rejected() -> None: + with pytest.raises(ValueError, match="never fires"): + Workload( + prompt="p", + image_reference=_IMAGE, + camera_script=build_camera_script(2, "forward"), + prompt_updates=(PromptUpdate(after_chunk=5, prompt="later"),), + ) + + +@pytest.mark.parametrize("from_file", [False, True]) +def test_duplicate_prompt_update_boundaries_are_rejected(tmp_path: Path, from_file: bool) -> None: + updates = [ + {"after_chunk": 1, "prompt": "Rain begins"}, + {"after_chunk": 0, "prompt": "Clouds gather"}, + {"after_chunk": 1, "prompt": "Rain stops"}, + ] + with pytest.raises(ValueError, match="Duplicate prompt update after chunk 1"): + if from_file: + path = tmp_path / "rollout.json" + path.write_text(json.dumps({"image": _IMAGE, "num_chunks": 3, "prompt_updates": updates})) + build_workload(parse_args(["--workload", str(path)])) + else: + Workload( + prompt="p", + image_reference=_IMAGE, + camera_script=build_camera_script(3, "forward"), + prompt_updates=( + PromptUpdate(after_chunk=1, prompt="Rain begins"), + PromptUpdate(after_chunk=0, prompt="Clouds gather"), + PromptUpdate(after_chunk=1, prompt="Rain stops"), + ), + ) + + +def test_workload_file_is_loaded_and_cli_overrides_win(tmp_path: Path) -> None: + path = tmp_path / "rollout.json" + path.write_text( + json.dumps( + { + "prompt": "A quiet street", + "image": "https://example.invalid/frame.png", + "num_chunks": 4, + "camera_pattern": "hold", + "fps": 12, + "seed": 7, + "prompt_updates": [{"after_chunk": 1, "prompt": "Rain begins", "transition_chunks": 2}], + } + ) + ) + workload = load_workload(path, overrides={"fps": 24, "seed": None}) + assert workload.prompt == "A quiet street" + assert workload.num_chunks == 4 + assert workload.fps == 24, "an explicit CLI value must beat the file" + assert workload.seed == 7, "a None override must leave the file value alone" + assert workload.prompt_updates[0].transition_chunks == 2 + + +def test_workload_file_without_a_rollout_length_is_rejected(tmp_path: Path) -> None: + path = tmp_path / "rollout.json" + path.write_text(json.dumps({"image": "https://example.invalid/frame.png"})) + with pytest.raises(ValueError, match="camera_action_script or num_chunks"): + load_workload(path) + + +def test_local_image_is_inlined_as_a_data_url(tmp_path: Path) -> None: + image = tmp_path / "frame.png" + image.write_bytes(b"\x89PNG\r\n\x1a\n") + reference = resolve_image_reference(str(image)) + assert reference.startswith("data:image/png;base64,") + assert resolve_image_reference("https://example.invalid/a.png") == "https://example.invalid/a.png" + with pytest.raises(BenchmarkError, match="neither a URL nor a readable file"): + resolve_image_reference(str(tmp_path / "missing.png")) + + +# --------------------------------------------------------------------------- # +# Metric math +# --------------------------------------------------------------------------- # + + +def test_rtf_is_a_cost_not_a_speed() -> None: + """RTF must match vLLM-Omni's own formula, where lower is better. + + ``vllm_omni/metrics/definitions.py::compute_audio_rtf`` is + ``latency / duration`` with the docstring "SLO red line < 1", and + ``benchmarks/patch/patch.py`` sets ``video_rtf = generation_s / + video_duration``. Emitting the reciprocal under the same field name would + silently invert every dashboard that aggregates it, so the direction is + pinned here rather than left to a comment. + """ + # Two chunks, 9 + 12 = 21 frames at 12 fps = 1.75 s of video, generated in 3.5 s. + slow = compute_metrics(_records([1.0, 2.5]), fps=12, warmup_chunks=0, session_wall_s=3.5) + assert slow["video_seconds"] == pytest.approx(1.75) + assert slow["video_rtf"] == pytest.approx(2.0), "twice as slow as real time" + assert slow["video_rtfx"] == pytest.approx(0.5) + + fast = compute_metrics(_records([0.4, 0.475]), fps=12, warmup_chunks=0, session_wall_s=0.875) + assert fast["video_rtf"] == pytest.approx(0.5), "twice as fast as real time" + assert fast["video_rtf"] < 1.0 < slow["video_rtf"] + assert fast["video_rtf"] * fast["video_rtfx"] == pytest.approx(1.0) + + +def test_steady_rtf_is_the_chunk_interval_over_its_deadline() -> None: + """A steady chunk's RTF is just its interval divided by its video duration.""" + metrics = compute_metrics(_records([1.0] + [1.357] * 15), fps=12, warmup_chunks=6) + # 12 frames at 12 fps = 1000 ms of video per steady chunk. + assert metrics["chunk_deadline_ms"] == pytest.approx(1000.0) + assert metrics["steady_video_rtf"] == pytest.approx(1.357, abs=1e-3) + + +def test_percentile_interpolates_so_short_runs_still_report_tails() -> None: + values = [10.0, 20.0, 30.0, 40.0] + assert percentile(values, 0.0) == 10.0 + assert percentile(values, 1.0) == 40.0 + assert percentile(values, 0.5) == 25.0 + assert percentile([5.0], 0.99) == 5.0 + + +def _records(intervals: list[float]) -> list[ChunkRecord]: + """One record per entry; ``intervals[0]`` is the TTFC, the rest are gaps.""" + records: list[ChunkRecord] = [] + arrival = 0.0 + for index, interval in enumerate(intervals): + arrival += interval + records.append( + ChunkRecord( + index=index, + arrival_s=arrival, + inter_arrival_s=interval, + byte_length=1000, + num_frames=pixel_frames_in_chunk(index), + ) + ) + return records + + +def test_steady_state_excludes_the_unsaturated_attention_window() -> None: + """Warmup chunks must not drag the steady-state mean down. + + The first six chunks attend over a growing window, so they are genuinely + cheaper. A benchmark that averages them in reports a cadence the session + will never sustain. + """ + intervals = [1.0] + [0.2] * (DEFAULT_WARMUP_CHUNKS - 1) + [0.8] * 4 + metrics = compute_metrics(_records(intervals), fps=16, warmup_chunks=DEFAULT_WARMUP_CHUNKS) + assert metrics["interval_steady"]["count"] == 3 + assert metrics["interval_steady"]["mean_ms"] == pytest.approx(800.0) + assert metrics["interval_all"]["mean_ms"] < metrics["interval_steady"]["mean_ms"] + + +def test_short_rollouts_report_that_steady_state_was_never_reached() -> None: + metrics = compute_metrics(_records([1.0, 0.5, 0.5]), fps=16, warmup_chunks=DEFAULT_WARMUP_CHUNKS) + assert metrics["steady_state_reached"] is False + assert metrics["interval_steady"] == {} + assert "slo_attainment" not in metrics + + +def test_real_time_factor_and_deadline_use_the_true_frame_counts() -> None: + # Eight chunks: 9 + 7*12 = 93 frames at 16 fps = 5.8125 s of video. + metrics = compute_metrics(_records([1.0] + [0.5] * 7), fps=16, warmup_chunks=0, session_wall_s=5.8125) + assert metrics["total_frames"] == 93 + assert metrics["video_seconds"] == pytest.approx(5.8125) + assert metrics["video_rtf"] == pytest.approx(1.0) + assert metrics["video_rtfx"] == pytest.approx(1.0) + # A steady chunk carries 12 frames, so real time allows 750 ms at 16 fps. + assert metrics["chunk_deadline_ms"] == pytest.approx(750.0) + assert steady_chunk_deadline_ms(12) == pytest.approx(1000.0) + + +def test_slo_attainment_counts_only_steady_chunks() -> None: + intervals = [1.0] + [0.1] * 5 + [0.7, 0.9, 0.7, 0.9] + metrics = compute_metrics(_records(intervals), fps=16, warmup_chunks=6, slo_ms=750.0) + assert metrics["slo_attainment"] == pytest.approx(2 / 3) + assert metrics["slo_violations"] == 1 + + +def test_playback_simulation_finds_stalls_a_percentile_would_hide() -> None: + """One late chunk behind a one-chunk buffer is a visible stall.""" + # 12 frames at 16 fps = 750 ms of playback per chunk; chunk 3 takes 2 s. + intervals = [1.0, 0.5, 0.5, 2.0, 0.5] + result = simulate_playback(_records(intervals), fps=16, buffer_chunks=1) + assert result.underrun_count >= 1 + assert result.first_underrun_chunk == 3 + assert result.total_stall_s > 0.0 + + +def test_a_deeper_buffer_absorbs_the_same_jitter() -> None: + intervals = [1.0, 0.5, 0.5, 1.4, 0.2, 0.2] + fast = simulate_playback(_records(intervals), fps=16, buffer_chunks=1) + buffered = simulate_playback(_records(intervals), fps=16, buffer_chunks=3) + assert buffered.underrun_count < fast.underrun_count + + +def test_playback_rejects_a_buffer_deeper_than_the_rollout() -> None: + with pytest.raises(ValueError, match="exceeds"): + simulate_playback(_records([1.0, 0.5]), fps=16, buffer_chunks=9) + + +def test_aggregate_pools_sessions_and_reports_spread() -> None: + fast = compute_metrics(_records([1.0] + [0.1] * 5 + [0.6] * 4), fps=16, warmup_chunks=6) + slow = compute_metrics(_records([1.0] + [0.1] * 5 + [0.9] * 4), fps=16, warmup_chunks=6) + aggregate = aggregate_metrics([fast, slow], fps=16) + assert aggregate["sessions"] == 2 + assert aggregate["steady_interval_ms_mean"] == pytest.approx(750.0) + assert aggregate["steady_interval_ms_spread"] == pytest.approx(300.0) + assert aggregate["video_rtf_best"] <= aggregate["video_rtf_mean"] <= aggregate["video_rtf_worst"] + + +def test_single_session_aggregate_is_the_session_itself() -> None: + metrics = compute_metrics(_records([1.0] + [0.5] * 3), fps=16, warmup_chunks=0) + aggregate = aggregate_metrics([metrics], fps=16) + assert aggregate["sessions"] == 1 + assert aggregate["video_rtf"] == metrics["video_rtf"] + + +# --------------------------------------------------------------------------- # +# CLI +# --------------------------------------------------------------------------- # + + +def test_endpoint_url_normalizes_http_and_appends_the_ws_path() -> None: + assert endpoint_url(parse_args([])) == "ws://127.0.0.1:8000/v1/realtime/video" + assert endpoint_url(parse_args(["--host", "h", "--port", "9"])) == "ws://h:9/v1/realtime/video" + args = parse_args(["--base-url", "http://example.invalid:8099"]) + assert endpoint_url(args) == "ws://example.invalid:8099/v1/realtime/video" + args = parse_args(["--base-url", "ws://example.invalid:8099/v1/realtime/video"]) + assert endpoint_url(args) == "ws://example.invalid:8099/v1/realtime/video" + + +def test_cli_rejects_a_zero_chunk_buffer() -> None: + with pytest.raises(SystemExit): + parse_args(["--playback-buffer-chunks", "0"]) + with pytest.raises(SystemExit): + parse_args(["--target-fps", "0"]) + + +def test_the_real_time_basis_is_independent_of_the_mux_label() -> None: + """The checkpoint declares no frame rate, so the basis must be statable. + + The same measured cadence has RTF 1.809 at 16 fps and 1.357 at 12 fps; + tying that verdict to the fps sent to the muxer would let a playback label + silently decide whether a run passes. + """ + records = _records([1.0] + [1.357] * 15) + at_16 = compute_metrics(records, fps=16, warmup_chunks=6) + at_12 = compute_metrics(records, fps=12, warmup_chunks=6) + assert at_16["chunk_deadline_ms"] == pytest.approx(750.0) + assert at_12["chunk_deadline_ms"] == pytest.approx(1000.0) + # RTF is cost, so the harsher 16 fps basis scores WORSE (higher). + assert at_16["video_rtf"] > at_12["video_rtf"] + # Same wall clock, same bytes: only the basis moved. + assert at_16["video_rtf"] / at_12["video_rtf"] == pytest.approx(16 / 12) + assert parse_args([]).target_fps is None, "default defers to the workload fps" + + +# --------------------------------------------------------------------------- # +# Protocol handling against a scripted server +# --------------------------------------------------------------------------- # + + +def _serve(script): + """Return an async context manager yielding a ws:// URL for ``script``.""" + from websockets.asyncio.server import Server, serve + + class _Server: + def __init__(self) -> None: + self.server: Server | None = None + self.start_payloads: list[dict] = [] + + async def __aenter__(self) -> str: + async def handler(websocket) -> None: + self.start_payloads.append(json.loads(await websocket.recv())) + await script(websocket, self.start_payloads[-1]) + + self.server = await serve(handler, "127.0.0.1", 0) + port = next(iter(self.server.sockets)).getsockname()[1] + return f"ws://127.0.0.1:{port}/v1/realtime/video" + + async def __aexit__(self, *exc) -> None: + assert self.server is not None + self.server.close() + await self.server.wait_closed() + + return _Server() + + +async def _emit_rollout(websocket, payload, *, intervals=None, chunks=None, trailer=True) -> None: + """Replay the documented event order: start, metadata+binary per chunk, done.""" + num_chunks = chunks if chunks is not None else len(payload["extra_params"]["camera_action_script"]) + await websocket.send(json.dumps({"type": "video.start", "request_id": "req-1", "format": "m4s"})) + for index in range(num_chunks): + if intervals is not None: + await asyncio.sleep(intervals[index]) + frames = pixel_frames_in_chunk(index) + body = bytes([index % 256]) * (100 + index) + await websocket.send( + json.dumps( + { + "type": "video.chunk_metadata", + "request_id": "req-1", + "kind": "media", + "num_frames": frames, + "byte_length": len(body), + "generation_chunk_index": index, + } + ) + ) + await websocket.send(body) + if trailer: + tail = b"\x00" * 8 + await websocket.send( + json.dumps( + { + "type": "video.chunk_metadata", + "request_id": "req-1", + "kind": "trailer", + "num_frames": 0, + "byte_length": len(tail), + "generation_chunk_index": None, + } + ) + ) + await websocket.send(tail) + await websocket.send(json.dumps({"type": "session.done", "stopped": False, "chunks": num_chunks})) + + +def _run(coro): + return asyncio.run(asyncio.wait_for(coro, timeout=30)) + + +def test_session_records_one_chunk_per_block_and_ignores_the_trailer() -> None: + workload = _workload(num_chunks=4) + + async def scenario() -> None: + server = _serve(_emit_rollout) + url = await server.__aenter__() + try: + result = await run_session( + url, + "model", + workload, + first_chunk_timeout=10, + chunk_timeout=10, + ping_interval=0, + collect_media=True, + print_chunks=False, + ) + finally: + await server.__aexit__(None, None, None) + + assert [record.index for record in result.records] == [0, 1, 2, 3] + assert [record.num_frames for record in result.records] == [9, 12, 12, 12] + assert result.request_id == "req-1" + assert result.ttfc_s is not None and result.ttfc_s > 0 + # The trailer's bytes are kept for remuxing but carry no chunk record. + assert len(result.media) == sum(record.byte_length for record in result.records) + 8 + assert server.start_payloads[0]["num_frames"] == workload.num_frames + + _run(scenario()) + + +def test_measured_intervals_track_the_server_cadence() -> None: + workload = _workload(num_chunks=3) + + async def script(websocket, payload) -> None: + await _emit_rollout(websocket, payload, intervals=[0.05, 0.30, 0.05]) + + async def scenario() -> None: + server = _serve(script) + url = await server.__aenter__() + try: + result = await run_session( + url, + "model", + workload, + first_chunk_timeout=10, + chunk_timeout=10, + ping_interval=0, + collect_media=False, + print_chunks=False, + ) + finally: + await server.__aexit__(None, None, None) + + intervals = [record.inter_arrival_s for record in result.records] + assert intervals[1] == pytest.approx(0.30, abs=0.15) + assert intervals[2] < intervals[1], "a fast chunk must not inherit the slow one's interval" + assert result.records[0].arrival_s == pytest.approx(result.ttfc_s) + + _run(scenario()) + + +def test_a_short_rollout_fails_loudly_instead_of_reporting_a_partial_cadence() -> None: + workload = _workload(num_chunks=4) + + async def script(websocket, payload) -> None: + await _emit_rollout(websocket, payload, chunks=2) + + async def scenario() -> None: + server = _serve(script) + url = await server.__aenter__() + try: + with pytest.raises(BenchmarkError, match="Expected 4 chunks"): + await run_session( + url, + "model", + workload, + first_chunk_timeout=10, + chunk_timeout=10, + ping_interval=0, + collect_media=False, + print_chunks=False, + ) + finally: + await server.__aexit__(None, None, None) + + _run(scenario()) + + +def test_a_server_error_event_aborts_the_session() -> None: + workload = _workload(num_chunks=2) + + async def script(websocket, payload) -> None: + await websocket.send(json.dumps({"type": "error", "message": "resolution mismatch"})) + + async def scenario() -> None: + server = _serve(script) + url = await server.__aenter__() + try: + with pytest.raises(BenchmarkError, match="resolution mismatch.*requested 2 chunks"): + await run_session( + url, + "model", + workload, + first_chunk_timeout=10, + chunk_timeout=10, + ping_interval=0, + collect_media=False, + print_chunks=False, + ) + finally: + await server.__aexit__(None, None, None) + + _run(scenario()) + + +def test_a_stalled_server_times_out_rather_than_hanging() -> None: + workload = _workload(num_chunks=2) + + async def script(websocket, payload) -> None: + await websocket.send(json.dumps({"type": "video.start", "request_id": "req-1", "format": "m4s"})) + await asyncio.sleep(5) + + async def scenario() -> None: + server = _serve(script) + url = await server.__aenter__() + try: + with pytest.raises(BenchmarkError, match="No media chunk for"): + await run_session( + url, + "model", + workload, + first_chunk_timeout=0.4, + chunk_timeout=0.4, + ping_interval=0, + collect_media=False, + print_chunks=False, + ) + finally: + await server.__aexit__(None, None, None) + + _run(scenario()) + + +def test_keepalive_pings_reach_the_server_during_a_slow_first_chunk() -> None: + """A compiled first block can outlast the server's stall timeout.""" + workload = _workload(num_chunks=1) + seen: list[str] = [] + + async def script(websocket, payload) -> None: + await websocket.send(json.dumps({"type": "video.start", "request_id": "req-1", "format": "m4s"})) + + async def drain() -> None: + async for message in websocket: + event = json.loads(message) + seen.append(event["type"]) + if event["type"] == "session.ping": + await websocket.send(json.dumps({"type": "session.pong"})) + + drainer = asyncio.create_task(drain()) + await asyncio.sleep(0.35) + await _emit_rollout(websocket, payload, trailer=False) + drainer.cancel() + + async def scenario() -> None: + server = _serve(script) + url = await server.__aenter__() + try: + result = await run_session( + url, + "model", + workload, + first_chunk_timeout=10, + chunk_timeout=10, + ping_interval=0.1, + collect_media=False, + print_chunks=False, + ) + finally: + await server.__aexit__(None, None, None) + assert len(result.records) == 1 + assert "session.ping" in seen + + _run(scenario()) + + +def test_prompt_updates_are_sent_on_their_chunk_boundary() -> None: + workload = Workload( + prompt="start", + image_reference=_IMAGE, + camera_script=build_camera_script(3, "forward"), + prompt_updates=(PromptUpdate(after_chunk=0, prompt="Rain begins", transition_chunks=1),), + ) + received: list[dict] = [] + + async def script(websocket, payload) -> None: + async def drain() -> None: + async for message in websocket: + received.append(json.loads(message)) + + drainer = asyncio.create_task(drain()) + await _emit_rollout(websocket, payload, intervals=[0.02, 0.15, 0.02], trailer=False) + await asyncio.sleep(0.05) + drainer.cancel() + + async def scenario() -> None: + server = _serve(script) + url = await server.__aenter__() + try: + await run_session( + url, + "model", + workload, + first_chunk_timeout=10, + chunk_timeout=10, + ping_interval=0, + collect_media=False, + print_chunks=False, + ) + finally: + await server.__aexit__(None, None, None) + interactions = [event for event in received if event.get("type") == "session.interaction"] + assert len(interactions) == 1 + assert interactions[0]["interaction"]["event"]["prompt"] == "Rain begins" + assert interactions[0]["interaction"]["transition_chunks"] == 1 + + _run(scenario()) + + +@pytest.mark.parametrize("delivered", [0, 1]) +def test_pongs_do_not_extend_media_deadlines(delivered) -> None: + seen = [] + + async def script(websocket, payload) -> None: + if delivered: + await websocket.send( + json.dumps( + { + "type": "video.chunk_metadata", + "kind": "media", + "generation_chunk_index": 0, + "num_frames": 9, + } + ) + ) + await websocket.send(b"media") + async for message in websocket: + seen.append(json.loads(message)["type"]) + await websocket.send(json.dumps({"type": "session.pong"})) + + async def scenario() -> None: + async with _serve(script) as url: + with pytest.raises(BenchmarkError, match=f"after {delivered} chunk"): + await asyncio.wait_for( + run_session( + url, + "model", + _workload(num_chunks=2), + first_chunk_timeout=0.25, + chunk_timeout=0.25, + ping_interval=0.03, + collect_media=False, + print_chunks=False, + ), + timeout=2, + ) + assert "session.ping" in seen + + _run(scenario()) + + +def test_terminal_chunk_is_only_excluded_from_steady_metrics() -> None: + records = _records([1.0] + [0.1] * 5 + [1.2, 0.8, 0.1]) + metrics = compute_metrics(records, fps=12, warmup_chunks=6) + assert metrics["steady_chunks"] == 2 + assert metrics["interval_steady"]["mean_ms"] == pytest.approx(1000) + assert metrics["steady_video_rtf"] == pytest.approx(1) + assert metrics["slo_attainment"] == pytest.approx(0.5) + assert metrics["interval_all"]["count"] == 8 + assert metrics["total_frames"] == 105 + assert metrics["wall_seconds"] == pytest.approx(records[-1].arrival_s) + assert metrics["playback"]["total_stall_ms"] == pytest.approx( + simulate_playback(records, fps=12).total_stall_s * 1000 + ) + + +def test_terminal_chunk_alone_does_not_establish_steady_state() -> None: + metrics = compute_metrics(_records([1.0] * 7), fps=12, warmup_chunks=6) + assert not metrics["steady_state_reached"] + assert "steady_video_rtf" not in metrics + + +def test_aggregate_preserves_custom_deadline_and_weights_intervals() -> None: + fast = compute_metrics(_records([1.0, 0.5, 0.1]), fps=12, warmup_chunks=0, slo_ms=700) + slow = compute_metrics(_records([1.0, 0.9, 0.9, 0.9, 0.1]), fps=12, warmup_chunks=0, slo_ms=700) + aggregate = aggregate_metrics([fast, slow], fps=12) + assert aggregate["chunk_deadline_ms"] == 700 + assert aggregate["steady_interval_ms_mean"] == pytest.approx(800) + slow["chunk_deadline_ms"] = 800 + with pytest.raises(ValueError, match="different chunk deadlines"): + aggregate_metrics([fast, slow], fps=12) + + +@pytest.mark.parametrize( + "flags, expected_shift, expected_negative", + [ + ([], 7.0, "file negative"), + (["--flow-shift", "5", "--negative-prompt", "CLI negative"], 5.0, "CLI negative"), + (["--negative-prompt", ""], 7.0, ""), + ], +) +def test_cli_workload_generation_overrides(tmp_path, flags, expected_shift, expected_negative) -> None: + path = tmp_path / "workload.json" + path.write_text( + json.dumps( + { + "image": _IMAGE, + "num_chunks": 3, + "flow_shift": 7.0, + "negative_prompt": "file negative", + } + ) + ) + workload = build_workload(parse_args(["--workload", str(path), *flags])) + assert workload.flow_shift == expected_shift + assert workload.negative_prompt == expected_negative + + +def test_builtin_workload_keeps_default_flow_shift() -> None: + assert build_workload(parse_args(["--image", _IMAGE])).flow_shift == 5.0 + + +def test_short_sample_report_qualifies_tail_percentiles(capsys) -> None: + metrics = compute_metrics(_records([1.0] * 16), fps=12) + print_report(metrics, workload=_workload(num_chunks=16), endpoint="ws://test", sessions=1, target_fps=12) + assert "not reliable tail estimates" in capsys.readouterr().out diff --git a/tests/clients/test_duplex_client.py b/tests/clients/test_duplex_client.py index 75ef06e3f..7313828ce 100644 --- a/tests/clients/test_duplex_client.py +++ b/tests/clients/test_duplex_client.py @@ -13,6 +13,7 @@ from __future__ import annotations import asyncio import base64 import json +import struct import wave from collections.abc import Callable from typing import Any @@ -33,6 +34,7 @@ from vllm_omni.clients.duplex import ( ReconnectPolicy, SessionConfig, SessionResumed, + SessionUpdated, build_realtime_url, chunk_period_ms, duplex_unit_boundary_ms, @@ -797,6 +799,52 @@ async def test_inline_client_submits_typed_commands_in_order(): assert ack.played_ms == 120 and ack.response_id == "resp-1" +@pytest.mark.asyncio +@pytest.mark.parametrize("accepted", [True, False]) +@pytest.mark.parametrize( + "patch", + [ + {"input_audio_format": "pcm_f32le", "sample_rate_hz": 24_000}, + {"audio": {"input": {"format": "pcm_f32le", "sample_rate_hz": 24_000}}}, + ], +) +async def test_inline_input_defaults_change_only_after_session_updated(accepted, patch): + client, omni = make_inline_client() + patch = {**patch, "overlap_silence_rms": 0.1} + async with client: + handle = omni.handles[0] + previous_session = dict(client.session_info) + await client.send({"type": "session.update", "session": patch}) + # Sending a patch is not an acknowledgement of its acceptance. + assert client.session_info == previous_session + assert client._input_defaults().input_sample_rate_hz == 16_000 + updated_session = {**previous_session, **patch} + event: dict[str, object] = ( + {"type": "session.updated", "session": updated_session} + if accepted + else {"type": "error", "error": {"code": "invalid_session_update", "message": "rejected"}} + ) + handle.feed(event) + received = await client.wait_for("session.updated" if accepted else "error", timeout_s=2.0) + assert isinstance(received, SessionUpdated if accepted else ErrorEvent) + + pcm = struct.pack(" 0, f"[{test_name}] No requests completed" diff --git a/tests/dfx/stability/scripts/run_stability_minicpmo_4_5.py b/tests/dfx/stability/scripts/run_stability_minicpmo_4_5.py new file mode 100644 index 000000000..fcb0590ce --- /dev/null +++ b/tests/dfx/stability/scripts/run_stability_minicpmo_4_5.py @@ -0,0 +1,71 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + +""" +MiniCPM-o-4.5 stability: OmniServer + ``vllm bench serve --omni`` for a fixed duration. + +Configuration: ``tests/dfx/stability/tests/test_minicpmo_4_5.json``. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from tests.dfx.conftest import ( + create_benchmark_indices, + create_test_parameter_mapping, + create_unique_server_pytest_params, + load_configs, +) +from tests.dfx.stability.helpers import _run_one_vllm_bench_batch, run_stability_benchmark_loop + +STABILITY_DIR = Path(__file__).resolve().parent.parent +DEPLOY_CONFIGS_DIR = STABILITY_DIR / "deploy" +CONFIG_FILE_PATH = str(STABILITY_DIR / "tests" / "test_minicpmo_4_5.json") +DEFAULT_NUM_PROMPTS_PER_BATCH = 20 +STABILITY_SERVER_TIMEOUT_ARGS = ["--stage-init-timeout", "600"] + +try: + BENCHMARK_CONFIGS = load_configs(CONFIG_FILE_PATH) +except FileNotFoundError: + BENCHMARK_CONFIGS = [] + +test_params = create_unique_server_pytest_params(BENCHMARK_CONFIGS, DEPLOY_CONFIGS_DIR) if BENCHMARK_CONFIGS else [] +server_to_benchmark_mapping = create_test_parameter_mapping(BENCHMARK_CONFIGS) if BENCHMARK_CONFIGS else {} +benchmark_indices = create_benchmark_indices(BENCHMARK_CONFIGS, server_to_benchmark_mapping) + + +# Pytest marks (hardware, local_model, slow, omni) come from test_minicpmo_4_5.json. +@pytest.mark.parametrize("omni_server", test_params, indirect=True) +@pytest.mark.parametrize("stability_benchmark_params", benchmark_indices, indirect=True) +def test_stability_minicpmo_4_5(omni_server, stability_benchmark_params): + test_name = stability_benchmark_params["test_name"] + params = stability_benchmark_params["params"] + duration_sec = params.get("duration_sec", 300) + num_prompts_per_batch = params.get("num_prompts_per_batch", DEFAULT_NUM_PROMPTS_PER_BATCH) + request_rate = params.get("request_rate") + max_concurrency = params.get("max_concurrency") + + bench_params = { + k: v + for k, v in params.items() + if k not in ("duration_sec", "request_rate", "max_concurrency", "num_prompts_per_batch") + } + + result = run_stability_benchmark_loop( + host=omni_server.host, + port=omni_server.port, + model=omni_server.model, + duration_sec=duration_sec, + params=bench_params, + request_rate=request_rate, + max_concurrency=max_concurrency, + result_dir=str(STABILITY_DIR), + num_prompts_per_batch=num_prompts_per_batch, + run_one_batch=_run_one_vllm_bench_batch, + ) + + assert result.get("failed", 0) == 0, f"[{test_name}] Failed requests detected: {result.get('errors', [])}" + assert result.get("completed", 0) > 0, f"[{test_name}] No requests completed" diff --git a/tests/dfx/stability/scripts/run_stability_minimax_h3.py b/tests/dfx/stability/scripts/run_stability_minimax_h3.py new file mode 100644 index 000000000..252d830d5 --- /dev/null +++ b/tests/dfx/stability/scripts/run_stability_minimax_h3.py @@ -0,0 +1,91 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + +""" +MiniMax-H3 T2V stability: OmniServer (diffusion) + ``diffusion_benchmark_serving.py`` / ``v1/videos``. + +Configuration: ``tests/dfx/stability/tests/test_minimax_h3.json``. + +``--lora-path`` is resolved like e2e FastH3 / Buildkite ``test-merge.yml``: +set ``VLLM_TEST_MINIMAX_H3_FASTH3_LORA`` to the adapter file, or populate the local +HF cache for ``FastVideo/FastVideo-FastH3-4-step-Preview-v1-LoRA``. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from tests.dfx.conftest import ( + create_benchmark_indices, + create_test_parameter_mapping, + create_unique_server_pytest_params, + load_configs, +) +from tests.dfx.stability.helpers import _run_one_diffusion_batch, run_stability_benchmark_loop +from tests.e2e.online_serving.minimax_h3._common import FASTH3_LORA + +STABILITY_DIR = Path(__file__).resolve().parent.parent +DEPLOY_CONFIGS_DIR = STABILITY_DIR / "deploy" +CONFIG_FILE_PATH = str(STABILITY_DIR / "tests" / "test_minimax_h3.json") +DEFAULT_NUM_PROMPTS_PER_BATCH = 20 +STABILITY_SERVER_TIMEOUT_ARGS = ["--stage-init-timeout", "1800", "--init-timeout", "1800"] + + +def _inject_fasth3_lora_path(configs: list[dict]) -> list[dict]: + """Fill serve_args.lora-path from VLLM_TEST_MINIMAX_H3_FASTH3_LORA / HF cache.""" + if not FASTH3_LORA: + return configs + for config in configs: + serve_args = config.setdefault("server_params", {}).setdefault("serve_args", {}) + serve_args["lora-path"] = FASTH3_LORA + return configs + + +try: + BENCHMARK_CONFIGS = _inject_fasth3_lora_path(load_configs(CONFIG_FILE_PATH)) +except FileNotFoundError: + BENCHMARK_CONFIGS = [] + +test_params = create_unique_server_pytest_params(BENCHMARK_CONFIGS, DEPLOY_CONFIGS_DIR) if BENCHMARK_CONFIGS else [] +server_to_benchmark_mapping = create_test_parameter_mapping(BENCHMARK_CONFIGS) if BENCHMARK_CONFIGS else {} +benchmark_indices = create_benchmark_indices(BENCHMARK_CONFIGS, server_to_benchmark_mapping) + + +# Pytest marks (hardware, local_model, slow, diffusion) come from test_minimax_h3.json. +@pytest.mark.skipif( + FASTH3_LORA is None, + reason="set VLLM_TEST_MINIMAX_H3_FASTH3_LORA or populate the local HF cache", +) +@pytest.mark.parametrize("omni_server", test_params, indirect=True) +@pytest.mark.parametrize("stability_benchmark_params", benchmark_indices, indirect=True) +def test_stability_minimax_h3(omni_server, stability_benchmark_params): + test_name = stability_benchmark_params["test_name"] + params = stability_benchmark_params["params"] + duration_sec = params.get("duration_sec", 300) + num_prompts_per_batch = params.get("num_prompts_per_batch", DEFAULT_NUM_PROMPTS_PER_BATCH) + request_rate = params.get("request_rate") + max_concurrency = params.get("max_concurrency") + + bench_params = { + k: v + for k, v in params.items() + if k not in ("duration_sec", "request_rate", "max_concurrency", "num_prompts_per_batch") + } + + result = run_stability_benchmark_loop( + host=omni_server.host, + port=omni_server.port, + model=omni_server.model, + duration_sec=duration_sec, + params=bench_params, + request_rate=request_rate, + max_concurrency=max_concurrency, + result_dir=str(STABILITY_DIR), + num_prompts_per_batch=num_prompts_per_batch, + run_one_batch=_run_one_diffusion_batch, + ) + + assert result.get("failed", 0) == 0, f"[{test_name}] Failed requests detected: {result.get('errors', [])}" + assert result.get("completed", 0) > 0, f"[{test_name}] No requests completed" diff --git a/tests/dfx/stability/scripts/test_stability_qwen3_omni.py b/tests/dfx/stability/scripts/run_stability_qwen3_omni.py similarity index 92% rename from tests/dfx/stability/scripts/test_stability_qwen3_omni.py rename to tests/dfx/stability/scripts/run_stability_qwen3_omni.py index 98ab5ee89..ddb3339d1 100644 --- a/tests/dfx/stability/scripts/test_stability_qwen3_omni.py +++ b/tests/dfx/stability/scripts/run_stability_qwen3_omni.py @@ -1,3 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + """ Qwen3-Omni stability: OmniServer + ``vllm bench serve --omni`` for a fixed duration. @@ -34,8 +37,7 @@ server_to_benchmark_mapping = create_test_parameter_mapping(BENCHMARK_CONFIGS) i benchmark_indices = create_benchmark_indices(BENCHMARK_CONFIGS, server_to_benchmark_mapping) -@pytest.mark.slow -@pytest.mark.omni +# Pytest marks (hardware, local_model, slow, omni) come from test_qwen3_omni.json. @pytest.mark.parametrize("omni_server", test_params, indirect=True) @pytest.mark.parametrize("stability_benchmark_params", benchmark_indices, indirect=True) def test_stability_qwen3_omni(omni_server, stability_benchmark_params): diff --git a/tests/dfx/stability/scripts/test_stability_qwen3_tts.py b/tests/dfx/stability/scripts/run_stability_qwen3_tts.py similarity index 92% rename from tests/dfx/stability/scripts/test_stability_qwen3_tts.py rename to tests/dfx/stability/scripts/run_stability_qwen3_tts.py index 293b723ca..dde525b36 100644 --- a/tests/dfx/stability/scripts/test_stability_qwen3_tts.py +++ b/tests/dfx/stability/scripts/run_stability_qwen3_tts.py @@ -1,3 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + """ Qwen3-TTS stability: OmniServer + ``vllm bench serve --omni`` for a fixed duration. @@ -34,8 +37,7 @@ server_to_benchmark_mapping = create_test_parameter_mapping(BENCHMARK_CONFIGS) i benchmark_indices = create_benchmark_indices(BENCHMARK_CONFIGS, server_to_benchmark_mapping) -@pytest.mark.slow -@pytest.mark.tts +# Pytest marks (hardware, local_model, slow, tts) come from test_qwen3_tts.json. @pytest.mark.parametrize("omni_server", test_params, indirect=True) @pytest.mark.parametrize("stability_benchmark_params", benchmark_indices, indirect=True) def test_stability_qwen3_tts(omni_server, stability_benchmark_params): diff --git a/tests/dfx/stability/scripts/test_stability_qwen_image.py b/tests/dfx/stability/scripts/run_stability_qwen_image.py similarity index 92% rename from tests/dfx/stability/scripts/test_stability_qwen_image.py rename to tests/dfx/stability/scripts/run_stability_qwen_image.py index 62b0cd86c..84e15fccb 100644 --- a/tests/dfx/stability/scripts/test_stability_qwen_image.py +++ b/tests/dfx/stability/scripts/run_stability_qwen_image.py @@ -1,3 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + """ Qwen-Image stability: OmniServer (diffusion) + ``diffusion_benchmark_serving.py``. @@ -34,8 +37,7 @@ server_to_benchmark_mapping = create_test_parameter_mapping(BENCHMARK_CONFIGS) i benchmark_indices = create_benchmark_indices(BENCHMARK_CONFIGS, server_to_benchmark_mapping) -@pytest.mark.slow -@pytest.mark.diffusion +# Pytest marks (hardware, local_model, slow, diffusion) come from test_qwen_image.json. @pytest.mark.parametrize("omni_server", test_params, indirect=True) @pytest.mark.parametrize("stability_benchmark_params", benchmark_indices, indirect=True) def test_stability_qwen_image(omni_server, stability_benchmark_params): diff --git a/tests/dfx/stability/scripts/test_stability_wan22.py b/tests/dfx/stability/scripts/run_stability_wan22.py similarity index 92% rename from tests/dfx/stability/scripts/test_stability_wan22.py rename to tests/dfx/stability/scripts/run_stability_wan22.py index aae0ae429..08c711d85 100644 --- a/tests/dfx/stability/scripts/test_stability_wan22.py +++ b/tests/dfx/stability/scripts/run_stability_wan22.py @@ -1,3 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + """ Wan2.2 T2V stability: OmniServer (diffusion) + ``diffusion_benchmark_serving.py`` / ``v1/videos``. @@ -34,8 +37,7 @@ server_to_benchmark_mapping = create_test_parameter_mapping(BENCHMARK_CONFIGS) i benchmark_indices = create_benchmark_indices(BENCHMARK_CONFIGS, server_to_benchmark_mapping) -@pytest.mark.slow -@pytest.mark.diffusion +# Pytest marks (hardware, local_model, slow, diffusion) come from test_wan22.json. @pytest.mark.parametrize("omni_server", test_params, indirect=True) @pytest.mark.parametrize("stability_benchmark_params", benchmark_indices, indirect=True) def test_stability_wan22(omni_server, stability_benchmark_params): diff --git a/tests/dfx/stability/tests/test_hunyuan_image.json b/tests/dfx/stability/tests/test_hunyuan_image.json index b468ceec3..43c92d1e2 100644 --- a/tests/dfx/stability/tests/test_hunyuan_image.json +++ b/tests/dfx/stability/tests/test_hunyuan_image.json @@ -7,10 +7,11 @@ "res": { "cuda": "H800" }, - "num_cards": 2 + "num_cards": 4 } }, "local_model", + "slow", "diffusion" ], "server_params": { @@ -44,5 +45,53 @@ "enable_negative_prompt": true } ] + }, + { + "test_name": "test_hunyuan_image_stability_tp2_sp2_a3", + "description": "A3: same as perf test_hunyuan_image_tp2_sp2 — DiT-only, TP=2 USP=2, 4 dies, no fp8", + "mark": [ + { + "hardware_marks": { + "res": { + "npu": "A3" + }, + "num_cards": 4 + } + }, + "local_model", + "slow", + "diffusion" + ], + "server_params": { + "model": "tencent/HunyuanImage-3.0-Instruct", + "serve_args": { + "deploy-config": "vllm_omni/deploy/hunyuan_image3_dit.yaml", + "tensor-parallel-size": 2, + "ulysses-degree": 2, + "vae-patch-parallel-size": 4, + "vae-use-tiling": true, + "enable-diffusion-pipeline-profiler": true + } + }, + "benchmark_params": [ + { + "dataset": "random", + "task": "t2i", + "backend": "vllm-omni", + "duration_sec": 86400, + "max_concurrency": 1, + "num_prompts_per_batch": 50, + "width": { + "min": 512, + "max": 1024 + }, + "height": { + "min": 512, + "max": 1024 + }, + "num_inference_steps": 50, + "enable_negative_prompt": true + } + ] } ] diff --git a/tests/dfx/stability/tests/test_hunyuanvideo15.json b/tests/dfx/stability/tests/test_hunyuanvideo15.json new file mode 100644 index 000000000..1da2feec3 --- /dev/null +++ b/tests/dfx/stability/tests/test_hunyuanvideo15.json @@ -0,0 +1,57 @@ +[ + { + "test_name": "test_hunyuanvideo15_i2v_cachedit_tp2_vae_patch2_stability", + "description": "HunyuanVideo-1.5 I2V 24h: CacheDiT + TP=2 + VAE patch parallel=2 + VAE tiling", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": ["H800"] + }, + "num_cards": 2 + } + }, + "local_model", + "slow", + "diffusion" + ], + "server_params": { + "model": "hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_i2v", + "serve_args": { + "flow-shift": 5.0, + "cache-backend": "cache_dit", + "tensor-parallel-size": 2, + "vae-patch-parallel-size": 2, + "vae-use-tiling": true, + "enable-diffusion-pipeline-profiler": true + } + }, + "benchmark_params": [ + { + "dataset": "random", + "task": "i2v", + "backend": "v1/videos", + "duration_sec": 86400, + "max_concurrency": 1, + "num_prompts_per_batch": 50, + "num_input_images": 1, + "seed": 42, + "enable_negative_prompt": true, + "random_request_config": [ + { + "width": 832, + "height": 480, + "num_inference_steps": 4, + "num_frames": 33, + "fps": 24, + "weight": 1 + } + ], + "extra_body": { + "guidance_scale": 6.0, + "flow_shift": 5.0 + } + } + ] + } +] diff --git a/tests/dfx/stability/tests/test_minicpmo_4_5.json b/tests/dfx/stability/tests/test_minicpmo_4_5.json new file mode 100644 index 000000000..eaf38ec1d --- /dev/null +++ b/tests/dfx/stability/tests/test_minicpmo_4_5.json @@ -0,0 +1,53 @@ +[ + { + "test_name": "test_minicpmo_4_5_stability_ready", + "description": "MiniCPM-o-4.5 ready deploy + seed-tts chat/audio stability (from perf test_minicpmo_4_5_ready)", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H800" + }, + "num_cards": 1 + } + }, + "local_model", + "slow", + "omni" + ], + "server_params": { + "model": "openbmb/MiniCPM-o-4_5", + "extra_cli_args": [ + "--deploy-config", + "vllm_omni/deploy/minicpmo_4_5.yaml", + "--trust-remote-code" + ] + }, + "benchmark_params": [ + { + "dataset_name": "seed-tts", + "dataset_path": "zhaochenyang20/seed-tts-eval", + "backend": "openai-chat-omni", + "endpoint": "/v1/chat/completions", + "duration_sec": 86400, + "max_concurrency": 1, + "num_prompts_per_batch": 50, + "no_oversample": true, + "disable_shuffle": true, + "trust_remote_code": true, + "seed_tts_locale": "en", + "extra_body": { + "modalities": [ + "text", + "audio" + ], + "chat_template_kwargs": { + "enable_thinking": false, + "use_tts_template": true + } + }, + "percentile-metrics": "ttft,e2el,audio_ttfp,audio_rtf,audio_duration" + } + ] + } +] diff --git a/tests/dfx/stability/tests/test_minimax_h3.json b/tests/dfx/stability/tests/test_minimax_h3.json new file mode 100644 index 000000000..d671c48fd --- /dev/null +++ b/tests/dfx/stability/tests/test_minimax_h3.json @@ -0,0 +1,62 @@ +[ + { + "test_name": "test_minimax_h3_t2v_stability_4gpu", + "description": "MiniMax-H3 FastH3 T2VA on 4 GPUs (USP4 + HSDP4, encoder-TP4, VAE-PP4 tile); fused lora-path from VLLM_TEST_MINIMAX_H3_FASTH3_LORA; requests must use num_inference_steps=4", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H800" + }, + "num_cards": 4 + } + }, + "local_model", + "slow", + "diffusion" + ], + "server_params": { + "model": "MiniMaxAI/MiniMax-H3", + "serve_args": { + "trust-remote-code": true, + "task-type": "fl2va", + "num-gpus": 4, + "usp": 4, + "ring": 1, + "use-hsdp": true, + "hsdp-shard-size": 4, + "text-encoder-tp-size": 4, + "vae-patch-parallel-size": 4, + "vae-parallel-mode": "tile", + "vae-use-tiling": true, + "stage-init-timeout": 1800, + "init-timeout": 1800 + } + }, + "benchmark_params": [ + { + "dataset": "random", + "task": "t2v", + "backend": "v1/videos", + "duration_sec": 86400, + "max_concurrency": 1, + "num_prompts_per_batch": 50, + "seed": 42, + "random_request_config": [ + { + "width": 1344, + "height": 768, + "prompt": "Cinematic drone shot pushing through misty mountain ridges at sunrise, golden light catching the clouds, slow and smooth.", + "num_inference_steps": 4, + "num_frames": 209, + "fps": 24, + "weight": 1 + } + ], + "extra_body": { + "extra_params": "{\"task\":\"t2va\",\"duration\":8.7,\"aspect_ratio\":\"16:9\"}" + } + } + ] + } +] diff --git a/tests/dfx/stability/tests/test_qwen3_omni.json b/tests/dfx/stability/tests/test_qwen3_omni.json index 816e981b6..bc2f7dacd 100644 --- a/tests/dfx/stability/tests/test_qwen3_omni.json +++ b/tests/dfx/stability/tests/test_qwen3_omni.json @@ -5,12 +5,14 @@ { "hardware_marks": { "res": { - "cuda": "H800" + "cuda": "H800", + "npu": "A3" }, "num_cards": 2 } }, "local_model", + "slow", "omni" ], "server_params": { diff --git a/tests/dfx/stability/tests/test_qwen3_tts.json b/tests/dfx/stability/tests/test_qwen3_tts.json index 1c9356574..37b7cca52 100644 --- a/tests/dfx/stability/tests/test_qwen3_tts.json +++ b/tests/dfx/stability/tests/test_qwen3_tts.json @@ -12,6 +12,7 @@ } }, "local_model", + "slow", "tts" ], "server_params": { diff --git a/tests/dfx/stability/tests/test_qwen_image.json b/tests/dfx/stability/tests/test_qwen_image.json index 0c21d0ac1..9756f1299 100644 --- a/tests/dfx/stability/tests/test_qwen_image.json +++ b/tests/dfx/stability/tests/test_qwen_image.json @@ -11,6 +11,7 @@ } }, "local_model", + "slow", "diffusion" ], "server_params": { @@ -36,5 +37,44 @@ "enable_negative_prompt": true } ] + }, + { + "test_name": "test_qwen_image_stability_a3", + "mark": [ + { + "hardware_marks": { + "res": { + "npu": "A3" + }, + "num_cards": 1 + } + }, + "local_model", + "slow", + "diffusion" + ], + "server_params": { + "model": "Qwen/Qwen-Image" + }, + "benchmark_params": [ + { + "dataset": "random", + "task": "t2i", + "backend": "vllm-omni", + "duration_sec": 86400, + "max_concurrency": 1, + "num_prompts_per_batch": 50, + "width": { + "min": 512, + "max": 1000 + }, + "height": { + "min": 512, + "max": 1000 + }, + "num_inference_steps": 50, + "enable_negative_prompt": true + } + ] } ] diff --git a/tests/dfx/stability/tests/test_wan22.json b/tests/dfx/stability/tests/test_wan22.json index e6ab0b5cf..de6627de4 100644 --- a/tests/dfx/stability/tests/test_wan22.json +++ b/tests/dfx/stability/tests/test_wan22.json @@ -12,6 +12,7 @@ } }, "local_model", + "slow", "diffusion" ], "server_params": { @@ -54,6 +55,7 @@ } }, "local_model", + "slow", "diffusion" ], "server_params": { diff --git a/tests/diffusion/attention/test_attention_config.py b/tests/diffusion/attention/test_attention_config.py index 4a053b1bf..0b975f27c 100644 --- a/tests/diffusion/attention/test_attention_config.py +++ b/tests/diffusion/attention/test_attention_config.py @@ -42,6 +42,14 @@ from vllm_omni.diffusion.data import ( pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu] +@pytest.fixture(autouse=True) +def _clear_inherited_attention_backend(monkeypatch): + # Weekly CPU exports DIFFUSION_ATTENTION_BACKEND; OmniDiffusionConfig + # treats that as an explicit default so auto-backend cases break. + # Tests that setenv the same variable still work: this fixture runs first. + monkeypatch.delenv("DIFFUSION_ATTENTION_BACKEND", raising=False) + + class TestAttentionSpec: def test_construct_no_skip_softmax(self): spec = AttentionSpec(backend="FLASH_ATTN") @@ -416,13 +424,6 @@ class TestBuildAttentionConfig: class TestOmniDiffusionConfigAttentionParsing: """Test OmniDiffusionConfig attention shorthand and structured config.""" - @pytest.fixture(autouse=True) - def _clear_diffusion_attention_backend_env(self, monkeypatch): - # OmniDiffusionConfig.__post_init__ applies DIFFUSION_ATTENTION_BACKEND via - # build_attention_config(); clear it so these tests assert config defaults, - # not whatever the process inherited from CI / sibling tests. - monkeypatch.delenv("DIFFUSION_ATTENTION_BACKEND", raising=False) - def test_diffusion_attention_backend_sets_default(self): config = OmniDiffusionConfig.from_kwargs(diffusion_attention_backend="SAGE_ATTN") assert isinstance(config.diffusion_attention_config, AttentionConfig) diff --git a/tests/diffusion/distributed/test_pause_keep_two_ranks.py b/tests/diffusion/distributed/test_pause_keep_two_ranks.py new file mode 100644 index 000000000..6ebd4743f --- /dev/null +++ b/tests/diffusion/distributed/test_pause_keep_two_ranks.py @@ -0,0 +1,81 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + +""" +Two-rank ``pause_generation(mode="keep")``: every rank runs the pause barrier +before the ACK, and a barrier failure on a non-zero rank fails the pause +without aborting or dropping the running batch. +""" + +from __future__ import annotations + +import asyncio +from collections import defaultdict +from contextlib import ExitStack + +import pytest + +from tests.e2e.features.custom_pipeline.test_diffusion_pause_keep import ( + CUSTOM_PIPELINE_CLASS, + MODEL, + WORKER_EXTENSION_CLASS, + _generate, + _pause_round, + _wait_for, +) +from tests.helpers.mark import hardware_test +from vllm_omni.entrypoints.async_omni import AsyncOmni + + +def _all_true(results) -> bool: + flat = [] + stack = list(results) + while stack: + item = stack.pop() + if isinstance(item, list): + stack.extend(item) + else: + flat.append(item) + return bool(flat) and all(item is True for item in flat) + + +@pytest.mark.core_model +@pytest.mark.diffusion +@hardware_test(res={"cuda": "L4"}, num_cards=2) +@pytest.mark.asyncio +async def test_keep_pause_two_ranks_ack_and_rank_failure(): + with ExitStack() as after: + engine = AsyncOmni( + model=MODEL, + custom_pipeline_args={"pipeline_class": CUSTOM_PIPELINE_CLASS}, + worker_extension_cls=WORKER_EXTENSION_CLASS, + enforce_eager=True, + enable_sleep_mode=True, + num_gpus=2, + parallel_config={"ulysses_degree": 2, "ulysses_mode": "advanced_uaa"}, + ) + after.callback(engine.shutdown) + events: dict[str, dict[str, float]] = defaultdict(dict) + assert _all_true(await engine.collective_rpc(method="start_counting_synchronize_device")) + + pause_latencies = [await _pause_round(engine, events, round_index) for round_index in range(3)] + # The result is the all-rank AND, so True means both ranks ran the barrier. + assert _all_true(await engine.collective_rpc(method="synchronize_device_seen")) + + assert _all_true(await engine.collective_rpc(method="set_synchronize_device_failure", args=(1, True))) + a = asyncio.create_task(_generate(engine, "a-fail", events)) + await _wait_for(lambda: "started" in events["a-fail"]) + with pytest.raises(RuntimeError, match="pause_scheduler failed"): + await engine.pause_generation(mode="keep", clear_cache=False) + assert await engine.is_paused() + a_output = await asyncio.wait_for(a, 60.0) + assert a_output.images, "a failed barrier must not abort the running batch" + + assert _all_true(await engine.collective_rpc(method="set_synchronize_device_failure", args=(1, False))) + await engine.resume_generation() + assert not await engine.is_paused() + recovered = await asyncio.wait_for(_generate(engine, "after-recovery", events), 60.0) + assert recovered.images + + assert not engine.request_states + print(f"[sp2] rounds=3 pause_latency_s min/max={min(pause_latencies):.3f}/{max(pause_latencies):.3f}") diff --git a/tests/diffusion/distributed/test_wan_spatial_shard.py b/tests/diffusion/distributed/test_wan_spatial_shard.py index 23c29c938..27177745e 100644 --- a/tests/diffusion/distributed/test_wan_spatial_shard.py +++ b/tests/diffusion/distributed/test_wan_spatial_shard.py @@ -493,6 +493,84 @@ def test_dist_zero_pad_only_applies_global_width_edges(monkeypatch: pytest.Monke assert last.shape == (1, 1, 2, 3) +@pytest.mark.core_model +@pytest.mark.cpu +class _PlainDecoder(torch.nn.Module): + """A decoder with nothing to patch, so the install only wraps ``forward``.""" + + def __init__(self): + super().__init__() + self.calls: list[dict] = [] + + def forward(self, x, feat_cache=None, feat_idx=None, first_chunk=False): + self.calls.append({"feat_cache": feat_cache, "feat_idx": feat_idx, "first_chunk": first_chunk}) + return x + + +@pytest.mark.core_model +@pytest.mark.cpu +@pytest.mark.parametrize("world_size, dst", [(4, -1), (4, 4), (4, 5), (1, 1), (4, 1.5)]) +def test_install_rejects_invalid_destination_before_mutating_decoder(monkeypatch, world_size, dst): + monkeypatch.setattr(wan_spatial_shard, "_rank_world", lambda group: (0, world_size)) + vae = SimpleNamespace(decoder=torch.nn.Identity()) + original_forward = vae.decoder.forward + + def unexpected_patch(*args, **kwargs): + pytest.fail("invalid destination must fail before patching modules") + + monkeypatch.setattr(wan_spatial_shard, "_patch_decoder_modules", unexpected_patch) + with pytest.raises(ValueError, match="dst must be None or an integer"): + wan_spatial_shard.install_wan_spatial_shard_decode(vae, object(), dst=dst) + assert vae.decoder.forward == original_forward + assert not getattr(vae, "_vllm_omni_wan_spatial_shard_installed", False) + + +def _install_on_plain_decoder(monkeypatch: pytest.MonkeyPatch, *, dst): + monkeypatch.setattr(wan_spatial_shard, "_rank_world", lambda group: (1, 2)) + monkeypatch.setattr( + wan_spatial_shard, + "split_for_parallel_decode", + lambda x, *, upsample_count, split_dim, group: (x, x.shape[wan_spatial_shard._spatial_dim(split_dim)]), + ) + gathers: list[dict] = [] + + def gather(x, *, expected_extent, split_dim, group, dst=None): + gathers.append({"expected_extent": expected_extent, "split_dim": split_dim, "dst": dst}) + return x + + monkeypatch.setattr(wan_spatial_shard, "gather_and_trim_extent", gather) + vae = SimpleNamespace(decoder=_PlainDecoder()) + wan_spatial_shard.install_wan_spatial_shard_decode(vae, object(), split_dim="width", dst=dst) + return vae, gathers + + +@pytest.mark.core_model +@pytest.mark.cpu +@pytest.mark.parametrize("dst", [0, 1, None]) +def test_install_passes_the_assembling_rank_through_to_the_gather(monkeypatch: pytest.MonkeyPatch, dst): + vae, gathers = _install_on_plain_decoder(monkeypatch, dst=dst) + cache = [None] + + out = vae.decoder(torch.zeros(1, 3, 1, 4, 6), feat_cache=cache, feat_idx=[0], first_chunk=True) + + assert out.shape == (1, 3, 1, 4, 6) + assert gathers == [{"expected_extent": 6, "split_dim": "width", "dst": dst}] + # The streaming arguments reach the wrapped decoder untouched. + assert vae.decoder.calls == [{"feat_cache": cache, "feat_idx": [0], "first_chunk": True}] + assert vae._vllm_omni_wan_spatial_shard_dst == dst + + +@pytest.mark.core_model +@pytest.mark.cpu +def test_install_refuses_to_change_the_assembling_rank(monkeypatch: pytest.MonkeyPatch): + vae, _ = _install_on_plain_decoder(monkeypatch, dst=None) + + # Same settings again: a no-op, as before. + wan_spatial_shard.install_wan_spatial_shard_decode(vae, object(), split_dim="width", dst=None) + with pytest.raises(ValueError, match="already patched to assemble on None"): + wan_spatial_shard.install_wan_spatial_shard_decode(vae, object(), split_dim="width", dst=0) + + @pytest.mark.core_model @pytest.mark.cpu def test_spatial_shard_height_gate_falls_back_for_partial_group(monkeypatch: pytest.MonkeyPatch): @@ -626,6 +704,105 @@ def _spatial_shard_decode_worker(rank: int, split_dim: str, return_dict, master_ dist.destroy_process_group() +_STREAMING_SHARD_CHUNKS = 3 + + +def _streaming_shard_worker(rank: int, return_dict, master_port: str) -> None: + """Sharded streaming decode with every rank keeping the frame, against an unsharded streaming reference. + + Two sessions are interleaved chunk by chunk, so the per-session temporal cache + and the halo exchange are exercised across successive calls, then one session + is reset and must reproduce its opening chunk. Every rank checks every chunk. + """ + from vllm_omni.diffusion.distributed.parallel_state import ( + destroy_model_parallel, + get_sp_group, + init_distributed_environment, + initialize_model_parallel, + ) + from vllm_omni.experimental.ar_diffusion.streaming_decode import WanStreamingDecoder + + os.environ["MASTER_ADDR"] = "127.0.0.1" + os.environ["MASTER_PORT"] = master_port + device = current_omni_platform.get_torch_device(rank) + current_omni_platform.set_device(device) + dtype = torch.float32 + backend = current_omni_platform.dist_backend + init_distributed_environment(world_size=_SPATIAL_SHARD_WORLD_SIZE, rank=rank, local_rank=rank, backend=backend) + initialize_model_parallel( + sequence_parallel_size=_SPATIAL_SHARD_WORLD_SIZE, ulysses_degree=_SPATIAL_SHARD_WORLD_SIZE, backend=backend + ) + # VLLM_OMNI_TEST_WAN_VAE may point at a local diffusers checkpoint (with or + # without a ``vae`` subfolder) for machines without hub access. + model = os.environ.get("VLLM_OMNI_TEST_WAN_VAE", _SPATIAL_SHARD_MODEL) + local_vae_only = os.path.isdir(model) and not os.path.isdir(os.path.join(model, "vae")) + load = {"torch_dtype": dtype} if local_vae_only else {"subfolder": _SPATIAL_SHARD_SUBFOLDER, "torch_dtype": dtype} + try: + reference_vae = DistributedAutoencoderKLWan.from_pretrained(model, **load).to(device=device, dtype=dtype).eval() + sharded_vae = DistributedAutoencoderKLWan.from_pretrained(model, **load).to(device=device, dtype=dtype).eval() + wan_spatial_shard.install_wan_spatial_shard_decode( + sharded_vae, get_sp_group().device_group, split_dim="width", dst=None + ) + reference = WanStreamingDecoder(reference_vae, bytes_per_pixel_fp32=1.0) + sharded = WanStreamingDecoder(sharded_vae, bytes_per_pixel_fp32=1.0) + + generator = torch.Generator(device=device).manual_seed(0) + # Width 104 latents = 832 px: not divisible by two ranks' halo-padded shards without padding. + shape = (1, reference_vae.config.z_dim, 1, _SPATIAL_SHARD_LATENT_HEIGHT, _SPATIAL_SHARD_LATENT_WIDTH) + chunks = { + name: [ + torch.randn(shape, generator=generator, device=device, dtype=dtype) + for _ in range(_STREAMING_SHARD_CHUNKS) + ] + for name in ("a", "b") + } + max_diff = 0.0 + with torch.inference_mode(): + ref_states = {name: reference.new_decode_state(name) for name in chunks} + shard_states = {name: sharded.new_decode_state(name) for name in chunks} + for index in range(_STREAMING_SHARD_CHUNKS): + for name in ("a", "b"): + expected = reference.decode_chunk(chunks[name][index], ref_states[name]).float() + got = sharded.decode_chunk(chunks[name][index], shard_states[name]).float() + if got.shape != expected.shape: + raise AssertionError( + f"rank {rank} {name}[{index}]: {tuple(got.shape)} != {tuple(expected.shape)}" + ) + max_diff = max(max_diff, (got - expected).abs().max().item()) + # Reset session a: a fresh state must reproduce its opening chunk, on this rank too. + shard_states["a"].release() + ref_states["a"].release() + shard_states["a"] = sharded.new_decode_state("a") + ref_states["a"] = reference.new_decode_state("a") + expected = reference.decode_chunk(chunks["a"][0], ref_states["a"]).float() + got = sharded.decode_chunk(chunks["a"][0], shard_states["a"]).float() + max_diff = max(max_diff, (got - expected).abs().max().item()) + return_dict[f"max_abs_diff_rank{rank}"] = max_diff + return_dict[f"shape_rank{rank}"] = tuple(got.shape) + finally: + destroy_model_parallel() + if dist.is_initialized(): + dist.destroy_process_group() + + +@pytest.mark.full_model +@pytest.mark.diffusion +@pytest.mark.parallel +@hardware_test(res={"cuda": ["H100", "B200"]}, num_cards=_SPATIAL_SHARD_WORLD_SIZE) +def test_streaming_shard_decode_matches_unsharded_streaming_on_every_rank(): + manager = mp.get_context("spawn").Manager() + return_dict = manager.dict() + mp.spawn(_streaming_shard_worker, args=(return_dict, "29510"), nprocs=_SPATIAL_SHARD_WORLD_SIZE, join=True) + + for rank in range(_SPATIAL_SHARD_WORLD_SIZE): + assert f"max_abs_diff_rank{rank}" in return_dict, f"rank {rank} did not report" + diff = return_dict[f"max_abs_diff_rank{rank}"] + shape = return_dict[f"shape_rank{rank}"] + print(f"streaming shard rank {rank}: max_abs_diff={diff:.6e} shape={shape}") + assert shape[-1] == _SPATIAL_SHARD_LATENT_WIDTH * 8 and shape[-2] == _SPATIAL_SHARD_LATENT_HEIGHT * 8 + assert diff <= _SPATIAL_SHARD_TOLERANCE, f"rank {rank}: {diff} exceeds {_SPATIAL_SHARD_TOLERANCE}" + + @pytest.mark.full_model @pytest.mark.diffusion @pytest.mark.parallel diff --git a/tests/diffusion/model_loader/test_hub_prefetch.py b/tests/diffusion/model_loader/test_hub_prefetch.py index 94170edcc..b17189539 100644 --- a/tests/diffusion/model_loader/test_hub_prefetch.py +++ b/tests/diffusion/model_loader/test_hub_prefetch.py @@ -3,21 +3,21 @@ import contextlib -import huggingface_hub import pytest +from tests.helpers.mock import patch_hf_snapshot_download from vllm_omni.diffusion.model_loader import hub_prefetch pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu] -def test_prefetch_subfolders_propagates_revision(monkeypatch): +def test_prefetch_subfolders_propagates_revision(monkeypatch, tmp_path): calls = [] - def fake_snapshot_download(self, **kwargs): + def fake_snapshot_download(**kwargs): calls.append(kwargs) - monkeypatch.setattr(huggingface_hub.HfApi, "snapshot_download", fake_snapshot_download) + patch_hf_snapshot_download(monkeypatch, fake_snapshot_download, hf_home=tmp_path) monkeypatch.setattr(hub_prefetch, "_repo_prefetch_lock", lambda _model: contextlib.nullcontext()) hub_prefetch.prefetch_subfolders( diff --git a/tests/diffusion/models/bagel/test_step_execution.py b/tests/diffusion/models/bagel/test_step_execution.py index 40d4154e3..e5f8e5d5a 100644 --- a/tests/diffusion/models/bagel/test_step_execution.py +++ b/tests/diffusion/models/bagel/test_step_execution.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project """Focused CPU tests for BAGEL step execution and packed batching.""" from __future__ import annotations @@ -18,12 +18,14 @@ from vllm_omni.diffusion.models.bagel.bagel_transformer import Bagel, NaiveCache from vllm_omni.diffusion.models.bagel.pipeline_bagel import ( BagelGenParams, BagelPipeline, + _bagel_canvas_requested, get_bagel_pre_process_func, ) from vllm_omni.diffusion.request import OmniDiffusionRequest from vllm_omni.diffusion.sched import StepScheduler from vllm_omni.diffusion.worker.input_batch import InputBatch from vllm_omni.diffusion.worker.utils import StepRequestState +from vllm_omni.errors import OmniClientError from vllm_omni.inputs.data import OmniDiffusionSamplingParams pytestmark = [pytest.mark.core_model, pytest.mark.cpu] @@ -118,7 +120,7 @@ def test_bagel_step_preprocessor_separates_incompatible_cfg_settings(first_extra assert first.batch_compatibility_key != second.batch_compatibility_key -def test_bagel_step_preprocessor_buckets_effective_img2img_sizes(tmp_path): +def _pre_process_with_stub_checkpoint(tmp_path): (tmp_path / "config.json").write_text( '{"vae_config":{"downsample":8},"latent_patch_size":2,"max_latent_size":32}', encoding="utf-8", @@ -131,9 +133,22 @@ def test_bagel_step_preprocessor_buckets_effective_img2img_sizes(tmp_path): '{"weight_map":{"latent_pos_embed.pos_embed":"ema.safetensors"}}', encoding="utf-8", ) - pre_process = get_bagel_pre_process_func( - types.SimpleNamespace(model=str(tmp_path), revision=None, step_execution=True) + return get_bagel_pre_process_func(types.SimpleNamespace(model=str(tmp_path), revision=None, step_execution=True)) + + +def _edit_request(source_size, sampling_params, request_id="req"): + return OmniDiffusionRequest( + prompt={ + "prompt": "edit this image", + "multi_modal_data": {"image": [Image.new("RGB", source_size)]}, + }, + sampling_params=sampling_params, + request_id=request_id, ) + + +def test_bagel_step_preprocessor_buckets_effective_img2img_sizes(tmp_path): + pre_process = _pre_process_with_stub_checkpoint(tmp_path) requests = [ OmniDiffusionRequest( prompt={ @@ -163,6 +178,172 @@ def test_bagel_step_preprocessor_buckets_effective_img2img_sizes(tmp_path): assert schedule.num_waiting_reqs == 1 +def test_bagel_pre_process_keeps_explicitly_requested_canvas(tmp_path): + pre_process = _pre_process_with_stub_checkpoint(tmp_path) + request = _edit_request((512, 1024), OmniDiffusionSamplingParams(num_inference_steps=2, height=1024, width=1024)) + + pre_process(request) + + assert (request.sampling_params.height, request.sampling_params.width) == (1024, 1024) + assert _bagel_canvas_requested(request.sampling_params) + + +def test_bagel_pre_process_aligns_source_derived_canvas_for_size_auto(tmp_path): + pre_process = _pre_process_with_stub_checkpoint(tmp_path) + request = _edit_request( + (500, 700), + OmniDiffusionSamplingParams( + num_inference_steps=2, + height=700, + width=500, + height_not_provided=True, + width_not_provided=True, + ), + ) + + pre_process(request) + + assert (request.sampling_params.height, request.sampling_params.width) == (704, 496) + assert not _bagel_canvas_requested(request.sampling_params) + + +def test_bagel_pre_process_derives_canvas_when_caller_asked_for_none(tmp_path): + pre_process = _pre_process_with_stub_checkpoint(tmp_path) + request = _edit_request((800, 400), OmniDiffusionSamplingParams(num_inference_steps=2)) + + pre_process(request) + + assert (request.sampling_params.height, request.sampling_params.width) == (400, 800) + assert (request.sampling_params.height_not_provided, request.sampling_params.width_not_provided) == (True, True) + assert not _bagel_canvas_requested(request.sampling_params) + + +def test_bagel_pre_process_marks_kv_metadata_canvas_as_requested(tmp_path): + pre_process = _pre_process_with_stub_checkpoint(tmp_path) + request = _edit_request( + (500, 700), + OmniDiffusionSamplingParams( + num_inference_steps=2, + height=700, + width=500, + height_not_provided=True, + width_not_provided=True, + kv_metadata={"image_shape": (1024, 1024)}, + ), + ) + + pre_process(request) + + assert (request.sampling_params.height, request.sampling_params.width) == (1024, 1024) + assert (request.sampling_params.height_not_provided, request.sampling_params.width_not_provided) == (False, False) + assert _bagel_canvas_requested(request.sampling_params) + + +def test_bagel_pre_process_prefers_explicit_canvas_over_kv_metadata(tmp_path): + pre_process = _pre_process_with_stub_checkpoint(tmp_path) + request = _edit_request( + (1280, 720), + OmniDiffusionSamplingParams( + num_inference_steps=2, + height=1024, + width=1024, + kv_metadata={"image_shape": (576, 1024)}, + ), + ) + + pre_process(request) + + assert (request.sampling_params.height, request.sampling_params.width) == (1024, 1024) + assert _bagel_canvas_requested(request.sampling_params) + + +def test_bagel_pre_process_floors_unaligned_explicit_canvas_with_warning(tmp_path): + pre_process = _pre_process_with_stub_checkpoint(tmp_path) + request = _edit_request( + (1280, 720), + OmniDiffusionSamplingParams(num_inference_steps=2, height=700, width=1000), + ) + + with patch("vllm_omni.diffusion.models.bagel.pipeline_bagel.logger.warning") as warning: + pre_process(request) + + assert (request.sampling_params.height, request.sampling_params.width) == (688, 992) + assert _bagel_canvas_requested(request.sampling_params) + warning.assert_called_once() + assert "width 1000->992, height 700->688" in warning.call_args.args[-1] + + +def test_bagel_pre_process_keeps_aligned_explicit_canvas_silent(tmp_path): + pre_process = _pre_process_with_stub_checkpoint(tmp_path) + request = _edit_request( + (1280, 720), + OmniDiffusionSamplingParams(num_inference_steps=2, height=512, width=1024), + ) + + with patch("vllm_omni.diffusion.models.bagel.pipeline_bagel.logger.warning") as warning: + pre_process(request) + + assert (request.sampling_params.height, request.sampling_params.width) == (512, 1024) + warning.assert_not_called() + + +def test_bagel_pre_process_rejects_explicit_canvas_over_checkpoint_limit(tmp_path): + pre_process = _pre_process_with_stub_checkpoint(tmp_path) + request = _edit_request( + (1280, 720), + OmniDiffusionSamplingParams(num_inference_steps=2, height=512, width=2048), + ) + + with pytest.raises(OmniClientError, match="width=2048 exceeds the BAGEL checkpoint limit of 1024") as excinfo: + pre_process(request) + + assert excinfo.value.status_code == 400 + + +def test_bagel_pre_process_aligns_text2img_canvas_too(tmp_path): + pre_process = _pre_process_with_stub_checkpoint(tmp_path) + request = OmniDiffusionRequest( + prompt="a cat", + sampling_params=OmniDiffusionSamplingParams(num_inference_steps=2, height=700, width=1000), + request_id="req", + ) + + pre_process(request) + + assert (request.sampling_params.height, request.sampling_params.width) == (688, 992) + + +def test_bagel_pre_process_skips_canvas_alignment_for_text_output(tmp_path): + pre_process = _pre_process_with_stub_checkpoint(tmp_path) + request = OmniDiffusionRequest( + prompt={ + "prompt": "describe this image", + "modalities": ["text"], + "multi_modal_data": {"image": [Image.new("RGB", (1280, 720))]}, + }, + sampling_params=OmniDiffusionSamplingParams(num_inference_steps=2, height=700, width=1000), + request_id="req", + ) + + pre_process(request) + + assert (request.sampling_params.height, request.sampling_params.width) == (700, 1000) + assert request.use_step_execution is False + + +@pytest.mark.parametrize( + ("sampling_kwargs", "requested"), + [ + ({"height": 1024, "width": 1024}, True), + ({"height": 1024, "width": 512, "height_not_provided": True, "width_not_provided": True}, False), + ({}, False), + ], + ids=["explicit_size", "size_auto", "unset"], +) +def test_bagel_canvas_requested_separates_asked_for_from_derived(sampling_kwargs, requested): + assert _bagel_canvas_requested(OmniDiffusionSamplingParams(**sampling_kwargs)) is requested + + def _cache(length: int, value: float, num_layers: int = 1) -> NaiveCache: cache = NaiveCache(num_layers) if length: diff --git a/tests/diffusion/models/hunyuan_image3/test_hunyuan_image3_it2i_multi_image.py b/tests/diffusion/models/hunyuan_image3/test_hunyuan_image3_it2i_multi_image.py index 1e0fd1590..14a854a7a 100644 --- a/tests/diffusion/models/hunyuan_image3/test_hunyuan_image3_it2i_multi_image.py +++ b/tests/diffusion/models/hunyuan_image3/test_hunyuan_image3_it2i_multi_image.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project """Multi-image input regression for HunyuanImage3 IT2I prompt construction. The official HunyuanImage-3.0-Instruct supports up to 3 reference images @@ -41,7 +41,7 @@ pytestmark = [pytest.mark.core_model, pytest.mark.cpu] class FakeTokenizer: - """Recording fake tokenizer mirroring the one in test_prompt_utils. + """Recording fake tokenizer mirroring the one in test_hunyuan_image3_prompt_utils. Special token ids: `<|startoftext|>`=1, ``=2, ``=3, ``=4. encode() returns one id per character starting at diff --git a/tests/diffusion/models/hunyuan_image3/test_prompt_utils.py b/tests/diffusion/models/hunyuan_image3/test_hunyuan_image3_prompt_utils.py similarity index 99% rename from tests/diffusion/models/hunyuan_image3/test_prompt_utils.py rename to tests/diffusion/models/hunyuan_image3/test_hunyuan_image3_prompt_utils.py index 7a467e359..a2be63fd2 100644 --- a/tests/diffusion/models/hunyuan_image3/test_prompt_utils.py +++ b/tests/diffusion/models/hunyuan_image3/test_hunyuan_image3_prompt_utils.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project from __future__ import annotations import ast diff --git a/tests/diffusion/models/lingbot_world/test_lingbot_world_attention.py b/tests/diffusion/models/lingbot_world/test_lingbot_world_attention.py index 7e12b3db3..d2d11538c 100644 --- a/tests/diffusion/models/lingbot_world/test_lingbot_world_attention.py +++ b/tests/diffusion/models/lingbot_world/test_lingbot_world_attention.py @@ -134,6 +134,8 @@ def _install_vllm_stubs() -> None: **kwargs, ) -> None: super().__init__() + self.quant_config = kwargs.pop("quant_config", None) + self.prefix = kwargs.pop("prefix", "") del kwargs self.return_bias = return_bias self.weight = nn.Parameter(torch.empty(output_size, input_size)) @@ -164,6 +166,8 @@ def _install_vllm_stubs() -> None: **kwargs, ) -> None: super().__init__() + self.quant_config = kwargs.pop("quant_config", None) + self.prefix = kwargs.pop("prefix", "") del kwargs self.num_heads = total_num_heads self.num_kv_heads = total_num_kv_heads or total_num_heads diff --git a/tests/diffusion/models/lingbot_world/test_lingbot_world_transformer.py b/tests/diffusion/models/lingbot_world/test_lingbot_world_transformer.py index e11c34e35..7087e6171 100644 --- a/tests/diffusion/models/lingbot_world/test_lingbot_world_transformer.py +++ b/tests/diffusion/models/lingbot_world/test_lingbot_world_transformer.py @@ -35,6 +35,8 @@ def _tiny_model( num_layers: int = 2, num_frames_per_block: int = 1, sliding_window_num_frames: int = 3, + quant_config=None, + prefix: str = "", attention_head_dim: int = 2, rope_max_seq_len: int = 16, ): @@ -55,6 +57,8 @@ def _tiny_model( num_frames_per_block=num_frames_per_block, sliding_window_num_frames=sliding_window_num_frames, local_attn_size=-1, + quant_config=quant_config, + prefix=prefix, ) @@ -485,11 +489,52 @@ def test_constructor_rejects_non_null_image_embedding_fields(field: str) -> None @pytest.mark.cpu -def test_constructor_rejects_unsupported_quantization_with_runtime_error() -> None: +@pytest.mark.parametrize("prefix", ["", "transformer"]) +@pytest.mark.parametrize("quantized", [False, True]) +def test_constructor_propagates_quantization_to_parallel_linears(prefix: str, quantized: bool) -> None: module = attention_tests._load_module() + quant_config = object() if quantized else None + model = _tiny_model(module, num_layers=1, quant_config=quant_config, prefix=prefix) + block = model.blocks[0] + quantized_linears = [ + model.c2ws_hidden_states_layer1, + model.c2ws_hidden_states_layer2, + block.self_attn.qkv, + block.self_attn.o, + block.cross_attn.q, + block.cross_attn.k, + block.cross_attn.v, + block.cross_attn.o, + block.ffn[0], + block.ffn[2], + block.cam_injector_layer1, + block.cam_injector_layer2, + ] - with pytest.raises(RuntimeError, match="quant_config.*not supported"): - module.CausalLingBotWorldTransformer3DModel(quant_config=object()) + assert len(quantized_linears) == 12 + assert all(layer.quant_config is quant_config for layer in quantized_linears) + expected_names = { + "c2ws_hidden_states_layer1", + "c2ws_hidden_states_layer2", + *( + f"blocks.0.{name}" + for name in ( + "self_attn.qkv", + "self_attn.o", + "cross_attn.q", + "cross_attn.k", + "cross_attn.v", + "cross_attn.o", + "ffn.0", + "ffn.2", + "cam_injector_layer1", + "cam_injector_layer2", + ) + ), + } + assert {layer.prefix for layer in quantized_linears} == { + f"{prefix}.{name}" if prefix else name for name in expected_names + } @pytest.mark.cpu diff --git a/tests/diffusion/models/lingbot_world/test_pipeline_lingbot_world.py b/tests/diffusion/models/lingbot_world/test_pipeline_lingbot_world.py index 91ae00b7d..8c24b8562 100644 --- a/tests/diffusion/models/lingbot_world/test_pipeline_lingbot_world.py +++ b/tests/diffusion/models/lingbot_world/test_pipeline_lingbot_world.py @@ -737,6 +737,99 @@ def test_component_discovery_uses_official_checkpoint_contract() -> None: assert module._loader_state.prefetch_calls == [("checkpoint", ("tokenizer", "text_encoder", "vae"), False)] +def _ulysses_config(size: int): + parallel_config = _od_config().parallel_config + parallel_config.sequence_parallel_size = size + parallel_config.ulysses_degree = size + return _od_config(parallel_config=parallel_config) + + +def _record_shard_install(monkeypatch, module, *, world_size: int): + """Stand in for the Ulysses group and the decoder patch; return what the install was asked for.""" + installs: list[dict] = [] + group = object() + + def install(vae, group_arg, split_dim, *, dst): + installs.append({"vae": vae, "group": group_arg, "split_dim": split_dim, "dst": dst}) + + monkeypatch.setattr(module, "install_wan_spatial_shard_decode", install) + monkeypatch.setattr(module.LingBotWorldCausalDMDPipeline, "_vae_shard_group", lambda self: (group, world_size)) + return installs, group + + +def test_a_multi_rank_deployment_shards_the_decoder_across_the_ulysses_ranks(monkeypatch) -> None: + """Multi-rank deployments shard decode by default.""" + module = _load_pipeline_module() + installs, group = _record_shard_install(monkeypatch, module, world_size=2) + + pipeline = module.LingBotWorldCausalDMDPipeline(od_config=_ulysses_config(2)) + + # Along the width, and assembled on every rank: each rank's post_decode consumes the frame. + assert installs == [{"vae": pipeline.vae, "group": group, "split_dim": "width", "dst": None}] + assert pipeline._vae_shard_split_dim == "width" + + +def test_a_single_rank_deployment_leaves_the_decoder_alone(monkeypatch) -> None: + module = _load_pipeline_module() + installs, _ = _record_shard_install(monkeypatch, module, world_size=1) + + pipeline = module.LingBotWorldCausalDMDPipeline(od_config=_ulysses_config(1)) + + assert installs == [] and pipeline._vae_shard_split_dim is None + + +@pytest.mark.parametrize("enabled", [True, False]) +@pytest.mark.parametrize("width, local_width", [(832, 208), (840, 216)]) +def test_streaming_decode_reservation_uses_padded_shards(monkeypatch, enabled, width, local_width): + module = _load_pipeline_module() + installs, _ = _record_shard_install(monkeypatch, module, world_size=4) + config = _ulysses_config(4) + config.model_config.update(lingbot_vae_spatial_sharding=enabled, ar_diffusion_height=480, ar_diffusion_width=width) + pipeline = module.LingBotWorldCausalDMDPipeline(od_config=config) + from vllm_omni.experimental.ar_diffusion.streaming_decode import WanStreamingDecoder + + # Exercise the real byte estimator without constructing a checkpoint VAE. + decoder = object.__new__(WanStreamingDecoder) + decoder._bytes_per_pixel_fp32 = 16.0 + monkeypatch.setattr(pipeline, "_streaming_decoder", lambda: decoder) + expected_width = local_width if enabled else width + assert pipeline._streaming_decode_bytes_per_session() == 16 * 480 * expected_width + assert len(installs) == int(enabled) + assert pipeline._vae_shard_world_size == (4 if enabled else 1) + assert config.parallel_config.sequence_parallel_size == config.parallel_config.ulysses_degree == 4 + + +def test_disabling_vae_sharding_does_not_require_a_shard_group(monkeypatch): + module = _load_pipeline_module() + config = _ulysses_config(4) + config.model_config["lingbot_vae_spatial_sharding"] = False + + def unexpected_group(self): + pytest.fail("disabled VAE sharding must not access its process group") + + monkeypatch.setattr(module.LingBotWorldCausalDMDPipeline, "_vae_shard_group", unexpected_group) + pipeline = module.LingBotWorldCausalDMDPipeline(od_config=config) + assert pipeline._vae_shard_split_dim is None + + +@pytest.mark.parametrize("value", ["false", 0, None]) +def test_vae_sharding_switch_requires_a_boolean(value): + module = _load_pipeline_module() + config = _ulysses_config(4) + config.model_config["lingbot_vae_spatial_sharding"] = value + with pytest.raises(ValueError, match="lingbot_vae_spatial_sharding must be a boolean"): + module.LingBotWorldCausalDMDPipeline(od_config=config) + + +def test_vae_shard_refuses_a_group_of_the_wrong_size(monkeypatch) -> None: + module = _load_pipeline_module() + installs, _ = _record_shard_install(monkeypatch, module, world_size=4) + + with pytest.raises(RuntimeError, match="sequence_parallel_size=2 but the Ulysses group has 4 ranks"): + module.LingBotWorldCausalDMDPipeline(od_config=_ulysses_config(2)) + assert installs == [] + + @pytest.mark.parametrize( ("field", "value", "feature"), [ @@ -758,8 +851,9 @@ def test_unsupported_parallel_modes_fail_before_component_loading(field: str, va assert module._loader_state.prefetch_calls == [] -def test_pure_ulysses_parallel_config_is_supported() -> None: +def test_pure_ulysses_parallel_config_is_supported(monkeypatch) -> None: module = _load_pipeline_module() + _record_shard_install(monkeypatch, module, world_size=2) parallel_config = _od_config().parallel_config parallel_config.sequence_parallel_size = 2 parallel_config.ulysses_degree = 2 @@ -795,13 +889,14 @@ def test_unsupported_sp_config_fails_before_component_loading(overrides): assert module._loader_state.prefetch_calls == [] -def test_unsupported_quantization_fails_before_component_loading() -> None: +def test_quantization_reaches_transformer_factory() -> None: module = _load_pipeline_module() + quant_config = object() - with pytest.raises(NotImplementedError, match="quantization"): - module.LingBotWorldCausalDMDPipeline(od_config=_od_config(quantization_config=object())) + pipeline = module.LingBotWorldCausalDMDPipeline(od_config=_od_config(quantization_config=quant_config)) - assert module._loader_state.prefetch_calls == [] + assert pipeline.transformer is not None + assert _FakeTransformerFactory.last_call[1:] == (quant_config, "transformer") def test_official_scheduler_config_matches_fixed_dmd_contract() -> None: @@ -1358,10 +1453,34 @@ def test_first_frame_condition_and_camera_fold_match_transformer_contract() -> N assert expected_camera.shape == (1, 384, 3, 2, 2) -def test_fixed_dmd_transition_and_cache_commit_trace() -> None: +@pytest.mark.parametrize("value", ["true", "false", 1, 0, None]) +def test_reuse_last_step_kv_rejects_non_boolean_config(value: object) -> None: + config = _od_config() + config.model_config["lingbot_reuse_last_step_kv"] = value + with pytest.raises(ValueError, match="lingbot_reuse_last_step_kv must be a bool"): + _pipeline(_load_pipeline_module(), od_config=config) + + +def test_reuse_last_step_kv_is_fixed_per_instance(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("VLLM_OMNI_LINGBOT_REUSE_LAST_STEP_KV", "1") + module = _load_pipeline_module() + config = _od_config() + config.model_config["lingbot_reuse_last_step_kv"] = True + reuse = _pipeline(module, od_config=config) + default = _pipeline(module) + config.model_config["lingbot_reuse_last_step_kv"] = False + assert reuse._dmd_blocks.reuse_last_step_kv is True + assert default._dmd_blocks.reuse_last_step_kv is False + assert reuse._dmd_blocks is reuse._dmd_blocks + + +@pytest.mark.parametrize("reuse_last_step_kv", [False, True]) +def test_fixed_dmd_transition_and_cache_commit_trace(reuse_last_step_kv: bool) -> None: module = _load_pipeline_module() transformer = _RecordingTransformer() - pipeline = _pipeline(module, transformer=transformer) + config = _od_config() + config.model_config["lingbot_reuse_last_step_kv"] = reuse_last_step_kv + pipeline = _pipeline(module, transformer=transformer, od_config=config) result = pipeline(_request()) @@ -1383,14 +1502,18 @@ def test_fixed_dmd_transition_and_cache_commit_trace() -> None: current = x0 torch.testing.assert_close(result.output, current) + expected_timesteps = [timestep for timestep, _ in warped_schedule] + if not reuse_last_step_kv: + expected_timesteps.append(0.0) torch.testing.assert_close( - torch.cat([call["timestep"] for call in transformer.calls]), - torch.tensor([*(timestep for timestep, _ in warped_schedule), 0.0]), + torch.cat([call["timestep"] for call in transformer.calls]), torch.tensor(expected_timesteps) ) - assert [call["update_cache"] for call in transformer.calls] == [False, False, False, False, True] - assert [call["start_frame"] for call in transformer.calls] == [0, 0, 0, 0, 0] + commit_flags = [False] * (len(expected_timesteps) - 1) + [True] + assert [call["update_cache"] for call in transformer.calls] == commit_flags + assert [call["start_frame"] for call in transformer.calls] == [0] * len(expected_timesteps) assert len({call["cache_id"] for call in transformer.calls}) == 1 - torch.testing.assert_close(transformer.calls[-1]["hidden_states"][:, :16], result.output) + committed_latent = result.output + warped_schedule[-1][1] if reuse_last_step_kv else result.output + torch.testing.assert_close(transformer.calls[-1]["hidden_states"][:, :16], committed_latent) @pytest.mark.parametrize( @@ -2389,10 +2512,13 @@ def test_forward_rejects_a_stepwise_camera_action_script(extra_args) -> None: pipeline(_request(sampling=sampling)) -def test_stepwise_progress_metadata_and_commit_trace() -> None: +@pytest.mark.parametrize("reuse_last_step_kv", [False, True]) +def test_stepwise_progress_metadata_and_commit_trace(reuse_last_step_kv: bool) -> None: module = _load_pipeline_module() transformer = _RecordingTransformer() - pipeline = _pipeline(module, transformer=transformer) + config = _od_config() + config.model_config["lingbot_reuse_last_step_kv"] = reuse_last_step_kv + pipeline = _pipeline(module, transformer=transformer, od_config=config) pipeline._ar_height = 16 pipeline._ar_width = 16 state = _stepwise_state(num_frames=21) @@ -2406,8 +2532,9 @@ def test_stepwise_progress_metadata_and_commit_trace() -> None: assert [output.chunk_index for output in outputs] == [0, 1] assert all(output.total_chunks == 2 for output in outputs) assert outputs[-1].finished is True - assert [call["update_cache"] for call in transformer.calls] == [False, False, False, False, True] * 2 - assert [call["start_frame"] for call in transformer.calls] == [0] * 5 + [3] * 5 + calls_per_chunk = 4 if reuse_last_step_kv else 5 + assert [call["update_cache"] for call in transformer.calls] == ([False] * (calls_per_chunk - 1) + [True]) * 2 + assert [call["start_frame"] for call in transformer.calls] == [0] * calls_per_chunk + [3] * calls_per_chunk assert fake.commits == ["main", "main"] for chunk_index, output in enumerate(outputs): metadata = output.output["metadata"]["ar_diffusion"] diff --git a/tests/diffusion/models/magi2/test_pipeline_magi2.py b/tests/diffusion/models/magi2/test_pipeline_magi2.py index 2cda7e66e..4e6057457 100644 --- a/tests/diffusion/models/magi2/test_pipeline_magi2.py +++ b/tests/diffusion/models/magi2/test_pipeline_magi2.py @@ -171,7 +171,7 @@ def test_huggingface_url_resolves_pinned_snapshot(tmp_path, monkeypatch): else pytest.fail("unexpected snapshot request") ) monkeypatch.setattr( - "vllm.transformers_utils.repo_utils.hf_api", + "vllm_omni.transformers_utils.repo_utils.hf_api", lambda: api, ) assert _resolve_checkpoint_root( diff --git a/tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py b/tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py index aca8bdb83..8cc3f1a9e 100644 --- a/tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py +++ b/tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py @@ -3,7 +3,7 @@ import json import sys -from contextlib import contextmanager +from contextlib import contextmanager, nullcontext from multiprocessing.reduction import ForkingPickler from types import SimpleNamespace from typing import Any, TypeVar @@ -2837,41 +2837,63 @@ def test_dit_encoder_selection_keeps_vae_resident(): component.offload_to_cpu.assert_not_called() -def test_keyframe_encode_pins_and_restores_cudnn_settings(): +@pytest.mark.parametrize( + ("capability", "allow_tf32"), + [((8, 6), True), ((9, 0), False), ((10, 0), True), ((10, 3), True), (None, True)], +) +@pytest.mark.parametrize("initial_tf32", [False, True]) +@pytest.mark.parametrize("fail_encode", [False, True]) +def test_keyframe_encode_pins_and_restores_cudnn_settings( + monkeypatch, capability, allow_tf32, initial_tf32, fail_encode +): + from vllm.platforms.interface import DeviceCapability + from vllm_omni.diffusion.models.minimax_h3.vae import ( _minimax_h3_keyframe_encode_context, ) + from vllm_omni.platforms import current_omni_platform + + def get_capability(device_id): + assert device_id == 1 + return DeviceCapability(*capability) if capability is not None else None + + monkeypatch.setattr(current_omni_platform, "is_cuda", lambda: True) + monkeypatch.setattr(current_omni_platform, "get_device_capability", get_capability) cudnn = torch.backends.cudnn - original = ( - cudnn.enabled, - cudnn.benchmark, - cudnn.deterministic, - cudnn.allow_tf32, - ) - try: - cudnn.enabled = False - cudnn.benchmark = True - cudnn.deterministic = False - cudnn.allow_tf32 = False - - with _minimax_h3_keyframe_encode_context(torch.device("cuda")): + with cudnn.flags(enabled=False, benchmark=True, deterministic=False, allow_tf32=initial_tf32): + error_context = pytest.raises(RuntimeError, match="encode failed") if fail_encode else nullcontext() + with error_context, _minimax_h3_keyframe_encode_context(torch.device("cuda:1")): assert cudnn.enabled assert not cudnn.benchmark assert cudnn.deterministic - assert cudnn.allow_tf32 + assert cudnn.allow_tf32 is allow_tf32 + if fail_encode: + raise RuntimeError("encode failed") assert not cudnn.enabled assert cudnn.benchmark assert not cudnn.deterministic - assert not cudnn.allow_tf32 - finally: - ( - cudnn.enabled, - cudnn.benchmark, - cudnn.deterministic, - cudnn.allow_tf32, - ) = original + assert cudnn.allow_tf32 is initial_tf32 + + +def test_keyframe_encode_cpu_keeps_cudnn_settings(monkeypatch): + from vllm_omni.diffusion.models.minimax_h3.vae import ( + _minimax_h3_keyframe_encode_context, + ) + from vllm_omni.platforms import current_omni_platform + + def unexpected_capability_query(device_id): + pytest.fail("CPU keyframe encode must not query CUDA capability") + + monkeypatch.setattr(current_omni_platform, "get_device_capability", unexpected_capability_query) + cudnn = torch.backends.cudnn + with cudnn.flags(enabled=False, benchmark=True, deterministic=False, allow_tf32=False): + with _minimax_h3_keyframe_encode_context(torch.device("cpu")): + assert not cudnn.enabled + assert cudnn.benchmark + assert not cudnn.deterministic + assert not cudnn.allow_tf32 @pytest.mark.parametrize("fail_encode", [False, True]) diff --git a/tests/diffusion/models/minimax_h3/test_minimax_h3_modulation.py b/tests/diffusion/models/minimax_h3/test_minimax_h3_modulation.py index fcca6f7ad..df3b22689 100644 --- a/tests/diffusion/models/minimax_h3/test_minimax_h3_modulation.py +++ b/tests/diffusion/models/minimax_h3/test_minimax_h3_modulation.py @@ -10,7 +10,6 @@ from vllm_omni.diffusion.layers.indexed_modulation import ( _MAX_1D_GRID_SIZE, _iter_row_chunks, _launch_row_chunks, - _use_hopper_bf16_affine_semantics, indexed_gate, indexed_gate_rms_norm_scale_shift, indexed_scale_shift_, @@ -194,52 +193,28 @@ def test_fused_modulation_preserves_bf16_residual_boundary() -> None: @pytest.mark.cuda -@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required") -@pytest.mark.skipif( - torch.cuda.is_available() and torch.cuda.get_device_capability()[0] != 9, - reason="SM90-specific precision path", -) +@pytest.mark.skipif(not current_omni_platform.is_cuda(), reason="CUDA required") @pytest.mark.skipif(not HAS_TRITON, reason="Triton required") -def test_hopper_fused_modulation_matches_pytorch_reference() -> None: - torch.manual_seed(42) - rows, hidden_size, conditions = 128, 3072, 4 - residual = torch.randn(rows, hidden_size, device="cuda", dtype=torch.bfloat16) - gate = torch.randn(conditions, hidden_size, device="cuda", dtype=torch.bfloat16) - branch = torch.randn(rows, hidden_size, device="cuda", dtype=torch.bfloat16) - weight = torch.randn(hidden_size, device="cuda", dtype=torch.bfloat16) - shift = torch.randn(conditions, hidden_size, device="cuda", dtype=torch.bfloat16) - scale = torch.randn(conditions, hidden_size, device="cuda", dtype=torch.bfloat16) - indices = torch.arange(rows, device="cuda") % conditions +@pytest.mark.parametrize("fused_gate", [False, True]) +def test_fused_modulation_keeps_fp32_affine_intermediates(fused_gate: bool) -> None: + device = current_omni_platform.get_torch_device() + rows, hidden_size = 2, 5376 + x = torch.ones(rows, hidden_size, device=device, dtype=torch.bfloat16) + weight = torch.full((hidden_size,), 1.125, device=device, dtype=x.dtype) + shift = torch.zeros(1, hidden_size, device=device, dtype=x.dtype) + scale = torch.full_like(shift, 0.09375) + indices = torch.zeros(rows, device=device, dtype=torch.int64) eps = 1e-6 + if fused_gate: + residual, actual = indexed_gate_rms_norm_scale_shift( + x, torch.zeros_like(shift), x, weight, shift, scale, indices, eps + ) + assert torch.equal(residual, x) + else: + actual = rms_norm_indexed_scale_shift(x, weight, shift, scale, indices, eps) - reference_residual = (residual + gate.index_select(0, indices) * branch).to(torch.bfloat16) - normalized = reference_residual.float() - variance = normalized.pow(2).mean(-1, keepdim=True) - normalized = (weight.float() * (normalized * torch.rsqrt(variance + eps))).to(torch.bfloat16) - reference_modulated = (normalized * (1.0 + scale.index_select(0, indices)) + shift.index_select(0, indices)).to( - torch.bfloat16 - ) - - actual_residual, actual_modulated = indexed_gate_rms_norm_scale_shift( - residual, - gate, - branch, - weight, - shift, - scale, - indices, - eps, - ) - - torch.testing.assert_close(actual_residual, reference_residual, atol=5e-2, rtol=5e-2) - torch.testing.assert_close(actual_modulated, reference_modulated, atol=5e-2, rtol=5e-2) - - -@pytest.mark.cuda -@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required") -def test_hopper_precision_dispatch_is_architecture_scoped(monkeypatch) -> None: - tensor = torch.empty(0, device="cuda") - monkeypatch.setattr(torch.cuda, "get_device_capability", lambda _device: (9, 0)) - assert _use_hopper_bf16_affine_semantics(tensor) - monkeypatch.setattr(torch.cuda, "get_device_capability", lambda _device: (10, 3)) - assert not _use_hopper_bf16_affine_semantics(tensor) + # Without the RMS epsilon, 1.125 * 1.09375 = 1.23046875 is exactly + # halfway between two BF16 values. FP32 normalization keeps the epsilon + # and rounds down to 1.2265625. Premature BF16 rounding drops the epsilon + # and instead rounds the tie up to 1.234375, changing every output row. + assert torch.equal(actual, torch.full_like(x, 1.2265625)) diff --git a/tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py b/tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py index 3beda5acc..dd54f00ae 100644 --- a/tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py +++ b/tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py @@ -16,6 +16,7 @@ from tests.diffusion.quantization.test_quantization_quality import ( _maybe_save_output, ) from tests.helpers.mark import hardware_marks +from tests.helpers.mock import patch_hf_snapshot_download _MINIMAX_H3_REPO = "MiniMaxAI/MiniMax-H3" _MINIMAX_H3_REVISION = "48d93ede732756e404a3b1b2f3b3a9b5a22f6cfc" @@ -258,13 +259,13 @@ def test_resolve_fl2va_model_ref(tmp_path, monkeypatch): fl2va_root.mkdir() (fl2va_root / "model_index.json").write_text("{}", encoding="utf-8") - def fake_snapshot_download(self, *, repo_id, revision, allow_patterns): + def fake_snapshot_download(*, repo_id, revision, allow_patterns): assert repo_id == _MINIMAX_H3_REPO assert revision == _MINIMAX_H3_REVISION assert allow_patterns == ["FL2VA/**"] return str(tmp_path) - monkeypatch.setattr("huggingface_hub.HfApi.snapshot_download", fake_snapshot_download) + patch_hf_snapshot_download(monkeypatch, fake_snapshot_download, hf_home=tmp_path) assert _resolve_fl2va_model_ref() == str(fl2va_root) diff --git a/tests/diffusion/models/nextstep_1_1/test_nextstep_rectangular.py b/tests/diffusion/models/nextstep_1_1/test_nextstep_rectangular.py new file mode 100644 index 000000000..279feb445 --- /dev/null +++ b/tests/diffusion/models/nextstep_1_1/test_nextstep_rectangular.py @@ -0,0 +1,52 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + +from types import MethodType, SimpleNamespace +from unittest.mock import Mock + +import pytest +import torch + +from vllm_omni.diffusion.models.nextstep_1_1 import pipeline_nextstep_1_1 as pipeline_module +from vllm_omni.diffusion.models.nextstep_1_1.modeling_nextstep import NextStepModel +from vllm_omni.inputs.data import OmniDiffusionSamplingParams + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion] + + +@pytest.mark.parametrize("height,width", [(32, 32), (32, 48), (48, 32)]) +@pytest.mark.parametrize("batch", [1, 2]) +def test_forward_preserves_latent_grid(monkeypatch, height, width, batch): + pipeline = object.__new__(pipeline_module.NextStep11Pipeline) + torch.nn.Module.__init__(pipeline) + pipeline._NextStep11Pipeline__device = torch.device("cpu") + pipeline.config = SimpleNamespace(use_gen_pos_embed=False) + pipeline.down_factor = 16 + pipeline.scaling_factor = 2.0 + pipeline.shift_factor = 0.25 + grid = SimpleNamespace(config=SimpleNamespace(latent_channels=2, latent_patch_size=2)) + expected = torch.arange(batch * 2 * (height // 8) * (width // 8), dtype=torch.float32) + expected = expected.reshape(batch, 2, height // 8, width // 8) + tokens = NextStepModel.patchify(grid, expected) + pipeline.model = Mock( + return_value=SimpleNamespace(last_hidden_state=torch.zeros(batch, 1, 4), past_key_values=None) + ) + pipeline.model.forward_model.return_value = pipeline.model.return_value + pipeline.model.unpatchify = MethodType(NextStepModel.unpatchify, grid) + pipeline.tokenizer = Mock( + bos_token=None, + return_value=SimpleNamespace( + input_ids=torch.ones(batch, 2, dtype=torch.long), attention_mask=torch.ones(batch, 2, dtype=torch.long) + ), + ) + pipeline._check_input = Mock(return_value=(["teapot"], None)) + pipeline._build_captions = Mock(return_value=(["teapot"] * batch, None, 1, 1.0)) + pipeline._add_prefix_ids = lambda _hw, ids, mask: (ids, mask) + pipeline.decoding = Mock(return_value=tokens) + pipeline.vae = SimpleNamespace(dtype=torch.float32, decode=lambda latent: SimpleNamespace(sample=latent)) + monkeypatch.setattr(pipeline_module, "StaticCache", lambda **_kwargs: None) + params = OmniDiffusionSamplingParams( + height=height, width=width, guidance_scale=1.0, num_outputs_per_prompt=batch, seed=None + ) + output = pipeline.forward(SimpleNamespace(prompts=["teapot"], sampling_params=params)) + torch.testing.assert_close(output.output, expected / 2 + 0.25, rtol=0, atol=0) diff --git a/tests/diffusion/test_async_output_worker.py b/tests/diffusion/test_async_output_worker.py index 8a81f30d4..9465f94b3 100644 --- a/tests/diffusion/test_async_output_worker.py +++ b/tests/diffusion/test_async_output_worker.py @@ -497,3 +497,99 @@ class TestWorkerProcShutdown: result_mq.shutdown.assert_called_once_with() assert proc.mq is None assert proc.result_mq is None + + +class TestSynchronizeDeviceBarrier: + """The pause barrier must not report a quiet device while outputs still pack.""" + + @staticmethod + def _rpc(timeout): + return { + "method": "synchronize_device", + "args": (), + "kwargs": {"timeout": timeout}, + "output_rank": 0, + "exec_all_ranks": False, + "collect_rank_status": False, + } + + def test_rejects_undrained_output(self, mocker): + proc = _make_worker_proc(step_execution=False) + proc.worker = MagicMock() + mocker.patch.object(proc, "drain_async_outputs", return_value=False) + + with pytest.raises(TimeoutError, match="did not drain"): + proc._execute_rpc(self._rpc(0.01)) + + proc.worker.execute_method.assert_not_called() + + @pytest.mark.parametrize("timeout", [0.5, None]) + def test_drains_with_the_call_timeout_then_synchronizes(self, mocker, timeout): + proc = _make_worker_proc(step_execution=False) + proc.worker = MagicMock() + call_order: list = [] + mocker.patch.object( + proc, + "drain_async_outputs", + side_effect=lambda **kw: call_order.append(("drain", kw)) or True, + ) + proc.worker.execute_method.side_effect = lambda *a, **kw: call_order.append(("sync", a, kw)) + + _, should_reply = proc._execute_rpc(self._rpc(timeout)) + + assert call_order == [ + ("drain", {"timeout": timeout}), + ("sync", ("synchronize_device",), {"timeout": timeout}), + ] + assert should_reply is True + + def test_completes_only_after_output_ready_is_published(self, mocker): + proc = _make_worker_proc(step_execution=False) + proc.worker = MagicMock() + + mock_platform = mocker.MagicMock() + mock_platform.record_device_event.return_value = MagicMock() + mocker.patch( + "vllm_omni.diffusion.worker.diffusion_worker.current_omni_platform", + mock_platform, + ) + mock_torch = mocker.MagicMock() + mock_torch.accelerator.current_accelerator.return_value.type = "cpu" + mocker.patch("vllm_omni.diffusion.worker.diffusion_worker.torch", mock_torch) + release = threading.Event() + mocker.patch( + "vllm_omni.diffusion.worker.diffusion_worker.pack_diffusion_output_shm", + side_effect=lambda *a, **kw: release.wait(timeout=5.0), + ) + + proc._return_result(DiffusionOutput(output="data"), rpc_id="1") + loop_thread = threading.Thread(target=proc._async_output_loop, daemon=True) + loop_thread.start() + + barrier_done = threading.Event() + + def run_barrier(): + proc._execute_rpc(self._rpc(5.0)) + barrier_done.set() + + barrier_thread = threading.Thread(target=run_barrier, daemon=True) + barrier_thread.start() + try: + assert not barrier_done.wait(timeout=0.2) + proc.worker.execute_method.assert_not_called() + + release.set() + assert barrier_done.wait(timeout=5.0) + + kinds = [ + call.args[0].kind + for call in proc.result_mq.enqueue.call_args_list + if isinstance(call.args[0], AsyncDiffusionOutput) + ] + assert kinds == [AsyncOutputKind.COMPUTE_DONE, AsyncOutputKind.OUTPUT_READY] + proc.worker.execute_method.assert_called_once_with("synchronize_device", timeout=5.0) + finally: + release.set() + proc._async_output_queue.put(None) + loop_thread.join(timeout=2.0) + barrier_thread.join(timeout=2.0) diff --git a/tests/diffusion/test_diffusion_engine_rpc_routing.py b/tests/diffusion/test_diffusion_engine_rpc_routing.py index bb64e5f01..d26f20c1d 100644 --- a/tests/diffusion/test_diffusion_engine_rpc_routing.py +++ b/tests/diffusion/test_diffusion_engine_rpc_routing.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project """Regression tests for the busy-loop-based RPC routing in DiffusionEngine. These tests guard the invariants established when ``collective_rpc`` was @@ -60,6 +60,12 @@ class _ConcurrencyTrackingExecutor: self.is_failed = False self._closed = False self.od_config = SimpleNamespace() + # Pause tests: a call whose gate is registered (by request id, or by + # method name for ``synchronize_device``) blocks inside the executor + # until the gate is set; ``version`` tags each call so a test can tell + # which "weights" a request ran on. + self.gates: dict[str, threading.Event] = {} + self.version = "v1" def collective_rpc( self, @@ -81,16 +87,25 @@ class _ConcurrencyTrackingExecutor: "kwargs": kwargs, "unique_reply_rank": unique_reply_rank, "thread": threading.get_ident(), + "version": self.version, } ) try: if self.rpc_delay: time.sleep(self.rpc_delay) + if method == "synchronize_device": + gate = self.gates.get(method) + if gate is not None: + gate.wait(timeout=5.0) + return [None] # Distinguish per-request execution from raw RPC by method name. if method in {"execute_model", "execute_stepwise", "generate"}: # args[0] is the request-like object for execute_model. req = args[0] if args else None tag = req.request_id if req is not None and hasattr(req, "request_id") else "unknown" + gate = self.gates.get(tag) + if gate is not None: + gate.wait(timeout=5.0) return DiffusionOutput(error=f"result_for_{tag}", finished=True) tag = args[0] if args else method return DiffusionOutput(error=f"rpc_result_for_{tag}") @@ -133,21 +148,27 @@ def _make_request(tag: str): def _make_engine_with_loop( - loop: asyncio.AbstractEventLoop, + loop: asyncio.AbstractEventLoop | None, rpc_delay: float = 0.0, + *, + scheduler: RequestScheduler | None = None, + start_loop: bool = True, ): """Construct a ``DiffusionEngine`` skeleton with a real busy loop. The engine is wired with a fake executor that asserts no concurrent - calls and a real ``RequestScheduler``. + calls and a real ``RequestScheduler``. ``start_loop=False`` leaves the + engine in its bootstrap state (no busy-loop thread). """ engine = DiffusionEngine.__new__(DiffusionEngine) engine._closed = False engine.od_config = SimpleNamespace(streaming_output=False) engine.executor = _ConcurrencyTrackingExecutor(rpc_delay=rpc_delay) - sched = RequestScheduler() - sched.initialize(SimpleNamespace(max_num_seqs=1, request_batch_max_wait_ms=0.0)) + sched = scheduler + if sched is None: + sched = RequestScheduler() + sched.initialize(SimpleNamespace(max_num_seqs=1, request_batch_max_wait_ms=0.0)) engine.scheduler = sched engine.step_execution = False engine.supports_request_batch = False @@ -160,9 +181,14 @@ def _make_engine_with_loop( engine._closed = False engine.abort_queue = queue.Queue() engine._rpc_queue = queue.Queue() + engine._scheduling_paused = False engine.main_loop = loop engine.stop_event = threading.Event() + if not start_loop: + engine._loop_started = False + engine.worker_thread = None + return engine engine._loop_started = True engine.worker_thread = threading.Thread(target=engine._busy_loop, daemon=True) engine.worker_thread.start() @@ -170,6 +196,10 @@ def _make_engine_with_loop( def _stop_engine(engine: DiffusionEngine) -> None: + if engine.worker_thread is None: + return + for gate in engine.executor.gates.values(): + gate.set() with engine._cv: engine.stop_event.set() engine._cv.notify_all() @@ -565,3 +595,319 @@ async def test_rpc_and_request_results_do_not_swap(): assert req_res.error == "result_for_req1" # Belt and braces: never overlapped on the executor. assert engine.executor.max_concurrent == 1 + + +# ───────────────────────── scheduler pause (keep) ───────────────────────── + + +class _GatedAdmissionScheduler(RequestScheduler): + """Batching wait that only ends when the test opens admission.""" + + def __init__(self) -> None: + super().__init__() + self.admission_open = threading.Event() + + def get_admission_wait_decision(self, *, now: float, dp_concurrent: bool = False): + decision = super().get_admission_wait_decision(now=now, dp_concurrent=dp_concurrent) + return decision.__class__(should_wait=True, deadline=None, max_batch=self.max_num_running_reqs) + + def should_end_admission_wait(self, decision, *, now: float, stable_since: float) -> bool: + return self.admission_open.is_set() + + +def _pause(engine: DiffusionEngine, timeout: float | None = None): + return engine.async_collective_rpc("pause_scheduler", timeout=timeout, kwargs={"mode": "keep"}) + + +def _resume(engine: DiffusionEngine): + return engine.async_collective_rpc("resume_scheduler") + + +def _methods(engine: DiffusionEngine) -> list[str]: + return [call["method"] for call in engine.executor.calls] + + +def _executed(engine: DiffusionEngine) -> list[tuple[str, str]]: + return [ + (call["args"][0].request_id, call["version"]) + for call in engine.executor.calls + if call["method"] == "execute_model" + ] + + +def _submit(engine: DiffusionEngine, tag: str) -> asyncio.Task: + return asyncio.create_task(_consume_final_output(engine.async_add_req_and_stream_response(_make_request(tag)))) + + +async def _wait_until(predicate, timeout: float = 3.0) -> None: + deadline = time.monotonic() + timeout + while not predicate(): + if time.monotonic() > deadline: + raise AssertionError("condition not reached in time") + await asyncio.sleep(0.005) + + +@pytest.mark.asyncio +async def test_keep_pause_retains_waiting_request_and_serves_control_rpcs(): + loop = asyncio.get_running_loop() + engine = _make_engine_with_loop(loop) + try: + assert await _pause(engine) is None + queued = _submit(engine, "queued") + result = await engine.async_collective_rpc("list_loras", unique_reply_rank=0) + assert result.error == "rpc_result_for_list_loras" + await asyncio.sleep(0.05) + assert engine.scheduler.num_waiting_requests() == 1 + assert _executed(engine) == [] + assert not queued.done() + + await _resume(engine) + assert (await asyncio.wait_for(queued, 3.0)).error == "result_for_queued" + assert _methods(engine) == ["synchronize_device", "list_loras", "execute_model"] + finally: + _stop_engine(engine) + + +@pytest.mark.asyncio +async def test_keep_pause_acks_after_running_batch_and_freezes_queue(): + loop = asyncio.get_running_loop() + engine = _make_engine_with_loop(loop) + gate = engine.executor.gates["A"] = threading.Event() + try: + a = _submit(engine, "A") + await _wait_until(lambda: _executed(engine) == [("A", "v1")]) + b = _submit(engine, "B") + await _wait_until(lambda: engine.scheduler.num_waiting_requests() == 1) + + pause = asyncio.create_task(_pause(engine)) + await asyncio.sleep(0.1) + assert not pause.done() + + gate.set() + await asyncio.wait_for(pause, 3.0) + assert _methods(engine) == ["execute_model", "synchronize_device"] + assert (await asyncio.wait_for(a, 3.0)).error == "result_for_A" + assert engine.scheduler.num_waiting_requests() == 1 + assert not b.done() + + engine.executor.version = "v2" + await _resume(engine) + assert (await asyncio.wait_for(b, 3.0)).error == "result_for_B" + assert _executed(engine) == [("A", "v1"), ("B", "v2")] + finally: + _stop_engine(engine) + + +@pytest.mark.asyncio +async def test_keep_pause_ack_does_not_release_later_rpcs_before_the_batch_is_emitted(): + """A pause ACK must not open the queue to RPCs that touch device memory + (sleep) before the batch that just ran has been delivered. + """ + loop = asyncio.get_running_loop() + engine = _make_engine_with_loop(loop) + barrier_gate = engine.executor.gates["synchronize_device"] = threading.Event() + methods_at_emit: list[list[str]] = [] + emit_outputs = engine._emit_outputs + + def _recording_emit(*args, **kwargs): + methods_at_emit.append(_methods(engine)) + return emit_outputs(*args, **kwargs) + + engine._emit_outputs = _recording_emit + try: + gate = engine.executor.gates["A"] = threading.Event() + a = _submit(engine, "A") + await _wait_until(lambda: _executed(engine) == [("A", "v1")]) + pause = asyncio.create_task(_pause(engine)) + await _wait_until(lambda: engine._scheduling_paused) + gate.set() + # The barrier holds the busy loop inside the post-execute RPC drain, + # so the sleep is queued before the pause ACK is published. + await _wait_until(lambda: "synchronize_device" in _methods(engine)) + sleep_rpc = asyncio.create_task(engine.async_collective_rpc("handle_sleep_task", args=("t",))) + await _wait_until(lambda: not engine._rpc_queue.empty()) + barrier_gate.set() + + await asyncio.wait_for(pause, 3.0) + assert (await asyncio.wait_for(a, 3.0)).error == "result_for_A" + assert (await asyncio.wait_for(sleep_rpc, 3.0)).error == "rpc_result_for_t" + finally: + _stop_engine(engine) + + assert _methods(engine) == ["execute_model", "synchronize_device", "handle_sleep_task"] + assert methods_at_emit and "handle_sleep_task" not in methods_at_emit[0] + + +@pytest.mark.asyncio +async def test_timed_out_pause_still_ends_the_drain_before_the_batch_is_emitted(): + """A pause whose caller already gave up is still a drain barrier: its + cancelled task must not let a later RPC run before the batch is emitted. + """ + loop = asyncio.get_running_loop() + engine = _make_engine_with_loop(loop) + methods_at_emit: list[list[str]] = [] + emit_outputs = engine._emit_outputs + + def _recording_emit(*args, **kwargs): + methods_at_emit.append(_methods(engine)) + return emit_outputs(*args, **kwargs) + + engine._emit_outputs = _recording_emit + try: + gate = engine.executor.gates["A"] = threading.Event() + a = _submit(engine, "A") + await _wait_until(lambda: _executed(engine) == [("A", "v1")]) + + # The caller times out while A is still executing: the pause future is + # cancelled, but its task stays queued ahead of the sleep. + with pytest.raises(TimeoutError): + await _pause(engine, timeout=0.1) + sleep_rpc = asyncio.create_task(engine.async_collective_rpc("handle_sleep_task", args=("t",))) + await _wait_until(lambda: engine._rpc_queue.qsize() == 2) + gate.set() + + assert (await asyncio.wait_for(a, 3.0)).error == "result_for_A" + assert (await asyncio.wait_for(sleep_rpc, 3.0)).error == "rpc_result_for_t" + finally: + _stop_engine(engine) + + assert _methods(engine) == ["execute_model", "handle_sleep_task"] + assert methods_at_emit and "handle_sleep_task" not in methods_at_emit[0] + + +@pytest.mark.asyncio +async def test_pause_rejects_unsupported_modes_without_closing_the_gate(): + loop = asyncio.get_running_loop() + engine = _make_engine_with_loop(loop) + try: + for kwargs in ({"mode": "abort"}, {"mode": "wait"}, {}): + with pytest.raises(ValueError): + await engine.async_collective_rpc("pause_scheduler", kwargs=kwargs) + engine.execution_mode = DiffusionExecutionMode.STEP_BATCH + with pytest.raises(NotImplementedError): + await _pause(engine) + engine.execution_mode = DiffusionExecutionMode.REQUEST_BATCH + + assert (await asyncio.wait_for(_submit(engine, "still-runs"), 3.0)).error == "result_for_still-runs" + assert _methods(engine) == ["execute_model"] + finally: + _stop_engine(engine) + + +@pytest.mark.asyncio +async def test_keep_pause_during_batching_wait_holds_the_batch(): + loop = asyncio.get_running_loop() + sched = _GatedAdmissionScheduler() + sched.initialize(SimpleNamespace(max_num_seqs=4, request_batch_max_wait_ms=1000.0)) + engine = _make_engine_with_loop(loop, scheduler=sched) + try: + waiting = _submit(engine, "W") + await _wait_until(lambda: engine.scheduler.num_waiting_requests() == 1) + await asyncio.sleep(0.05) + + await asyncio.wait_for(_pause(engine), 3.0) + assert _methods(engine) == ["synchronize_device"] + assert engine.scheduler.num_waiting_requests() == 1 + + await _resume(engine) + await asyncio.sleep(0.05) + assert _executed(engine) == [] + sched.admission_open.set() + assert (await asyncio.wait_for(waiting, 3.0)).error == "result_for_W" + finally: + sched.admission_open.set() + _stop_engine(engine) + + +@pytest.mark.asyncio +async def test_pause_timeout_leaves_gate_closed_until_explicit_resume(): + loop = asyncio.get_running_loop() + engine = _make_engine_with_loop(loop) + gate = engine.executor.gates["A"] = threading.Event() + try: + a = _submit(engine, "A") + await _wait_until(lambda: _executed(engine) == [("A", "v1")]) + b = _submit(engine, "B") + await _wait_until(lambda: engine.scheduler.num_waiting_requests() == 1) + + with pytest.raises(TimeoutError): + await _pause(engine, timeout=0.1) + gate.set() + assert (await asyncio.wait_for(a, 3.0)).error == "result_for_A" + await asyncio.sleep(0.1) + assert _executed(engine) == [("A", "v1")] + assert not b.done() + + await _resume(engine) + assert (await asyncio.wait_for(b, 3.0)).error == "result_for_B" + assert "synchronize_device" not in _methods(engine) + finally: + _stop_engine(engine) + + +@pytest.mark.asyncio +async def test_repeated_pause_and_resume_are_idempotent(): + loop = asyncio.get_running_loop() + engine = _make_engine_with_loop(loop) + try: + await _resume(engine) + await _pause(engine) + await _pause(engine) + queued = _submit(engine, "Q") + await asyncio.sleep(0.05) + assert _executed(engine) == [] + await _resume(engine) + await _resume(engine) + assert (await asyncio.wait_for(queued, 3.0)).error == "result_for_Q" + assert _methods(engine) == ["synchronize_device", "synchronize_device", "execute_model"] + finally: + _stop_engine(engine) + + +@pytest.mark.asyncio +async def test_abort_of_waiting_request_while_paused_delivers_terminal(): + loop = asyncio.get_running_loop() + engine = _make_engine_with_loop(loop) + try: + await _pause(engine) + b = _submit(engine, "B") + await _wait_until(lambda: engine.scheduler.num_waiting_requests() == 1) + engine.abort("B") + + result = await asyncio.wait_for(b, 3.0) + assert result.aborted is True + assert engine.scheduler.get_request_state("B") is None + assert _executed(engine) == [] + + await _resume(engine) + assert (await asyncio.wait_for(_submit(engine, "C"), 3.0)).error == "result_for_C" + assert _executed(engine) == [("C", "v1")] + assert engine.worker_thread.is_alive() + finally: + _stop_engine(engine) + + +def test_pause_and_resume_before_loop_starts_use_engine_local_path(): + engine = _make_engine_with_loop(None, start_loop=False) + + assert engine.collective_rpc("pause_scheduler", kwargs={"mode": "keep"}) is None + assert _methods(engine) == ["synchronize_device"] + + assert engine.collective_rpc("resume_scheduler") is None + assert engine.add_req_and_wait_for_response(_make_request("X")).error == "result_for_X" + assert _executed(engine) == [("X", "v1")] + + +@pytest.mark.asyncio +async def test_requests_without_pause_never_trigger_the_barrier(): + loop = asyncio.get_running_loop() + engine = _make_engine_with_loop(loop) + try: + first = await asyncio.wait_for(_submit(engine, "r1"), 3.0) + await engine.async_collective_rpc("ping", args=("p",), unique_reply_rank=0) + second = await asyncio.wait_for(_submit(engine, "r2"), 3.0) + finally: + _stop_engine(engine) + + assert (first.error, second.error) == ("result_for_r1", "result_for_r2") + assert _methods(engine) == ["execute_model", "ping", "execute_model"] diff --git a/tests/diffusion/test_diffusion_scheduler.py b/tests/diffusion/test_diffusion_scheduler.py index 38da458b0..c0009c30f 100644 --- a/tests/diffusion/test_diffusion_scheduler.py +++ b/tests/diffusion/test_diffusion_scheduler.py @@ -2092,3 +2092,19 @@ class TestStepScheduler: with pytest.raises(ValueError): self.scheduler.add_request(request) + + +class TestPendingFinishedRequestIds: + def test_reports_finished_ids_that_still_hold_state_without_clearing_them(self): + sched = RequestScheduler() + sched.initialize(SimpleNamespace(max_num_seqs=1, request_batch_max_wait_ms=0.0)) + sched.add_request(_make_request("a")) + sched.add_request(_make_request("b")) + + sched.finish_requests("a", DiffusionRequestStatus.FINISHED_ABORTED) + assert sched.pending_finished_request_ids() == {"a"} + + sched.pop_request_state("a") + assert sched.pending_finished_request_ids() == set() + # The next wave still ships the id to the worker for its own cleanup. + assert sched.schedule().finished_req_ids == {"a"} diff --git a/tests/diffusion/test_diffusion_worker_synchronize_device.py b/tests/diffusion/test_diffusion_worker_synchronize_device.py new file mode 100644 index 000000000..6e4d3f4bf --- /dev/null +++ b/tests/diffusion/test_diffusion_worker_synchronize_device.py @@ -0,0 +1,53 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project +"""Unit tests for the device step of the diffusion pause barrier.""" + +from types import SimpleNamespace +from unittest.mock import MagicMock + +import pytest + +from vllm_omni.diffusion.worker.diffusion_worker import DiffusionWorker + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +def _make_worker(runner: object) -> DiffusionWorker: + worker = object.__new__(DiffusionWorker) + worker.model_runner = runner + worker.rank = 0 + return worker + + +def test_synchronize_device_waits_for_prefetch_before_device_sync(mocker): + platform = mocker.patch("vllm_omni.diffusion.worker.diffusion_worker.current_omni_platform") + order: list = [] + manager = MagicMock() + manager.wait_prefetch.side_effect = lambda timeout: order.append(("prefetch", timeout)) or True + platform.synchronize.side_effect = lambda: order.append("sync") + worker = _make_worker(SimpleNamespace(kv_transfer_manager=manager)) + + worker.synchronize_device(timeout=2.5) + + assert order == [("prefetch", 2.5), "sync"] + + +def test_synchronize_device_fails_when_prefetch_is_still_outstanding(mocker): + platform = mocker.patch("vllm_omni.diffusion.worker.diffusion_worker.current_omni_platform") + manager = MagicMock() + manager.wait_prefetch.return_value = False + worker = _make_worker(SimpleNamespace(kv_transfer_manager=manager)) + + with pytest.raises(TimeoutError, match="prefetch"): + worker.synchronize_device(timeout=0.01) + + platform.synchronize.assert_not_called() + + +def test_synchronize_device_without_transfer_manager_only_syncs(mocker): + platform = mocker.patch("vllm_omni.diffusion.worker.diffusion_worker.current_omni_platform") + worker = _make_worker(SimpleNamespace(kv_transfer_manager=None)) + + worker.synchronize_device(timeout=None) + + platform.synchronize.assert_called_once_with() diff --git a/tests/distributed/omni_connectors/test_kv_async_prefetch.py b/tests/distributed/omni_connectors/test_kv_async_prefetch.py index 6ef5a3f8c..b596e97bb 100644 --- a/tests/distributed/omni_connectors/test_kv_async_prefetch.py +++ b/tests/distributed/omni_connectors/test_kv_async_prefetch.py @@ -5,6 +5,7 @@ mock connector, dedup, per-call sender_info isolation, role classification, and miss/abort paths. GPU H2D and D2D paths need CUDA (integration tests). """ +import threading from types import SimpleNamespace import pytest @@ -354,3 +355,41 @@ def test_consume_then_apply_attaches_payload(): receiver._record_stream_for_prefetched(data) receiver.apply_kv_cache_to_request(req, data) assert req.past_key_values is not None + + +# --------------------------------------------------------------------------- # +# wait_prefetch: the pause barrier's view of outstanding background loads +# --------------------------------------------------------------------------- # + + +def test_wait_prefetch_is_true_when_nothing_outstanding(): + _, receiver, _ = _make_sender_receiver() + assert receiver.wait_prefetch(timeout=0.01) is True + + +def test_wait_prefetch_is_true_when_prefetch_disabled(): + _, receiver, _ = _make_sender_receiver(async_prefetch=False) + assert receiver.wait_prefetch(timeout=0.01) is True + + +def test_wait_prefetch_reports_a_running_prefetch_and_keeps_its_payload(): + sender, receiver, connector = _make_sender_receiver() + _seed_payload(sender, "rid-w") + gate = threading.Event() + real_get = connector.get + + def slow_get(*args, **kwargs): + gate.wait(timeout=5.0) + return real_get(*args, **kwargs) + + connector.get = slow_get + receiver.start_prefetch({"request_id": "rid-w", "kv_sender_info": _SENDER_INFO}) + try: + assert receiver.wait_prefetch(timeout=0.05) is False + gate.set() + assert receiver.wait_prefetch(timeout=5.0) is True + data, _ = receiver.consume_prefetched_kv(_req("rid-w")) + assert data is not None + finally: + gate.set() + receiver.shutdown_prefetch() diff --git a/tests/e2e/accuracy/ltx/test_ltx25_official_similarity.py b/tests/e2e/accuracy/ltx/test_ltx25_official_similarity.py index c8ce29c37..ba39fb204 100644 --- a/tests/e2e/accuracy/ltx/test_ltx25_official_similarity.py +++ b/tests/e2e/accuracy/ltx/test_ltx25_official_similarity.py @@ -32,7 +32,6 @@ from huggingface_hub import hf_hub_download, snapshot_download from torchmetrics.image import PeakSignalNoiseRatio, StructuralSimilarityIndexMeasure from tests.e2e.accuracy.helpers import reset_artifact_dir -from tests.helpers import skip_if_gated_repo_inaccessible from tests.helpers.mark import hardware_test OFFICIAL_REPOSITORY = "https://github.com/Lightricks/LTX-2.git" @@ -303,32 +302,6 @@ def _resolve_ltx25_image() -> Path: ) -def _require_ltx25_model_access() -> None: - configured_omni_model = os.environ.get("VLLM_TEST_LTX25_MODEL") - if not (configured_omni_model and Path(configured_omni_model).exists()): - omni_model = configured_omni_model or LTX25_OMNI_MODEL_ID - omni_revision = os.environ.get("VLLM_TEST_LTX25_MODEL_REVISION") - if omni_revision is None and omni_model == LTX25_OMNI_MODEL_ID: - omni_revision = LTX25_OMNI_MODEL_REVISION - skip_if_gated_repo_inaccessible( - omni_model, - revision=omni_revision, - filename="model_index.json", - ) - - configured_official_model = os.environ.get("VLLM_TEST_LTX25_OFFICIAL_MODEL") - if not (configured_official_model and Path(configured_official_model).exists()): - official_model = configured_official_model or LTX25_OFFICIAL_MODEL_ID - official_revision = os.environ.get("VLLM_TEST_LTX25_OFFICIAL_MODEL_REVISION") - if official_revision is None and official_model == LTX25_OFFICIAL_MODEL_ID: - official_revision = LTX25_OFFICIAL_MODEL_REVISION - skip_if_gated_repo_inaccessible( - official_model, - revision=official_revision, - filename=LTX25_OFFICIAL_COMMON_FILES["text_encoder"], - ) - - def _ltx25_request(image: Path | None = None) -> dict[str, object]: request: dict[str, object] = { "prompt": LTX25_PROMPT, @@ -475,7 +448,6 @@ def _assert_strict_similarity( @pytest.mark.parametrize("task", ["t2v", "i2v"]) def test_ltx25_distilled_two_stage_matches_official(accuracy_artifact_root: Path, task: str) -> None: """Compare Omni with official canonical DiffVAE and matching connector weights.""" - _require_ltx25_model_access() artifact_parent = accuracy_artifact_root / "ltx_official" output_root = reset_artifact_dir(artifact_parent / f"ltx2_5_distilled_{task}") official_root, official_revision = _ltx25_official_source(artifact_parent) @@ -600,7 +572,6 @@ def test_ltx25_distilled_two_stage_matches_official(accuracy_artifact_root: Path @pytest.mark.parametrize("pipeline_mode", ["full_one_stage", "full_two_stage"]) def test_ltx25_full_matches_official(accuracy_artifact_root: Path, task: str, pipeline_mode: str) -> None: """Compare raw official Full/SFT and DiffVAE weights with Omni.""" - _require_ltx25_model_access() artifact_parent = accuracy_artifact_root / "ltx_official" output_root = reset_artifact_dir(artifact_parent / f"ltx2_5_{pipeline_mode}_{task}") model_class_name = { diff --git a/tests/e2e/features/comfyui/mock_videos_server.py b/tests/e2e/features/comfyui/mock_videos_server.py new file mode 100644 index 000000000..c2ddfbb5e --- /dev/null +++ b/tests/e2e/features/comfyui/mock_videos_server.py @@ -0,0 +1,115 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + +"""Minimal mock of vLLM-Omni's ``/v1/videos`` API for e2e tests. + +Records every received multipart field to ``MOCK_STATE_FILE`` (JSON) so a test +can assert exactly what the client serialized, and returns a synthetic MP4. +""" + +import asyncio +import json +import os +import subprocess +import uuid + +from fastapi import FastAPI, Request +from fastapi.responses import Response +from starlette.datastructures import UploadFile + +app = FastAPI() +_JOBS: dict[str, dict] = {} +_STATE_FILE = os.environ.get("MOCK_STATE_FILE") + + +def _record(fields: dict) -> None: + if _STATE_FILE: + with open(_STATE_FILE, "w") as f: + json.dump(fields, f) + + +def _synthetic_mp4() -> bytes: + return subprocess.run( + [ + "ffmpeg", + "-hide_banner", + "-loglevel", + "error", + "-y", + "-f", + "lavfi", + "-i", + "testsrc2=size=320x240:rate=24:duration=1", + "-f", + "lavfi", + "-i", + "sine=frequency=440:sample_rate=32000:duration=1", + "-map", + "0:v", + "-map", + "1:a", + "-shortest", + "-c:v", + "libx264", + "-pix_fmt", + "yuv420p", + "-c:a", + "aac", + "-ar", + "32000", + "-ac", + "2", + "-movflags", + "frag_keyframe+empty_moov", + "-f", + "mp4", + "pipe:1", + ], + check=True, + capture_output=True, + ).stdout + + +@app.post("/v1/videos") +async def create_video(request: Request): + form = await request.form() + fields = {} + for key, value in form.items(): + if isinstance(value, UploadFile): + data = await value.read() + fields[key] = { + "file": True, + "size": len(data), + "filename": value.filename, + "content_type": value.content_type, + } + if value.content_type == "application/json": + fields[key]["json"] = json.loads(data) + else: + fields[key] = str(value) + _record(fields) + + job_id = str(uuid.uuid4()) + _JOBS[job_id] = {"status": "queued"} + return {"id": job_id, "status": "queued"} + + +@app.get("/v1/videos/{job_id}") +async def get_video(job_id: str): + if job_id not in _JOBS: + return {"status": "failed", "error": "unknown job"} + return {"id": job_id, "status": "completed"} + + +@app.get("/v1/videos/{job_id}/content") +async def get_video_content(job_id: str): + if job_id not in _JOBS: + return Response(status_code=404) + mp4 = await asyncio.to_thread(_synthetic_mp4) + return Response(content=mp4, media_type="video/mp4") + + +@app.delete("/v1/videos/{job_id}") +async def delete_video(job_id: str): + _JOBS.pop(job_id, None) + return {"deleted": True} diff --git a/tests/e2e/features/comfyui/test_latent_mask.py b/tests/e2e/features/comfyui/test_latent_mask.py new file mode 100644 index 000000000..e208fd5b4 --- /dev/null +++ b/tests/e2e/features/comfyui/test_latent_mask.py @@ -0,0 +1,80 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + +"""Unit tests for latent-mask serialization helpers.""" + +import json + +import pytest +import torch +from comfyui_vllm_omni.utils.latent_mask import scalar_mask_to_json, video_mask_to_grid_json + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +def test_scalar_mask_to_json(): + assert scalar_mask_to_json(0.5) == "0.5" + assert scalar_mask_to_json(1.0) == "1.0" + with pytest.raises(ValueError): + scalar_mask_to_json(1.5) + with pytest.raises(ValueError): + scalar_mask_to_json(-0.1) + + +def test_video_mask_grid_shape(): + mask = torch.zeros(256, 448) + grid = json.loads(video_mask_to_grid_json(mask, width=448, height=256, num_frames=107)) + assert len(grid) == 2 + 5 * ((107 - 5) // 17) + assert len(grid[0]) == 256 // 16 + assert len(grid[0][0]) == 448 // 16 + + +def test_video_mask_grid_shape_non_aligned_frames(): + # 96 snaps up to 107 (17n+5), so Tv == 32, matching the server. + grid = json.loads(video_mask_to_grid_json(torch.zeros(256, 448), width=448, height=256, num_frames=96)) + assert len(grid) == 32 + + +def test_video_mask_grid_shape_short_clip(): + # num_frames <= 5 -> Tv == 2. + grid = json.loads(video_mask_to_grid_json(torch.zeros(64, 64), width=64, height=64, num_frames=1)) + assert len(grid) == 2 + + +def test_video_mask_grid_shape_unaligned_width(): + # width=500 floors to 480 (multiple of 32) -> gw == 30, not 31. + grid = json.loads(video_mask_to_grid_json(torch.zeros(64, 64), width=500, height=256, num_frames=22)) + assert len(grid[0][0]) == 30 + + +def test_video_mask_3d_input(): + grid = json.loads(video_mask_to_grid_json(torch.ones(1, 256, 448), width=448, height=256, num_frames=107)) + assert grid[0][0][0] == 1.0 + + +def test_video_mask_temporal(): + # 3D mask: first half 0 (preserve), second half 1 (regenerate). + mask = torch.zeros(7, 64, 64) + mask[4:] = 1.0 + grid = json.loads(video_mask_to_grid_json(mask, width=64, height=64, num_frames=22)) + assert len(grid) == 7 + assert all(v == 0.0 for row in grid[0] for v in row) + assert all(v == 1.0 for row in grid[-1] for v in row) + + +def test_video_mask_temporal_resize(): + # 2 temporal slices resize to tv == 7. + mask = torch.zeros(2, 64, 64) + mask[1] = 1.0 + grid = json.loads(video_mask_to_grid_json(mask, width=64, height=64, num_frames=22)) + assert len(grid) == 7 + + +def test_video_mask_uniform_preserved(): + grid = json.loads(video_mask_to_grid_json(torch.full((64, 64), 0.5), width=64, height=64, num_frames=107)) + assert all(v == 0.5 for row in grid for cell in row for v in cell) + + +def test_video_mask_values_in_range(): + grid = json.loads(video_mask_to_grid_json(torch.rand(128, 128), width=128, height=128, num_frames=22)) + assert all(0.0 <= v <= 1.0 for row in grid for cell in row for v in cell) diff --git a/tests/e2e/features/comfyui/test_latent_mask_editing.py b/tests/e2e/features/comfyui/test_latent_mask_editing.py new file mode 100644 index 000000000..9162974b6 --- /dev/null +++ b/tests/e2e/features/comfyui/test_latent_mask_editing.py @@ -0,0 +1,99 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + +"""End-to-end test for latent-mask editing serialization. + +Exercises ``VLLMOmniClient.generate_video`` with a ``latent_edit`` payload +against a mock ``/v1/videos`` server and asserts the multipart fields the +client produced. Runs on CPU: the ComfyUI ``comfy_api`` / ``comfy_extras`` +modules are mocked by this directory's ``conftest.py``. +""" + +import asyncio +import json +import os +import socket +import subprocess +import sys +import time + +import pytest +import torch +from comfy_api.input import VideoInput +from comfyui_vllm_omni.utils.api_client import VLLMOmniClient + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + +_TESTS_DIR = os.path.dirname(os.path.abspath(__file__)) + + +def _free_port() -> int: + with socket.socket() as s: + s.bind(("127.0.0.1", 0)) + return s.getsockname()[1] + + +@pytest.fixture +def mock_server(tmp_path): + port = _free_port() + state_file = tmp_path / "state.json" + env = dict(os.environ, MOCK_STATE_FILE=str(state_file)) + proc = subprocess.Popen( + [sys.executable, "-m", "uvicorn", "mock_videos_server:app", "--host", "127.0.0.1", "--port", str(port)], + cwd=_TESTS_DIR, + env=env, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + try: + deadline = time.time() + 30 + while time.time() < deadline: + try: + with socket.create_connection(("127.0.0.1", port), timeout=1): + break + except OSError: + time.sleep(0.2) + yield f"http://127.0.0.1:{port}/v1", state_file + finally: + proc.terminate() + proc.wait(timeout=10) + + +def test_latent_edit_serialization(mock_server): + base_url, state_file = mock_server + + # A non-trivial video mask requires a source video. The mocked VideoInput + # only needs to provide ``save_to`` for the client's multipart upload. + source_video = VideoInput(b"mock_source_video") + mask = torch.zeros(1, 120, 160) + latent_edit = {"source_video": source_video, "video_mask": mask, "audio_mask": 0.5} + + async def run(): + client = VLLMOmniClient(base_url) + return await client.generate_video( + model="MiniMaxAI/MiniMax-H3", + prompt="restyle the clip", + width=160, + height=120, + num_frames=22, + fps=24, + latent_edit=latent_edit, + ) + + out = asyncio.run(run()) + assert out is not None + + with open(state_file) as f: + fields = json.load(f) + + assert fields["source_video"]["file"] is True + assert fields["source_video"]["content_type"] == "video/mp4" + for name in ("video_noise_mask", "audio_noise_mask"): + assert fields[name]["file"] is True + assert fields[name]["content_type"] == "application/json" + assert fields["audio_noise_mask"]["json"] == 0.5 + + video_mask = fields["video_noise_mask"]["json"] + assert len(video_mask) == 7 # num_frames=22 aligns to 22 (17n+5) -> Tv = 7 + assert len(video_mask[0]) == 6 # height 120 floors to 96 (multiple of 32) -> gh 6 + assert len(video_mask[0][0]) == 10 # width 160 -> gw 10 diff --git a/tests/e2e/features/custom_pipeline/test_diffusion_pause_keep.py b/tests/e2e/features/custom_pipeline/test_diffusion_pause_keep.py new file mode 100644 index 000000000..cd8231473 --- /dev/null +++ b/tests/e2e/features/custom_pipeline/test_diffusion_pause_keep.py @@ -0,0 +1,183 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + +""" +Lifecycle test for ``AsyncOmni.pause_generation(mode="keep")`` on a diffusion stage. + +Per round, request A is started, request B is queued behind it, and the pause +is issued while A is still running. The pause must return only after A has +been delivered, B must stay queued through ``sleep`` / ``wake_up``, and B may +start only after ``resume_generation``. + +A second test follows the order the docs recommend for a weight update: sleep +is issued as soon as the pause returns, without awaiting the request that was +running, which must still be delivered intact. + +Both single-GPU executor backends are covered explicitly: ``uni`` (in-process +worker) and ``mp`` (worker process with the asynchronous D2H output thread). + +Usage: + pytest tests/e2e/features/custom_pipeline/test_diffusion_pause_keep.py -v -s +""" + +from __future__ import annotations + +import asyncio +import os +import time +from collections import defaultdict +from contextlib import ExitStack + +import pytest + +from tests.helpers.mark import hardware_test +from vllm_omni.diffusion.data import is_diffusion_request_started_output +from vllm_omni.entrypoints.async_omni import AsyncOmni +from vllm_omni.inputs.data import OmniDiffusionSamplingParams + +MODEL = "tiny-random/Qwen-Image" +CUSTOM_PIPELINE_CLASS = "tests.e2e.features.helpers.custom_pipeline.QwenImagePipelineWithLogProbForTest" +WORKER_EXTENSION_CLASS = "tests.e2e.features.helpers.custom_pipeline.vLLMOmniColocateWorkerExtensionForTest" + +# CI runs a couple of rounds; set PAUSE_KEEP_ROUNDS=10 for a soak run. +ROUNDS = int(os.environ.get("PAUSE_KEEP_ROUNDS", "2")) +# Enough denoising steps that the pause lands while A is still executing. +NUM_INFERENCE_STEPS = 40 + + +def _sampling_params() -> OmniDiffusionSamplingParams: + return OmniDiffusionSamplingParams( + num_inference_steps=NUM_INFERENCE_STEPS, + guidance_scale=0.0, + height=256, + width=256, + seed=42, + emit_request_lifecycle=True, + ) + + +async def _generate(engine: AsyncOmni, request_id: str, events: dict[str, dict[str, float]]): + final = None + async for output in engine.generate( + prompt={"prompt_ids": list(range(50))}, + request_id=request_id, + sampling_params_list=[_sampling_params()], + output_modalities=["image"], + ): + if is_diffusion_request_started_output(output): + events[request_id]["started"] = time.monotonic() + continue + final = output + events[request_id]["finished"] = time.monotonic() + return final + + +async def _wait_for(predicate, timeout: float = 30.0) -> None: + deadline = time.monotonic() + timeout + while not predicate(): + assert time.monotonic() < deadline, "condition not reached in time" + await asyncio.sleep(0.005) + + +async def _pause_round(engine: AsyncOmni, events: dict[str, dict[str, float]], round_index: int) -> float: + """A runs, B queues, keep-pause lands while A runs; B starts only after resume. + + Returns the pause latency. + """ + a_id, b_id = f"a{round_index}", f"b{round_index}" + a = asyncio.create_task(_generate(engine, a_id, events)) + await _wait_for(lambda: "started" in events[a_id]) + b = asyncio.create_task(_generate(engine, b_id, events)) + await asyncio.sleep(0.05) + + assert not a.done(), "A finished before the pause was issued; raise NUM_INFERENCE_STEPS" + pause_start = time.monotonic() + await engine.pause_generation(mode="keep", clear_cache=False) + pause_latency = time.monotonic() - pause_start + assert await engine.is_paused() + + a_output = await asyncio.wait_for(a, 60.0) + assert a_output.images, "A must be delivered, not aborted, by a keep pause" + assert "started" not in events[b_id], "B must not start while paused" + + # Control RPCs and the sleep cycle stay available on the paused stage. + assert await engine.list_loras() == [] + await engine.sleep(level=1) + assert await engine.is_sleeping() + await engine.wake_up() + assert not await engine.is_sleeping() + assert await engine.is_paused() + assert not b.done() and "started" not in events[b_id] + + resume_at = time.monotonic() + await engine.resume_generation() + b_output = await asyncio.wait_for(b, 60.0) + assert b_output.images, "B must run after resume" + assert events[b_id]["started"] >= resume_at + return pause_latency + + +@pytest.mark.core_model +@pytest.mark.diffusion +@hardware_test(res={"cuda": "L4"}, num_cards=1) +@pytest.mark.asyncio +@pytest.mark.parametrize("backend", ["uni", "mp"]) +async def test_keep_pause_sleep_wake_resume_lifecycle(backend: str): + with ExitStack() as after: + engine = AsyncOmni( + model=MODEL, + custom_pipeline_args={"pipeline_class": CUSTOM_PIPELINE_CLASS}, + worker_extension_cls=WORKER_EXTENSION_CLASS, + enforce_eager=True, + enable_sleep_mode=True, + distributed_executor_backend=backend, + ) + after.callback(engine.shutdown) + events: dict[str, dict[str, float]] = defaultdict(dict) + + pause_latencies = [await _pause_round(engine, events, round_index) for round_index in range(ROUNDS)] + + assert not engine.request_states, "every request must be finalized" + print( + f"[{backend}] rounds={ROUNDS} steps={NUM_INFERENCE_STEPS} " + f"pause_latency_s min/median/max=" + f"{min(pause_latencies):.3f}/{sorted(pause_latencies)[ROUNDS // 2]:.3f}/{max(pause_latencies):.3f}" + ) + + +@pytest.mark.core_model +@pytest.mark.diffusion +@hardware_test(res={"cuda": "L4"}, num_cards=1) +@pytest.mark.asyncio +@pytest.mark.parametrize("backend", ["uni", "mp"]) +async def test_sleep_right_after_keep_pause_keeps_the_running_batch(backend: str): + """The documented weight-update order: sleep is issued as soon as the pause + returns, with the request that was running still un-awaited. + """ + with ExitStack() as after: + engine = AsyncOmni( + model=MODEL, + custom_pipeline_args={"pipeline_class": CUSTOM_PIPELINE_CLASS}, + worker_extension_cls=WORKER_EXTENSION_CLASS, + enforce_eager=True, + enable_sleep_mode=True, + distributed_executor_backend=backend, + ) + after.callback(engine.shutdown) + events: dict[str, dict[str, float]] = defaultdict(dict) + + a = asyncio.create_task(_generate(engine, "a", events)) + await _wait_for(lambda: "started" in events["a"]) + await asyncio.sleep(0.05) + assert not a.done(), "A finished before the pause was issued; raise NUM_INFERENCE_STEPS" + + await engine.pause_generation(mode="keep", clear_cache=False) + await engine.sleep(level=1) + + a_output = await asyncio.wait_for(a, 60.0) + assert a_output.images, "A must survive a sleep issued right after the pause ACK" + + await engine.wake_up() + await engine.resume_generation() + b_output = await asyncio.wait_for(_generate(engine, "b", events), 120.0) + assert b_output.images, "the stage must generate again after wake and resume" diff --git a/tests/e2e/features/helpers/custom_pipeline.py b/tests/e2e/features/helpers/custom_pipeline.py index 63cb21056..3f1b48563 100644 --- a/tests/e2e/features/helpers/custom_pipeline.py +++ b/tests/e2e/features/helpers/custom_pipeline.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project """Test doubles for custom-pipeline E2E and RLHF integration. @@ -290,6 +290,36 @@ class vLLMOmniColocateWorkerExtensionForTest(CustomPipelineWorkerExtension): """Return a stable identifier for assertions in unit tests.""" return "vllm-omni-colocate-worker-extension-for-test" + def start_counting_synchronize_device(self) -> bool: + """Count pause-barrier runs on this rank (test-only).""" + self._synchronize_device_calls = 0 + inner = type(self).synchronize_device + + def _counted(timeout=None): + self._synchronize_device_calls += 1 + return inner(self, timeout=timeout) + + self.synchronize_device = _counted + return True + + def synchronize_device_seen(self) -> bool: + """True from every rank only when each rank ran the barrier at least once.""" + return getattr(self, "_synchronize_device_calls", 0) > 0 + + def set_synchronize_device_failure(self, rank: int, fail: bool) -> bool: + """Make (or stop making) the pause barrier fail on ``rank`` (test-only).""" + if self.rank != rank: + return True + if fail: + + def _fail(timeout=None): + raise RuntimeError(f"injected barrier failure on rank {rank}") + + self.synchronize_device = _fail + else: + self.__dict__.pop("synchronize_device", None) + return True + # --------------------------------------------------------------------------- # Pipeline layer diff --git a/tests/e2e/offline_inference/test_hunyuanimage3.py b/tests/e2e/offline_inference/test_hunyuanimage3.py index 21bfe9b39..ab1bb5701 100644 --- a/tests/e2e/offline_inference/test_hunyuanimage3.py +++ b/tests/e2e/offline_inference/test_hunyuanimage3.py @@ -558,7 +558,7 @@ def test_shared_script_ar_path_reaches_generation( # safe against MODEL_NAME's base checkpoint missing the Instruct-only # / tokens. Wiring it here exercises the # real production path (get_ar_tokenizer_validator via the registry), - # not just the isolated fake-tokenizer coverage in test_prompt_utils.py + # not just the isolated fake-tokenizer coverage in test_hunyuan_image3_prompt_utils.py # / test_shared_script_ar_integration.py. validate_tokenizer=get_ar_tokenizer_validator(model_class_name), ) diff --git a/tests/e2e/online_serving/test_bagel.py b/tests/e2e/online_serving/test_bagel.py index 84c2302cc..1a11dc037 100644 --- a/tests/e2e/online_serving/test_bagel.py +++ b/tests/e2e/online_serving/test_bagel.py @@ -25,9 +25,11 @@ import os from io import BytesIO import pytest +from PIL import Image from vllm.assets.image import ImageAsset from tests.helpers.mark import hardware_test +from tests.helpers.media import generate_synthetic_image from tests.helpers.runtime import OmniServerParams from tests.helpers.stage_config import get_deploy_config_path @@ -120,3 +122,55 @@ def test_bagel_img2img_online(omni_server, online_client) -> None: } online_client.send_diffusion_request(request_config) + + +EDIT_SOURCE_SIZE = (1280, 720) +EDIT_REQUESTED_SIZE = "1024x1024" + + +def _edit_source_jpeg() -> tuple[str, bytes, str]: + synthetic = generate_synthetic_image(*EDIT_SOURCE_SIZE, seed=7287) + return ("source.jpg", base64.b64decode(synthetic["base64"]), "image/jpeg") + + +def _build_edit_request(size: str) -> dict: + return { + "files": [("image", _edit_source_jpeg())], + "data": { + "prompt": IMG2IMG_PROMPT, + "size": size, + "num_inference_steps": 2, + "seed": 42, + "response_format": "b64_json", + }, + } + + +@pytest.mark.slow +@pytest.mark.diffusion +@hardware_test(res={"cuda": "H100"}) +@pytest.mark.parametrize("omni_server", test_params, indirect=True) +def test_bagel_image_edit_honors_explicit_size_two_stage(omni_server, online_client) -> None: + """An explicit ``size`` wins over the AR stage's source-derived KV image shape (#7283).""" + (response,) = online_client.send_images_edits_http_request(_build_edit_request(EDIT_REQUESTED_SIZE)) + + payload = response.json_body + assert isinstance(payload, dict) + assert payload["size"] == EDIT_REQUESTED_SIZE + image = Image.open(BytesIO(base64.b64decode(payload["data"][0]["b64_json"]))) + assert image.size == (1024, 1024) + + +@pytest.mark.slow +@pytest.mark.diffusion +@hardware_test(res={"cuda": "H100"}) +@pytest.mark.parametrize("omni_server", test_params, indirect=True) +def test_bagel_image_edit_floors_unaligned_size_two_stage(omni_server, online_client) -> None: + """A side that is not a multiple of the latent stride is floored to one (1000x700 -> 992x688).""" + (response,) = online_client.send_images_edits_http_request(_build_edit_request("1000x700")) + + payload = response.json_body + assert isinstance(payload, dict) + assert payload["size"] == "992x688" + image = Image.open(BytesIO(base64.b64decode(payload["data"][0]["b64_json"]))) + assert image.size == (992, 688) diff --git a/tests/e2e/online_serving/test_lingbot_video.py b/tests/e2e/online_serving/test_lingbot_video.py index e6f275bb9..0da9c4dab 100644 --- a/tests/e2e/online_serving/test_lingbot_video.py +++ b/tests/e2e/online_serving/test_lingbot_video.py @@ -1,7 +1,19 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project -"""Online serving smoke coverage for the dense LingBot-Video checkpoint.""" +""" +Online serving smoke for ``robbyant/lingbot-video-dense-1.3b``. + +``test_text_to_image_001`` and ``test_video_generation_modes_001`` are the +baseline T2I / T2V+TI2V cases (``core_model`` + ``advanced_model``) shared by +ready (L2) and merge (L3). Heavier MoE and World coverage stays on +``full_model`` in ``test_lingbot_video_moe.py`` / ``test_lingbot_world_v2.py``. + +From ``tests/``:: + + pytest -s -v e2e/online_serving/test_lingbot_video.py -m "core_model and diffusion" --run-level=core_model + pytest -s -v e2e/online_serving/test_lingbot_video.py -m "advanced_model and diffusion" --run-level=advanced_model +""" import json import os @@ -41,7 +53,8 @@ def _get_diffusion_feature_cases(model: str): ] -@pytest.mark.full_model +@pytest.mark.core_model +@pytest.mark.advanced_model @pytest.mark.diffusion @pytest.mark.parametrize("omni_server", _get_diffusion_feature_cases(MODEL), indirect=True) def test_text_to_image_001(omni_server: OmniServer, openai_client: OpenAIClientHandler) -> None: @@ -63,7 +76,8 @@ def test_text_to_image_001(omni_server: OmniServer, openai_client: OpenAIClientH ) -@pytest.mark.full_model +@pytest.mark.core_model +@pytest.mark.advanced_model @pytest.mark.diffusion @pytest.mark.parametrize("omni_server", _get_diffusion_feature_cases(MODEL), indirect=True) def test_video_generation_modes_001(omni_server: OmniServer, openai_client: OpenAIClientHandler) -> None: diff --git a/tests/e2e/online_serving/test_lingbot_video_expansion.py b/tests/e2e/online_serving/test_lingbot_video_expansion.py index 4a411ddcf..74c7c143b 100644 --- a/tests/e2e/online_serving/test_lingbot_video_expansion.py +++ b/tests/e2e/online_serving/test_lingbot_video_expansion.py @@ -4,9 +4,9 @@ """ L4 expansion coverage for ``robbyant/lingbot-video-dense-1.3b``. -This file remains dense-only. Basic T2I/T2V/TI2V serving for both dense and MoE -checkpoints is covered by ``test_lingbot_video.py`` and -``test_lingbot_video_moe.py``. Distributed feature rows belong in follow-up +This file remains dense-only. Baseline T2I/T2V/TI2V for the dense checkpoint +lives in ``test_lingbot_video.py`` (L2/L3). MoE smoke stays in +``test_lingbot_video_moe.py`` (L4). Distributed feature rows belong in follow-up PRs. """ diff --git a/tests/e2e/online_serving/test_ltx25.py b/tests/e2e/online_serving/test_ltx25.py index 4975abcc0..400210a7d 100644 --- a/tests/e2e/online_serving/test_ltx25.py +++ b/tests/e2e/online_serving/test_ltx25.py @@ -7,7 +7,6 @@ import os import pytest -from tests.helpers import skip_if_gated_repo_inaccessible from tests.helpers.mark import hardware_marks from tests.helpers.media import generate_synthetic_image from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler @@ -24,16 +23,6 @@ pytestmark = [pytest.mark.diffusion, pytest.mark.slow] SINGLE_CARD_MARKS = hardware_marks(res={"cuda": "H100"}) -@pytest.fixture(scope="module", autouse=True) -def require_ltx25_model_access() -> None: - if not os.path.isdir(MODEL): - skip_if_gated_repo_inaccessible( - MODEL, - revision=MODEL_REVISION or None, - filename="model_index.json", - ) - - def _server(model_class_name: str) -> OmniServerParams: return OmniServerParams( model=MODEL, diff --git a/tests/e2e/online_serving/test_stable_audio_online_expansion.py b/tests/e2e/online_serving/test_stable_audio_online_expansion.py index fbd6bdbfd..beb51f7c9 100644 --- a/tests/e2e/online_serving/test_stable_audio_online_expansion.py +++ b/tests/e2e/online_serving/test_stable_audio_online_expansion.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project """ E2E online serving test for Stable Audio Open text-to-audio diffusion. @@ -11,7 +11,6 @@ import os import pytest -from tests.helpers import skip_if_gated_repo_inaccessible from tests.helpers.mark import hardware_marks from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler @@ -32,17 +31,10 @@ def _stable_audio_server_cases(model: str): ] -@pytest.fixture -def _require_stable_audio_access() -> None: - """Skip cleanly (before the server boots) if the gated checkpoint is inaccessible.""" - skip_if_gated_repo_inaccessible(STABLE_AUDIO_TEST_MODEL) - - @pytest.mark.slow @pytest.mark.diffusion @pytest.mark.parametrize("omni_server", _stable_audio_server_cases(STABLE_AUDIO_TEST_MODEL), indirect=True) def test_stable_audio_t2a_online( - _require_stable_audio_access: None, omni_server: OmniServer, openai_client: OpenAIClientHandler, ) -> None: diff --git a/tests/engine/duplex/test_aura_resumable_hooks.py b/tests/engine/duplex/test_aura_resumable_hooks.py new file mode 100644 index 000000000..e2ed43372 --- /dev/null +++ b/tests/engine/duplex/test_aura_resumable_hooks.py @@ -0,0 +1,725 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + +"""Public duplex hook: resumable defaults True; ephemeral ids when False.""" + +from __future__ import annotations + +import pytest +from vllm.sampling_params import SamplingParams + +from vllm_omni.engine.duplex.config import DuplexCapabilities +from vllm_omni.engine.duplex.contracts import ( + DuplexFence, + DuplexStageRequestContext, + DuplexStageSubmission, + duplex_ephemeral_stage_request_id, +) +from vllm_omni.engine.duplex.session.manager import DuplexSessionManager + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +def test_submission_resumable_default_preserves_minicpm() -> None: + ctx = DuplexStageRequestContext( + request_id="r", + session_id="s", + fence=DuplexFence("s"), + stage_id=0, + final_stage_id=2, + config_generation=0, + sampling_params=(SamplingParams(max_tokens=1),), + ) + submission = DuplexStageSubmission( + context=ctx, + prompt={"prompt_token_ids": [1, 2]}, + already_submitted=False, + ) + assert submission.resumable is True + + +def test_stage_request_id_respects_resumable_flag() -> None: + fence = DuplexFence("sess", epoch=3, turn_id=9) + resumable_id = DuplexSessionManager.stage_request_id(fence, stage_id=0, resumable=True) + ephemeral_id = DuplexSessionManager.stage_request_id(fence, stage_id=0, resumable=False) + assert ephemeral_id == duplex_ephemeral_stage_request_id(fence, stage_id=0) + assert resumable_id != ephemeral_id + assert ephemeral_id.endswith("stage0-turn9") + from vllm_omni.engine.duplex.contracts import duplex_turn_id_from_request_id + + assert duplex_turn_id_from_request_id(ephemeral_id) == 9 + assert duplex_turn_id_from_request_id(resumable_id) is None + + +def test_capabilities_expose_concurrent_turn_requests_default_false() -> None: + caps = DuplexCapabilities() + assert caps.supports_concurrent_turn_requests is False + assert caps.allows_video_without_audio() is False + assert caps.required_input_modalities == frozenset({"audio"}) + assert caps.optional_input_modalities == frozenset({"video"}) + assert caps.supports_core_resumable_request is False + payload = caps.as_dict() + assert payload["supports_concurrent_turn_requests"] is False + assert payload["required_input_modalities"] == ["audio"] + assert payload["optional_input_modalities"] == ["video"] + + +def test_next_commit_allowed_soft_opens_on_r4_release() -> None: + from types import SimpleNamespace + + from vllm_omni.engine.duplex.session import helpers + + caps = DuplexCapabilities(supports_concurrent_turn_requests=True) + session = SimpleNamespace(active_response_id="resp", capabilities=caps) + tasks = SimpleNamespace( + active_response_task=None, + has_response_bound_append_tasks=lambda: False, + ) + # Monkeypatch assistant_playback_active via response_in_progress path: active_response_id set. + assert helpers.response_in_progress(session, tasks) is True + assert helpers.next_commit_allowed(session, tasks, concurrent_turn_requests_released=False) is False + assert helpers.next_commit_allowed(session, tasks, concurrent_turn_requests_released=True) is True + + caps_off = DuplexCapabilities(supports_concurrent_turn_requests=False) + session_off = SimpleNamespace(active_response_id="resp", capabilities=caps_off) + assert helpers.next_commit_allowed(session_off, tasks, concurrent_turn_requests_released=True) is False + + +def test_active_response_accepts_own_turn_when_concurrent_turn_requests() -> None: + """Approach A: each response owns its turn; draining TTS uses request→response map.""" + from vllm_omni.engine.duplex.config import DuplexCapabilities, DuplexSessionConfig + from vllm_omni.engine.duplex.session.engine_session import DuplexEngineSession + + session = DuplexEngineSession( + session_id="s", + config=DuplexSessionConfig(model="m", modalities=["text"]), + capabilities=DuplexCapabilities(supports_concurrent_turn_requests=True), + ) + session.begin_response(turn_id=3) + assert session.active_response_accepts_model_turn(3) + assert not session.active_response_accepts_model_turn(4) + assert not session.active_response_accepts_model_turn(2) + session.bind_draining_request("req-old", "resp-old") + assert session.response_id_for_request("req-old") == "resp-old" + assert session.is_draining_request("req-old") + + # Ending the active response must not wipe older draining bindings. + session.bind_draining_request("req-newer-tts", session.active_response_id or "resp-active") + ended = session.active_response_id + session.end_response(commit_text=False) + assert session.is_draining_request("req-old") + assert session.response_id_for_request("req-old") == "resp-old" + assert not session.is_draining_request("req-newer-tts") + session.clear_draining_for_response("resp-old") + assert not session.is_draining_request("req-old") + del ended + + strict = DuplexEngineSession( + session_id="s2", + config=DuplexSessionConfig(model="m", modalities=["text"]), + capabilities=DuplexCapabilities(supports_concurrent_turn_requests=False), + ) + strict.begin_response(turn_id=3) + assert strict.active_response_accepts_model_turn(3) + assert not strict.active_response_accepts_model_turn(4) + + +def test_draining_request_exempt_from_completed_turn_filter() -> None: + """Draining ownership must win over the model_turn_id < turn_id late-audio guard.""" + from vllm_omni.engine.duplex.config import DuplexCapabilities, DuplexSessionConfig + from vllm_omni.engine.duplex.session.engine_session import DuplexEngineSession + + session = DuplexEngineSession( + session_id="s-drain", + config=DuplexSessionConfig(model="m", modalities=["text", "audio"]), + capabilities=DuplexCapabilities(supports_concurrent_turn_requests=True), + ) + session.begin_response(turn_id=1) + r1 = session.active_response_id + assert r1 is not None + session.bind_draining_request("req-r1-tts", r1) + # Overlapping R2 becomes active, then finishes — must not wipe R1 draining. + session.begin_response(turn_id=2) + session.end_response(commit_text=False) + session.turn_id = 2 + assert session.active_response_id is None + assert session.is_draining_request("req-r1-tts") + assert session.response_id_for_request("req-r1-tts") == r1 + draining_response_id = session.response_id_for_request("req-r1-tts") + model_turn_id = 1 + drop = ( + draining_response_id is None + and session.active_response_id is None + and model_turn_id is not None + and model_turn_id < session.turn_id + ) + assert drop is False + + +def test_ephemeral_turn_id_parser_accepts_stage_turn_ids() -> None: + from vllm_omni.engine.duplex.contracts import duplex_turn_id_from_request_id + + assert duplex_turn_id_from_request_id("duplex-s.x.e.0.r.stage0-turn12") == 12 + assert duplex_turn_id_from_request_id("duplex-s.x.e.0.r.stage2-turn3") == 3 + assert duplex_turn_id_from_request_id("duplex-s.x.e.0.r.stage0_t9") is None + assert duplex_turn_id_from_request_id("duplex-s.x.e.0.r.stage0") is None + + +def test_same_turn_request_ids_skip_other_turns_and_epochs() -> None: + from vllm_omni.engine.duplex.contracts import duplex_same_turn_request_ids + + stage3 = "duplex-s.x.e.0.r.stage3-turn4" + found = duplex_same_turn_request_ids( + stage3, + [ + "duplex-s.x.e.0.r.stage0-turn4", + "duplex-s.x.e.0.r.stage1-turn4", + "duplex-s.x.e.0.r.stage2-turn4", + stage3, + "duplex-s.x.e.0.r.stage2-turn5", + "duplex-s.x.e.1.r.stage0-turn4", + ], + ) + assert found == [ + "duplex-s.x.e.0.r.stage0-turn4", + "duplex-s.x.e.0.r.stage1-turn4", + "duplex-s.x.e.0.r.stage2-turn4", + ] + from vllm_omni.engine.duplex.contracts import duplex_session_id_from_request_id + + fence_session = "sess id" + import base64 + + encoded = base64.urlsafe_b64encode(fence_session.encode()).decode("ascii").rstrip("=") + assert duplex_session_id_from_request_id(f"duplex-s.{encoded}.e.0.r.stage1-turn2") == fence_session + assert duplex_session_id_from_request_id("not-a-request") is None + + +def test_stale_keys_skip_already_draining_request_ids() -> None: + """Rebinding must not move an older draining Stage2/3 id onto the newer response.""" + from vllm_omni.engine.duplex.config import DuplexCapabilities, DuplexSessionConfig + from vllm_omni.engine.duplex.session.engine_session import DuplexEngineSession + + session = DuplexEngineSession( + session_id="s-stale", + config=DuplexSessionConfig(model="m", modalities=["text", "audio"]), + capabilities=DuplexCapabilities(supports_concurrent_turn_requests=True), + ) + session.begin_response(turn_id=1) + r1 = session.active_response_id + assert r1 is not None + session.bind_draining_request("req-r1-talker", r1) + session.begin_response(turn_id=2) + r2 = session.active_response_id + assert r2 is not None and r2 != r1 + # Simulate the concurrent-turn rebind loop: already-draining ids keep R1. + for rid in ("req-r1-talker", "req-r2-talker"): + if session.is_draining_request(rid): + continue + session.bind_draining_request(rid, r2) + assert session.response_id_for_request("req-r1-talker") == r1 + assert session.response_id_for_request("req-r2-talker") == r2 + + +def test_end_response_clears_only_own_draining_entries() -> None: + from vllm_omni.engine.duplex.config import DuplexCapabilities, DuplexSessionConfig + from vllm_omni.engine.duplex.session.engine_session import DuplexEngineSession + + session = DuplexEngineSession( + session_id="s-end", + config=DuplexSessionConfig(model="m", modalities=["text", "audio"]), + capabilities=DuplexCapabilities(supports_concurrent_turn_requests=True), + ) + session.begin_response(turn_id=1) + r1 = session.active_response_id + session.bind_draining_request("req-r1", r1) + session.begin_response(turn_id=2) + r2 = session.active_response_id + session.bind_draining_request("req-r2", r2) + session.end_response(commit_text=False) + assert session.is_draining_request("req-r1") + assert session.response_id_for_request("req-r1") == r1 + assert not session.is_draining_request("req-r2") + + +def test_on_stage_failure_resolves_draining_response_before_active() -> None: + """Mirrors runner.on_stage_failure: draining request_id owns the failed done.""" + from vllm_omni.engine.duplex.config import DuplexCapabilities, DuplexSessionConfig + from vllm_omni.engine.duplex.session.engine_session import DuplexEngineSession + + session = DuplexEngineSession( + session_id="s-fail", + config=DuplexSessionConfig(model="m", modalities=["text", "audio"]), + capabilities=DuplexCapabilities(supports_concurrent_turn_requests=True), + ) + session.begin_response(turn_id=1) + r1 = session.active_response_id + assert r1 is not None + session.bind_draining_request("req-r1-talker", r1) + session.begin_response(turn_id=2) + r2 = session.active_response_id + assert r2 is not None and r2 != r1 + + request_id = "req-r1-talker" + draining_response_id = ( + session.response_id_for_request(request_id) if session.is_draining_request(request_id) else None + ) + response_id = draining_response_id or session.active_response_id + assert response_id == r1 + assert response_id != r2 + session.clear_draining_for_response(r1) + assert not session.is_draining_request(request_id) + assert session.active_response_id == r2 + + +def test_concurrent_turn_requests_released_resets_when_new_stage0_binds() -> None: + """After R4 release, a new ephemeral Stage0 bind must clear the gate flag.""" + from types import SimpleNamespace + + run = SimpleNamespace(concurrent_turn_requests_released=True) + # Simulate the concurrent-turn rebind arm in model_channel._append_via_data_plane. + concurrent_turn = True + if concurrent_turn: + run.concurrent_turn_requests_released = False + assert run.concurrent_turn_requests_released is False + + +def test_draining_stage_ids_come_from_the_plugin() -> None: + from types import SimpleNamespace + + from vllm_omni.engine.duplex.session.model_channel import ModelChannel + + channel = ModelChannel.__new__(ModelChannel) + channel._ctx = SimpleNamespace( + plugin=SimpleNamespace(draining_stage_ids=lambda *, stage_count: {stage_count - 1}), + stage_port=SimpleNamespace(stage_count=5), + ) + assert channel._draining_stage_ids() == frozenset({4}) + + channel._ctx.plugin = SimpleNamespace() + assert channel._draining_stage_ids() == frozenset() + + +def test_draining_empty_eos_completes_owning_response_before_shortcut() -> None: + """R2 finishing first must not swallow R1's empty Stage3 EOS.""" + import asyncio + from types import SimpleNamespace + + from vllm_omni.engine.duplex.config import DuplexCapabilities, DuplexSessionConfig + from vllm_omni.engine.duplex.session.engine_session import DuplexEngineSession + from vllm_omni.engine.duplex.session.model_channel import ModelChannel + + session = DuplexEngineSession( + session_id="s-empty-eos", + config=DuplexSessionConfig(model="m", modalities=["text", "audio"], extra_body={"auto_response": True}), + capabilities=DuplexCapabilities(supports_concurrent_turn_requests=True, supports_core_resumable_request=False), + ) + session.begin_response(turn_id=1) + r1 = session.active_response_id + assert r1 is not None + r1_request = "duplex-s.x.e.0.r.stage3-turn1" + still_running = "duplex-s.x.e.0.r.stage1-turn2" + session.bind_draining_request(r1_request, r1) + session.begin_response(turn_id=2) + session.end_response(commit_text=False) + session.turn_id = 2 + assert session.active_response_id is None + + class _Plane: + def is_terminal(self, request_id: str | None) -> bool: + del request_id + return False + + def close_stream(self, request_id: str) -> None: + del request_id + + def mark_terminal(self, request_id: str) -> None: + del request_id + + class _Port: + def __init__(self) -> None: + self.cleanups: list[list[str]] = [] + + async def cleanup(self, request_ids: list[str], *, abort: bool = False) -> None: + del abort + self.cleanups.append(list(request_ids)) + + class _Out: + def __init__(self) -> None: + self.events: list[dict[str, object]] = [] + + def auto_responds(self) -> bool: + return True + + def emit(self, payload: dict[str, object]) -> None: + self.events.append(payload) + + port = _Port() + out = _Out() + + async def _noop(*_args: object, **_kwargs: object) -> None: + return None + + async def _schedule(*_args: object, **_kwargs: object) -> bool: + return False + + channel = ModelChannel( + SimpleNamespace( + session=session, + plugin=SimpleNamespace(data_plane=_Plane()), + stage_port=port, + model_state=SimpleNamespace(clear_continuation=lambda: None), + services=SimpleNamespace(spawn=lambda *_a, **_k: None), + ), + out, + close_from_runtime=_noop, + schedule_silence_continuation=_schedule, + abort_request=_noop, + ) + asyncio.run( + channel._send_one_model_output_event( + { + "data_plane_request_id": r1_request, + "end_of_turn": True, + "model_turn_id": 1, + "text": "", + "stage_role": "tts", + } + ) + ) + done = [event for event in out.events if event.get("type") == "response.done"] + assert done and done[-1].get("response_id") == r1 + assert port.cleanups == [[r1_request]] + assert all(still_running not in ids for ids in port.cleanups) + assert not session.is_draining_request(r1_request) + + +def test_draining_completion_drops_finished_response_books_only() -> None: + """R1 drain finish drops R1 books and leaves the live R2 snapshot.""" + import asyncio + from types import SimpleNamespace + + from vllm_omni.engine.duplex.config import DuplexCapabilities, DuplexSessionConfig + from vllm_omni.engine.duplex.session.engine_session import DuplexEngineSession + from vllm_omni.engine.duplex.session.model_channel import ModelChannel + + session = DuplexEngineSession( + session_id="s-drain-books", + config=DuplexSessionConfig(model="m", modalities=["text", "audio"], extra_body={"auto_response": True}), + capabilities=DuplexCapabilities(supports_concurrent_turn_requests=True, supports_core_resumable_request=False), + ) + session.begin_response(turn_id=1) + r1 = session.active_response_id + assert r1 is not None + session.append_assistant_text("from-r1") + session.snapshot_active_response_for_drain() + r1_request = "duplex-s.x.e.0.r.stage3-turn1" + session.bind_draining_request(r1_request, r1) + session.begin_response(turn_id=2) + r2 = session.active_response_id + assert r2 is not None and r2 != r1 + session.append_assistant_text("from-r2") + session.snapshot_active_response_for_drain() + + class _Plane: + def is_terminal(self, request_id: str | None) -> bool: + del request_id + return False + + def close_stream(self, request_id: str) -> None: + del request_id + + def mark_terminal(self, request_id: str) -> None: + del request_id + + class _Port: + async def cleanup(self, request_ids: list[str], *, abort: bool = False) -> None: + del request_ids, abort + + class _Out: + def __init__(self) -> None: + self.events: list[dict[str, object]] = [] + + def auto_responds(self) -> bool: + return True + + def emit(self, payload: dict[str, object]) -> None: + self.events.append(payload) + + out = _Out() + + async def _noop(*_args: object, **_kwargs: object) -> None: + return None + + async def _schedule(*_args: object, **_kwargs: object) -> bool: + return False + + channel = ModelChannel( + SimpleNamespace( + session=session, + plugin=SimpleNamespace(data_plane=_Plane()), + stage_port=_Port(), + model_state=SimpleNamespace(clear_continuation=lambda: None), + services=SimpleNamespace(spawn=lambda *_a, **_k: None), + ), + out, + close_from_runtime=_noop, + schedule_silence_continuation=_schedule, + abort_request=_noop, + ) + asyncio.run( + channel._send_one_model_output_event( + { + "data_plane_request_id": r1_request, + "end_of_turn": True, + "model_turn_id": 1, + "text": "", + "stage_role": "tts", + } + ) + ) + assert r1 not in session._conversation.assistant_response_snapshots + assert r1 not in session._playback.by_response + assert f"item_{r1}" not in session._conversation.history_item_placeholders + assert r2 in session._conversation.assistant_response_snapshots + assert session.active_response_id == r2 + done = [event for event in out.events if event.get("type") == "response.done"] + assert done and done[-1].get("response_id") == r1 + + +def test_stale_continue_does_not_close_the_new_response() -> None: + import asyncio + from types import SimpleNamespace + + from vllm_omni.engine.duplex.config import DuplexCapabilities, DuplexSessionConfig + from vllm_omni.engine.duplex.session.engine_session import DuplexEngineSession + from vllm_omni.engine.duplex.session.model_channel import ModelChannel + + session = DuplexEngineSession( + session_id="s-stale-continue", + config=DuplexSessionConfig(model="m", modalities=["text", "audio"], extra_body={"auto_response": True}), + capabilities=DuplexCapabilities(supports_concurrent_turn_requests=True, supports_core_resumable_request=False), + ) + session.begin_response(turn_id=2) + live = session.active_response_id + session.epoch = 1 + + class _Out: + def __init__(self) -> None: + self.events: list[dict[str, object]] = [] + + def auto_responds(self) -> bool: + return True + + def emit(self, payload: dict[str, object]) -> None: + self.events.append(payload) + + out = _Out() + cleared: list[bool] = [] + + async def _noop(*_args: object, **_kwargs: object) -> None: + return None + + async def _schedule(*_args: object, **_kwargs: object) -> bool: + return False + + channel = ModelChannel( + SimpleNamespace( + session=session, + plugin=SimpleNamespace(data_plane=SimpleNamespace()), + stage_port=SimpleNamespace(), + model_state=SimpleNamespace(clear_continuation=lambda: cleared.append(True)), + services=SimpleNamespace(spawn=lambda *_a, **_k: None), + run=SimpleNamespace(closing=False), + ), + out, + close_from_runtime=_noop, + schedule_silence_continuation=_schedule, + abort_request=_noop, + ) + asyncio.run(channel.maybe_continue_response(expected_epoch=0)) + assert session.active_response_id == live + assert out.events == [] + assert cleared == [] + + session.bind_draining_request("duplex-s.x.e.1.r.stage3-turn2", live or "") + asyncio.run(channel.maybe_continue_response(expected_epoch=1)) + assert session.active_response_id == live + assert out.events == [] + + session.clear_draining_requests() + asyncio.run(channel.maybe_continue_response(expected_epoch=1)) + assert session.active_response_id is None + assert [event.get("type") for event in out.events] == ["response.done"] + assert out.events[0].get("response_id") == live + + +def test_silent_listen_with_continuation_releases_ephemeral_request() -> None: + import asyncio + from collections.abc import Coroutine + from types import SimpleNamespace + from typing import Any + + from vllm_omni.engine.duplex.config import DuplexCapabilities, DuplexSessionConfig + from vllm_omni.engine.duplex.session.engine_session import DuplexEngineSession + from vllm_omni.engine.duplex.session.model_channel import ModelChannel + + session = DuplexEngineSession( + session_id="s-silent-continue", + config=DuplexSessionConfig(model="m", modalities=["text", "audio"]), + capabilities=DuplexCapabilities(supports_concurrent_turn_requests=True, supports_core_resumable_request=False), + ) + session.begin_response(turn_id=1) + response_id = session.active_response_id + request_id = "duplex-s.x.e.0.r.stage1-turn1" + session.bind_request(request_id) + + class _Plane: + def __init__(self) -> None: + self.terminal: list[str] = [] + + def is_terminal(self, request_id: str | None) -> bool: + return request_id in self.terminal + + def mark_terminal(self, request_id: str) -> None: + self.terminal.append(request_id) + + class _Port: + def __init__(self) -> None: + self.cleanups: list[list[str]] = [] + + async def cleanup(self, request_ids: list[str], *, abort: bool = False) -> None: + del abort + self.cleanups.append(list(request_ids)) + + class _Out: + def __init__(self) -> None: + self.events: list[dict[str, object]] = [] + + def auto_responds(self) -> bool: + return False + + def emit(self, payload: dict[str, object]) -> None: + self.events.append(payload) + + port = _Port() + out = _Out() + spawned: list[Coroutine[Any, Any, None]] = [] + + async def _noop(*_args: object, **_kwargs: object) -> None: + return None + + async def _schedule(*_args: object, **_kwargs: object) -> bool: + return False + + channel = ModelChannel( + SimpleNamespace( + session=session, + plugin=SimpleNamespace(data_plane=_Plane()), + stage_port=port, + model_state=SimpleNamespace( + continuation_owner_id=None, continuation_units=0, clear_continuation=lambda: None + ), + services=SimpleNamespace(spawn=lambda coro, **_k: spawned.append(coro)), + run=SimpleNamespace(closing=False), + ), + out, + close_from_runtime=_noop, + schedule_silence_continuation=_schedule, + abort_request=_noop, + ) + asyncio.run( + channel._send_one_model_output_event( + { + "data_plane_request_id": request_id, + "is_listen": True, + "end_of_turn": True, + "model_turn_id": 1, + "reason": "model_listen", + } + ) + ) + assert port.cleanups == [[request_id]] + assert session.active_request_id is None + assert spawned, "silent listen with unused continuation must schedule maybe_continue" + asyncio.run(spawned[0]) + assert port.cleanups == [[request_id]] + assert session.active_response_id is None + assert [event.get("type") for event in out.events] == ["response.listen", "response.done"] + assert out.events[0].get("response_id") == response_id + assert out.events[1].get("response_id") == response_id + + +def test_silent_listen_aborts_data_plane_request_with_full_id() -> None: + """abort_data_plane_request must pass [request_id], not a bare str.""" + import asyncio + from types import SimpleNamespace + + from vllm_omni.engine.duplex.config import DuplexCapabilities, DuplexSessionConfig + from vllm_omni.engine.duplex.session.engine_session import DuplexEngineSession + from vllm_omni.engine.duplex.session.model_channel import ModelChannel + + session = DuplexEngineSession( + session_id="s-silent-abort", + config=DuplexSessionConfig(model="m", modalities=["text", "audio"]), + capabilities=DuplexCapabilities(supports_concurrent_turn_requests=True, supports_core_resumable_request=False), + ) + request_id = "duplex-s.x.e.0.r.stage0-turn7" + session.bind_request(request_id) + aborts: list[list[str]] = [] + + class _Plane: + def is_terminal(self, request_id: str | None) -> bool: + return False + + def mark_terminal(self, request_id: str) -> None: + del request_id + + class _Port: + async def cleanup(self, request_ids: list[str], *, abort: bool = False) -> None: + del request_ids, abort + + class _Out: + def auto_responds(self) -> bool: + return True + + def emit(self, payload: dict[str, object]) -> None: + del payload + + async def _noop(*_args: object, **_kwargs: object) -> None: + return None + + async def _abort(request_ids: list[str], *, notify: bool = False) -> None: + del notify + aborts.append(list(request_ids)) + + channel = ModelChannel( + SimpleNamespace( + session=session, + plugin=SimpleNamespace(data_plane=_Plane()), + stage_port=_Port(), + model_state=SimpleNamespace( + continuation_owner_id=None, continuation_units=0, clear_continuation=lambda: None + ), + services=SimpleNamespace(spawn=lambda coro, **_k: None), + run=SimpleNamespace(closing=False), + ), + _Out(), + close_from_runtime=_noop, + schedule_silence_continuation=_noop, + abort_request=_abort, + ) + asyncio.run( + channel._send_one_model_output_event( + { + "data_plane_request_id": request_id, + "is_listen": True, + "end_of_turn": True, + "model_turn_id": 1, + "reason": "model_listen", + "abort_data_plane_request": True, + } + ) + ) + assert aborts == [[request_id]] + assert all(len(item) > 1 for batch in aborts for item in batch), "must not split request id into chars" diff --git a/tests/engine/duplex/test_engine_session.py b/tests/engine/duplex/test_engine_session.py index 5ef4ab6b9..358ee2d9c 100644 --- a/tests/engine/duplex/test_engine_session.py +++ b/tests/engine/duplex/test_engine_session.py @@ -242,6 +242,19 @@ def test_history_commit_with_ack_only_policy_defers_unacknowledged_text(): assert f"item_{response_id}" in session.pending_history_item_ids +def test_history_commit_with_ack_only_policy_keeps_only_acknowledged_prefix(): + session = _session(config=DuplexSessionConfig(playback_commit_policy="ack_only")) + session.begin_response() + session.append_assistant_text("hello world") + session.mark_audio_sent(duration_ms=1_000, text_chars=6) + session.mark_audio_sent(duration_ms=2_000, text_chars=11) + session.acknowledge_playback(played_ms=1_000, committed_ms=1_000) + + committed = session.end_response(commit_text=True) + + assert committed == {"role": "assistant", "content": "hello"} + + @pytest.mark.parametrize("audio_complete", [False, True]) def test_unaligned_response_keeps_empty_turn_before_later_user_input(audio_complete): session = _session(config=DuplexSessionConfig(playback_commit_policy="ack_only")) @@ -469,6 +482,36 @@ def test_cancel_fence_releases_stage_requests_and_advances_identity(): session.cancel_fence(DuplexFence("sid-other", epoch=1, turn_id=0), DuplexFence("sid-cancel", epoch=2)) +def test_request_resource_keys_are_stage_id_and_request_id(): + session = _session("sid-keys") + session.bind_stage_request(0, "req-a", fence=session.fence) + session.bind_stage_request(1, "req-b", fence=session.fence) + session.bind_stage_request(2, "req-tts", fence=session.fence) + stale_keys = list(session.request_resources.keys()) + stale_ids = list(dict.fromkeys(rid for _, rid in stale_keys)) + assert stale_ids == ["req-a", "req-b", "req-tts"] + for sid, rid in stale_keys: + if sid < 2: + session.request_resources.pop((sid, rid), None) + assert session.resource_request_ids() == ["req-tts"] + with pytest.raises(TypeError, match="unhashable"): + dict.fromkeys(rid for _, rid in session.request_resources.items()) + + +def test_release_resources_for_request_ids_keeps_other_ids_on_the_same_fence(): + session = _session("sid-drain-release") + fence = session.fence + session.bind_stage_request(2, "drain", fence=fence) + session.bind_stage_request(3, "drain", fence=fence) + session.bind_stage_request(0, "live", fence=fence) + + released = session.release_resources_for_request_ids(["drain"]) + + assert released == ["drain"] + assert session.resource_request_ids() == ["live"] + assert session.release_resources_for_request_ids([]) == [] + + # ---- public view ---- @@ -688,3 +731,65 @@ def test_minicpmo_native_capabilities_do_not_overclaim_single_session_deployment assert caps["supports_multi_session"] is False assert caps["supports_multi_session_same_replica"] is False + + +def test_draining_response_stays_ack_admissible_after_next_begin_response() -> None: + from vllm_omni.engine.duplex.events import ErrorEvent + from vllm_omni.engine.duplex.session.playback_ledger import apply_playback_ack + + session = _session() + first = session.begin_response(turn_id=1) + session.append_assistant_text("hello") + session.mark_audio_sent(400, text_chars=5) + session.snapshot_active_response_for_drain() + second = session.begin_response(turn_id=2) + assert second != first + assert session.playback.sent_ms == 0 + events = apply_playback_ack( + session, + { + "type": "playback.ack", + "response_id": first, + "item_id": f"item_{first}", + "played_ms": 100, + "committed_ms": 100, + }, + ) + assert not any(isinstance(event, ErrorEvent) and event.code == "playback_item_not_found" for event in events) + session.mark_audio_sent(900, text_chars=5, response_id=first) + assert session.playback_for_response(first).sent_ms == 900 + assert session.playback.sent_ms == 0 + + +def test_finished_drain_keeps_sent_audio_ackable() -> None: + from vllm_omni.engine.duplex.events import ErrorEvent + from vllm_omni.engine.duplex.session.playback_ledger import apply_playback_ack + + session = _session() + first = session.begin_response(turn_id=1) + session.append_assistant_text("hello") + session.mark_audio_sent(400, text_chars=5) + session.snapshot_active_response_for_drain() + second = session.begin_response(turn_id=2) + session.release_finished_drain_response(first) + assert first in session._conversation.assistant_response_snapshots + assert f"item_{first}" in session._conversation.history_item_placeholders + events = apply_playback_ack( + session, + { + "type": "playback.ack", + "response_id": first, + "item_id": f"item_{first}", + "played_ms": 400, + "committed_ms": 400, + }, + ) + assert not any(isinstance(event, ErrorEvent) for event in events) + assert session.active_response_id == second + + silent = session.begin_response(turn_id=3) + session.snapshot_active_response_for_drain() + session.begin_response(turn_id=4) + session.release_finished_drain_response(silent) + assert silent not in session._conversation.assistant_response_snapshots + assert f"item_{silent}" not in session._conversation.history_item_placeholders diff --git a/tests/engine/duplex/test_input_transcribed.py b/tests/engine/duplex/test_input_transcribed.py new file mode 100644 index 000000000..98914b899 --- /dev/null +++ b/tests/engine/duplex/test_input_transcribed.py @@ -0,0 +1,49 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + +import pytest + +from vllm_omni.engine.duplex.realtime_events import ( + RealtimeProjectionState, + project_internal_event, +) + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +def test_input_transcribed_is_a_user_transcript_event() -> None: + events = project_internal_event( + RealtimeProjectionState(session_id="s"), + {"type": "input.transcribed", "transcript": " 你好 "}, + ) + assert len(events) == 1 + assert events[0].wire_type == "conversation.item.input_audio_transcription.completed" + assert events[0].transcript == "你好" + + +def test_input_transcribed_updates_the_committed_item() -> None: + state = RealtimeProjectionState(session_id="s") + project_internal_event( + state, + { + "type": "input.committed", + "realtime_item_id": "item_user_1", + "is_speech": True, + "message": {"role": "user"}, + }, + ) + events = project_internal_event( + state, + {"type": "input.transcribed", "transcript": "你好", "realtime_item_id": "item_user_1"}, + ) + assert events[0].item_id == "item_user_1" + item = state.conversation_items["item_user_1"] + assert item["content"][0]["transcript"] == "你好" + + +def test_blank_input_transcribed_emits_nothing() -> None: + events = project_internal_event( + RealtimeProjectionState(session_id="s"), + {"type": "input.transcribed", "transcript": " "}, + ) + assert events == [] diff --git a/tests/engine/duplex/test_overlap_policy.py b/tests/engine/duplex/test_overlap_policy.py index 096b7c7c9..61840e2bb 100644 --- a/tests/engine/duplex/test_overlap_policy.py +++ b/tests/engine/duplex/test_overlap_policy.py @@ -79,6 +79,39 @@ def test_an_explicit_client_overlap_action_requests_barge_in() -> None: assert decision["reason"] == "client_overlap_action" +def test_turn_mode_keeps_silent_vision_follow_during_overlap() -> None: + """Vision-follow is is_speech=False + a frame; must not drop while TTS plays.""" + session = _session() + session.capabilities = type(session.capabilities)( + required_input_modalities=frozenset({"video"}), + optional_input_modalities=frozenset({"audio"}), + ) + payload = {"format": "pcm16", "audio": _pcm16(0.0), "video_frames": ["frame"], "is_speech": False} + decision = overlap_policy.decide(session, {"is_speech": False}, payload, auto_responds=False) + assert decision["action"] == "listen" + assert decision["reason"] == "vision_follow" + assert decision["buffer_audio"] is True + assert decision["defer_runtime_append"] is False + assert decision["force_listen"] is False + + +def test_turn_mode_drops_silent_vision_without_capability() -> None: + session = _session() + assert session.capabilities.allows_video_without_audio() is False + payload = {"format": "pcm16", "audio": _pcm16(0.0), "video_frames": ["frame"], "is_speech": False} + decision = overlap_policy.decide(session, {"is_speech": False}, payload, auto_responds=False) + assert decision["action"] == "drop" + assert decision["reason"] == "silence_or_noise" + + +def test_turn_mode_still_drops_silent_overlap_without_vision() -> None: + session = _session() + payload = {"format": "pcm16", "audio": _pcm16(0.0), "is_speech": False} + decision = overlap_policy.decide(session, {"is_speech": False}, payload, auto_responds=False) + assert decision["action"] == "drop" + assert decision["reason"] == "silence_or_noise" + + def test_barge_in_is_deferred_when_the_session_cannot_support_it() -> None: session = _session() session.capabilities = type(session.capabilities)(supports_barge_in=False) diff --git a/tests/engine/duplex/test_session_manager.py b/tests/engine/duplex/test_session_manager.py index bccce18f1..cb00737d9 100644 --- a/tests/engine/duplex/test_session_manager.py +++ b/tests/engine/duplex/test_session_manager.py @@ -196,7 +196,11 @@ class FakePlugin(DuplexModelPlugin): def capabilities(self, *, max_sessions: int) -> DuplexCapabilities: del max_sessions - return DuplexCapabilities(supports_input_append=True, supports_core_resumable_request=True) + # Resident Stage0 ids (MiniCPM-shaped); AURA opts out via supports_core_resumable_request=False. + return DuplexCapabilities( + supports_input_append=True, + supports_core_resumable_request=True, + ) def validate_client_extra_body(self, extra_body: object) -> None: pass @@ -409,7 +413,8 @@ async def test_open_answers_with_capabilities_and_emits_session_created() -> Non assert result.session_id == "sid-open" assert result.lease_generation == 0 assert result.capabilities == DuplexCapabilities( - supports_input_append=True, supports_core_resumable_request=True + supports_input_append=True, + supports_core_resumable_request=True, ) assert result.public_session is not None assert result.public_session["id"] == "sid-open" @@ -1028,6 +1033,40 @@ async def test_append_bytes_are_reserved_at_admission_until_the_runner_dequeues( gate.set() +async def test_append_admission_counts_audio_and_video_frame_bytes() -> None: + """Manager reserves len(audio)+Σlen(frame); video bytes count toward the same limit.""" + async with Harness.create(max_sessions=1, max_pending_input_bytes_per_session=20) as harness: + await harness.open("sid-av") + harness.events() + session = harness.session("sid-av") + runner = harness.manager.runners["sid-av"] + gate = asyncio.Event() + runner._mailbox.put_nowait(_Internal("wait", {"gate": gate})) + runner._on_internal = lambda item: gate.wait() # type: ignore[method-assign] + await asyncio.sleep(0) + + # Default caps require audio; attach video to a non-empty audio unit. + frame_a = "aaaa" + frame_b = "bbbbbb" + audio = b"1234" + expected = len(audio) + len(frame_a) + len(frame_b) + harness.command( + "sid-av", + AppendAudio(audio=audio, video_frames=(frame_a, frame_b), event_id="evt-av"), + ) + assert session.pending_input_bytes == expected + # Second append that would exceed the limit is backpressured. + harness.command( + "sid-av", + AppendAudio(audio=b"x" * 10, video_frames=("yyyyyyyyyy",), event_id="evt-over"), + ) + errors = [event for event in harness.events("sid-av") if isinstance(event, ErrorEvent)] + assert [error.code for error in errors] == ["input_backpressure"] + assert errors[0].related_event_id == "evt-over" + assert session.pending_input_bytes == expected + gate.set() + + async def test_expired_session_retains_the_admission_slot_until_cleanup_succeeds() -> None: async with Harness.create(max_sessions=1, idle_ttl_s=1.0) as harness: await harness.open("sid-expired") diff --git a/tests/engine/duplex/test_session_runner.py b/tests/engine/duplex/test_session_runner.py index 51711f52a..f48154a48 100644 --- a/tests/engine/duplex/test_session_runner.py +++ b/tests/engine/duplex/test_session_runner.py @@ -180,7 +180,10 @@ class Harness: segment_token_ids=tuple(segment_token_ids), segment_output_metadata=dict(segment_output_metadata or {}), ) - return self.runner.on_stage_output(stage_id, output, metrics, request_id=output.request_id, context=context) + request_id = getattr(output, "request_id", None) + if not isinstance(request_id, str): + raise AssertionError("stage output is missing request_id") + return self.runner.on_stage_output(stage_id, output, metrics, request_id=request_id, context=context) async def deliver_and_settle(self, output: object, **kwargs: Any) -> list[DuplexEvent]: self.deliver(output, **kwargs) @@ -520,6 +523,28 @@ async def test_commit_without_audio_is_rejected() -> None: await close_harness(h) +@pytest.mark.asyncio +async def test_empty_silent_commit_acks_without_opening_response() -> None: + """#3: empty is_speech=False commit keeps the silent-ack pair, even auto-respond.""" + h = await open_harness(auto_response=True) + try: + # resolve_commit only admits empty silent commits after non-speech was seen. + projector = h.runner._require_projector() + projector.input_audio_buffer_had_non_speech = True + events = await h.run(commands.Commit()) + assert "error" not in types(events), types(events) + assert "input_audio_buffer.committed" in types(events), types(events) + assert "response.listen" in types(events), types(events) + listen = next(event for event in events if event.type == "response.listen") + assert listen.details["reason"] == "silence_or_noise" + committed = next(event for event in events if event.type == "input_audio_buffer.committed") + assert committed.details.get("empty") is True + assert committed.details.get("no_response") is True + assert h.port.submissions == [] + finally: + await close_harness(h) + + @pytest.mark.asyncio async def test_commit_projects_the_user_item_and_does_not_resubmit_consumed_audio() -> None: h = await open_harness() @@ -674,6 +699,45 @@ async def test_stale_epoch_output_is_dropped_after_barge_in() -> None: await close_harness(h) +async def test_barge_in_aborts_draining_tts_as_well_as_the_active_request() -> None: + h = await open_harness() + try: + await h.run(append_audio()) + request_id = h.stage0_request_id() + await h.deliver_and_settle(tts_output(request_id, samples=24000, text="he")) + h.session.bind_draining_request("duplex-drain-tts", "resp-old") + # Draining TTS sits on an older turn fence. cancel_fence only drops + # the fence being cancelled, so these bindings must be popped by id. + from vllm_omni.engine.duplex.contracts import DuplexFence + from vllm_omni.engine.duplex.session.engine_session import DuplexRequestResource + + older = DuplexFence(h.session.session_id, epoch=h.session.epoch, turn_id=h.session.turn_id + 5) + h.session.request_resources[(2, "duplex-drain-tts")] = DuplexRequestResource( + stage_id=2, request_id="duplex-drain-tts", fence=older, submitted=True + ) + h.session.request_resources[(3, "duplex-drain-tts")] = DuplexRequestResource( + stage_id=3, request_id="duplex-drain-tts", fence=older, submitted=True + ) + h.session.request_resources[(1, "still-live")] = DuplexRequestResource( + stage_id=1, request_id="still-live", fence=older, submitted=True + ) + events = await h.run(commands.BargeIn()) + assert h.port.aborts == [[request_id, "duplex-drain-tts"]] + assert not h.session.is_draining_request("duplex-drain-tts") + done_ids = [ + event.response["id"] + for event in events + if getattr(event, "wire_type", "") == "response.done" and isinstance(getattr(event, "response", None), dict) + ] + assert "resp-old" in done_ids + assert (2, "duplex-drain-tts") not in h.session.request_resources + assert (3, "duplex-drain-tts") not in h.session.request_resources + assert (0, request_id) not in h.session.request_resources + assert (1, "still-live") in h.session.request_resources + finally: + await close_harness(h) + + @pytest.mark.asyncio async def test_cancel_response_without_active_response_is_rejected() -> None: h = await open_harness() @@ -778,6 +842,46 @@ async def test_listen_decision_on_a_resumable_request_closes_the_bounded_respons await close_harness(h) +@pytest.mark.asyncio +async def test_direct_response_listen_still_emits_response_done_after_continuation_clear() -> None: + """AURA-like silent DIRECT_RESPONSE: after continuation budget is spent, emit response.done. + + Non-resumable turn-commit still closes the active response on a listen/direct + decision so the client is not left without a terminal event. + """ + from dataclasses import replace + + h = await open_harness() + try: + h.session.capabilities = replace(h.session.capabilities, supports_core_resumable_request=False) + await h.run(append_audio()) + assert h.port.submissions, "ephemeral Stage0 must submit" + request_id = h.port.submissions[-1].context.request_id + assert "-turn" in request_id + await h.deliver_and_settle(tts_output(request_id, samples=24000, text="ok")) + response_id = h.session.active_response_id + assert response_id is not None + model_state = h.runner.model_state + model_state.continuation_owner_id = f"response:{response_id}" + model_state.continuation_units = h.runner.model._AUTO_RESPONSE_MAX_CONTINUATION_UNITS + + listen = listen_output(request_id) + listen.finished = True + events = await h.deliver_and_settle( + listen, + stage_id=0, + segment_finished=True, + segment_token_ids=[11, 12, LISTEN_TOKEN_ID], + segment_output_metadata={"meta.listen_token_id": LISTEN_TOKEN_ID}, + ) + assert "response.listen" in types(events) + done = find(events, "response.done") + assert done.response_id == response_id + assert h.session.active_response_id is None + finally: + await close_harness(h) + + @pytest.mark.asyncio async def test_listen_decision_on_a_resumable_request_keeps_the_turn_going() -> None: """Same unfinished listen, budget left: it must schedule the next unit. @@ -860,6 +964,38 @@ async def test_stage_failure_fails_the_active_response() -> None: await close_harness(h) +@pytest.mark.asyncio +async def test_stage_failure_on_draining_request_fails_that_response_only() -> None: + """Overlapped: a draining older request failure must not wipe the active response.""" + from dataclasses import replace + + h = await open_harness() + try: + h.session.capabilities = replace(h.session.capabilities, supports_concurrent_turn_requests=True) + await h.run(append_audio()) + request_id = h.stage0_request_id() + await h.deliver_and_settle(tts_output(request_id, samples=24000, text="hello")) + r1 = h.session.active_response_id + assert r1 is not None + draining_req = "req-r1-talker-drain" + h.session.bind_draining_request(draining_req, r1) + # Open a newer active response while R1 TTS is still draining. + r2 = h.session.begin_response(turn_id=(h.session.turn_id or 0) + 1) + assert h.session.active_response_id == r2 + assert r2 != r1 + + h.runner.on_stage_failure(2, RuntimeError("drain-boom"), request_id=draining_req) + events = await h.settle() + assert types(events)[0] == "error" + done = find(events, "response.done") + assert done.response_id == r1 + assert done.status == "failed" + assert h.session.active_response_id == r2 + assert not h.session.is_draining_request(draining_req) + finally: + await close_harness(h) + + # --------------------------------------------------------------------------- # # Turn mode (no auto response) # # --------------------------------------------------------------------------- # @@ -877,6 +1013,91 @@ async def test_turn_mode_skips_silent_chunks() -> None: await close_harness(h) +@pytest.mark.asyncio +async def test_turn_mode_keeps_silent_chunks_with_video_frames() -> None: + """Engine vision-follow buffers silent+frames; MiniCPM commit stays speech-gated.""" + from dataclasses import replace + + h = await open_harness(auto_response=False) + try: + h.session.capabilities = replace( + h.session.capabilities, + required_input_modalities=frozenset({"video"}), + optional_input_modalities=frozenset({"audio"}), + ) + # Minimal 1x1 JPEG (base64) — wire validation only checks non-empty str. + frame = ( + "/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkS" + "Ew8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/2wBDAQkJ" + "CQwLDBgNDRgyIRwhMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIy" + "MjIyMjIyMjIyMjIyMjL/wAARCAABAAEDASIAAhEBAxEB/8QAFQABAQAAAAAAAAAA" + "AAAAAAAAAAj/xAAUEAEAAAAAAAAAAAAAAAAAAAAA/8QAFQEBAQAAAAAAAAAAAAAA" + "AAAAAAD/xAAUEQEAAAAAAAAAAAAAAAAAAAAA/9oADAMBAAIQAxAAAAGfAP/EABQQ" + "AQAAAAAAAAAAAAAAAAAAAAD/2gAIAQEAAQUCf//EABQRAQAAAAAAAAAAAAAAAAAA" + "AAD/2gAIAQMBAT8Bf//EABQRAQAAAAAAAAAAAAAAAAAAAAD/2gAIAQIBAT8Bf//E" + "ABQQAQAAAAAAAAAAAAAAAAAAAAD/2gAIAQEABj8Cf//EABQQAQAAAAAAAAAAAAAA" + "AAAAAAD/2gAIAQEAAT8hf//Z" + ) + cmd = commands.AppendAudio( + audio=pcm_f32(160, value=0.0), + format="pcm_f32le", + sample_rate_hz=16000, + is_speech=False, + video_frames=(frame,), + ) + events = await h.run(cmd) + assert "response.listen" not in types(events) + assert h.runner.model_state.audio_buffer.has_pending() + events = await h.run(commands.Commit(create_response=True)) + # MiniCPM prepare_commit is speech-gated (no frames-only Stage0). + assert "input_audio_buffer.committed" in types(events), types(events) + assert h.port.submissions == [] + finally: + await close_harness(h) + + +@pytest.mark.asyncio +async def test_turn_mode_keeps_silent_vision_while_response_in_progress() -> None: + """TTS still playing: vision-follow must buffer, not drop as silence_or_noise.""" + from dataclasses import replace + + h = await open_harness(auto_response=False) + try: + h.session.capabilities = replace( + h.session.capabilities, + required_input_modalities=frozenset({"video"}), + optional_input_modalities=frozenset({"audio"}), + ) + h.session._response.active_response_id = "resp-tts" + frame = ( + "/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkS" + "Ew8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/2wBDAQkJ" + "CQwLDBgNDRgyIRwhMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIy" + "MjIyMjIyMjIyMjIyMjL/wAARCAABAAEDASIAAhEBAxEB/8QAFQABAQAAAAAAAAAA" + "AAAAAAAAAAj/xAAUEAEAAAAAAAAAAAAAAAAAAAAA/8QAFQEBAQAAAAAAAAAAAAAA" + "AAAAAAD/xAAUEQEAAAAAAAAAAAAAAAAAAAAA/9oADAMBAAIQAxAAAAGfAP/EABQQ" + "AQAAAAAAAAAAAAAAAAAAAAD/2gAIAQMBAT8Bf//EABQRAQAAAAAAAAAAAAAAAAAA" + "AAD/2gAIAQIBAT8Bf//EABQQAQAAAAAAAAAAAAAAAAAAAAD/2gAIAQEABj8Cf//E" + "ABQQAQAAAAAAAAAAAAAAAAAAAAD/2gAIAQEAAT8hf//Z" + ) + events = await h.run( + commands.AppendAudio( + audio=pcm_f32(160, value=0.0), + format="pcm_f32le", + sample_rate_hz=16000, + is_speech=False, + video_frames=(frame,), + ) + ) + listen_reasons = [ + getattr(event, "details", {}) or {} for event in events if getattr(event, "type", None) == "response.listen" + ] + assert all(details.get("reason") != "silence_or_noise" for details in listen_reasons) + assert h.runner.model_state.audio_buffer.has_pending() + finally: + await close_harness(h) + + @pytest.mark.asyncio async def test_turn_mode_commit_with_response_create_starts_one_response() -> None: h = await open_harness(auto_response=False) @@ -1033,7 +1254,7 @@ async def test_conversation_items_can_be_injected_and_deleted() -> None: await close_harness(h) -def _stage_metrics_of(event: object) -> dict[str, dict[str, object]]: +def _stage_metrics_of(event: DuplexEvent) -> dict[str, dict[str, object]]: """Per-stage engine metrics as the client reads them off one wire event.""" payload = event.to_realtime() metadata = payload.get("metadata") @@ -1081,6 +1302,40 @@ async def test_stage0_metrics_reach_the_response_even_though_its_output_feeds_tt await close_harness(h) +@pytest.mark.asyncio +async def test_projected_stage1_metrics_reach_the_spoken_audio_event() -> None: + """Thinker text is projected, but its TTFT/TPOT still have to ride the audio event.""" + h = await open_harness(stage_count=3) + try: + plugin = h.runner.model._ctx.plugin + + def project_stage1(*, stage_id: int, output: object, context: object) -> bool: + del output, context + return stage_id == 1 + + plugin.project_intermediate_output = project_stage1 + await h.run(append_audio()) + request_id = h.stage0_request_id() + thinker = SimpleNamespace( + request_id=request_id, + finished=True, + outputs=[SimpleNamespace(text="hello", token_ids=[7, 8], multimodal_output={})], + multimodal_output={}, + ) + forwarded = h.deliver( + thinker, + stage_id=1, + segment_finished=False, + metrics=stage_stats(stage_id=1, request_id=request_id, num_tokens_out=2, itls_ms=[6.0]), + ) + assert forwarded is False + events = await h.deliver_and_settle(tts_output(request_id, samples=24000, text="hello"), stage_id=2) + stage_metrics = _stage_metrics_of(find(events, "response.output_audio.delta")) + assert stage_metrics["1"]["num_tokens_out"] == 2 + finally: + await close_harness(h) + + @pytest.mark.asyncio async def test_stage0_metrics_from_several_units_are_summed_into_one_response() -> None: """Two pass-through segments before the first audio are one response's tokens.""" diff --git a/tests/engine/duplex/test_session_runner_ephemeral_bind.py b/tests/engine/duplex/test_session_runner_ephemeral_bind.py new file mode 100644 index 000000000..cbde18ec3 --- /dev/null +++ b/tests/engine/duplex/test_session_runner_ephemeral_bind.py @@ -0,0 +1,28 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Ephemeral Stage0 bind uses r.stage0-turn{N}, not resumable r.stage0.""" + +from __future__ import annotations + +from vllm_omni.engine.duplex.contracts import ( + DuplexFence, + duplex_ephemeral_stage_request_id, + duplex_resource_request_id, +) +from vllm_omni.engine.duplex.session.manager import DuplexSessionManager + + +def test_stage0_request_id_ephemeral_when_not_resumable() -> None: + fence = DuplexFence("duplex-test", epoch=0, turn_id=3) + rid = DuplexSessionManager.stage_request_id(fence, stage_id=0, resumable=False) + expected = duplex_ephemeral_stage_request_id(fence, stage_id=0) + assert rid == expected + assert rid.endswith(".r.stage0-turn3") + assert not rid.endswith(".r.stage0") + + +def test_stage0_request_id_resumable_keeps_stable_role() -> None: + fence = DuplexFence("duplex-test", epoch=1, turn_id=3) + rid = DuplexSessionManager.stage_request_id(fence, stage_id=0, resumable=True) + expected = duplex_resource_request_id(fence, "stage0") + assert rid == expected + assert rid.endswith(".r.stage0") diff --git a/tests/engine/test_arg_utils.py b/tests/engine/test_arg_utils.py index 7132d7512..142affab7 100644 --- a/tests/engine/test_arg_utils.py +++ b/tests/engine/test_arg_utils.py @@ -22,6 +22,7 @@ from pydantic import ValidationError from transformers import PretrainedConfig, Qwen3OmniMoeConfig from vllm.engine.arg_utils import EngineArgs +from tests.helpers.mock import patch_hf_snapshot_download from vllm_omni.config.model import OmniModelConfig from vllm_omni.engine.arg_utils import OmniEngineArgs from vllm_omni.engine.stage_init_utils import build_engine_args_dict @@ -296,7 +297,7 @@ def test_remote_tokenizer_subfolder_download_does_not_report_failure(tmp_path, m baseline_config = Mock() warning = mocker.patch("vllm_omni.engine.arg_utils.logger.warning") - monkeypatch.setattr("huggingface_hub.HfApi.snapshot_download", lambda *args, **kwargs: str(tmp_path)) + patch_hf_snapshot_download(monkeypatch, lambda *args, **kwargs: str(tmp_path), hf_home=tmp_path) monkeypatch.setattr(OmniEngineArgs, "_patch_empty_hf_config", lambda *args, **kwargs: None) monkeypatch.setattr(EngineArgs, "create_model_config", lambda _self: baseline_config) monkeypatch.setattr( @@ -543,3 +544,40 @@ def test_tensor_parallel_size_none_is_handled(): ) assert isinstance(args, dict) assert "tensor_parallel_size" not in args + + +# For https://github.com/vllm-project/vllm-omni/issues/7564 +def test_from_cli_args_preserves_text_encoder_tp_size(): + """`--text-encoder-tp-size` must survive from_cli_args field filtering. + + Library callers build engine args via ``OmniEngineArgs.from_cli_args``; + the dataclass field filter drops any namespace attribute the dataclass + does not declare, silently resetting the diffusion text-encoder TP to 1. + """ + engine_args = OmniEngineArgs.from_cli_args( + SimpleNamespace(text_encoder_tp_size=2), + ) + assert engine_args.text_encoder_tp_size == 2 + + +# For https://github.com/vllm-project/vllm-omni/issues/7564 +def test_text_encoder_tp_size_reaches_default_diffusion_parallel_config(): + """The preserved CLI value must land in DiffusionParallelConfig. + + Forward the preserved explicit override to the generic diffusion + fallback, which resolves it through + ``DiffusionParallelConfig.from_stage_overrides``. Serializing all engine + defaults would also forward unrelated LLM-only fields to strict diffusion + ingress, unlike the explicit-kwargs library entrypoint. + """ + from vllm_omni.config.config_factory import StageConfigFactory + + engine_args = OmniEngineArgs.from_cli_args( + SimpleNamespace(text_encoder_tp_size=2), + ) + stage_cfg = StageConfigFactory.create_default_diffusion( + {"text_encoder_tp_size": engine_args.text_encoder_tp_size}, + )[0] + + parallel_config = stage_cfg["engine_args"]["parallel_config"] + assert parallel_config["text_encoder_tp_size"] == 2 diff --git a/tests/engine/test_async_omni_engine_stage_init.py b/tests/engine/test_async_omni_engine_stage_init.py index ec897bd7d..a8255f81a 100644 --- a/tests/engine/test_async_omni_engine_stage_init.py +++ b/tests/engine/test_async_omni_engine_stage_init.py @@ -14,6 +14,7 @@ import pytest from omegaconf import OmegaConf from vllm.v1.engine.utils import EngineZmqAddresses +from tests.helpers.mock import patch_hf_snapshot_download from vllm_omni.config.omni_config import OmniStageRuntimeConfig from vllm_omni.diffusion.data import AttentionConfig from vllm_omni.engine import omni_engine_base as async_omni_engine_module @@ -1566,22 +1567,20 @@ def test_build_engine_args_pulls_stage_subdirs_missing_from_cached_snapshot(monk never materialized. Stage init must fetch exactly those subfolders instead of joining onto a path that exists nowhere. """ - from huggingface_hub import HfApi - from vllm_omni.engine.stage_init_utils import build_engine_args_dict snapshot = tmp_path / "snapshots" / "deadbeef" _make_snapshot(snapshot, ["tokenizer"]) calls = [] - def fake_snapshot_download(self, repo_id, **kwargs): + def fake_snapshot_download(repo_id, **kwargs): calls.append(kwargs) if kwargs.get("local_files_only"): return str(snapshot) _make_snapshot(snapshot, ["language_model"]) return str(snapshot) - monkeypatch.setattr(HfApi, "snapshot_download", fake_snapshot_download) + patch_hf_snapshot_download(monkeypatch, fake_snapshot_download, hf_home=tmp_path) stage_cfg = types.SimpleNamespace( stage_id=0, @@ -1601,18 +1600,16 @@ def test_build_engine_args_pulls_stage_subdirs_missing_from_cached_snapshot(monk def test_build_engine_args_skips_hub_call_when_cached_snapshot_is_complete(monkeypatch, tmp_path): """A warm cache stays offline-friendly: no outgoing Hub request.""" - from huggingface_hub import HfApi - from vllm_omni.engine.stage_init_utils import build_engine_args_dict snapshot = tmp_path / "snapshots" / "deadbeef" _make_snapshot(snapshot, ["language_model", "tokenizer"]) - def fake_snapshot_download(self, repo_id, **kwargs): + def fake_snapshot_download(repo_id, **kwargs): assert kwargs.get("local_files_only"), "warm cache must not reach the Hub" return str(snapshot) - monkeypatch.setattr(HfApi, "snapshot_download", fake_snapshot_download) + patch_hf_snapshot_download(monkeypatch, fake_snapshot_download, hf_home=tmp_path) stage_cfg = types.SimpleNamespace( stage_id=0, @@ -1633,8 +1630,6 @@ def test_build_engine_args_redownloads_a_partial_subdir(monkeypatch, tmp_path): that local path leaves vLLM's loader without any fallback for the missing weights. """ - from huggingface_hub import HfApi - from vllm_omni.engine.stage_init_utils import build_engine_args_dict snapshot = tmp_path / "snapshots" / "deadbeef" @@ -1644,14 +1639,14 @@ def test_build_engine_args_redownloads_a_partial_subdir(monkeypatch, tmp_path): _make_snapshot(snapshot, ["tokenizer"]) calls = [] - def fake_snapshot_download(self, repo_id, **kwargs): + def fake_snapshot_download(repo_id, **kwargs): calls.append(kwargs) if kwargs.get("local_files_only"): return str(snapshot) _make_snapshot(snapshot, ["language_model"]) return str(snapshot) - monkeypatch.setattr(HfApi, "snapshot_download", fake_snapshot_download) + patch_hf_snapshot_download(monkeypatch, fake_snapshot_download, hf_home=tmp_path) stage_cfg = types.SimpleNamespace( stage_id=0, @@ -1672,8 +1667,6 @@ def test_build_engine_args_redownloads_when_index_lists_missing_shards(monkeypat The index downloads early; accepting it as proof of weights converts the Hub ID into a local path vLLM cannot fetch the remaining shards for. """ - from huggingface_hub import HfApi - from vllm_omni.engine.stage_init_utils import build_engine_args_dict snapshot = tmp_path / "snapshots" / "deadbeef" @@ -1693,13 +1686,13 @@ def test_build_engine_args_redownloads_when_index_lists_missing_shards(monkeypat _make_snapshot(snapshot, ["tokenizer"]) calls = [] - def fake_snapshot_download(self, repo_id, **kwargs): + def fake_snapshot_download(repo_id, **kwargs): calls.append(kwargs) if not kwargs.get("local_files_only"): (partial / "model-00002-of-00002.safetensors").write_text("x") return str(snapshot) - monkeypatch.setattr(HfApi, "snapshot_download", fake_snapshot_download) + patch_hf_snapshot_download(monkeypatch, fake_snapshot_download, hf_home=tmp_path) stage_cfg = types.SimpleNamespace( stage_id=0, @@ -1716,8 +1709,6 @@ def test_build_engine_args_redownloads_when_index_lists_missing_shards(monkeypat def test_build_engine_args_redownloads_shards_without_their_index(monkeypatch, tmp_path): """Shard-named weights always ship an index; one without it is partial.""" - from huggingface_hub import HfApi - from vllm_omni.engine.stage_init_utils import build_engine_args_dict snapshot = tmp_path / "snapshots" / "deadbeef" @@ -1727,7 +1718,7 @@ def test_build_engine_args_redownloads_shards_without_their_index(monkeypatch, t _make_snapshot(snapshot, ["tokenizer"]) calls = [] - def fake_snapshot_download(self, repo_id, **kwargs): + def fake_snapshot_download(repo_id, **kwargs): calls.append(kwargs) if not kwargs.get("local_files_only"): shards = {f"w{i}.weight": f"model-0000{i}-of-00004.safetensors" for i in range(1, 5)} @@ -1736,7 +1727,7 @@ def test_build_engine_args_redownloads_shards_without_their_index(monkeypatch, t (partial / shard).write_text("x") return str(snapshot) - monkeypatch.setattr(HfApi, "snapshot_download", fake_snapshot_download) + patch_hf_snapshot_download(monkeypatch, fake_snapshot_download, hf_home=tmp_path) stage_cfg = types.SimpleNamespace( stage_id=0, @@ -1752,8 +1743,6 @@ def test_build_engine_args_redownloads_shards_without_their_index(monkeypatch, t def test_build_engine_args_redownloads_a_tokenizer_folder_without_vocabulary(monkeypatch, tmp_path): """Templates and configs download first; alone they are not a tokenizer.""" - from huggingface_hub import HfApi - from vllm_omni.engine.stage_init_utils import build_engine_args_dict snapshot = tmp_path / "snapshots" / "deadbeef" @@ -1763,13 +1752,13 @@ def test_build_engine_args_redownloads_a_tokenizer_folder_without_vocabulary(mon (partial / "chat_template.jinja").write_text("{{ messages }}") calls = [] - def fake_snapshot_download(self, repo_id, **kwargs): + def fake_snapshot_download(repo_id, **kwargs): calls.append(kwargs) if not kwargs.get("local_files_only"): _make_snapshot(snapshot, ["tokenizer"]) return str(snapshot) - monkeypatch.setattr(HfApi, "snapshot_download", fake_snapshot_download) + patch_hf_snapshot_download(monkeypatch, fake_snapshot_download, hf_home=tmp_path) stage_cfg = types.SimpleNamespace( stage_id=0, @@ -1787,19 +1776,17 @@ def test_build_engine_args_redownloads_a_tokenizer_folder_without_vocabulary(mon def test_build_engine_args_forwards_revision_and_download_dir(monkeypatch, tmp_path): """revision/download_dir must shape snapshot selection (they cannot be corrected downstream once the repo ID is a local path).""" - from huggingface_hub import HfApi - from vllm_omni.engine.stage_init_utils import build_engine_args_dict snapshot = tmp_path / "snapshots" / "pinned" _make_snapshot(snapshot, ["language_model", "tokenizer"]) seen = [] - def fake_snapshot_download(self, repo_id, **kwargs): + def fake_snapshot_download(repo_id, **kwargs): seen.append(kwargs) return str(snapshot) - monkeypatch.setattr(HfApi, "snapshot_download", fake_snapshot_download) + patch_hf_snapshot_download(monkeypatch, fake_snapshot_download, hf_home=tmp_path) stage_cfg = types.SimpleNamespace( stage_id=0, @@ -1824,8 +1811,6 @@ def test_build_engine_args_forwards_revision_and_download_dir(monkeypatch, tmp_p def test_build_engine_args_resolves_tokenizer_revision_separately(monkeypatch, tmp_path): """A tokenizer pinned to a different revision resolves against its own snapshot.""" - from huggingface_hub import HfApi - from vllm_omni.engine.stage_init_utils import build_engine_args_dict model_snapshot = tmp_path / "snapshots" / "model-rev" @@ -1833,12 +1818,12 @@ def test_build_engine_args_resolves_tokenizer_revision_separately(monkeypatch, t tokenizer_snapshot = tmp_path / "snapshots" / "tok-rev" _make_snapshot(tokenizer_snapshot, ["tokenizer"]) - def fake_snapshot_download(self, repo_id, **kwargs): + def fake_snapshot_download(repo_id, **kwargs): if kwargs.get("revision") == "tok-rev": return str(tokenizer_snapshot) return str(model_snapshot) - monkeypatch.setattr(HfApi, "snapshot_download", fake_snapshot_download) + patch_hf_snapshot_download(monkeypatch, fake_snapshot_download, hf_home=tmp_path) stage_cfg = types.SimpleNamespace( stage_id=0, @@ -1861,8 +1846,6 @@ def test_build_engine_args_resolves_tokenizer_revision_separately(monkeypatch, t def test_build_engine_args_resolves_root_tokenizer_revision_separately(monkeypatch, tmp_path): """An empty tokenizer_subdir targets the snapshot root and still honors its own revision.""" - from huggingface_hub import HfApi - from vllm_omni.engine.stage_init_utils import build_engine_args_dict model_snapshot = tmp_path / "snapshots" / "model-rev" @@ -1875,12 +1858,12 @@ def test_build_engine_args_resolves_root_tokenizer_revision_separately(monkeypat tokenizer_snapshot.mkdir(parents=True, exist_ok=True) (tokenizer_snapshot / _SUBDIR_ARTIFACT["tokenizer"]).write_text("x") - def fake_snapshot_download(self, repo_id, **kwargs): + def fake_snapshot_download(repo_id, **kwargs): if kwargs.get("revision") == "tok-rev": return str(tokenizer_snapshot) return str(model_snapshot) - monkeypatch.setattr(HfApi, "snapshot_download", fake_snapshot_download) + patch_hf_snapshot_download(monkeypatch, fake_snapshot_download, hf_home=tmp_path) stage_cfg = types.SimpleNamespace( stage_id=0, @@ -1902,19 +1885,17 @@ def test_build_engine_args_resolves_root_tokenizer_revision_separately(monkeypat def test_build_engine_args_fails_closed_when_subdir_cannot_be_downloaded(monkeypatch, tmp_path): """An undownloadable subfolder raises here instead of reaching HuggingFace.""" - from huggingface_hub import HfApi - from vllm_omni.engine.stage_init_utils import build_engine_args_dict snapshot = tmp_path / "snapshots" / "deadbeef" _make_snapshot(snapshot, ["tokenizer"]) - def fake_snapshot_download(self, repo_id, **kwargs): + def fake_snapshot_download(repo_id, **kwargs): if kwargs.get("local_files_only"): return str(snapshot) raise OSError("offline") - monkeypatch.setattr(HfApi, "snapshot_download", fake_snapshot_download) + patch_hf_snapshot_download(monkeypatch, fake_snapshot_download, hf_home=tmp_path) stage_cfg = types.SimpleNamespace( stage_id=0, @@ -1927,16 +1908,14 @@ def test_build_engine_args_fails_closed_when_subdir_cannot_be_downloaded(monkeyp build_engine_args_dict(stage_cfg, "MiniMaxAI/MiniMax-Music3") -def test_build_engine_args_fails_closed_on_cold_cache_instead_of_joining_repo_id(monkeypatch): +def test_build_engine_args_fails_closed_on_cold_cache_instead_of_joining_repo_id(monkeypatch, tmp_path): """With nothing cached, the stage must not join a subdir onto the repo id.""" - from huggingface_hub import HfApi - from vllm_omni.engine.stage_init_utils import build_engine_args_dict - def fake_snapshot_download(self, repo_id, **kwargs): + def fake_snapshot_download(repo_id, **kwargs): raise OSError("offline") - monkeypatch.setattr(HfApi, "snapshot_download", fake_snapshot_download) + patch_hf_snapshot_download(monkeypatch, fake_snapshot_download, hf_home=tmp_path) stage_cfg = types.SimpleNamespace( stage_id=0, diff --git a/tests/engine/test_duplex_orchestrator.py b/tests/engine/test_duplex_orchestrator.py index 703af6a35..1bffda919 100644 --- a/tests/engine/test_duplex_orchestrator.py +++ b/tests/engine/test_duplex_orchestrator.py @@ -55,6 +55,7 @@ def _build( stages: int = 1, running_counter: FakeRunningCounter | None = None, runtime_config: DuplexSessionRuntimeConfig | None = None, + plugin=None, ) -> tuple[DuplexOrchestrator, list[FakeStageClient], asyncio.Queue, asyncio.Queue]: clients = [FakeStageClient(stage_type="llm", final_output=index == stages - 1) for index in range(stages)] pools = _build_stage_pools( @@ -70,7 +71,7 @@ def _build( rpc_async_queue=rpc_q, stage_pools=pools, running_counter=running_counter, - plugin=MiniCPMO45DuplexPlugin(_encode_audio), + plugin=plugin or MiniCPMO45DuplexPlugin(_encode_audio), duplex_session_config=runtime_config or DuplexSessionRuntimeConfig(reaper_interval_s=0.01), model_config=None, ) @@ -197,7 +198,8 @@ async def test_open_preregisters_the_stage0_request_and_close_releases_it() -> N assert request_state.session_owned is True assert request_state.session_id == SESSION_ID assert request_state.fence == DuplexFence(SESSION_ID) - assert request_state.streaming.enabled is True + # Set on submit from submission.resumable, not again at preregister. + assert request_state.streaming.enabled is False # Preregistration reserves the id; nothing is running until an append submits. assert counter.value == 0 assert clients[0].add_request_calls == [] @@ -211,6 +213,18 @@ async def test_open_preregisters_the_stage0_request_and_close_releases_it() -> N assert orchestrator.session_manager.active_count() == 0 +@pytest.mark.asyncio +async def test_aura_ephemeral_preregister_disables_streaming() -> None: + from vllm_omni.model_executor.models.aura_omni.duplex.plugin import AuraDuplexPlugin + + orchestrator, _, rpc_q, _ = _build(plugin=AuraDuplexPlugin(_encode_audio)) + result = await _open(orchestrator, rpc_q) + assert result.ok is True + request_state = next(iter(orchestrator.request_states.values())) + assert request_state.streaming.enabled is False + await orchestrator.session_manager.shutdown() + + @pytest.mark.asyncio async def test_open_failure_rolls_back_session_and_reserved_request() -> None: orchestrator, _, rpc_q, _ = _build() @@ -275,6 +289,7 @@ async def test_append_submits_the_resumable_stage0_request_and_counts_it_running assert submitted.resumable is True assert submitted.sampling_params.max_tokens == 20 request_state = orchestrator.request_states[request_id] + assert request_state.streaming.enabled is True assert request_state.stage_fences[0] == DuplexFence(SESSION_ID) assert 0 in request_state.stage_submit_ts assert counter.value == 1 @@ -390,6 +405,31 @@ async def test_forward_failure_closes_the_owning_session() -> None: assert types[-1] == "session.expired" +@pytest.mark.asyncio +async def test_aura_forward_failure_keeps_the_session() -> None: + from vllm_omni.model_executor.models.aura_omni.duplex.plugin import AuraDuplexPlugin + + orchestrator, clients, rpc_q, output_q = _build(stages=2, plugin=AuraDuplexPlugin(_encode_audio)) + await _open(orchestrator, rpc_q) + request_id = next(iter(orchestrator.request_states)) + await _submit(orchestrator, _append_audio()) + request_state = orchestrator.request_states[request_id] + session = orchestrator.session_manager.get(SESSION_ID) + assert session is not None + + absorbed = await orchestrator._handle_forward_failure(request_id, 1, request_state, ValueError("stale talker")) + await _settle(orchestrator) + + assert absorbed is True + assert request_id not in orchestrator.request_states + assert SESSION_ID in orchestrator.session_manager.runners + assert session.state != DuplexSessionState.CLOSED + types = [message.event.type for message in [output_q.get_nowait() for _ in range(output_q.qsize())]] + assert "error" in types + assert "session.expired" not in types + await orchestrator.session_manager.shutdown() + + @pytest.mark.asyncio async def test_request_cleanup_failure_is_deferred_and_retried_by_the_reaper(monkeypatch) -> None: orchestrator, clients, rpc_q, _ = _build() @@ -560,3 +600,53 @@ async def test_turn_plugin_processes_multimodal_prompt_before_stage_submission(m assert not orchestrator.request_states[submitted.request_id].streaming.enabled finally: await _close(orchestrator, rpc_q) + + +@pytest.mark.asyncio +async def test_sentence_partial_does_not_legacy_forward_the_full_stage_output() -> None: + from vllm_omni.engine.duplex.plugin import PartialStageForward + + orchestrator, *_ = _build(stages=4) + forwarded: list[object] = [] + + async def record_forward(req_id, stage_id, output, req_state, **kwargs): + del req_id, stage_id, req_state, kwargs + forwarded.append(output) + + async def no_intercept(*args, **kwargs): + del args, kwargs + return False + + orchestrator._forward_to_next_stage = record_forward # type: ignore[method-assign] + orchestrator._intercept_stage_output = no_intercept # type: ignore[method-assign] + orchestrator.async_chunk = True + orchestrator._stage_receives_async_chunks = lambda stage_id: False # type: ignore[method-assign] + + full = SimpleNamespace(request_id="r", finished=True, text="FULL") + chunk = SimpleNamespace(request_id="r", finished=True, text="CHUNK") + req_state = DuplexOrchestratorRequestState( + request_id="r", + final_stage_id=3, + session_owned=True, + sampling_params_list=[SimpleNamespace() for _ in range(4)], + ) + orchestrator.plugin.plan_partial_stage_output = lambda *args, **kwargs: PartialStageForward( # type: ignore[method-assign] + output=chunk, is_final_update=True + ) + orchestrator.plugin.partial_stage_followup = lambda *args, **kwargs: None # type: ignore[method-assign] + + await orchestrator._route_output(1, 0, full, req_state, None) + + assert forwarded == [chunk] + assert req_state.skip_legacy_stage_forward is False + + forwarded.clear() + orchestrator.plugin.plan_partial_stage_output = lambda *args, **kwargs: None # type: ignore[method-assign] + await orchestrator._route_output(1, 0, full, req_state, None) + + assert forwarded == [full] + + +def test_default_plugin_declares_no_draining_stages() -> None: + plugin = MiniCPMO45DuplexPlugin(_encode_audio) + assert plugin.draining_stage_ids(stage_count=4) == frozenset() diff --git a/tests/engine/test_orchestrator_error_handling.py b/tests/engine/test_orchestrator_error_handling.py index b8949786e..21a26279d 100644 --- a/tests/engine/test_orchestrator_error_handling.py +++ b/tests/engine/test_orchestrator_error_handling.py @@ -13,6 +13,7 @@ import asyncio import queue import time from types import SimpleNamespace +from unittest.mock import AsyncMock import janus import pytest @@ -22,6 +23,7 @@ from vllm.v1.serial_utils import MsgpackEncoder from vllm_omni.engine.messages import ( AddCompanionRequestMessage, + CollectiveRPCRequestMessage, EngineQueueMessage, ErrorMessage, ShutdownRequestMessage, @@ -1143,3 +1145,64 @@ async def test_diffusion_client_error_output_propagates_non_400_status( finally: orchestrator_fixture.request_sync_q.put_nowait(ShutdownRequestMessage()) orchestrator_fixture.thread.join(timeout=5) + + +@pytest.mark.asyncio +async def test_control_rpc_failure_is_reported_to_the_caller_not_fatal() -> None: + """A control RPC that fails on one replica must come back as that + replica's error result instead of raising out of _request_handler. + """ + + class _FailingClient(FakeStageClient): + async def collective_rpc_async(self, *args, **kwargs): + raise TimeoutError("outputs did not drain") + + ok = FakeStageClient(stage_type="diffusion") + ok.collective_rpc_async = AsyncMock(return_value=None) + orchestrator, queues = _build_bare_orchestrator( + _build_stage_pools([[ok], [_FailingClient(stage_type="diffusion")]]) + ) + try: + await orchestrator._handle_collective_rpc( + CollectiveRPCRequestMessage( + rpc_id="rpc-1", + method="pause_scheduler", + args=(), + kwargs={"mode": "keep"}, + stage_ids=[0, 1], + ) + ) + + result = queues[2].async_q.get_nowait() + assert result.rpc_id == "rpc-1" + assert result.stage_ids == [0, 1] + assert result.results[0] is None + assert result.results[1]["supported"] is False + assert "outputs did not drain" in result.results[1]["error"] + finally: + for q in queues: + q.close() + + +@pytest.mark.asyncio +async def test_rpc_failure_capture_is_limited_to_pause_and_resume() -> None: + """Only pause/resume failures are reported as a result; every other RPC + keeps propagating the way it does today. + """ + orchestrator, queues = _build_bare_orchestrator(_build_stage_pools([[FakeStageClient(stage_type="diffusion")]])) + orchestrator.stage_pools[0].collective_rpc = AsyncMock(side_effect=TimeoutError("worker died")) + + def _msg(rpc_id: str, method: str) -> CollectiveRPCRequestMessage: + return CollectiveRPCRequestMessage(rpc_id=rpc_id, method=method, args=(), kwargs={}, stage_ids=[0]) + + try: + await orchestrator._handle_collective_rpc(_msg("rpc-resume", "resume_scheduler")) + result = queues[2].async_q.get_nowait() + assert result.results[0]["supported"] is False + assert "worker died" in result.results[0]["error"] + + with pytest.raises(TimeoutError): + await orchestrator._handle_collective_rpc(_msg("rpc-sleep", "sleep")) + finally: + for q in queues: + q.close() diff --git a/tests/engine/test_orchestrator_event_driven.py b/tests/engine/test_orchestrator_event_driven.py index 284dde55d..109721446 100644 --- a/tests/engine/test_orchestrator_event_driven.py +++ b/tests/engine/test_orchestrator_event_driven.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project """Event-driven orchestration loop (``VLLM_OMNI_EVENT_DRIVEN_ORCH=1``) tests. Parity suite: re-runs the legacy orchestration scenarios from @@ -19,7 +19,10 @@ import pytest from vllm_omni.engine.async_omni_engine import AsyncOmniEngine from vllm_omni.engine.messages import ShutdownRequestMessage -from vllm_omni.engine.orchestrator import _event_driven_orch_enabled +from vllm_omni.engine.orchestrator import ( + _event_driven_orch_default_for_pipeline, + _event_driven_orch_enabled, +) from . import test_orchestrator as legacy from . import test_orchestrator_error_handling as legacy_errors @@ -125,6 +128,19 @@ def test_flag_parsing(monkeypatch) -> None: assert _event_driven_orch_enabled() is False +def test_only_qwen3_tts_has_a_pipeline_default() -> None: + assert _event_driven_orch_default_for_pipeline("qwen3_tts") is True + for model_type in (None, "qwen3_omni_moe", "minicpmo_4_5", "moss_tts_delay"): + assert _event_driven_orch_default_for_pipeline(model_type) is False + + +def test_explicit_event_driven_value_overrides_pipeline_default(monkeypatch) -> None: + monkeypatch.delenv("VLLM_OMNI_EVENT_DRIVEN_ORCH", raising=False) + assert _event_driven_orch_enabled(default=True) is True + monkeypatch.setenv("VLLM_OMNI_EVENT_DRIVEN_ORCH", "0") + assert _event_driven_orch_enabled(default=True) is False + + def test_default_is_legacy_loop(monkeypatch) -> None: """Without the env flag, the harness runs the legacy poll loop.""" monkeypatch.delenv("VLLM_OMNI_EVENT_DRIVEN_ORCH", raising=False) diff --git a/tests/engine/test_orchestrator_monitor.py b/tests/engine/test_orchestrator_monitor.py index ee9126cf4..7ac3f3958 100644 --- a/tests/engine/test_orchestrator_monitor.py +++ b/tests/engine/test_orchestrator_monitor.py @@ -1,3 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + import json import time @@ -45,6 +48,8 @@ def test_note_loop_and_flush_json(tmp_path, monkeypatch): monitor.register_replica(0, 0) monitor.note_loop(idle=False) + monitor.set_dispatch_queue_size(9) + monitor.set_dispatch_queue_size(7) monitor.note_loop(idle=True) now[0] = 1.5 @@ -54,10 +59,37 @@ def test_note_loop_and_flush_json(tmp_path, monkeypatch): assert payload["configured_window_s"] == monitor_mod._WINDOW_S assert payload["windows"]["loop_idle"] == [1] assert payload["windows"]["loop_active"] == [1] + assert payload["windows"]["dispatch_queue_size"] == [7] + assert payload["windows"]["dispatch_queue_high_water"] == [9] assert payload["replicas"]["stage=0,replica=0"]["outputs_queue_size"] == [5] assert payload["replicas"]["stage=0,replica=0"]["inflight"] == [10] +@pytest.mark.parametrize("drain", [False, True]) +def test_dispatch_backlog_carries_across_windows(tmp_path, monkeypatch, drain): + out_path = tmp_path / "orch_monitor.json" + monkeypatch.setenv("VLLM_OMNI_ORCH_MONITOR_PATH", str(out_path)) + now = [0.0] + monkeypatch.setattr(time, "monotonic", lambda: now[0]) + monitor = OrchestratorMonitor(replica_sampler=lambda: {}) + monitor.set_dispatch_queue_size(3) + monitor.set_dispatch_queue_size(1) + + now[0] = monitor_mod._WINDOW_S + monitor.note_loop(idle=False) + if drain: + monitor.set_dispatch_queue_size(0) + now[0] += monitor_mod._WINDOW_S + monitor.note_loop(idle=False) + now[0] += monitor_mod._WINDOW_S + monitor.flush() + + windows = json.loads(out_path.read_text())["windows"] + remaining = 0 if drain else 1 + assert windows["dispatch_queue_size"] == [1, remaining, remaining] + assert windows["dispatch_queue_high_water"] == [3, 1, remaining] + + def test_register_replica_backfills_prior_windows(tmp_path, monkeypatch): out_path = tmp_path / "orch_monitor.json" monkeypatch.setenv("VLLM_OMNI_ORCH_MONITOR_PATH", str(out_path)) diff --git a/tests/engine/test_orchestrator_stage_input_bridge.py b/tests/engine/test_orchestrator_stage_input_bridge.py index 842824c2d..3c1dd6d2b 100644 --- a/tests/engine/test_orchestrator_stage_input_bridge.py +++ b/tests/engine/test_orchestrator_stage_input_bridge.py @@ -14,6 +14,7 @@ import pytest from vllm.outputs import CompletionOutput, RequestOutput from vllm.sampling_params import SamplingParams +from vllm_omni.engine.duplex_orchestrator import DuplexOrchestrator from vllm_omni.engine.orchestrator import ( Orchestrator, OrchestratorRequestState, @@ -249,6 +250,41 @@ async def test_async_prewarm_skips_outgoing_only_stage(payload_sender_info) -> N ) +@pytest.mark.asyncio +async def test_async_prewarm_skips_stage_with_custom_process_input_func() -> None: + """AURA Stage1 has asr2aura: must not be zero-prewarmed under async_chunk. + + The custom-process-input gate lives on DuplexOrchestrator (AURA path), not + the turn-based Orchestrator base. + """ + orchestrator = object.__new__(DuplexOrchestrator) + stage0 = FakePrewarmPool("sender") + stage1 = FakePrewarmPool("receiver") # would receive chunks if role alone decided + stage1.stage_client = SimpleNamespace(custom_process_input_func=lambda *a, **k: None) + stage2 = FakePrewarmPool("receiver") + orchestrator.stage_pools = [stage0, stage1, stage2] + orchestrator._emit_tx_edge = lambda **_kwargs: None + orchestrator._on_stage_submitted = MagicMock() + req_state = OrchestratorRequestState( + request_id="req-prewarm-custom", + prompt={"prompt_token_ids": [1, 2]}, + sampling_params_list=[SamplingParams(max_tokens=1) for _ in range(3)], + final_stage_id=2, + ) + + prewarmed = await orchestrator._prewarm_async_chunk_stages( + "req-prewarm-custom", + SimpleNamespace(prompt_token_ids=[1, 2], resumable=True), + req_state, + ) + + assert prewarmed is True + assert stage1.submitted == [] + assert len(stage2.submitted) == 1 + assert 1 not in req_state.stage_submit_ts + assert 2 in req_state.stage_submit_ts + + @pytest.mark.asyncio async def test_async_route_forwards_to_outgoing_only_stage() -> None: orchestrator = object.__new__(Orchestrator) diff --git a/tests/engine/test_single_stage_mode.py b/tests/engine/test_single_stage_mode.py index 2e01e910b..0d5e47960 100644 --- a/tests/engine/test_single_stage_mode.py +++ b/tests/engine/test_single_stage_mode.py @@ -17,7 +17,12 @@ from vllm.v1.engine.utils import EngineZmqAddresses from vllm_omni.config.config_factory import StageConfigFactory from vllm_omni.config.omni_config import VllmOmniARStageConfig, VllmOmniDiffusionStageConfig -from vllm_omni.config.stage_config import DuplexSessionRuntimeConfig, StageExecutionType, StagePipelineConfig +from vllm_omni.config.stage_config import ( + DuplexSessionRuntimeConfig, + PipelineConfig, + StageExecutionType, + StagePipelineConfig, +) from vllm_omni.engine.async_omni_engine import AsyncOmniEngine from vllm_omni.engine.stage_engine_core_client import StageEngineCoreClientBase from vllm_omni.engine.stage_engine_startup import ( @@ -393,6 +398,7 @@ class TestSingleStageModeDetection: stage_cfgs: list[Any] | None = None, resolved_config_path: str = "/fake/path", patch_deploy_config: bool = True, + resolved_pipeline: PipelineConfig | None = None, **kwargs: Any, ) -> AsyncOmniEngine: mock_stage_configs = stage_cfgs or [_make_stage_cfg(0)] @@ -407,10 +413,19 @@ class TestSingleStageModeDetection: "vllm_omni.engine.omni_engine_base.load_deploy_config", return_value=SimpleNamespace(duplex_session=DuplexSessionRuntimeConfig()), ) + + def resolve_stage_configs(engine, *args, **kwargs): + if resolved_pipeline is not None: + engine._config_resolution = SimpleNamespace( + pipeline_config=resolved_pipeline, config_path=resolved_config_path + ) + return resolved_config_path, mock_stage_configs + mocker.patch.object( AsyncOmniEngine, "_resolve_stage_configs", - return_value=(resolved_config_path, mock_stage_configs), + autospec=True, + side_effect=resolve_stage_configs, ) mocker.patch.object(AsyncOmniEngine, "_bootstrap_orchestrator") mock_thread_cls = mocker.patch("threading.Thread") @@ -435,6 +450,19 @@ class TestSingleStageModeDetection: ) assert engine.single_stage_mode is True + @pytest.mark.parametrize("model_type, expected", [("qwen3_tts", True), ("qwen3_omni_moe", False)]) + def test_event_driven_default_uses_resolved_pipeline(self, mocker, model_type, expected): + pipeline = PipelineConfig( + model_type=model_type, + stages=(StagePipelineConfig(stage_id=0, model_stage="a", final_output=True),), + ) + engine = self._make_engine_no_thread(mocker, resolved_pipeline=pipeline) + + StageConfigFactory.get_pipeline_config.assert_called_once_with( + model="fake-model", trust_remote_code=False, deploy_config_path=None + ) + assert engine._event_driven_orch_default is expected + def test_stage_id_kwarg_promotes_to_single_stage_mode(self, mocker: MockerFixture): engine = self._make_engine_no_thread( mocker, diff --git a/tests/entrypoints/duplex/test_startup_warmup.py b/tests/entrypoints/duplex/test_startup_warmup.py new file mode 100644 index 000000000..0971e317d --- /dev/null +++ b/tests/entrypoints/duplex/test_startup_warmup.py @@ -0,0 +1,56 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + +import pybase64 as base64 + +from vllm_omni.engine.duplex.config import DuplexCapabilities +from vllm_omni.entrypoints.duplex.warmup import ( + _WARMUP_AUDIO_SECONDS, + _WARMUP_FRAME_HEIGHT, + _WARMUP_FRAME_WIDTH, + _WARMUP_SAMPLE_RATE_HZ, + _short_pcm_f32le_b64, + _warmup_jpeg_b64, + startup_warmup_kind, +) + + +class _Plugin: + def __init__(self, required: frozenset[str]) -> None: + self._required = required + + def capabilities(self, *, max_sessions: int) -> DuplexCapabilities: + del max_sessions + return DuplexCapabilities(required_input_modalities=self._required) + + +def test_video_required_model_warms_up_one_turn_by_default() -> None: + plugin = _Plugin(frozenset({"video"})) + assert startup_warmup_kind(plugin, 0) == "video_turn" + + +def test_negative_warmup_frames_disables_the_video_turn() -> None: + plugin = _Plugin(frozenset({"video"})) + assert startup_warmup_kind(plugin, -1) is None + + +def test_audio_primary_model_stays_opt_in() -> None: + plugin = _Plugin(frozenset({"audio"})) + assert startup_warmup_kind(plugin, 0) is None + assert startup_warmup_kind(plugin, 4) == "silent_frames" + + +def test_startup_turn_audio_is_short_and_not_silent() -> None: + raw = base64.b64decode(_short_pcm_f32le_b64()) + assert len(raw) == int(_WARMUP_SAMPLE_RATE_HZ * _WARMUP_AUDIO_SECONDS) * 4 + assert any(raw) + + +def test_startup_turn_frame_is_a_camera_sized_jpeg() -> None: + from io import BytesIO + + from PIL import Image + + image = Image.open(BytesIO(base64.b64decode(_warmup_jpeg_b64()))) + assert image.size == (_WARMUP_FRAME_WIDTH, _WARMUP_FRAME_HEIGHT) + assert image.format == "JPEG" diff --git a/tests/entrypoints/openai_api/test_image_server.py b/tests/entrypoints/openai_api/test_image_server.py index 2f23d7c8d..3b9314edf 100644 --- a/tests/entrypoints/openai_api/test_image_server.py +++ b/tests/entrypoints/openai_api/test_image_server.py @@ -1719,7 +1719,7 @@ def test_image_edit_parameter_pass(async_omni_test_client): img = Image.open(io.BytesIO(img_bytes)) assert img.format.lower() == "jpeg" assert data["output_format"] == "jpeg" - assert data["size"] == f"{img.width}x{img.height}" + assert data["size"] == "64x64" def test_image_edit_layers_and_resolution(async_omni_test_client): @@ -1896,6 +1896,7 @@ def test_image_edit_parameter_default_single_stage(test_client): assert captured_sampling_params.width == 24 assert captured_sampling_params.height == 16 + assert (captured_sampling_params.height_not_provided, captured_sampling_params.width_not_provided) == (True, True) assert captured_sampling_params.num_outputs_per_prompt == 1 assert captured_sampling_params.num_inference_steps == 4 assert captured_sampling_params.guidance_scale == 7.5 @@ -1913,6 +1914,40 @@ def test_image_edit_parameter_default_single_stage(test_client): assert response.status_code == 400 +def test_image_edit_explicit_size_marks_canvas_provided_single_stage(test_client): + response = test_client.post( + "/v1/images/edits", + files=[("image", make_test_image_bytes((24, 16)))], + data={"prompt": "hello world.", "size": "16x24"}, + ) + + assert response.status_code == 200 + sampling = test_client.app.state.engine_client.captured_sampling_params_list[0] + assert (sampling.height, sampling.width) == (24, 16) + assert (sampling.height_not_provided, sampling.width_not_provided) == (False, False) + + +def test_image_edit_response_size_reports_generated_image_single_stage(test_client): + response = test_client.post( + "/v1/images/edits", + files=[("image", make_test_image_bytes((24, 16)))], + data={"prompt": "hello world.", "size": "16x24"}, + ) + + assert response.status_code == 200 + assert response.json()["size"] == "64x64" + + +def test_generate_images_response_size_reports_generated_image(test_client): + response = test_client.post( + "/v1/images/generations", + json={"prompt": "a cat", "n": 1, "size": "1024x1024"}, + ) + + assert response.status_code == 200 + assert response.json()["size"] == "64x64" + + def test_image_edit_compression_jpeg(test_client): img_bytes_1 = make_test_image_bytes((16, 16)) # uploadfile with image key diff --git a/tests/entrypoints/openai_api/test_serving_speech.py b/tests/entrypoints/openai_api/test_serving_speech.py index 146488aa2..9a2f9e1fc 100644 --- a/tests/entrypoints/openai_api/test_serving_speech.py +++ b/tests/entrypoints/openai_api/test_serving_speech.py @@ -25,6 +25,7 @@ from fastapi.responses import JSONResponse from fastapi.testclient import TestClient from pydantic import ValidationError from pytest_mock import MockerFixture +from vllm.entrypoints.serve import create_error_response from vllm.entrypoints.serve.engine.protocol import ErrorInfo, ErrorResponse from vllm_omni.config.stage_config import StagePipelineConfig @@ -33,7 +34,6 @@ from vllm_omni.diffusion.sched.request_scheduler import RequestScheduler from vllm_omni.diffusion.sched.step_scheduler import StepScheduler from vllm_omni.entrypoints.omni_base import OmniEngineDeadError from vllm_omni.entrypoints.openai import api_server as api_server_module -from vllm_omni.entrypoints.openai import errors as openai_errors from vllm_omni.entrypoints.openai import serving_speech as serving_speech_module from vllm_omni.entrypoints.openai.audio_utils_mixin import AudioMixin from vllm_omni.entrypoints.openai.protocol.audio import ( @@ -3875,6 +3875,7 @@ def _make_api_server_request(handler, *, method: str = "POST", path: str = "/v1/ app = FastAPI() app.state.openai_serving_speech = handler app.state.api_server_count = 1 + app.state.serving_tokenization = None scope = { "type": "http", "app": app, @@ -3889,23 +3890,6 @@ def _make_api_server_request(handler, *, method: str = "POST", path: str = "/v1/ return Request(scope) -def _patch_api_server_base(mocker: MockerFixture): - def _fake_create_error_response(message, err_type="BadRequestError", status_code=400, param=None): - return ErrorResponse( - error=ErrorInfo( - message=message, - type=err_type, - param=param, - code=getattr(status_code, "value", status_code), - ) - ) - - fake_base = mocker.MagicMock() - fake_base.create_error_response.side_effect = _fake_create_error_response - mocker.patch.object(openai_errors, "base", return_value=fake_base) - return fake_base - - def _assert_openai_error_response( response: JSONResponse, *, @@ -3921,8 +3905,7 @@ def _assert_openai_error_response( assert message in body["error"]["message"] -def test_api_server_list_voices_without_speech_handler_returns_404(mocker: MockerFixture): - _patch_api_server_base(mocker) +def test_api_server_list_voices_without_speech_handler_returns_404(): raw_request = _make_api_server_request(None, method="GET") response = asyncio.run(api_server_module.list_voices(raw_request)) @@ -3933,7 +3916,6 @@ def test_api_server_list_voices_without_speech_handler_returns_404(mocker: Mocke def test_api_server_upload_voice_value_error_returns_400(mocker: MockerFixture): - _patch_api_server_base(mocker) handler = mocker.MagicMock() handler.upload_voice = mocker.AsyncMock(side_effect=ValueError("Unsupported MIME type: audio/x-m4a")) raw_request = _make_api_server_request(handler) @@ -3951,8 +3933,7 @@ def test_api_server_upload_voice_value_error_returns_400(mocker: MockerFixture): _assert_openai_error_response(response, status_code=400, message="Unsupported MIME type") -def test_api_server_upload_voice_without_speech_handler_returns_404(mocker: MockerFixture): - _patch_api_server_base(mocker) +def test_api_server_upload_voice_without_speech_handler_returns_404(): raw_request = _make_api_server_request(None) response = asyncio.run( @@ -3968,8 +3949,50 @@ def test_api_server_upload_voice_without_speech_handler_returns_404(mocker: Mock ) +@pytest.mark.parametrize("has_speech_handler", [True, False]) +@pytest.mark.parametrize("method", ["GET", "POST", "DELETE"]) +def test_voice_routes_without_tokenization(mocker: MockerFixture, method: str, has_speech_handler: bool): + handler = mocker.MagicMock() if has_speech_handler else None + if handler is not None: + handler._get_available_voices.return_value = [] + handler.uploaded_speakers = {} + handler.delete_voice = mocker.AsyncMock(return_value=False) + app = _make_api_server_request(handler).app + app.add_api_route("/v1/audio/voices", api_server_module.list_voices, methods=["GET"]) + app.add_api_route("/v1/audio/voices", api_server_module.upload_voice, methods=["POST"]) + app.add_api_route("/v1/audio/voices/{name}", api_server_module.delete_voice, methods=["DELETE"]) + + with TestClient(app) as client: + if method == "POST": + response = client.post( + "/v1/audio/voices", + files={"consent": (None, "cons_test"), "name": (None, "probe")}, + ) + elif method == "DELETE": + response = client.delete("/v1/audio/voices/missing") + else: + response = client.get("/v1/audio/voices") + + if not has_speech_handler: + status_code, err_type, message = 404, "NotFoundError", "The model does not support Speech API" + elif method == "POST": + status_code, err_type, message = ( + 400, + "BadRequestError", + "Either 'audio_sample' or 'speaker_embedding' must be provided", + ) + elif method == "DELETE": + status_code, err_type, message = 404, "NotFoundError", "Voice 'missing' not found" + else: + assert response.status_code == 200 + assert response.json() == {"voices": [], "uploaded_voices": []} + return + + assert response.status_code == status_code + assert response.json() == {"error": {"message": message, "type": err_type, "param": None, "code": status_code}} + + def test_api_server_upload_voice_without_input_returns_400(mocker: MockerFixture): - _patch_api_server_base(mocker) raw_request = _make_api_server_request(mocker.MagicMock()) response = asyncio.run( @@ -3986,7 +4009,6 @@ def test_api_server_upload_voice_without_input_returns_400(mocker: MockerFixture def test_api_server_upload_voice_with_audio_and_embedding_returns_400(mocker: MockerFixture): - _patch_api_server_base(mocker) raw_request = _make_api_server_request(mocker.MagicMock()) response = asyncio.run( @@ -4003,7 +4025,6 @@ def test_api_server_upload_voice_with_audio_and_embedding_returns_400(mocker: Mo def test_api_server_upload_voice_exception_returns_500(mocker: MockerFixture): - _patch_api_server_base(mocker) handler = mocker.MagicMock() handler.upload_voice = mocker.AsyncMock(side_effect=RuntimeError("disk failed")) raw_request = _make_api_server_request(handler) @@ -4032,7 +4053,6 @@ def test_api_server_upload_voice_with_no_adapter( tmp_path: Path, ): monkeypatch.setenv("SPEAKER_SAMPLES_DIR", str(tmp_path)) - _patch_api_server_base(mocker) engine_client = mocker.MagicMock() engine_client.errored = False engine_client.stage_configs = [] @@ -4059,8 +4079,7 @@ def test_api_server_upload_voice_with_no_adapter( assert response.status_code == 200 -def test_api_server_delete_voice_without_speech_handler_returns_404(mocker: MockerFixture): - _patch_api_server_base(mocker) +def test_api_server_delete_voice_without_speech_handler_returns_404(): raw_request = _make_api_server_request(None, method="DELETE", path="/v1/audio/voices/probe") response = asyncio.run(api_server_module.delete_voice("probe", raw_request)) @@ -4071,7 +4090,6 @@ def test_api_server_delete_voice_without_speech_handler_returns_404(mocker: Mock def test_api_server_delete_voice_value_error_returns_400(mocker: MockerFixture): - _patch_api_server_base(mocker) handler = mocker.MagicMock() handler.delete_voice = mocker.AsyncMock(side_effect=ValueError("Invalid voice name")) raw_request = _make_api_server_request(handler, method="DELETE", path="/v1/audio/voices/probe") @@ -4082,7 +4100,6 @@ def test_api_server_delete_voice_value_error_returns_400(mocker: MockerFixture): def test_api_server_delete_voice_not_found_returns_404(mocker: MockerFixture): - _patch_api_server_base(mocker) handler = mocker.MagicMock() handler.delete_voice = mocker.AsyncMock(return_value=False) raw_request = _make_api_server_request(handler, method="DELETE", path="/v1/audio/voices/missing") @@ -4098,7 +4115,6 @@ def test_api_server_delete_voice_not_found_returns_404(mocker: MockerFixture): def test_api_server_delete_voice_exception_returns_500(mocker: MockerFixture): - _patch_api_server_base(mocker) handler = mocker.MagicMock() handler.delete_voice = mocker.AsyncMock(side_effect=RuntimeError("disk failed")) raw_request = _make_api_server_request(handler, method="DELETE", path="/v1/audio/voices/probe") @@ -4113,10 +4129,9 @@ def test_api_server_delete_voice_exception_returns_500(mocker: MockerFixture): ) -def test_api_server_create_speech_without_handler_returns_404(mocker: MockerFixture): - fake_base = _patch_api_server_base(mocker) +def test_api_server_create_speech_without_handler_returns_404(): raw_request = _make_api_server_request(None, path="/v1/audio/speech") - raw_request.app.state.serving_tokenization = fake_base + raw_request.app.state.serving_tokenization = SimpleNamespace(create_error_response=create_error_response) request = OpenAICreateSpeechRequest(input="Hello") response = asyncio.run(api_server_module.create_speech(request, raw_request)) @@ -4126,10 +4141,9 @@ def test_api_server_create_speech_without_handler_returns_404(mocker: MockerFixt ) -def test_api_server_create_speech_batch_without_handler_returns_404(mocker: MockerFixture): - fake_base = _patch_api_server_base(mocker) +def test_api_server_create_speech_batch_without_handler_returns_404(): raw_request = _make_api_server_request(None, path="/v1/audio/speech/batch") - raw_request.app.state.serving_tokenization = fake_base + raw_request.app.state.serving_tokenization = SimpleNamespace(create_error_response=create_error_response) request = BatchSpeechRequest(items=[SpeechBatchItem(input="hi")]) response = asyncio.run(api_server_module.create_speech_batch(request, raw_request)) @@ -4194,11 +4208,10 @@ def test_api_server_create_speech_batch_omits_null_fields(mocker: MockerFixture) assert errored["error"] == "Input text cannot be empty" -def test_api_server_create_audio_generate_without_handler_returns_404(mocker: MockerFixture): - fake_base = _patch_api_server_base(mocker) +def test_api_server_create_audio_generate_without_handler_returns_404(): raw_request = _make_api_server_request(None, path="/v1/audio/generate") raw_request.app.state.openai_serving_audio_generate = None - raw_request.app.state.serving_tokenization = fake_base + raw_request.app.state.serving_tokenization = SimpleNamespace(create_error_response=create_error_response) request = OpenAICreateAudioGenerateRequest(input="a bird singing") response = asyncio.run(api_server_module.create_audio_generate(request, raw_request)) diff --git a/tests/entrypoints/test_async_omni_pause_sleep_routing.py b/tests/entrypoints/test_async_omni_pause_sleep_routing.py index 38ec40163..d3a60e5a6 100644 --- a/tests/entrypoints/test_async_omni_pause_sleep_routing.py +++ b/tests/entrypoints/test_async_omni_pause_sleep_routing.py @@ -25,6 +25,7 @@ def _make_omni(*, stage_types: list[str]) -> AsyncOmni: omni._paused = False omni._admitting = 0 omni._hold_admission_until_resume = False + omni._paused_stage_ids = set() omni._sleeping_tags = set() omni._stage_sleeping_tags = {} omni._level2_sleeping = False @@ -460,3 +461,225 @@ def test_partial_wake_does_not_skip_remaining_sleeping_stage(): assert not omni._stage_sleeping_tags asyncio.run(run()) + + +# ───────────────────── diffusion pause_generation(mode="keep") ───────────────────── + + +def _rpc_methods(omni: AsyncOmni) -> list[tuple[str, list[int] | None]]: + return [(call.kwargs["method"], call.kwargs.get("stage_ids")) for call in omni.collective_rpc.await_args_list] + + +@pytest.mark.cpu +def test_diffusion_keep_routes_to_backend(): + async def run() -> None: + omni = _make_omni(stage_types=["diffusion"]) + + await omni.pause_generation(mode="keep", clear_cache=False) + + omni.collective_rpc.assert_awaited_once_with( + method="pause_scheduler", + args=(), + kwargs={"mode": "keep"}, + stage_ids=[0], + ) + assert await omni.is_paused() is True + assert omni._hold_admission_until_resume is True + + asyncio.run(run()) + + +@pytest.mark.cpu +@pytest.mark.parametrize( + "pause_kwargs", + [ + {}, + {"mode": "abort"}, + {"mode": "wait"}, + {"mode": "keep", "wait_for_inflight_requests": True}, + ], +) +def test_diffusion_non_keep_modes_pause_frontend_only(pause_kwargs): + async def run() -> None: + omni = _make_omni(stage_types=["diffusion"]) + omni.reset_prefix_cache = AsyncMock(return_value=True) + omni.reset_mm_cache = AsyncMock() + omni.reset_encoder_cache = AsyncMock() + + await omni.pause_generation(**pause_kwargs) + assert await omni.is_paused() is True + omni.collective_rpc.assert_not_awaited() + + await omni.resume_generation() + omni.collective_rpc.assert_not_awaited() + assert await omni.is_paused() is False + + asyncio.run(run()) + + +@pytest.mark.cpu +def test_mixed_engine_keep_sends_engine_core_and_diffusion_pauses(): + async def run() -> None: + omni = _make_omni(stage_types=["llm", "diffusion"]) + omni.reset_prefix_cache = AsyncMock(return_value=True) + omni.reset_mm_cache = AsyncMock() + omni.reset_encoder_cache = AsyncMock() + + await omni.pause_generation(mode="keep", clear_cache=True) + + assert omni.collective_rpc.await_args_list[0].kwargs == { + "method": "pause_scheduler", + "args": (), + "kwargs": {"mode": "keep", "clear_cache": True}, + "stage_ids": [0], + } + assert omni.collective_rpc.await_args_list[1].kwargs == { + "method": "pause_scheduler", + "args": (), + "kwargs": {"mode": "keep"}, + "stage_ids": [1], + } + # The frontend cache reset still runs after the backend pauses. + omni.reset_prefix_cache.assert_awaited_once() + + await omni.resume_generation() + assert _rpc_methods(omni)[2:] == [("resume_scheduler", [0]), ("resume_scheduler", [1])] + + asyncio.run(run()) + + +@pytest.mark.cpu +def test_keep_targets_only_requested_diffusion_stages(): + async def run() -> None: + omni = _make_omni(stage_types=["diffusion", "diffusion"]) + + await omni.pause_generation(mode="keep", clear_cache=False, stage_ids=[1]) + assert _rpc_methods(omni) == [("pause_scheduler", [1])] + + await omni.resume_generation(stage_ids=[0]) + assert _rpc_methods(omni) == [("pause_scheduler", [1])] + # Stage 1's scheduler is still closed: admission must stay closed too, + # otherwise new requests queue on a stage that will not schedule them. + assert await omni.is_paused() is True + + await omni.resume_generation(stage_ids=[1]) + assert _rpc_methods(omni) == [("pause_scheduler", [1]), ("resume_scheduler", [1])] + assert await omni.is_paused() is False + + await omni.resume_generation() + assert len(_rpc_methods(omni)) == 2 + + asyncio.run(run()) + + +@pytest.mark.cpu +def test_keep_then_sleep_wake_resume_keeps_admission_paused_until_resume(): + async def run() -> None: + omni = _make_omni(stage_types=["diffusion"]) + omni.reset_prefix_cache = AsyncMock(return_value=True) + omni.reset_encoder_cache = AsyncMock() + omni.collective_rpc = AsyncMock( + side_effect=lambda **kw: [OmniACK(task_id="t", status="SUCCESS", stage_id=0, rank=0)] + if kw["method"] in {"handle_sleep_task", "handle_wake_task"} + else [None] + ) + + await omni.pause_generation(mode="keep", clear_cache=False) + await omni.sleep(level=1) + await omni.wake_up() + assert await omni.is_paused() is True + + await omni.resume_generation() + assert await omni.is_paused() is False + assert [m for m, _ in _rpc_methods(omni)] == [ + "pause_scheduler", + "handle_sleep_task", + "handle_wake_task", + "resume_scheduler", + ] + + asyncio.run(run()) + + +@pytest.mark.cpu +@pytest.mark.parametrize( + "failure", + [ + AsyncMock(side_effect=NotImplementedError("request-level execution only")), + AsyncMock(return_value=[{"supported": False, "error": "TimeoutError: outputs did not drain"}]), + ], +) +def test_keep_failure_surfaces_and_leaves_explicit_resume_possible(failure): + async def run() -> None: + omni = _make_omni(stage_types=["diffusion"]) + omni.collective_rpc = failure + + with pytest.raises((NotImplementedError, RuntimeError)): + await omni.pause_generation(mode="keep", clear_cache=False) + + assert await omni.is_paused() is True + assert omni._paused_stage_ids == {0} + + omni.collective_rpc = AsyncMock(return_value=[None]) + await omni.resume_generation() + omni.collective_rpc.assert_awaited_once_with( + method="resume_scheduler", + args=(), + kwargs=None, + stage_ids=[0], + ) + assert await omni.is_paused() is False + assert omni._paused_stage_ids == set() + + asyncio.run(run()) + + +@pytest.mark.cpu +def test_resume_failure_keeps_admission_paused_and_target_recorded(): + async def run() -> None: + omni = _make_omni(stage_types=["diffusion"]) + await omni.pause_generation(mode="keep", clear_cache=False) + + omni.collective_rpc = AsyncMock(return_value=[{"error": "replica gone"}]) + with pytest.raises(RuntimeError, match="resume_scheduler failed"): + await omni.resume_generation() + + assert await omni.is_paused() is True + assert omni._paused_stage_ids == {0} + + asyncio.run(run()) + + +@pytest.mark.cpu +@pytest.mark.parametrize("first, second", [([1], [0]), ([0], [1])]) +def test_mixed_engine_partial_resume_keeps_admission_paused(first, second): + async def run() -> None: + omni = _make_omni(stage_types=["llm", "diffusion"]) + await omni.pause_generation(mode="keep", clear_cache=False) + + await omni.resume_generation(stage_ids=first) + assert await omni.is_paused() is True + + await omni.resume_generation(stage_ids=second) + assert await omni.is_paused() is False + assert _rpc_methods(omni)[2:] == [("resume_scheduler", first), ("resume_scheduler", second)] + + asyncio.run(run()) + + +@pytest.mark.cpu +def test_two_ar_stages_partial_resume_keeps_admission_paused(): + async def run() -> None: + omni = _make_omni(stage_types=["llm", "llm"]) + omni.reset_prefix_cache = AsyncMock(return_value=True) + omni.reset_mm_cache = AsyncMock() + omni.reset_encoder_cache = AsyncMock() + await omni.pause_generation(mode="abort") + + await omni.resume_generation(stage_ids=[0]) + assert await omni.is_paused() is True + + await omni.resume_generation(stage_ids=[1]) + assert await omni.is_paused() is False + + asyncio.run(run()) diff --git a/tests/entrypoints/test_serve.py b/tests/entrypoints/test_serve.py index 0f51a4320..ad5e33dc1 100644 --- a/tests/entrypoints/test_serve.py +++ b/tests/entrypoints/test_serve.py @@ -15,7 +15,10 @@ import pytest from pytest_mock import MockerFixture from vllm.v1.engine.utils import EngineZmqAddresses -from vllm_omni.config.omni_config import VllmOmniDiffusionStageConfig +from vllm_omni.config.omni_config import ( + VllmOmniDiffusionStageConfig, + normalize_and_validate_diffusion_engine_ingress_kwargs, +) from vllm_omni.config.resolver import OmniConfigResolution from vllm_omni.engine.stage_engine_startup import StageReplicaResources from vllm_omni.engine.stage_runtime import StageEngineLaunch @@ -87,6 +90,26 @@ def _parse_serve_args(argv: list[str]) -> TrackingNamespace: return parser.parse_args(argv) +def test_no_guardrails_is_only_forwarded_as_model_config(mocker: MockerFixture) -> None: + """The CLI alias must not reach the strict diffusion config validator.""" + parser = TrackingArgumentParser() + subparsers = parser.add_subparsers(dest="subparser") + OmniServeCommand().subparser_init(subparsers) + args = parser.parse_args(["serve", "fake-model", "--omni", "--no-guardrails"]) + assert args.get_explicit_kwargs_dict()["no_guardrails"] is True + + # Keep the real CLI parser and config validator, but do not start a server. + args.headless = True + mocker.patch.dict("os.environ", {"VLLM_DISABLE_LOG_LOGO": "1"}) + mocker.patch("vllm_omni.entrypoints.cli.serve.run_headless") + OmniServeCommand.cmd(args) + + explicit = args.get_explicit_kwargs_dict() + normalized = normalize_and_validate_diffusion_engine_ingress_kwargs(explicit, stage_id=0) + assert normalized["model_config"]["guardrails"] is False + assert "no_guardrails" not in explicit + + def test_omni_serve_requires_model_when_none_provided() -> None: """Regression for https://github.com/vllm-project/vllm-omni/issues/4158: ``vllm serve --omni`` with no model must fail fast instead of silently diff --git a/tests/examples/test_minicpmo_realtime_web_server.py b/tests/examples/test_minicpmo_realtime_web_server.py index 44f04eb1b..0f7149a34 100644 --- a/tests/examples/test_minicpmo_realtime_web_server.py +++ b/tests/examples/test_minicpmo_realtime_web_server.py @@ -113,12 +113,23 @@ def test_qwen_config_does_not_inject_reference_voice(): assert '"profile": "qwen3-turn"' in page assert '"adapter": "vad"' in page assert '"refAudio": null' in page - for name in ("common", "minicpm_native", "qwen3_turn"): + for name in ("common", "minicpm_native", "qwen3_turn", "aura_ptt"): assert client.get(f"/static/profiles/{name}.js").status_code == 200 with pytest.raises(ValueError, match="ref-audio"): build_app(profile="qwen3-turn", ref_audio="unused.wav") +def test_aura_ptt_config_defaults_and_rejects_ref_audio(): + client = TestClient(build_app(profile="aura-ptt", model="aurateam/AURA")) + page = client.get("/").text + assert '"profile": "aura-ptt"' in page + assert '"refAudio": null' in page + assert "aura_ptt.js" in page + assert 'id="pttButton"' in page + with pytest.raises(ValueError, match="ref-audio"): + build_app(profile="aura-ptt", ref_audio="unused.wav") + + def test_config_cannot_close_its_script_element(): page = TestClient(build_app(model="")).get("/").text assert "