From 68a992060b34e9617da6311416a3eafce61d2214 Mon Sep 17 00:00:00 2001 From: Ricardo Noriega Date: Thu, 16 Jul 2026 08:17:02 +0200 Subject: [PATCH 001/240] [Bugfix] Fix audio.format parameter silently ignored in chat completions (#4718) Signed-off-by: Ricardo Noriega De Soto --- docs/serving/audio_generate_api.md | 5 +- docs/serving/speech_api.md | 2 +- .../examples/online_serving/text_to_audio.md | 2 +- .../test_qwen3_omni_expansion.py | 19 ++ .../openai_api/test_audio_format.py | 175 ++++++++++++++++++ .../entrypoints/openai/audio_utils_mixin.py | 7 +- .../entrypoints/openai/protocol/audio.py | 14 +- vllm_omni/entrypoints/openai/serving_chat.py | 39 +++- 8 files changed, 246 insertions(+), 17 deletions(-) create mode 100644 tests/entrypoints/openai_api/test_audio_format.py diff --git a/docs/serving/audio_generate_api.md b/docs/serving/audio_generate_api.md index afe693ae4..3109446c1 100644 --- a/docs/serving/audio_generate_api.md +++ b/docs/serving/audio_generate_api.md @@ -66,7 +66,7 @@ Content-Type: application/json |-----------|------|---------|-------------| | `input` | string | **required** | Text prompt describing the audio to generate | | `model` | string | server's model | Model to use (optional, should match server if specified) | -| `response_format` | string | "wav" | Audio format: wav, mp3, flac, pcm, aac, opus | +| `response_format` | string | "wav" | Audio format: wav, mp3, flac, pcm, opus | | `speed` | float | 1.0 | Playback speed (0.25 - 4.0) | #### Diffusion Parameters @@ -90,7 +90,6 @@ Returns binary audio data with the appropriate `Content-Type` header: | `mp3` | `audio/mpeg` | | `flac` | `audio/flac` | | `pcm` | `audio/pcm` | -| `aac` | `audio/aac` | | `opus` | `audio/opus` | ## Examples @@ -285,7 +284,7 @@ Pydantic validation failure (e.g. invalid `response_format`, `speed` out of rang "detail": [ { "type": "literal_error", - "msg": "Input should be 'wav', 'pcm', 'flac', 'mp3', 'aac' or 'opus'", + "msg": "Input should be 'wav', 'pcm', 'flac', 'mp3' or 'opus'", ... } ] diff --git a/docs/serving/speech_api.md b/docs/serving/speech_api.md index 27dfda95f..2bf630476 100644 --- a/docs/serving/speech_api.md +++ b/docs/serving/speech_api.md @@ -109,7 +109,7 @@ Content-Type: application/json | `input` | string | **required** | The text to synthesize into speech | | `model` | string | server's model | Model to use (optional, should match server if specified) | | `voice` | string | "vivian" | Speaker name (e.g., vivian, ryan, aiden) | -| `response_format` | string | "wav" | Audio format: wav, mp3, flac, pcm, aac, opus | +| `response_format` | string | "wav" | Audio format: wav, mp3, flac, pcm, opus | | `speed` | float | 1.0 | Playback speed (0.25-4.0) | #### vLLM-Omni Extension Parameters diff --git a/docs/user_guide/examples/online_serving/text_to_audio.md b/docs/user_guide/examples/online_serving/text_to_audio.md index b1ec33a20..a4833aef6 100644 --- a/docs/user_guide/examples/online_serving/text_to_audio.md +++ b/docs/user_guide/examples/online_serving/text_to_audio.md @@ -139,7 +139,7 @@ Content-Type: application/json |-----------|------|---------|-------------| | `input` | string | **required** | Text prompt describing the audio to generate | | `model` | string | server's model | Model to use (optional, should match server if specified) | -| `response_format` | string | "wav" | Audio format: wav, mp3, flac, pcm, aac, opus | +| `response_format` | string | "wav" | Audio format: wav, mp3, flac, pcm, opus | | `speed` | float | 1.0 | Playback speed (0.25 - 4.0) | | `audio_length` | float | null | Audio duration in seconds (max ~47s for `stable-audio-open-1.0`) | | `audio_start` | float | 0.0 | Audio start time in seconds | diff --git a/tests/e2e/online_serving/test_qwen3_omni_expansion.py b/tests/e2e/online_serving/test_qwen3_omni_expansion.py index 25ba4f284..3ded285ce 100644 --- a/tests/e2e/online_serving/test_qwen3_omni_expansion.py +++ b/tests/e2e/online_serving/test_qwen3_omni_expansion.py @@ -7,6 +7,7 @@ E2E Online tests for Qwen3-Omni model. import os import pytest +from openai import BadRequestError from tests.helpers.mark import hardware_test from tests.helpers.media import generate_synthetic_audio, generate_synthetic_image, generate_synthetic_video @@ -610,3 +611,21 @@ def test_text_to_audio_long_output_001(omni_server, openai_client) -> None: text = responses[0].text_content if responses else "" word_count = len(text.split()) assert word_count >= 200, f"Expected at least 200 words in long output, got {word_count}" + + +@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2) +@pytest.mark.parametrize("omni_server", test_params[:1], indirect=True) +def test_invalid_audio_format_rejected(omni_server, openai_client) -> None: + """Ensure invalid audio format is rejected with 400 before streaming starts.""" + messages = dummy_messages_from_mix_data( + system_prompt=get_system_prompt(), + content_text=get_prompt(), + ) + with pytest.raises(BadRequestError, match="audio format"): + openai_client.client.chat.completions.create( + model=omni_server.model, + messages=messages, + modalities=["text", "audio"], + audio={"voice": "alloy", "format": "aac"}, + stream=True, + ) diff --git a/tests/entrypoints/openai_api/test_audio_format.py b/tests/entrypoints/openai_api/test_audio_format.py new file mode 100644 index 000000000..2f1f487d9 --- /dev/null +++ b/tests/entrypoints/openai_api/test_audio_format.py @@ -0,0 +1,175 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Tests for audio output format handling in chat completions. + +Covers: +- #4716: audio.format parameter must be respected, not hardcoded to WAV +- Format validation rejects unsupported formats +- pcm16 is mapped to pcm for soundfile compatibility +- Default format is WAV when not specified +- create_audio encodes correctly for each supported format +""" + +from __future__ import annotations + +import numpy as np +import pytest + +from vllm_omni.entrypoints.openai.audio_utils_mixin import AudioMixin +from vllm_omni.entrypoints.openai.protocol.audio import ( + DEFAULT_AUDIO_FORMAT, + SUPPORTED_AUDIO_FORMATS, + SUPPORTED_CHAT_AUDIO_FORMATS, + CreateAudio, +) + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +class TestAudioFormatConstants: + def test_default_format_is_wav(self): + assert DEFAULT_AUDIO_FORMAT == "wav" + + def test_supported_formats_no_aac(self): + assert "aac" not in SUPPORTED_AUDIO_FORMATS + + def test_chat_formats_include_pcm16(self): + assert "pcm16" in SUPPORTED_CHAT_AUDIO_FORMATS + + def test_chat_formats_superset_of_audio_formats(self): + assert SUPPORTED_AUDIO_FORMATS <= SUPPORTED_CHAT_AUDIO_FORMATS + + +class TestCreateAudio: + @pytest.fixture + def mixin(self): + return AudioMixin() + + @pytest.fixture + def audio_tensor(self): + return np.sin(np.linspace(0, 2 * np.pi, 24000)).astype(np.float32) + + @pytest.mark.parametrize("fmt", ["wav", "mp3", "flac", "opus", "pcm"]) + def test_create_audio_supported_formats(self, mixin, audio_tensor, fmt): + audio_obj = CreateAudio( + audio_tensor=audio_tensor, + sample_rate=24000, + response_format=fmt, + speed=1.0, + base64_encode=False, + ) + response = mixin.create_audio(audio_obj) + assert len(response.audio_data) > 0 + + def test_wav_magic_bytes(self, mixin, audio_tensor): + audio_obj = CreateAudio( + audio_tensor=audio_tensor, + sample_rate=24000, + response_format="wav", + speed=1.0, + base64_encode=False, + ) + response = mixin.create_audio(audio_obj) + assert response.audio_data[:4] == b"RIFF" + assert response.media_type == "audio/wav" + + def test_mp3_encoding(self, mixin, audio_tensor): + audio_obj = CreateAudio( + audio_tensor=audio_tensor, + sample_rate=24000, + response_format="mp3", + speed=1.0, + base64_encode=False, + ) + response = mixin.create_audio(audio_obj) + assert response.media_type == "audio/mpeg" + assert response.audio_data[:4] != b"RIFF" + + def test_flac_magic_bytes(self, mixin, audio_tensor): + audio_obj = CreateAudio( + audio_tensor=audio_tensor, + sample_rate=24000, + response_format="flac", + speed=1.0, + base64_encode=False, + ) + response = mixin.create_audio(audio_obj) + assert response.audio_data[:4] == b"fLaC" + assert response.media_type == "audio/flac" + + def test_unsupported_format_falls_back_to_default(self, mixin, audio_tensor): + audio_obj = CreateAudio( + audio_tensor=audio_tensor, + sample_rate=24000, + response_format="aac", + speed=1.0, + base64_encode=False, + ) + response = mixin.create_audio(audio_obj) + assert response.audio_data[:4] == b"RIFF" + assert response.media_type == "audio/wav" + + def test_base64_encoding(self, mixin, audio_tensor): + import base64 + + audio_obj = CreateAudio( + audio_tensor=audio_tensor, + sample_rate=24000, + response_format="wav", + speed=1.0, + base64_encode=True, + ) + response = mixin.create_audio(audio_obj) + decoded = base64.b64decode(response.audio_data) + assert decoded[:4] == b"RIFF" + + +class TestResolveAudioFormat: + """Test _resolve_audio_format via the serving chat class.""" + + @pytest.fixture + def serving_chat(self): + from vllm_omni.entrypoints.openai.serving_chat import OmniOpenAIServingChat + + return object.__new__(OmniOpenAIServingChat) + + def _make_request(self, audio_params=None): + from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest + + req = ChatCompletionRequest( + model="test-model", + messages=[{"role": "user", "content": "hello"}], + ) + if audio_params is not None: + req.audio = audio_params + return req + + def test_default_format_when_no_audio_params(self, serving_chat): + request = self._make_request() + result = serving_chat._resolve_audio_format(request) + assert result == "wav" + + def test_extracts_mp3_format(self, serving_chat): + request = self._make_request({"format": "mp3", "voice": "alloy"}) + result = serving_chat._resolve_audio_format(request) + assert result == "mp3" + + def test_pcm16_mapped_to_pcm(self, serving_chat): + request = self._make_request({"format": "pcm16", "voice": "alloy"}) + result = serving_chat._resolve_audio_format(request) + assert result == "pcm" + + def test_invalid_format_returns_error(self, serving_chat): + from vllm.entrypoints.openai.engine.protocol import ErrorResponse + + request = self._make_request({"format": "aac", "voice": "alloy"}) + result = serving_chat._resolve_audio_format(request) + assert isinstance(result, ErrorResponse) + assert "aac" in result.error.message + + def test_all_supported_formats_accepted(self, serving_chat): + from vllm.entrypoints.openai.engine.protocol import ErrorResponse + + for fmt in SUPPORTED_CHAT_AUDIO_FORMATS: + request = self._make_request({"format": fmt, "voice": "alloy"}) + result = serving_chat._resolve_audio_format(request) + assert not isinstance(result, ErrorResponse), f"Format {fmt} should be accepted" diff --git a/vllm_omni/entrypoints/openai/audio_utils_mixin.py b/vllm_omni/entrypoints/openai/audio_utils_mixin.py index 12024013d..6c7043201 100644 --- a/vllm_omni/entrypoints/openai/audio_utils_mixin.py +++ b/vllm_omni/entrypoints/openai/audio_utils_mixin.py @@ -5,7 +5,7 @@ import torch import torchaudio from vllm.logger import init_logger -from vllm_omni.entrypoints.openai.protocol.audio import AudioResponse, CreateAudio +from vllm_omni.entrypoints.openai.protocol.audio import DEFAULT_AUDIO_FORMAT, AudioResponse, CreateAudio try: import soundfile @@ -49,13 +49,12 @@ class AudioMixin: "pcm": ("RAW", "audio/pcm", {"subtype": "PCM_16"}), "flac": ("FLAC", "audio/flac", {}), "mp3": ("MP3", "audio/mpeg", {}), - "aac": ("AAC", "audio/aac", {}), "opus": ("OGG", "audio/ogg", {"subtype": "OPUS"}), } if response_format not in supported_formats: - logger.warning(f"Unsupported response format '{response_format}', defaulting to 'wav'.") - response_format = "wav" + logger.warning(f"Unsupported response format '{response_format}', defaulting to '{DEFAULT_AUDIO_FORMAT}'.") + response_format = DEFAULT_AUDIO_FORMAT soundfile_format, media_type, kwargs = supported_formats[response_format] diff --git a/vllm_omni/entrypoints/openai/protocol/audio.py b/vllm_omni/entrypoints/openai/protocol/audio.py index ad7bf8fae..92b2efd14 100644 --- a/vllm_omni/entrypoints/openai/protocol/audio.py +++ b/vllm_omni/entrypoints/openai/protocol/audio.py @@ -6,6 +6,10 @@ from pydantic import AliasChoices, BaseModel, Field, field_validator, model_vali _MAX_EMBEDDING_DIM = 8192 +SUPPORTED_AUDIO_FORMATS: frozenset[str] = frozenset({"wav", "pcm", "flac", "mp3", "opus"}) +SUPPORTED_CHAT_AUDIO_FORMATS: frozenset[str] = SUPPORTED_AUDIO_FORMATS | {"pcm16"} +DEFAULT_AUDIO_FORMAT: str = "wav" + def _normalize_ref_audio_value(value): if value is None: @@ -59,7 +63,7 @@ class OpenAICreateSpeechRequest(BaseModel): default=None, description="Instructions for voice style/emotion (maps to 'instruct' for Qwen3-TTS)", ) - response_format: Literal["wav", "pcm", "flac", "mp3", "aac", "opus"] = "wav" + response_format: Literal["wav", "pcm", "flac", "mp3", "opus"] = DEFAULT_AUDIO_FORMAT speed: float | None = Field( default=1.0, ge=0.25, @@ -325,7 +329,7 @@ class OpenAICreateAudioGenerateRequest(BaseModel): description="Text prompt describing the audio to generate", ) model: str | None = None - response_format: Literal["wav", "pcm", "flac", "mp3", "aac", "opus"] = "wav" + response_format: Literal["wav", "pcm", "flac", "mp3", "opus"] = DEFAULT_AUDIO_FORMAT speed: float | None = Field( default=1.0, ge=0.25, @@ -391,7 +395,7 @@ class SpeechBatchItem(BaseModel): input: str voice: str | None = Field(default=None, validation_alias=AliasChoices("voice", "speaker")) instructions: str | None = None - response_format: Literal["wav", "pcm", "flac", "mp3", "aac", "opus"] | None = None + response_format: Literal["wav", "pcm", "flac", "mp3", "opus"] | None = None speed: float | None = Field(default=None, ge=0.25, le=4.0) task_type: Literal["CustomVoice", "VoiceDesign", "Base"] | None = None language: str | None = None @@ -411,7 +415,7 @@ class BatchSpeechRequest(BaseModel): items: list[SpeechBatchItem] = Field(..., min_length=1) voice: str | None = Field(default=None, validation_alias=AliasChoices("voice", "speaker")) instructions: str | None = None - response_format: Literal["wav", "pcm", "flac", "mp3", "aac", "opus"] = "wav" + response_format: Literal["wav", "pcm", "flac", "mp3", "opus"] = DEFAULT_AUDIO_FORMAT speed: float | None = Field(default=1.0, ge=0.25, le=4.0) task_type: Literal["CustomVoice", "VoiceDesign", "Base"] | None = None language: str | None = None @@ -500,7 +504,7 @@ class StreamingSpeechSessionConfig(BaseModel): task_type: Literal["CustomVoice", "VoiceDesign", "Base"] | None = None language: str | None = None instructions: str | None = None - response_format: Literal["wav", "pcm", "flac", "mp3", "aac", "opus"] = "wav" + response_format: Literal["wav", "pcm", "flac", "mp3", "opus"] = DEFAULT_AUDIO_FORMAT speed: float | None = Field(default=1.0, ge=0.25, le=4.0) max_new_tokens: int | None = Field(default=None, ge=1) initial_codec_chunk_frames: int | None = Field( diff --git a/vllm_omni/entrypoints/openai/serving_chat.py b/vllm_omni/entrypoints/openai/serving_chat.py index a1a51c303..eec6a03b9 100755 --- a/vllm_omni/entrypoints/openai/serving_chat.py +++ b/vllm_omni/entrypoints/openai/serving_chat.py @@ -97,7 +97,12 @@ from vllm.v1.engine.exceptions import EngineDeadError from vllm_omni.entrypoints.openai.audio_utils_mixin import AudioMixin from vllm_omni.entrypoints.openai.image_api_utils import encode_image_base64_with_compression, validate_layered_layers from vllm_omni.entrypoints.openai.protocol import OmniChatCompletionStreamResponse -from vllm_omni.entrypoints.openai.protocol.audio import AudioResponse, CreateAudio +from vllm_omni.entrypoints.openai.protocol.audio import ( + DEFAULT_AUDIO_FORMAT, + SUPPORTED_CHAT_AUDIO_FORMATS, + AudioResponse, + CreateAudio, +) from vllm_omni.entrypoints.openai.protocol.images import ( ImageData, ImageEditARDeltaChunk, @@ -450,6 +455,11 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin): output_modalities = getattr(request, "modalities", engine_output_modalities) request.modalities = output_modalities if output_modalities is not None else engine_output_modalities + if request.modalities and "audio" in request.modalities: + audio_format_check = self._resolve_audio_format(request) + if isinstance(audio_format_check, ErrorResponse): + return audio_format_check + num_inference_steps = None extra_body: dict[str, Any] = {} # Omni multistage image generation: Stage-0 (AR) should receive a clean @@ -2561,10 +2571,14 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin): else: sample_rate = int(sr_raw) + audio_format = self._resolve_audio_format(request) + if isinstance(audio_format, ErrorResponse): + return audio_format + audio_obj = CreateAudio( audio_tensor=audio_tensor, sample_rate=sample_rate, - response_format="wav", + response_format=audio_format, speed=1.0, base64_encode=True, ) @@ -3515,10 +3529,14 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin): raise ValueError(f"Unexpected audio tensor rank {audio_tensor.ndim}; expected 1-3 dims.") audio_array = audio_tensor.numpy() + audio_format = self._resolve_audio_format(request) + if isinstance(audio_format, ErrorResponse): + return audio_format + audio_obj = CreateAudio( audio_tensor=audio_array, sample_rate=sample_rate, - response_format="wav", + response_format=audio_format, speed=1.0, base64_encode=True, ) @@ -3734,6 +3752,21 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin): return height, width + def _resolve_audio_format(self, request: ChatCompletionRequest) -> str | ErrorResponse: + """Extract and validate the audio output format from a chat completion request.""" + audio_params = getattr(request, "audio", None) + if isinstance(audio_params, dict): + audio_format = audio_params.get("format", DEFAULT_AUDIO_FORMAT) + else: + audio_format = DEFAULT_AUDIO_FORMAT + if audio_format not in SUPPORTED_CHAT_AUDIO_FORMATS: + return self._create_error_response( + f"Invalid audio format '{audio_format}'. Supported formats: {sorted(SUPPORTED_CHAT_AUDIO_FORMATS)}", + ) + if audio_format == "pcm16": + audio_format = "pcm" + return audio_format + def _create_error_response( self, message: str, From 5c3900960bc0d500c05619f02ee99211ecb81d73 Mon Sep 17 00:00:00 2001 From: Canlin Guo Date: Thu, 16 Jul 2026 15:02:15 +0800 Subject: [PATCH 002/240] [Feature] Support MOSS-TTS-Local-v1.5 vocoder graph (#4929) Signed-off-by: gcanlin Signed-off-by: sunnyxyli <1550693033@qq.com> Co-authored-by: sunnyxyli <1550693033@qq.com> --- .../benchmarks/data_modules/seed_tts_eval.py | 7 +- .../cuda_graph_streaming_decoder_wrapper.py | 164 ++++++++++++++++++ .../moss_tts/modeling_moss_tts_codec.py | 113 ++++++++++-- .../models/moss_tts/moss_codec_cudagraph.py | 19 +- .../worker/gpu_generation_model_runner.py | 11 +- 5 files changed, 283 insertions(+), 31 deletions(-) create mode 100644 vllm_omni/model_executor/models/moss_tts/cuda_graph_streaming_decoder_wrapper.py diff --git a/vllm_omni/benchmarks/data_modules/seed_tts_eval.py b/vllm_omni/benchmarks/data_modules/seed_tts_eval.py index 70ed7cfef..13c1f2c21 100644 --- a/vllm_omni/benchmarks/data_modules/seed_tts_eval.py +++ b/vllm_omni/benchmarks/data_modules/seed_tts_eval.py @@ -613,7 +613,12 @@ def compute_seed_tts_wer_metrics( e, ) - wav_16k = _pcm_s16le_to_f32_16k(pcm) + # Request functions normalize ``tts_output_pcm_bytes`` to Seed-TTS WER + # format before it reaches this evaluator: 24 kHz mono int16 PCM. + # Do not apply the streamed-response env format here again; for MOSS + # local that would reinterpret already-normalized 24 kHz mono audio as + # 48 kHz stereo and corrupt ASR/WER. + wav_16k = _pcm_s16le_to_f32_16k(pcm, pcm_sample_rate=24000, channels=1) if len(wav_16k) == 0: asr_failed += 1 if include_per_item: diff --git a/vllm_omni/model_executor/models/moss_tts/cuda_graph_streaming_decoder_wrapper.py b/vllm_omni/model_executor/models/moss_tts/cuda_graph_streaming_decoder_wrapper.py new file mode 100644 index 000000000..1daa5081e --- /dev/null +++ b/vllm_omni/model_executor/models/moss_tts/cuda_graph_streaming_decoder_wrapper.py @@ -0,0 +1,164 @@ +# Copyright 2026 OpenMOSS and the vLLM-Omni team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +"""CUDA graph wrapper for MOSS-TTS Local streaming codec decode.""" + +from __future__ import annotations + +import time +from collections.abc import Callable +from dataclasses import dataclass + +import torch +import torch.nn as nn +from torch.cuda import CUDAGraph +from vllm.logger import init_logger +from vllm.platforms import current_platform + +logger = init_logger(__name__) + + +@dataclass +class _CapturedStreamingDecodeGraph: + graph: CUDAGraph + static_codes: torch.Tensor + static_lengths: torch.Tensor + static_audio: torch.Tensor + static_audio_lengths: torch.Tensor + + +class CUDAGraphStreamingDecoderWrapper: + """Replay fixed-size streaming ``_decode_frame`` steps with CUDA graphs. + + The wrapped codec must already be inside ``codec.streaming(batch_size)``. + Graphs are keyed by the streaming step frame count ``T``. The batch width is + fixed to the codec stream slot count. + """ + + def __init__( + self, + codec: nn.Module, + *, + batch_size: int, + num_quantizers: int, + reset_streaming_state: Callable[[], None] | None = None, + ) -> None: + self.codec = codec + self.batch_size = int(batch_size) + self.num_quantizers = int(num_quantizers) + self.reset_streaming_state = reset_streaming_state + self.graphs: dict[int, _CapturedStreamingDecodeGraph] = {} + self._pool = None + self._warmed_up = False + + @property + def is_ready(self) -> bool: + return bool(self.graphs) + + @property + def capture_sizes(self) -> list[int]: + return sorted(self.graphs) + + @torch.no_grad() + def warmup(self, device: torch.device, capture_sizes: list[int]) -> None: + if self._warmed_up: + return + self._warmed_up = True + if device.type != "cuda": + return + sizes = sorted({int(size) for size in capture_sizes if int(size) > 0}, reverse=True) + if not sizes: + return + + logger.info( + "MOSS-TTS streaming decoder CUDA graph warmup: batch_size=%d n_vq=%d capture_sizes=%s", + self.batch_size, + self.num_quantizers, + list(reversed(sizes)), + ) + start_s = time.perf_counter() + with torch.cuda.device(device): + for size in sizes: + try: + self._capture(size, device) + logger.info(" Captured MOSS-TTS streaming decoder CUDA graph for size=%d", size) + except Exception: + self.graphs.pop(size, None) + logger.warning( + " Failed to capture MOSS-TTS streaming decoder CUDA graph for size=%d; " + "this size will use eager decode", + size, + exc_info=True, + ) + logger.info( + "MOSS-TTS streaming decoder CUDA graph warmup complete: %d/%d captured in %.1f ms", + len(self.graphs), + len(sizes), + (time.perf_counter() - start_s) * 1000.0, + ) + + @torch.no_grad() + def _capture(self, size: int, device: torch.device) -> None: + codes = torch.zeros( + self.num_quantizers, + self.batch_size, + size, + dtype=torch.long, + device=device, + ) + lengths = torch.full((self.batch_size,), size, dtype=torch.long, device=device) + exec_mask = torch.ones(self.batch_size, dtype=torch.bool, device=device) + + self.codec._set_streaming_exec_mask(exec_mask) + stream = torch.cuda.Stream() + stream.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(stream): + _ = self.codec._decode_frame(codes, lengths) + torch.cuda.current_stream().wait_stream(stream) + torch.accelerator.synchronize(device) + + if self.reset_streaming_state is not None: + self.reset_streaming_state() + self.codec._set_streaming_exec_mask(exec_mask) + if self._pool is None: + self._pool = current_platform.get_global_graph_pool() + graph = CUDAGraph() + with torch.cuda.graph(graph, pool=self._pool, capture_error_mode="thread_local"): + output = self.codec._decode_frame(codes, lengths) + + if output.audio is None or output.audio_lengths is None: + raise RuntimeError("MOSS-TTS streaming decoder CUDA graph capture produced empty audio.") + + self.graphs[size] = _CapturedStreamingDecodeGraph( + graph=graph, + static_codes=codes, + static_lengths=lengths, + static_audio=output.audio, + static_audio_lengths=output.audio_lengths, + ) + + @torch.no_grad() + def decode( + self, + codes_step: torch.Tensor, + exec_mask: torch.Tensor, + ) -> tuple[torch.Tensor, torch.Tensor] | None: + if not codes_step.is_cuda or torch.cuda.is_current_stream_capturing(): + return None + n_vq, batch_size, step_t = codes_step.shape + if int(n_vq) != self.num_quantizers or int(batch_size) != self.batch_size: + return None + # The streaming codec advances its internal KV offsets by tensor T, so + # bucket-padding a smaller step into a larger captured graph would write + # padded frames into state. Capture exact step sizes instead. + entry = self.graphs.get(int(step_t)) + if entry is None: + return None + + self.codec._set_streaming_exec_mask(exec_mask) + entry.static_codes.copy_(codes_step) + entry.graph.replay() + return entry.static_audio, entry.static_audio_lengths + + +__all__ = ["CUDAGraphStreamingDecoderWrapper"] diff --git a/vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_codec.py b/vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_codec.py index 60f455240..8f1a72e9b 100644 --- a/vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_codec.py +++ b/vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_codec.py @@ -26,6 +26,9 @@ from vllm_omni.model_executor.models.moss_tts.audio_tokenizer_v2 import ( from vllm_omni.model_executor.models.moss_tts.configuration_moss_audio_tokenizer_v2 import ( MossAudioTokenizerConfig as MossAudioTokenizerV2Config, ) +from vllm_omni.model_executor.models.moss_tts.cuda_graph_streaming_decoder_wrapper import ( + CUDAGraphStreamingDecoderWrapper, +) from vllm_omni.model_executor.models.moss_tts.moss_codec_cudagraph import ( MossTTSCUDAGraphCodecWrapper, ) @@ -43,6 +46,7 @@ class _MossCodecStreamSession: *, stream_slots: int, n_vq: int, + cudagraph_capture_sizes: list[int] | None = None, ) -> None: self._codec = codec self._stream_slots = int(stream_slots) @@ -54,6 +58,19 @@ class _MossCodecStreamSession: self._closed = False with torch.no_grad(): self._exit_stack.enter_context(codec.streaming(self._batch_size)) + self._cudagraph_wrapper: CUDAGraphStreamingDecoderWrapper | None = None + capture_sizes = sorted({int(size) for size in (cudagraph_capture_sizes or []) if int(size) > 0}) + if capture_sizes and self._device.type == "cuda": + self._cudagraph_wrapper = CUDAGraphStreamingDecoderWrapper( + codec, + batch_size=self._batch_size, + num_quantizers=self._n_vq, + reset_streaming_state=lambda: self.reset_slots(list(range(self._batch_size))), + ) + self._cudagraph_wrapper.warmup(self._device, capture_sizes) + self.reset_slots(list(range(self._batch_size))) + if not self._cudagraph_wrapper.is_ready: + self._cudagraph_wrapper = None def acquire(self) -> int | None: if not self._free_stream_slots: @@ -109,12 +126,23 @@ class _MossCodecStreamSession: codes_lengths[slot] = int(codes.shape[1]) exec_mask[slot] = True - self._codec._set_streaming_exec_mask(exec_mask) - result = self._codec._decode_frame(codes_step, codes_lengths) - if result.audio is None: - return {} - audio = result.audio.detach().to("cpu", torch.float32) - lengths = result.audio_lengths.detach().to("cpu") if result.audio_lengths is not None else None + graph_output: tuple[torch.Tensor, torch.Tensor] | None = None + if self._cudagraph_wrapper is not None: + graph_output = self._cudagraph_wrapper.decode(codes_step, exec_mask) + + used_cudagraph = graph_output is not None + if used_cudagraph: + audio_tensor, lengths_tensor = graph_output + else: + self._codec._set_streaming_exec_mask(exec_mask) + result = self._codec._decode_frame(codes_step, codes_lengths) + if result.audio is None: + return {} + audio_tensor = result.audio + lengths_tensor = result.audio_lengths + + audio = audio_tensor.detach().to("cpu", torch.float32) + lengths = lengths_tensor.detach().to("cpu") if lengths_tensor is not None else None out: dict[int, torch.Tensor] = {} for slot in slot_codes: wav = audio[slot] @@ -174,10 +202,17 @@ class MossTTSCodecDecoder(nn.Module): self._sr_tensor = torch.tensor(self._OUTPUT_SAMPLE_RATE, dtype=torch.int32) self._stream_session: _MossCodecStreamSession | None = None self._stream_slots: int = self._connector_int("codec_stream_slots", default=0) - self._stream_max_step_frames: int = self._connector_int("codec_max_step_frames", default=100) + self._stream_chunk_frames: int = self._connector_int("codec_chunk_frames", default=0) + self._stream_max_step_frames: int = self._stream_chunk_frames or 100 self._stream_req_slots: dict[str, int] = {} self._stream_pending_codes: dict[str, list[torch.Tensor]] = {} self._stream_starved_reqs: set[str] = set() + self._codec_streaming: bool = self._connector_bool("codec_streaming", default=False) + default_graph_capture_max = self._stream_chunk_frames or min(self._stream_max_step_frames, 32) + default_graph_capture_sizes = list(range(1, max(1, default_graph_capture_max) + 1)) + self._streaming_cudagraph_capture_sizes = self._streaming_cudagraph_capture_sizes_from_compilation_config( + default_graph_capture_sizes + ) # ------------------------------------------------------------------ # vLLM-Omni stubs (codec has no AR loop) @@ -240,6 +275,15 @@ class MossTTSCodecDecoder(nn.Module): }, ) + if self._codec_streaming and runtime_additional_information is None: + return OmniOutput( + text_hidden_states=None, + multimodal_outputs={ + "model_outputs": [empty] * num_req, + "sr": [sr_tensor] * num_req, + }, + ) + audios: list[torch.Tensor] = [empty] * num_req srs: list[torch.Tensor] = [sr_tensor] * num_req device = next(self._codec.parameters()).device @@ -293,7 +337,7 @@ class MossTTSCodecDecoder(nn.Module): continue meta = (info.get("meta", {}) if isinstance(info, dict) else {}) or {} finished = bool(meta.get("stream_finished", meta.get("finished", False))) - streaming_enabled = bool(meta.get("codec_streaming", False)) + streaming_enabled = bool(meta.get("codec_streaming", self._codec_streaming)) code_flat_numel = meta.get("code_flat_numel") if streaming_enabled and finished and code_flat_numel is not None and int(code_flat_numel) == 0: for _, wav in self._finish_empty_streaming_requests([info]).items(): @@ -382,7 +426,7 @@ class MossTTSCodecDecoder(nn.Module): if not isinstance(info, dict): continue meta = (info.get("meta", {}) or {}) if isinstance(info.get("meta", {}), dict) else {} - if not bool(meta.get("codec_streaming", False)): + if not bool(meta.get("codec_streaming", self._codec_streaming)): continue finished = bool(meta.get("stream_finished", meta.get("finished", False))) if not finished: @@ -446,6 +490,7 @@ class MossTTSCodecDecoder(nn.Module): self._codec, stream_slots=max(1, slots), n_vq=self._n_vq, + cudagraph_capture_sizes=self._streaming_cudagraph_capture_sizes, ) return self._stream_session @@ -597,6 +642,30 @@ class MossTTSCodecDecoder(nn.Module): return int(extra_cfg[name]) return default + def _connector_bool(self, name: str, default: bool = False) -> bool: + model_cfg = getattr(self.vllm_config, "model_config", None) + connector_cfg = getattr(model_cfg, "stage_connector_config", None) + if isinstance(connector_cfg, dict): + extra_cfg: dict | None = connector_cfg.get("extra", connector_cfg) + else: + extra_cfg = getattr(connector_cfg, "extra", None) + if isinstance(extra_cfg, dict) and name in extra_cfg: + return bool(extra_cfg[name]) + return default + + def _streaming_cudagraph_capture_sizes_from_compilation_config( + self, + default: list[int], + ) -> list[int]: + if getattr(self.vllm_config.model_config, "enforce_eager", True): + return [] + compilation_config = getattr(self.vllm_config, "compilation_config", None) + capture_sizes = getattr(compilation_config, "cudagraph_capture_sizes", None) + if capture_sizes: + max_step_frames = max(1, int(self._stream_max_step_frames)) + return sorted({int(size) for size in capture_sizes if 0 < int(size) <= max_step_frames}) + return list(default) + # ------------------------------------------------------------------ # Weight loading # ------------------------------------------------------------------ @@ -716,7 +785,9 @@ class MossTTSCodecDecoder(nn.Module): self._n_channels, ) - self._maybe_enable_decoder_cudagraph(device) + self._configure_decoder_cudagraph(device) + if self._codec_streaming and self._streaming_cudagraph_capture_sizes: + self._ensure_stream_session() # vLLM's track_weights_loading() compares the returned set against # ``self.named_parameters()``. After ``self._codec = codec`` above, @@ -739,12 +810,30 @@ class MossTTSCodecDecoder(nn.Module): codec = MossAudioTokenizerModel(codec_cfg) return codec_cfg, codec - def _maybe_enable_decoder_cudagraph(self, device: torch.device) -> None: - """Capture CUDA Graphs for the codec decoder if enforce_eager is False.""" + def _configure_decoder_cudagraph(self, device: torch.device) -> None: + """Select the codec CUDA Graph path. + + ``enforce_eager`` is the single graph on/off switch. If graphing is + enabled, ``codec_streaming`` decides whether decode uses the persistent + streaming-state wrapper or the offline full-chunk wrapper. + """ if getattr(self.vllm_config.model_config, "enforce_eager", True): + self._streaming_cudagraph_capture_sizes = [] return if self._codec is None: return + if self._codec_streaming: + logger.info( + "MOSS-TTS codec CUDA Graph selected streaming wrapper: capture_sizes=%s", + self._streaming_cudagraph_capture_sizes, + ) + return + + self._enable_non_streaming_decoder_cudagraph(device) + + def _enable_non_streaming_decoder_cudagraph(self, device: torch.device) -> None: + if self._codec is None: + return # Read capture sizes from the connector's extra config (same convention # as Qwen3-TTS), falling back to a sensible default covering common diff --git a/vllm_omni/model_executor/models/moss_tts/moss_codec_cudagraph.py b/vllm_omni/model_executor/models/moss_tts/moss_codec_cudagraph.py index 71585f1c0..87f255072 100644 --- a/vllm_omni/model_executor/models/moss_tts/moss_codec_cudagraph.py +++ b/vllm_omni/model_executor/models/moss_tts/moss_codec_cudagraph.py @@ -1,6 +1,6 @@ """CUDA Graph acceleration for the MOSS Audio Tokenizer codec decoder. -Captures the codec decode method (_decode_frame, or _decode on the vendored codec) for a set of fixed frame-count +Captures MossAudioTokenizerModel._decode_frame for a set of fixed frame-count bucket sizes, then replays the captured graph at inference time to eliminate kernel-launch overhead. Inputs that exceed all captured sizes fall back to eager execution transparently. @@ -24,7 +24,7 @@ logger = init_logger(__name__) class MossTTSCUDAGraphCodecWrapper: - """CUDA Graph wrapper for the codec decode method (_decode_frame, or _decode on the vendored codec). + """CUDA Graph wrapper for MossAudioTokenizerModel._decode_frame. Graphs are keyed by padded_T (int). On each call the actual T is bucket-matched to the smallest pre-captured size >= T. The static code @@ -51,15 +51,6 @@ class MossTTSCUDAGraphCodecWrapper: enabled: bool = True, ) -> None: self.model = model - # Upstream remote-code tokenizer exposes _decode_frame; the vendored - # MossAudioTokenizerModel still exposes _decode. Resolve once here so - # both codec implementations work. - decode_fn = getattr(model, "_decode_frame", None) or getattr(model, "_decode", None) - if decode_fn is None: - raise AttributeError( - f"{type(model).__module__}.{type(model).__name__} exposes neither _decode_frame nor _decode" - ) - self._decode_fn = decode_fn self.capture_sizes: list[int] = sorted(capture_sizes) self.num_quantizers = num_quantizers self.enabled = enabled @@ -108,7 +99,7 @@ class MossTTSCUDAGraphCodecWrapper: dummy_codes = torch.zeros(nq, 1, size, dtype=torch.long, device=device) dummy_lengths = torch.tensor([size], dtype=torch.long, device=device) with torch.no_grad(): - _ = self._decode_fn(dummy_codes, dummy_lengths) + _ = self.model._decode_frame(dummy_codes, dummy_lengths) torch.accelerator.synchronize(device) @@ -141,13 +132,13 @@ class MossTTSCUDAGraphCodecWrapper: # Extra eager warmup inside capture to ensure all kernels are compiled. with torch.no_grad(): - _ = self._decode_fn(static_codes, static_lengths) + _ = self.model._decode_frame(static_codes, static_lengths) torch.accelerator.synchronize(device) graph = CUDAGraph() with torch.no_grad(): with torch.cuda.graph(graph, pool=current_platform.get_global_graph_pool()): - static_out = self._decode_fn(static_codes, static_lengths) + static_out = self.model._decode_frame(static_codes, static_lengths) self.graphs[size] = graph self.static_codes[size] = static_codes diff --git a/vllm_omni/worker/gpu_generation_model_runner.py b/vllm_omni/worker/gpu_generation_model_runner.py index c8fe92eba..ec41900b1 100644 --- a/vllm_omni/worker/gpu_generation_model_runner.py +++ b/vllm_omni/worker/gpu_generation_model_runner.py @@ -76,10 +76,6 @@ class GPUGenerationModelRunner(OmniGPUModelRunner, OmniConnectorModelRunnerMixin def _update_request_states(self, scheduler_output: SchedulerOutput): # remove requests - # Some stateful vocoder model may need to clean the state - # to avoid the leak of slots when the requests have been aborted. - if scheduler_output.finished_req_ids and hasattr(self.model, "on_requests_finished"): - self.model.on_requests_finished(scheduler_output.finished_req_ids) for req_id in scheduler_output.finished_req_ids: self.input_batch.remove_request(req_id) scheduled_req_ids = scheduler_output.num_scheduled_tokens.keys() @@ -157,6 +153,13 @@ class GPUGenerationModelRunner(OmniGPUModelRunner, OmniConnectorModelRunnerMixin if self.model_config.async_chunk and num_scheduled_tokens: self._update_request_states(scheduler_output) deferred_state_corrections_fn = self._update_states(scheduler_output) + + # Notify stateful models of finished requests before the + # zero-token early return. Request cleanup is a model lifecycle + # concern and does not depend on the inter-stage transfer mode. + if scheduler_output.finished_req_ids and hasattr(self.model, "on_requests_finished"): + self.model.on_requests_finished(scheduler_output.finished_req_ids) + if not scheduler_output.total_num_scheduled_tokens: return self.attach_omni_connector_output(EMPTY_MODEL_RUNNER_OUTPUT) From 2a49b2b21722df40efdfaa4a6ce1f4dd5f802d05 Mon Sep 17 00:00:00 2001 From: wangyu <53896905+yenuo26@users.noreply.github.com> Date: Thu, 16 Jul 2026 22:45:00 +0800 Subject: [PATCH 003/240] =?UTF-8?q?[CI][Perf]=20Add=20JSON=20mark=20suppor?= =?UTF-8?q?t=20and=20paired=20parametrize=20for=20L4=20perf,=20so=20as=20t?= =?UTF-8?q?o=20further=20support=20performance=E2=80=91case=20selection=20?= =?UTF-8?q?for=20NPU=E2=80=91CI.=20(#5084)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: wangyu <410167048@qq.com> Signed-off-by: wangyu <53896905+yenuo26@users.noreply.github.com> --- .claude/skills/add-diffusion-model/SKILL.md | 2 +- .claude/skills/vllm-omni-test/SKILL.md | 18 +- .../vllm-omni-test/references/test-routing.md | 31 +- docs/contributing/ci/CI_5levels.md | 17 +- .../test_examples/l4_performance_tests.inc.md | 122 +++- docs/contributing/ci/test_guide.md | 14 +- docs/contributing/ci/tests_markers.md | 31 +- recipes/LTX/LTX-2.3.md | 4 +- tests/dfx/conftest.py | 352 ++++++++- tests/dfx/perf/scripts/run_benchmark.py | 49 +- .../perf/scripts/run_diffusion_benchmark.py | 189 +++-- .../dfx/perf/tests/test_bagel_vllm_omni.json | 48 ++ .../perf/tests/test_cosmos3_vllm_omni.json | 458 ++++++------ tests/dfx/perf/tests/test_higgs_audio_v3.json | 12 + .../perf/tests/test_hunyuan_image3_it2i.json | 284 ++++---- .../tests/test_hunyuan_image_tp2_cfgp2.json | 78 +- .../tests/test_hunyuan_image_tp2_sp2.json | 78 +- .../perf/tests/test_hunyuan_image_tp4.json | 78 +- tests/dfx/perf/tests/test_ltx2_vllm_omni.json | 478 +++++++------ .../tests/test_qwen3_omni_async_chunk.json | 668 +++++++++--------- .../tests/test_qwen3_omni_multi_replicas.json | 236 ++++--- .../tests/test_qwen3_omni_no_async_chunk.json | 346 ++++----- .../perf/tests/test_qwen3_omni_vllm_text.json | 366 +++++----- .../test_qwen_image_edit_2511_vllm_omni.json | 324 +++++---- .../tests/test_qwen_image_edit_vllm_omni.json | 314 ++++---- .../test_qwen_image_layered_vllm_omni.json | 106 +-- .../perf/tests/test_qwen_image_vllm_omni.json | 484 +++++++------ tests/dfx/perf/tests/test_runner_metadata.py | 199 ++++++ tests/dfx/perf/tests/test_tts.json | 36 + tests/dfx/perf/tests/test_voxcpm2.json | 12 + .../perf/tests/test_wan22_i2v_vllm_omni.json | 232 +++--- 31 files changed, 3475 insertions(+), 2191 deletions(-) diff --git a/.claude/skills/add-diffusion-model/SKILL.md b/.claude/skills/add-diffusion-model/SKILL.md index 906f3c485..76c894473 100644 --- a/.claude/skills/add-diffusion-model/SKILL.md +++ b/.claude/skills/add-diffusion-model/SKILL.md @@ -334,7 +334,7 @@ Classify the model's **CI priority** first: | **L1** | `tests/diffusion/models/{slug}/`, `tests/diffusion/cache/`, transformer unit tests | `core_model` + `cpu` | `test-ready.yml` | Weight remap, `_sp_plan`, cache enabler registration, shape contracts | | **L2** | `tests/e2e/online_serving/test_{slug}.py` (and offline if the category is offline-first) | **`core_model` + `advanced_model`** (both on baseline smoke) + `diffusion` + `@hardware_test` / `hardware_marks` | `test-ready.yml` | Default deploy smoke — minimal `num_inference_steps`, single prompt | | **L3** | `tests/e2e/online_serving/test_{slug}.py` **and** `tests/e2e/offline_inference/test_{slug}.py` when offline matters | Baseline smoke: **`core_model` + `advanced_model`**; heavier cases: `advanced_model` only (+ `diffusion`) | `test-merge.yml` **or** merged into nightly diffusion function job | Real weights, streaming/API paths, LoRA/offload smoke | -| **L4** | `tests/e2e/online_serving/test_{slug}_expansion.py` (+ offline expansion if needed) | `full_model` + `diffusion` | `test-nightly.yml` (X2I / X2V / X2A function groups) | Feature combos per [#1832](https://github.com/vllm-project/vllm-omni/issues/1832); perf → `tests/dfx/perf/tests/test_{model}_vllm_omni.json` | +| **L4** | `tests/e2e/online_serving/test_{slug}_expansion.py` (+ offline expansion if needed) | `full_model` + `diffusion` | `test-nightly.yml` (X2I / X2V / X2A function groups) | Feature combos per [#1832](https://github.com/vllm-project/vllm-omni/issues/1832); perf → `tests/dfx/perf/tests/test_{model}_vllm_omni.json` with per-case **`mark`** (`hardware_marks` + `full_model` + `diffusion`) | **L2 & L3 online — same file, dual marks on the baseline smoke:** The **first / simplest** case in `test_{slug}.py` (default deploy, minimal steps, single prompt) should carry **both** `@pytest.mark.core_model` and `@pytest.mark.advanced_model` on the **same** function so L2 (`test-ready.yml`) and L3 (`test-merge.yml`) share one smoke test. Heavier deploy variants or API paths in the same file use **`advanced_model` only**. When L3 moves to nightly, migrate those heavier cases into `test_{slug}_expansion.py` with `full_model` and remove the dedicated `test-merge.yml` job (see `test_longcat_image_expansion.py`, `test_qwen_image_expansion.py`). diff --git a/.claude/skills/vllm-omni-test/SKILL.md b/.claude/skills/vllm-omni-test/SKILL.md index 40a81f4cf..dcc2506da 100644 --- a/.claude/skills/vllm-omni-test/SKILL.md +++ b/.claude/skills/vllm-omni-test/SKILL.md @@ -97,7 +97,7 @@ Existing references: `tests/e2e/offline_inference/test_qwen2_5_omni.py` (L2-styl | **Offline inference e2e** | `tests/e2e/offline_inference/` | **Module (default):** `omni_runner` + `omni_runner_handler`. **Function (isolation only):** `omni_runner_function` + `omni_runner_handler_function`. Diffusion/TTS may use `Omni(...).generate` directly | L2: `core_model` + **one of** `omni` / `tts` / `diffusion`; `@hardware_test(...)` when GPU/NPU is required | | **Online serving e2e** | `tests/e2e/online_serving/` | **Module (default):** `omni_server` + `openai_client`. **Function (isolation only):** `omni_server_function` + `openai_client_function`. Clients: `send_omni_request` (omni), `send_audio_speech_request` (tts), `send_diffusion_request` / `send_video_diffusion_request` / `send_images_generations_request` (diffusion) | Baseline smoke: **`core_model` + `advanced_model`**; heavier paths: `advanced_model` only; L4 expansion: `full_model` | | **Documentation / runnable examples** | `tests/examples/offline_inference/`, `tests/examples/online_serving/` | **Offline docs (preferred):** extract Python/Bash blocks from the doc README (e.g. `ReadmeSnippet.extract_readme_snippets`), `pytest.mark.parametrize` each snippet, run via `example_runner.run` with a stable `output_subfolder`. **Online docs:** copy client/request scripts into dedicated tests and keep them in sync with the doc page. | Usually **L4**: `advanced_model`, often `example` plus hardware marks matching the nightly docs-example job (see `.buildkite/test-nightly.yml`). Full conventions: [docs/contributing/ci/test_examples/l4_doc_example_tests.inc.md](../../../docs/contributing/ci/test_examples/l4_doc_example_tests.inc.md) (introduced in [PR #1910](https://github.com/vllm-project/vllm-omni/pull/1910): naming, output directory layout, skip rules, avoid trimming `num_inference_steps` without a strong CI reason). | -| **Performance / benchmark** | `tests/dfx/perf/tests/*.json` + `run_*_benchmark.py` | JSON or script-driven server + load config; assert explicit metrics / baselines | L4 Perf: `full_model` + `benchmark`; wire `test-nightly.yml` Perf steps | +| **Performance / benchmark** | `tests/dfx/perf/tests/*.json` + `run_*_benchmark.py` | JSON or script-driven server + load config; assert explicit metrics / baselines | L4 Perf: JSON `mark` with `full_model` + `omni`/`tts`/`diffusion`; wire `test-nightly.yml` Perf steps | | **Invalid parameter / negative HTTP validation** | `tests/dfx/reliability/invalid_param_test/` | Live `omni_server` + low-level `send_*_http_request` with `err_code` / `err_message` | `pytest.mark.slow` + `omni` / `tts` / `diffusion` + `@hardware_marks` (`H100` or `L4`); CI in **`test-weekly.yml`** (not ready/merge/nightly) | **If the user’s test plan includes invalid parameter validation / invalid params / negative HTTP / 400 validation:** do **not** add those `test_*` functions to `tests/e2e/online_serving/test_*.py` or `*_expansion.py`. **Move or author them** under `tests/dfx/reliability/invalid_param_test/` in the **endpoint-matching script** (see **Invalid parameter validation** below). Success-path e2e and invalid-param dfx tests must stay in separate modules. @@ -270,7 +270,7 @@ def test_text_to_video_001(omni_server, openai_client) -> None: *Documentation example tests:* follow the **Preferred Test Strategy** in [l4_doc_example_tests.inc.md](../../../docs/contributing/ci/test_examples/l4_doc_example_tests.inc.md): dynamic extraction for offline READMEs; explicit copied client code for online pages until extraction is justified; use the documented **naming**, **output directory** (page folder + case id), and **skipping** rules (e.g. Gradio-only scripts). -*Performance tests:* add or extend entries under `tests/perf/` (and JSON configs where the project uses them), with **explicit baselines** and the same marker/run-level pairing as the CI step that will execute them. +*Performance tests:* add or extend entries under `tests/dfx/perf/tests/` (and JSON configs where the project uses them), with **explicit baselines**, **`mark`** on each case (`hardware_marks` + `full_model` + type marker), and the same nightly Perf step pattern as in-tree configs. **3. Cross-cutting rules** @@ -592,27 +592,31 @@ Implementation strategy: **When the user explicitly asks for L4 perf (or full L4 including perf):** 1. **Do not** put throughput/latency assertions inside `test_*_expansion.py` — perf uses the **dfx benchmark harness**, not `omni_server` e2e fixtures. -2. Add or extend a **JSON config** under `tests/dfx/perf/tests/` (mirror in-tree names: `test_qwen_image_vllm_omni.json`, `test_bagel_vllm_omni.json`). -3. Each JSON entry: `test_name`, `server_params` (`model`, `serve_args`), `benchmark_params[]` with explicit **`baseline`** (`throughput_qps`, `latency_mean`, `peak_memory_mb_mean`, …). +2. Add or extend a **JSON config** under `tests/dfx/perf/tests/` (mirror in-tree names: `test_qwen_image_vllm_omni.json`, `test_cosmos3_vllm_omni.json`, `test_qwen3_omni_async_chunk.json`). +3. Each JSON **case**: `test_name`, optional **`mark`** (`hardware_marks` required when present; `marks`: `full_model` + `omni`/`tts`/`diffusion`), `server_params` (`model`, `serve_args` or `stage_config_name`), `benchmark_params[]` with **`name`** and explicit **`baseline`** (`throughput_qps`, `latency_mean`, `peak_memory_mb_mean`, …). Diffusion cases also set `server_type` (e.g. `"vllm-omni"`) and usually `benchmark_endpoint`. 4. Run locally via the matching script (model type → runner): | Model type | Runner script | Example config | |------------|---------------|----------------| | **Diffusion X2I/X2V** | `tests/dfx/perf/scripts/run_diffusion_benchmark.py` | `tests/dfx/perf/tests/test_qwen_image_vllm_omni.json` | | **TTS** | `tests/dfx/perf/scripts/run_benchmark.py` | `tests/dfx/perf/tests/test_tts.json` (shared) or `test_{slug}.json` (dedicated; use when the model must not join the shared nightly matrix before integration — e.g. `test_voxcpm2.json`, `test_coqui_tts.json`) | -| **Omni** | (see existing Omni Perf steps — `run_benchmark.py` / dedicated omni configs in-tree) | `tests/dfx/perf/tests/test_omni*.json` | +| **Omni** | `tests/dfx/perf/scripts/run_benchmark.py` | `test_qwen3_omni_no_async_chunk.json`, `test_qwen3_omni_async_chunk.json`, `test_qwen3_omni_vllm_text.json`, `test_qwen3_omni_multi_replicas.json` | -5. Wire **`test-nightly.yml` · Perf Test** step for that model (separate from Function Test): export `DIFFUSION_BENCHMARK_DIR` / `BENCHMARK_DIR`, run pytest on the script with `--test-config-file`, **upload artifacts** (`buildkite-agent artifact upload`), often **multi-GPU H100** for diffusion perf. +5. Wire **`test-nightly.yml` · Perf Test** step for that model (separate from Function Test): export `DIFFUSION_BENCHMARK_DIR` / `BENCHMARK_DIR`, run pytest on the script with **`--test-config-file`** (nightly perf steps do not use `-m`), **upload artifacts** (`buildkite-agent artifact upload`), often **multi-GPU H100** for diffusion perf. 6. **One benchmark scenario → one `test_name` block** in JSON (same spirit as one `test_*` per function case). Combine server `serve_args` + workload in one entry; use multiple `benchmark_params` rows for size/step sweeps under the same server config. -**Perf local command (diffusion example):** +**Perf local commands:** ```bash cd tests export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN +# Single file (CI-like) pytest -s -v dfx/perf/scripts/run_diffusion_benchmark.py \ --test-config-file dfx/perf/tests/test__vllm_omni.json +# Bulk load + filter by JSON mark +pytest -sv dfx/perf/scripts/run_diffusion_benchmark.py -m "full_model and diffusion and H100" +pytest -sv dfx/perf/scripts/run_benchmark.py -m "full_model and omni and H100" ``` **Clarify in the test plan** which L4 pillars you deliver: `Function only` | `Function + Perf` | `Function + Accuracy + Perf`. diff --git a/.claude/skills/vllm-omni-test/references/test-routing.md b/.claude/skills/vllm-omni-test/references/test-routing.md index 445a78399..f78e1c77d 100644 --- a/.claude/skills/vllm-omni-test/references/test-routing.md +++ b/.claude/skills/vllm-omni-test/references/test-routing.md @@ -127,8 +127,12 @@ pytest -s -v e2e/online_serving/test_qwen_image_expansion.py -m "full_model and cd tests export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN +# CI-like: single JSON file (nightly Perf Test steps) pytest -s -v dfx/perf/scripts/run_diffusion_benchmark.py \ --test-config-file dfx/perf/tests/test_qwen_image_vllm_omni.json +# Local bulk load: all *.json under tests/dfx/perf/tests/, filter by JSON mark +pytest -sv dfx/perf/scripts/run_diffusion_benchmark.py -m "full_model and diffusion and H100" +pytest -sv dfx/perf/scripts/run_benchmark.py -m "full_model and omni and H100" ``` Broad marker sweep (when no explicit file shard): @@ -150,7 +154,26 @@ Nightly **L4** for a model is often **multiple jobs** in `test-nightly.yml`, not **Rule:** “L4 functional cases” → **Function only** by default. State that Perf/Accuracy are separate pillars; add them only when requested. -**Diffusion perf JSON** (`tests/dfx/perf/tests/test__vllm_omni.json`): array of objects with `test_name`, `server_params` (`model`, `serve_args`), `benchmark_params[]`, and per-workload **`baseline`** (`throughput_qps`, `latency_mean`, `peak_memory_mb_mean`). Copy structure from `test_qwen_image_vllm_omni.json` in-tree. +**Diffusion perf JSON** (`tests/dfx/perf/tests/test__vllm_omni.json`): array of **case objects**, each with: + +| Field | Required | Notes | +|-------|----------|-------| +| `test_name` | Yes | Unique server/workload id | +| `mark` | Recommended | When present: **`hardware_marks` required** + optional `marks` (`full_model`, `diffusion`, `local_model`, …). Parsed by `resolve_pytest_marks`; enables local `-m` filtering. | +| `server_type` | Diffusion | e.g. `"vllm-omni"` — routes case to `run_diffusion_benchmark.py` | +| `benchmark_endpoint` | Diffusion | e.g. `/v1/videos`, `/v1/images/generations` | +| `server_params` | Yes | `model`, `serve_args`, … | +| `benchmark_params[]` | Yes | Each row: **`name`** (pytest id suffix), workload fields, **`baseline`** | + +Copy structure from `test_qwen_image_vllm_omni.json` or `test_cosmos3_vllm_omni.json` (2-GPU mark example). + +**Omni perf JSON**: `test_qwen3_omni_*.json` — same `mark` shape with `"marks": ["full_model", "omni"]`; run via `run_benchmark.py`. + +**Runner split**: `is_diffusion_perf_config()` → diffusion when `server_type` is set or `"diffusion"` ∈ `mark.marks`; otherwise omni/tts → `run_benchmark.py`. + +**Param ids**: `{test_name}-{benchmark_params.name}` (e.g. `test_tts-p0`, not `test_tts-0`). + +**Result filenames**: runtime hardware from `get_runtime_resource_label()`; `H100` omitted on default CI pool. **Diffusion perf nightly step** (under **Diffusion X2I(&A&T)**; copy full `kubernetes` plugins from `· Perf Test · Qwen-Image`): @@ -177,9 +200,9 @@ Nightly **L4** for a model is often **multiple jobs** in `test-nightly.yml`, not | Model type | Perf runner | Config example | |------------|-------------|----------------| -| Diffusion X2I/X2V | `dfx/perf/scripts/run_diffusion_benchmark.py` | `test_qwen_image_vllm_omni.json` | -| TTS | `dfx/perf/scripts/run_benchmark.py` | `test_tts.json` (`BENCHMARK_DIR`) | -| Omni | `run_benchmark.py` / omni configs in-tree | `test_omni*.json` | +| Diffusion X2I/X2V | `dfx/perf/scripts/run_diffusion_benchmark.py` | `test_qwen_image_vllm_omni.json`, `test_cosmos3_vllm_omni.json` | +| TTS | `dfx/perf/scripts/run_benchmark.py` | `test_tts.json`, `test_voxcpm2.json`, `test_higgs_audio_v3.json` | +| Omni | `dfx/perf/scripts/run_benchmark.py` | `test_qwen3_omni_async_chunk.json`, `test_qwen3_omni_no_async_chunk.json`, … | Do **not** put throughput/latency baselines inside `test_*_expansion.py` — that belongs in the dfx perf JSON + nightly Perf job. diff --git a/docs/contributing/ci/CI_5levels.md b/docs/contributing/ci/CI_5levels.md index 17b4a3792..c9d86e1c4 100644 --- a/docs/contributing/ci/CI_5levels.md +++ b/docs/contributing/ci/CI_5levels.md @@ -126,11 +126,12 @@ Through five levels (L1-L5) and common (Common) specifications, the system clari Interface tests:
/tests/entrypoints/test_xxx
Performance:
- /tests/dfx/perf/tests/test_qwen_omni.json (Omni), test_tts.json (TTS),
- and /tests/dfx/perf/tests/test_{diffusion_model}_vllm_omni.json (Diffusion)
+ /tests/dfx/perf/tests/test_qwen3_omni_*.json (Omni), test_tts.json (TTS),
+ test_voxcpm2.json, test_higgs_audio_v3.json, and
+ /tests/dfx/perf/tests/test_{diffusion_model}_vllm_omni.json (Diffusion)
Doc Test:
- tests/example/online_serving/test_{model_name}.py
- tests/example/offline_inference/test_{model_name}.py
+ tests/examples/online_serving/test_{model_name}.py
+ tests/examples/offline_inference/test_{model_name}.py
Accuracy Test:
/tests/e2e/accuracy/test_{model_name}.py @@ -708,16 +709,16 @@ L4 level testing is a comprehensive quality audit before a version release. It e ### 3.2 Testing Content and Scope - ***Full Functionality Testing***: Executes all test cases defined in `test_{model_name}_expansion.py`, covering all implemented features, positive flows, boundary conditions, and exception handling. -- ***Performance Testing***: Uses `tests/dfx/perf/tests/test_qwen_omni.json`, `tests/dfx/perf/tests/test_tts.json`, and diffusion configs in the form `tests/dfx/perf/tests/test_*_vllm_omni.json` (passed to `run_benchmark.py` via `--test-config-file`) to drive performance testing tools for stress, load, and endurance tests, collecting metrics like throughput, response time, and resource utilization. +- ***Performance Testing***: Uses `tests/dfx/perf/tests/test_qwen3_omni_*.json` (Omni), `test_tts.json` / `test_voxcpm2.json` / `test_higgs_audio_v3.json` (TTS), and diffusion configs `tests/dfx/perf/tests/test_*_vllm_omni.json` (passed to `run_benchmark.py` or `run_diffusion_benchmark.py` via `--test-config-file` in nightly **Perf Test** steps) to drive throughput, latency, and memory benchmarks. Each JSON **case** may declare an optional top-level **`mark`** array: exactly one ``hardware_marks`` object plus pytest marker name strings (`full_model`, `omni` / `tts` / `diffusion`, …). Runners attach those marks to each parametrized `(server, benchmark index)` pair so **local** bulk runs can filter with `-m` (for example `-m "full_model and H100 and diffusion"`). Nightly CI perf jobs select workloads by **`--test-config-file`**, not `-m`. Details are in the Performance Tests example below. - ***Documentation Testing***: Verifies whether the example code provided to users is runnable and its results match the description. ### 3.3 Test Directory and Execution Files - ***Functional Testing***: Same directories as L3. -- ***Performance Test Configuration***: `tests/dfx/perf/tests/test_qwen_omni.json`, `tests/dfx/perf/tests/test_tts.json`, and diffusion configs `tests/dfx/perf/tests/test_*_vllm_omni.json` (e.g. `test_qwen_image_vllm_omni.json`) +- ***Performance Test Configuration***: `tests/dfx/perf/tests/test_qwen3_omni_*.json`, `test_tts.json`, `test_voxcpm2.json`, `test_higgs_audio_v3.json`, and diffusion configs `tests/dfx/perf/tests/test_*_vllm_omni.json` (e.g. `test_qwen_image_vllm_omni.json`, `test_cosmos3_vllm_omni.json`). Optional per-case `mark` for local `-m` filtering is documented in Section 3.4 Performance Tests. - ***Documentation Example Tests***: -- - `tests/example/online_serving/test_{model_name}.py` - - `tests/example/offline_inference/test_{model_name}.py` + - `tests/examples/online_serving/test_{model_name}.py` + - `tests/examples/offline_inference/test_{model_name}.py` ### 3.4 Execution Method and Example diff --git a/docs/contributing/ci/test_examples/l4_performance_tests.inc.md b/docs/contributing/ci/test_examples/l4_performance_tests.inc.md index f1f3073dc..5c59ba499 100644 --- a/docs/contributing/ci/test_examples/l4_performance_tests.inc.md +++ b/docs/contributing/ci/test_examples/l4_performance_tests.inc.md @@ -1,14 +1,41 @@ -When you want to add L4-level ***performance test*** cases, you can refer to the following format for case addition in `tests/dfx/perf/tests/test_qwen_omni.json`, `tests/dfx/perf/tests/test_tts.json`, or diffusion configs such as `tests/dfx/perf/tests/test_*_vllm_omni.json` (selected via `pytest ... run_benchmark.py --test-config-file `): +When you want to add L4-level ***performance test*** cases, add entries to JSON files under `tests/dfx/perf/tests/` and run them via `tests/dfx/perf/scripts/run_benchmark.py` (omni / TTS) or `run_diffusion_benchmark.py` (diffusion). + +**Config file layout (in-tree examples)** + +| Model type | Runner | Example JSON files | +| ---------- | ------ | ------------------ | +| Omni | `run_benchmark.py` | `test_qwen3_omni_no_async_chunk.json`, `test_qwen3_omni_async_chunk.json`, `test_qwen3_omni_vllm_text.json`, `test_qwen3_omni_multi_replicas.json` | +| TTS | `run_benchmark.py` | `test_tts.json`, `test_voxcpm2.json`, `test_higgs_audio_v3.json` | +| Diffusion | `run_diffusion_benchmark.py` | `test_qwen_image_vllm_omni.json`, `test_bagel_vllm_omni.json`, `test_wan22_i2v_vllm_omni.json`, `test_cosmos3_vllm_omni.json`, … | + +**How runners pick cases** + +Without **`--test-config-file`**, each runner scans all `*.json` under `tests/dfx/perf/tests/` but only keeps its own model type: + +- **`run_benchmark.py`**: omni and TTS cases only (skips diffusion JSON). +- **`run_diffusion_benchmark.py`**: diffusion cases only (skips omni / TTS JSON). + +Diffusion cases are detected when the JSON has `server_type` (typically `"vllm-omni"`) or `"diffusion"` in the `mark` array. Omni / TTS JSON has neither. + +**Running perf cases** + +Pass **`--test-config-file`** to run one JSON file as-is, or omit it for the bulk scan above and filter with pytest **`-m`** on each case's JSON `mark` (for example `pytest … run_benchmark.py -m "full_model and tts and H100"`). ```JSON { - "test_name": "test_qwen3_omni", + "test_name": "test_qwen3_omni_async_chunk", + "mark": [ + {"hardware_marks": {"res": {"cuda": "H100"}, "num_cards": 2}}, + "full_model", + "omni" + ], "server_params": { "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct", "stage_config_name": "qwen3_omni.yaml" }, "benchmark_params": [ { + "name": "random_10p", "dataset_name": "random", "num_prompts": [10, 20], "max_concurrency": [1, 4], @@ -33,8 +60,95 @@ When you want to add L4-level ***performance test*** cases, you can refer to the | Field | Required | Description | | ---------------- | -------- | --------------------------------------------------------------- | | test_name | Yes | Unique identifier for the test case | +| mark | No | Pytest marks for this case (see **`mark` field** below). Omit only for configs not meant to be filtered by `-m`. | | server_params | Yes | Server-side configuration parameters | | benchmark_params | Yes | Benchmark running parameters (supports multiple configurations) | +| server_type | Diffusion only | When set (e.g. `"vllm-omni"`), routes the case to `run_diffusion_benchmark.py`. | +| benchmark_endpoint | Diffusion only | API path for the benchmark client (e.g. `/v1/videos`, `/v1/images/generations`). | + +**`mark` field** + +Optional top-level field on each perf JSON **case object** (one per `test_name`). `run_benchmark.py` and `run_diffusion_benchmark.py` read it via `tests.dfx.conftest.resolve_pytest_marks` and attach the marks to the corresponding `pytest.param`, so you can filter locally with `-m`. + +When `mark` is present, it must be an **array** with exactly one ``hardware_marks`` object (same shape as `@hardware_test` / `hardware_marks()` in `tests/helpers/mark.py`), followed by registered pytest marker name strings such as `full_model`, `omni`, `tts`, `diffusion`, or `local_model`. + +Supported form: + +| Form | Example | Effect | +| ---- | ------- | ------ | +| Array (single platform) | `"mark": [{"hardware_marks": {"res": {"cuda": "H100"}, "num_cards": 2}}, "full_model", "diffusion"]` | Hardware marks from `hardware_marks(...)`; `num_cards` > 1 adds `distributed_*` (+ CUDA `skipif_cuda` when applicable). String entries become extra pytest marks. | +| Array (multi-platform) | `"mark": [{"hardware_marks": {"res": {"cuda": "H100", "rocm": "MI325", "npu": "A2"}, "num_cards": {"cuda": 2, "rocm": 2, "npu": 1}}}, "full_model", "omni"]` | Same as above, but declares **multiple platforms** in one case. Filter examples: `-m "full_model and H100 and cuda"`, `-m "full_model and MI325 and rocm"`. | + +Recommended for L4 perf cases: + +```JSON +{ + "test_name": "test_bagel_single_device_single_stage_t2i", + "mark": [ + {"hardware_marks": {"res": {"cuda": "H100"}, "num_cards": 1}}, + "full_model", + "diffusion" + ], + "server_type": "vllm-omni", + "server_params": { "...": "..." }, + "benchmark_params": [ { "name": "1024x1024_steps20", "...": "..." } ] +} +``` + +Multi-GPU diffusion (example: Cosmos3 with `cfg-parallel-size=2`): + +```JSON +{ + "test_name": "test_cosmos3_t2i_official_demo_2gpu", + "mark": [ + {"hardware_marks": {"res": {"cuda": "H100"}, "num_cards": 2}}, + "full_model", + "diffusion" + ], + "server_type": "vllm-omni", + "benchmark_endpoint": "/v1/images/generations", + "server_params": { "...": "..." }, + "benchmark_params": [ { "name": "1024x1024_steps4", "...": "..." } ] +} +``` + +- Omni perf: include `"omni"` in the `mark` array +- TTS perf: include `"tts"` in the `mark` array +- Diffusion perf: include `"diffusion"` in the `mark` array +- HunyuanImage local-weight cases: add `"local_model"` to the `mark` array + +**Parametrization IDs** + +Each `(server, benchmark index)` pair becomes one `pytest.param` with id `{test_name}-{suffix}`. The suffix comes from `benchmark_params[].name` when set (for example `test_tts-p0`, `test_omni-p1`); otherwise it is derived from fields like `task` / `eval_phase`. + +**Benchmark result filenames** + +Result files use the **runtime** hardware label from `get_runtime_resource_label()` (detected GPU/NPU name on the machine that ran the job), **not** `mark.hardware_marks.res`. On the default H100 CI pool, `H100` is **omitted** from filenames (`resource_label_for_filename`). + +Examples: + +- Omni/TTS: `result_{test_name}_{optional_hw}_{dataset}_....json` under `BENCHMARK_DIR` +- Diffusion: one aggregate `diffusion_result_{config_stem}_{optional_hw}_{timestamp}.json` per source JSON file (array of all runs from that file) + +**Local commands** + +```bash +# Bulk load + filter by JSON mark +pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py -m "full_model and H100 and diffusion" +pytest -s -v tests/dfx/perf/scripts/run_benchmark.py -m "full_model and omni and H100" + +# Single file (same as nightly CI Perf steps) +pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py \ + --test-config-file tests/dfx/perf/tests/test_bagel_vllm_omni.json +pytest -s -v tests/dfx/perf/scripts/run_benchmark.py \ + --test-config-file tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json + +# Optional baseline assertion (default off) +pytest -sv tests/dfx/perf/scripts/run_diffusion_benchmark.py --assert-baseline \ + --test-config-file tests/dfx/perf/tests/test_qwen_image_vllm_omni.json +``` + +See also [Markers for Tests](./tests_markers.md) for registered hardware markers (`H100`, `L4`, `cuda`, `distributed_cuda`, …). **`server_params` Configuration** @@ -79,10 +193,12 @@ You can add any benchmark running parameters you need here. For all optional par 1. Change the --xxx-xx-xx running parameters to xxx_xx_xx format and fill them as keys in the JSON file. 2. For boolean variables in the running parameters, modify them to forms such as ignore_eos: true/false and fill them into the JSON file. 3. Optionally add a `baseline` object (see **Baseline thresholds** below). If you omit `baseline` or leave it empty, the performance test still runs but does not assert metric thresholds from this field. -4. The qps and concurrency modes are recommended to be mutually exclusive. For detailed explanations, see the table below: +4. Set `"name"` on each `benchmark_params` entry for stable pytest ids and readable result keys. +5. The qps and concurrency modes are recommended to be mutually exclusive. For detailed explanations, see the table below: | Parameter | Type | Required | Example/Values | Description | | --------------- | ----------- | -------- | --------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| name | string | No | `"1024x1024_steps4"` | Stable suffix for pytest param id and logs. | | num_prompts | int / array | Yes | 10,[10, 20, 30] | Number of requests. Supports single values or arrays. If a single value is used, it will be automatically expanded to match the number of qps or max_concurrency, e.g., [10,10,10]. If an array is used, its length must match the number of qps or max_concurrency. | | request_rate | float / array | No | 0.5, [0.5, 1, inf] | Queries per second. Supports single values or arrays. If a single value is used, it will be automatically expanded to match the number of num_prompts, e.g., [1,1,1]. If an array is used, its length must match the number of num_prompts. | | max_concurrency | int / array | No | 1, [1, 2, 3] | Maximum concurrent in-flight requests. Same array / expansion rules as `request_rate` (mutually exclusive with QPS mode). | diff --git a/docs/contributing/ci/test_guide.md b/docs/contributing/ci/test_guide.md index 26996129e..ad6a63963 100644 --- a/docs/contributing/ci/test_guide.md +++ b/docs/contributing/ci/test_guide.md @@ -149,12 +149,18 @@ Failed jobs: 1/2 pytest -s -v -m "full_model and distributed_cuda and L4" --run-level=full_model pytest -s -v -m "full_model and (omni or tts) and H100" --run-level=full_model ``` - - To run a standalone perf benchmark (defaults to ``test_qwen_omni.json``; use ``--test-config-file tests/dfx/perf/tests/test_tts.json`` for TTS): - + If you only want to run specific test cases on a particular platform, you can use: ```bash - pytest -s -v tests/dfx/perf/scripts/run_benchmark.py + pytest -s -v -m "full_model and distributed_cuda and L4" --run-level=full_model ``` + Note: ``run_benchmark.py`` and ``run_diffusion_benchmark.py`` accept an optional ``--test-config-file``. If omitted, each loads every ``*.json`` under ``tests/dfx/perf/tests/`` (omni/tts vs diffusion split by ``is_diffusion_perf_config``) and pytest ``-m`` filters by each case's JSON ``mark``: + ```bash + pytest -sv tests/dfx/perf/scripts/run_benchmark.py -m "full_model and tts and H100" + pytest -sv tests/dfx/perf/scripts/run_diffusion_benchmark.py -m "full_model and diffusion and H100" + pytest -sv tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_tts.json + pytest -sv tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_cosmos3_vllm_omni.json + ``` + Nightly **Perf Test** jobs in ``test-nightly.yml`` use ``--test-config-file`` only (no ``-m``). E2e L4 function tests still use ``full_model`` + ``--run-level full_model`` (see [test-nightly.yml](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/test-nightly.yml)). Example: === "L5 level" diff --git a/docs/contributing/ci/tests_markers.md b/docs/contributing/ci/tests_markers.md index cc89cee47..0a0a974e6 100644 --- a/docs/contributing/ci/tests_markers.md +++ b/docs/contributing/ci/tests_markers.md @@ -33,7 +33,8 @@ Defined in `pyproject.toml`: | `skipif_rocm` | Skip if the num of ROCm cards is less than the required * | | `skipif_npu` | Skip if the num of NPU cards is less than the required * | | `slow` | Slow tests (may skip in quick CI) | -| `benchmark` | Benchmark tests | +| `benchmark` | Benchmark tests (decorator on runner test functions; perf JSON uses `full_model` + type marker instead) | +| `local_model` | Tests requiring local / non-HF-hub model weights | \* Means those markers are auto-added by `@hardware_test` (parametrization decorator) or `hardware_marks` (only returning the list of marks for flexibility). @@ -94,7 +95,7 @@ This decorator is intended to make hardware-aware, cross-platform test authoring num_cards=2, ) ``` -- `res` must be a dict; supported resources: CUDA (L4/H100), ROCm (MI325), NPU (A2/A3) +- `res` must be a dict; supported resources: CUDA (L4/H100), ROCm (MI325), XPU (B60), MUSA (S5000), NPU (A2/A3) - `num_cards` can be int (all platforms) or dict (per platform); defaults to 1 when missing - Distributed markers (`distributed_cuda`, `distributed_rocm`, `distributed_npu`) are auto-added for multi-card cases - Filtering examples: @@ -120,6 +121,32 @@ def test_feature(omni_server): ... ``` +### JSON `mark` field (L4 perf configs) + +Perf JSON under `tests/dfx/perf/tests/` can attach pytest marks per **case** (`test_name` block). Parsed by `tests.dfx.conftest.resolve_pytest_marks` and applied to each parametrized run in `run_benchmark.py` / `run_diffusion_benchmark.py`. + +When `mark` is present, it must be an **array** with exactly one ``hardware_marks`` object (same semantics as `hardware_marks()` above), followed by registered pytest marker name strings such as `full_model`, `omni`, `tts`, or `diffusion`. + +```json +{ + "test_name": "test_cosmos3_t2i_official_demo_2gpu", + "mark": [ + {"hardware_marks": {"res": {"cuda": "H100"}, "num_cards": 2}}, + "full_model", + "diffusion" + ], + "server_type": "vllm-omni", + "server_params": { "...": "..." }, + "benchmark_params": [{ "name": "1024x1024_steps4", "...": "..." }] +} +``` + +- Local bulk load: `pytest -sv tests/dfx/perf/scripts/run_diffusion_benchmark.py -m "full_model and diffusion and H100"` +- Nightly CI perf steps: `--test-config-file tests/dfx/perf/tests/test__vllm_omni.json` (file selects cases; no `-m`) +- Result filenames use **runtime** GPU detection (`get_runtime_resource_label`); `H100` is omitted on the default CI pool + +See [L4 performance test examples](./test_examples/l4_performance_tests.inc.md) for the full schema. + ## Add Support for a New Platform If you want to add support for a new platform (e.g., "tpu" for a new accelerator), follow these steps: diff --git a/recipes/LTX/LTX-2.3.md b/recipes/LTX/LTX-2.3.md index c83a14883..7a4f1507c 100644 --- a/recipes/LTX/LTX-2.3.md +++ b/recipes/LTX/LTX-2.3.md @@ -148,7 +148,7 @@ Do not copy latency or VRAM numbers between commits or machines. Record them only after running a latest-head sweep on the target hardware. For formal PR benchmarking, reuse -`tests/dfx/perf/tests/test_ltx2_3_vllm_omni.json` with -`tests/dfx/perf/scripts/run_diffusion_benchmark.py`. That config captures the +`tests/dfx/perf/tests/test_ltx2_vllm_omni.json` with +`tests/dfx/perf/scripts/run_diffusion_benchmark.py`. Each case in that file includes a JSON **`mark`** (`hardware_marks` + `full_model` + `diffusion`) for local `-m` filtering. That config captures the single-device eager baseline and CFG-parallel=2 case for a small 384x512, 25-frame, 20-step workload. diff --git a/tests/dfx/conftest.py b/tests/dfx/conftest.py index a22f760b2..2eb88bbd6 100644 --- a/tests/dfx/conftest.py +++ b/tests/dfx/conftest.py @@ -11,11 +11,180 @@ from typing import Any import pytest from tests.dfx.reliability.helpers import list_remote_process_pids_by_pattern, post_chat_completions_raw +from tests.helpers.mark import hardware_marks from tests.helpers.runtime import OmniServerParams from tests.helpers.stage_config import modify_stage_config from vllm_omni.platforms import current_omni_platform +def _named_pytest_marks(names: list[str]) -> list[pytest.MarkDecorator]: + marks: list[pytest.MarkDecorator] = [] + for name in names: + name = name.strip() + if not name: + raise ValueError("mark name must be a non-empty string") + marks.append(getattr(pytest.mark, name)) + return marks + + +def _hardware_marks_from_dict(hw: Any) -> list[pytest.MarkDecorator]: + if not isinstance(hw, dict): + raise ValueError(f"mark.hardware_marks must be a dict, got {type(hw).__name__}") + res = hw.get("res") + if not isinstance(res, dict): + raise ValueError(f"mark.hardware_marks.res must be a dict, got {type(res).__name__}") + num_cards = hw.get("num_cards", 1) + return list(hardware_marks(res=res, num_cards=num_cards)) + + +def resolve_pytest_marks(mark_field: Any) -> list[pytest.MarkDecorator]: + """Convert a JSON ``mark`` field into pytest mark decorators. + + Supported form (per test-case object in perf/stability JSON):: + + "mark": [ + {"hardware_marks": {"res": {"cuda": "H100"}, "num_cards": 2}}, + "full_model", + "diffusion" + ] + + Exactly one ``hardware_marks`` object is required when ``mark`` is present. + ``hardware_marks`` delegates to :func:`tests.helpers.mark.hardware_marks` + (same shape as ``@hardware_test``). Additional array entries are registered + pytest marker names (strings). + """ + if mark_field is None: + return [] + + if isinstance(mark_field, list): + marks: list[pytest.MarkDecorator] = [] + hw_seen = False + for item in mark_field: + if isinstance(item, dict) and "hardware_marks" in item: + if hw_seen: + raise ValueError("mark array must contain at most one hardware_marks object") + hw_seen = True + marks.extend(_hardware_marks_from_dict(item["hardware_marks"])) + unknown_keys = set(item) - {"hardware_marks"} + if unknown_keys: + raise ValueError( + f"mark hardware_marks object only allows hardware_marks; unknown keys: {sorted(unknown_keys)}" + ) + elif isinstance(item, str): + item = item.strip() + if not item: + raise ValueError("mark name must be a non-empty string") + marks.extend(_named_pytest_marks([item])) + else: + raise ValueError( + f"mark array entries must be hardware_marks objects or marker name strings; got {type(item).__name__}" + ) + if not hw_seen: + raise ValueError("mark array must contain a hardware_marks object") + return marks + + raise ValueError(f"mark must be a list; got {type(mark_field).__name__}") + + +def _mark_names(mark_field: Any) -> set[str]: + if isinstance(mark_field, list): + return {str(item) for item in mark_field if isinstance(item, str)} + return set() + + +def is_diffusion_perf_config(cfg: dict[str, Any]) -> bool: + """True for perf JSON cases intended for ``run_diffusion_benchmark.py``.""" + if cfg.get("server_type") is not None: + return True + return "diffusion" in _mark_names(cfg.get("mark")) + + +def _marks_by_test_name(configs: list[dict[str, Any]]) -> dict[str, list[pytest.MarkDecorator]]: + return {str(cfg["test_name"]): resolve_pytest_marks(cfg.get("mark")) for cfg in configs} + + +def create_unique_server_pytest_params( + configs: list[dict[str, Any]], + stage_configs_dir: Path, +) -> list[Any]: + """Like :func:`create_unique_server_params`, but wrap each row in ``pytest.param`` with JSON marks.""" + marks_by_name = _marks_by_test_name(configs) + return [ + pytest.param( + row, + marks=marks_by_name.get(row[0], []), + id=row[0], + ) + for row in create_unique_server_params(configs, stage_configs_dir) + ] + + +def create_benchmark_pytest_params( + benchmark_configs: list[dict[str, Any]], + server_to_benchmark_mapping: dict[str, dict], +) -> list[Any]: + """Like :func:`create_benchmark_indices`, but wrap each index in ``pytest.param`` with JSON marks.""" + marks_by_name = _marks_by_test_name(benchmark_configs) + params: list[Any] = [] + seen: set[str] = set() + for config in benchmark_configs: + test_name = config["test_name"] + if test_name in seen: + continue + seen.add(test_name) + params_list = get_benchmark_params_for_server(test_name, server_to_benchmark_mapping) + id_suffixes = _unique_benchmark_param_id_suffixes(params_list) + for idx, id_suffix in enumerate(id_suffixes): + params.append( + pytest.param( + (test_name, idx), + marks=marks_by_name.get(test_name, []), + id=f"{test_name}-{id_suffix}", + ) + ) + return params + + +def create_paired_benchmark_pytest_params( + server_entries: list[tuple[Any, str]], + params_by_test_name: dict[str, list[Any]], + marks_by_name: dict[str, list[pytest.MarkDecorator]], +) -> list[Any]: + """One ``pytest.param`` per ``(server entry, benchmark index)`` pair. + + Pass ``server_entry`` and ``(test_name, idx)`` as separate ``pytest.param`` + arguments for ``@pytest.mark.parametrize("server_fixture,benchmark_params", ...)``. + """ + pairs: list[Any] = [] + for server_entry, test_name in server_entries: + params_list = params_by_test_name.get(test_name, []) + id_suffixes = _unique_benchmark_param_id_suffixes(params_list) + for idx, id_suffix in enumerate(id_suffixes): + pairs.append( + pytest.param( + server_entry, + (test_name, idx), + marks=marks_by_name.get(test_name, []), + id=f"{test_name}-{id_suffix}", + ) + ) + return pairs + + +def create_paired_omni_benchmark_pytest_params( + configs: list[dict[str, Any]], + stage_configs_dir: Path, +) -> list[Any]: + """Paired params for ``run_benchmark.py`` (omni/tts).""" + mapping = create_test_parameter_mapping(configs) + marks_by_name = _marks_by_test_name(configs) + server_entries = [(row, row[0]) for row in create_unique_server_params(configs, stage_configs_dir)] + params_by_test_name = { + test_name: get_benchmark_params_for_server(test_name, mapping) for _, test_name in server_entries + } + return create_paired_benchmark_pytest_params(server_entries, params_by_test_name, marks_by_name) + + def load_configs(config_path: str) -> list[dict[str, Any]]: try: abs_path = Path(config_path).resolve() @@ -32,6 +201,29 @@ def load_configs(config_path: str) -> list[dict[str, Any]]: raise RuntimeError(f"Failed to load configuration file: {str(e)}") +def load_benchmark_configs( + config_path: str | None = None, + *, + config_dir: Path | None = None, +) -> list[dict[str, Any]]: + """Load one benchmark JSON file, or merge all ``*.json`` under *config_dir*. + + When *config_path* is omitted, every ``*.json`` in *config_dir* is loaded and + concatenated. Pytest ``-m`` expressions then select cases via each entry's + ``mark`` field (e.g. ``-m tts`` after bulk load). + """ + if config_path is not None: + return load_configs(config_path) + if config_dir is None: + raise ValueError("load_benchmark_configs requires config_path or config_dir") + configs: list[dict[str, Any]] = [] + for path in sorted(config_dir.glob("*.json")): + configs.extend(load_configs(str(path))) + if not configs: + raise ValueError(f"No benchmark JSON files found under {config_dir}") + return configs + + def modify_stage(default_path: str, updates: dict[str, Any] | None, deletes: dict[str, Any] | None) -> str: kwargs: dict[str, Any] = {} if updates is not None: @@ -293,6 +485,158 @@ def _safe_filename_token(value: Any | None, *, default: str = "na") -> str: return s if s else default +def _benchmark_param_id_suffix(param: dict[str, Any], *, idx: int) -> str: + """Derive a readable pytest id suffix from one ``benchmark_params`` entry.""" + name = param.get("name") + if isinstance(name, str) and name.strip(): + return _safe_filename_token(name.strip()) + + parts: list[str] = [] + for key in ("task", "eval_phase", "dataset_name", "dataset"): + value = param.get(key) + if value is None or value == "": + continue + token = _safe_filename_token(str(value)) + if token != "na": + parts.append(token) + if parts: + return "_".join(parts) + + return f"case{idx}" + + +def _unique_benchmark_param_id_suffixes(params_list: list[dict[str, Any]]) -> list[str]: + """Return unique pytest id suffixes for a server's benchmark param list.""" + raw = [_benchmark_param_id_suffix(param, idx=idx) for idx, param in enumerate(params_list)] + seen: dict[str, int] = {} + unique: list[str] = [] + for suffix in raw: + count = seen.get(suffix, 0) + seen[suffix] = count + 1 + if count == 0: + unique.append(suffix) + else: + unique.append(f"{suffix}_{count}") + return unique + + +def extract_mark_resource_label(mark_field: Any) -> str: + """Return a filename-safe hardware label from ``mark.hardware_marks.res`` values. + + Example: ``{"cuda": "H100"}`` -> ``"H100"``; multiple platforms join with ``-``. + + Prefer :func:`get_runtime_resource_label` for perf result filenames so labels + reflect the machine that actually ran the benchmark. + """ + if isinstance(mark_field, list): + for item in mark_field: + if isinstance(item, dict) and "hardware_marks" in item: + return extract_mark_resource_label(item) + return "na" + if not isinstance(mark_field, dict): + return "na" + hw = mark_field.get("hardware_marks") + if not isinstance(hw, dict): + return "na" + res = hw.get("res") + if not isinstance(res, dict) or not res: + return "na" + labels = [_safe_filename_token(value) for value in res.values()] + return "-".join(labels) if labels else "na" + + +_KNOWN_RUNTIME_RESOURCE_TOKENS: tuple[str, ...] = ( + "H100", + "H800", + "H200", + "H20", + "L40S", + "L40", + "L4", + "A100", + "A800", + "A10G", + "A10", + "A30", + "MI325", + "MI300", + "MI250", + "B60", + "S5000", + "910B4", + "910B", + "910", + "310P", + "A2", + "A3", +) +_RUNTIME_RESOURCE_LABEL: str | None = None + + +def _normalize_runtime_device_label(raw: str) -> str: + """Map a platform device name to a short filename-safe resource token.""" + if not raw or not str(raw).strip(): + return "na" + upper = str(raw).upper() + for token in _KNOWN_RUNTIME_RESOURCE_TOKENS: + if token.upper() in upper: + return _safe_filename_token(token) + compact = re.sub(r"[^a-zA-Z0-9]+", "", str(raw)) + for prefix in ("NVIDIA", "AMD", "ASCEND", "HUAWEI"): + if compact.upper().startswith(prefix): + compact = compact[len(prefix) :] + break + return _safe_filename_token(compact[:48]) if compact else "na" + + +def _read_runtime_device_name(*, device_id: int = 0) -> str | None: + """Device name from the active Omni platform.""" + if current_omni_platform.device_count() <= device_id: + return None + get_name = getattr(current_omni_platform, "get_device_name", None) + if not callable(get_name): + return None + raw = get_name(device_id) + if isinstance(raw, str) and raw.strip(): + return raw.strip() + return None + + +def get_runtime_resource_label(*, device_id: int = 0, refresh: bool = False) -> str: + """Return a filename-safe hardware label detected on the running machine.""" + global _RUNTIME_RESOURCE_LABEL + if not refresh and _RUNTIME_RESOURCE_LABEL is not None: + return _RUNTIME_RESOURCE_LABEL + raw = _read_runtime_device_name(device_id=device_id) + label = _normalize_runtime_device_label(raw) if raw else "na" + if not refresh: + _RUNTIME_RESOURCE_LABEL = label + return label + + +_FILENAME_OMIT_RESOURCE_LABELS = frozenset({"H100"}) + + +def hardware_json_value(resource_label: str | None) -> str: + """Hardware token stored in perf result JSON (empty when unknown).""" + token = _safe_filename_token(resource_label) + return "" if token == "na" else token + + +def resource_label_for_filename(resource_label: str | None) -> str: + """Hardware token embedded in result filenames (H100 omitted on default CI pool).""" + token = _safe_filename_token(resource_label) + if token in _FILENAME_OMIT_RESOURCE_LABELS: + return "" + return token + + +def extract_configs_resource_label(configs: list[dict[str, Any]]) -> str: + """Return runtime hardware label for perf result filenames.""" + del configs + return get_runtime_resource_label() + + def resolve_baseline_value( baseline_raw: Any, *, @@ -357,6 +701,7 @@ def run_benchmark( max_concurrency: Any | None = None, random_input_len: Any | None = None, random_output_len: Any | None = None, + resource_label: str | None = None, ) -> dict[str, Any]: """Run one ``vllm bench serve --omni`` iteration and return parsed metrics. @@ -367,7 +712,11 @@ def run_benchmark( current_dt = datetime.now().strftime("%Y%m%d-%H%M%S") ri = _safe_filename_token(random_input_len) ro = _safe_filename_token(random_output_len) - result_filename = f"result_{test_name}_{dataset_name}_{flow}_{num_prompt}_in{ri}_out{ro}_{current_dt}.json" + hw = resource_label_for_filename(resource_label) + if hw: + result_filename = f"result_{test_name}_{hw}_{dataset_name}_{flow}_{num_prompt}_in{ri}_out{ro}_{current_dt}.json" + else: + result_filename = f"result_{test_name}_{dataset_name}_{flow}_{num_prompt}_in{ri}_out{ro}_{current_dt}.json" if "--result-filename" in args: print(f"The result file will be overwritten by {result_filename}") command = ( @@ -437,6 +786,7 @@ def run_benchmark( result["random_input_len"] = random_input_len if random_output_len is not None: result["random_output_len"] = random_output_len + result["Hardware"] = hardware_json_value(resource_label) with open(result_path, "w", encoding="utf-8") as f: json.dump(result, f, ensure_ascii=False, indent=2) return result diff --git a/tests/dfx/perf/scripts/run_benchmark.py b/tests/dfx/perf/scripts/run_benchmark.py index 13329a209..21167cb07 100644 --- a/tests/dfx/perf/scripts/run_benchmark.py +++ b/tests/dfx/perf/scripts/run_benchmark.py @@ -7,21 +7,24 @@ from typing import Any import pytest from tests.dfx.conftest import ( - create_benchmark_indices, + create_paired_omni_benchmark_pytest_params, create_test_parameter_mapping, - create_unique_server_params, get_benchmark_params_for_server, - load_configs, + get_runtime_resource_label, + is_diffusion_perf_config, + load_benchmark_configs, resolve_baseline_value, run_benchmark, ) from tests.helpers.runtime import OmniServer -pytestmark = [pytest.mark.full_model] - # Compare metrics to each test JSON ``baseline`` block only when pytest is run with ``--assert-baseline`` # (registered in ``tests/dfx/conftest.py``; default: off). ``run_benchmark`` and ``_resolve_baseline_value`` are # defined in the same module. +# +# Optional JSON field ``mark`` is applied as pytest marks via +# ``create_paired_omni_benchmark_pytest_params`` (e.g. ``"mark": [{"hardware_marks": +# {"res": {"cuda": "H100"}, "num_cards": 2}}, "full_model", "omni"]``). os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" @@ -40,22 +43,22 @@ def _get_config_file_from_argv() -> str | None: _PERF_TESTS_DIR = Path(__file__).resolve().parent.parent / "tests" -_DEFAULT_CONFIG_FILE = str(_PERF_TESTS_DIR / "test_qwen_omni.json") CONFIG_FILE_PATH = _get_config_file_from_argv() if CONFIG_FILE_PATH is None: + _all_configs = load_benchmark_configs(config_dir=_PERF_TESTS_DIR) + BENCHMARK_CONFIGS = [cfg for cfg in _all_configs if not is_diffusion_perf_config(cfg)] print( - "No --test-config-file in argv, using default: tests/dfx/perf/tests/test_qwen_omni.json " - "(override with e.g. --test-config-file tests/dfx/perf/tests/test_tts.json)" + f"No --test-config-file: loaded {len(BENCHMARK_CONFIGS)} omni/tts case(s) from " + f"{_PERF_TESTS_DIR}/*.json (skipped {len(_all_configs) - len(BENCHMARK_CONFIGS)} diffusion; " + f"use -m to filter, e.g. -m tts)" ) - CONFIG_FILE_PATH = _DEFAULT_CONFIG_FILE - -BENCHMARK_CONFIGS = load_configs(CONFIG_FILE_PATH) - +else: + BENCHMARK_CONFIGS = load_benchmark_configs(CONFIG_FILE_PATH) DEPLOY_CONFIGS_DIR = Path(__file__).parent.parent / "deploy" -test_params = create_unique_server_params(BENCHMARK_CONFIGS, DEPLOY_CONFIGS_DIR) server_to_benchmark_mapping = create_test_parameter_mapping(BENCHMARK_CONFIGS) +paired_benchmark_params = create_paired_omni_benchmark_pytest_params(BENCHMARK_CONFIGS, DEPLOY_CONFIGS_DIR) _omni_server_lock = threading.Lock() @@ -91,17 +94,11 @@ def omni_server(request): print("OmniServer stopped") -benchmark_indices = create_benchmark_indices(BENCHMARK_CONFIGS, server_to_benchmark_mapping) - - @pytest.fixture -def benchmark_params(request, omni_server): - """Benchmark parameters fixture with proper parametrization""" +def benchmark_params(request): + """Benchmark parameters fixture; paired with ``omni_server`` via parametrization.""" test_name, param_index = request.param - if test_name != omni_server.test_name: - pytest.skip(f"Skipping parameter for {test_name} - current server is {omni_server.test_name}") - all_params = get_benchmark_params_for_server(test_name, server_to_benchmark_mapping) if not all_params: @@ -153,8 +150,11 @@ def assert_result( @pytest.mark.benchmark -@pytest.mark.parametrize("omni_server", test_params, indirect=True) -@pytest.mark.parametrize("benchmark_params", benchmark_indices, indirect=True) +@pytest.mark.parametrize( + "omni_server,benchmark_params", + paired_benchmark_params, + indirect=["omni_server", "benchmark_params"], +) def test_performance_benchmark(omni_server, benchmark_params, request): test_name = benchmark_params["test_name"] params = benchmark_params["params"] @@ -168,6 +168,7 @@ def test_performance_benchmark(omni_server, benchmark_params, request): print(f"Benchmark parameters: {benchmark_params}") assert_baseline = request.config.getoption("--assert-baseline", default=False) + resource_label = get_runtime_resource_label() def to_list(value, default=None): if value is None: @@ -239,6 +240,7 @@ def test_performance_benchmark(omni_server, benchmark_params, request): max_concurrency=None, random_input_len=params.get("random_input_len"), random_output_len=params.get("random_output_len"), + resource_label=resource_label, ) assert_result( result, @@ -264,6 +266,7 @@ def test_performance_benchmark(omni_server, benchmark_params, request): max_concurrency=concurrency, random_input_len=params.get("random_input_len"), random_output_len=params.get("random_output_len"), + resource_label=resource_label, ) assert_result( result, diff --git a/tests/dfx/perf/scripts/run_diffusion_benchmark.py b/tests/dfx/perf/scripts/run_diffusion_benchmark.py index b0860978b..d19c0900e 100644 --- a/tests/dfx/perf/scripts/run_diffusion_benchmark.py +++ b/tests/dfx/perf/scripts/run_diffusion_benchmark.py @@ -8,15 +8,21 @@ This runner separates two concepts: 2. ``benchmark_endpoint``: which serving API the benchmark client calls. Examples: ``/v1/chat/completions`` and ``/v1/videos``. -A config JSON file is REQUIRED via --test-config-file: +A config JSON file may be passed via --test-config-file. If omitted, every ``*.json`` under +``tests/dfx/perf/tests/`` is loaded and pytest ``-m`` filters by each case's ``mark``: + pytest run_diffusion_benchmark.py -m "diffusion" pytest run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_vllm_omni.json Optional: ``--assert-baseline`` compares metrics to the ``baseline`` block in each benchmark entry (default: off). -All benchmark results for a session are consolidated into a single JSON file under -BENCHMARK_RESULT_DIR (override via the DIFFUSION_BENCHMARK_DIR environment variable). -Each entry in the file contains the test metadata (test_name, endpoint, benchmark_params, -timestamp) together with the raw metrics returned by the benchmark script. +Optional JSON field ``mark`` is applied as pytest marks on that case via +``pytest.param`` (e.g. ``"mark": [{"hardware_marks": {"res": {"cuda": "H100"}, "num_cards": 1}}, "full_model", "diffusion"]``). + +All benchmark results are written under BENCHMARK_RESULT_DIR (override via the +DIFFUSION_BENCHMARK_DIR environment variable). Each source JSON file gets one +aggregated ``diffusion_result_{config_stem}_{hardware}_{timestamp}.json`` (JSON array +of all runs from cases in that file). Bulk load without ``--test-config-file`` uses +the same per-file aggregation; ``-m`` only selects which cases run. """ import json @@ -35,8 +41,15 @@ import psutil import pytest from benchmarks.diffusion.backends import endpoint_filename_token, normalize_endpoint - -pytestmark = [pytest.mark.diffusion, pytest.mark.full_model, pytest.mark.local_model] +from tests.dfx.conftest import ( + create_paired_benchmark_pytest_params, + get_runtime_resource_label, + hardware_json_value, + is_diffusion_perf_config, + resolve_pytest_marks, + resource_label_for_filename, +) +from tests.helpers.runtime import get_open_port os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" os.environ.setdefault("DIFFUSION_ATTENTION_BACKEND", "FLASH_ATTN") @@ -126,6 +139,10 @@ _BRANCHPOINT_COMMIT_SHA: str | None = None DIFFUSION_RESULT_TEMPLATE_PATH = Path(__file__).parent / "diffusion_result_template.json" +_DIFFUSION_SOURCE_CONFIG_KEY = "_source_config_file" +_PERF_TESTS_DIR = Path(__file__).resolve().parent.parent / "tests" + + def _get_config_file_from_argv() -> str | None: """Read --test-config-file from sys.argv at import time so pytest parametrize can use it. @@ -142,9 +159,6 @@ def _get_config_file_from_argv() -> str | None: CONFIG_FILE_PATH = _get_config_file_from_argv() -if CONFIG_FILE_PATH is None: - print("No config file provided, using default config file: tests/dfx/perf/tests/test_qwen_image_vllm_omni.json") - CONFIG_FILE_PATH = "tests/dfx/perf/tests/test_qwen_image_vllm_omni.json" # --------------------------------------------------------------------------- # Config loading @@ -184,28 +198,85 @@ def load_configs(config_path: str) -> list[dict[str, Any]]: raise RuntimeError(f"Failed to load configuration file: {str(e)}") -BENCHMARK_CONFIGS = load_configs(CONFIG_FILE_PATH) - -_config_stem = Path(CONFIG_FILE_PATH).stem # e.g. "test_qwen_image_vllm_omni" -AGGREGATED_RESULT_FILE = BENCHMARK_RESULT_DIR / f"diffusion_result_{_config_stem}_{_SESSION_TIMESTAMP}.json" +def load_diffusion_benchmark_configs( + config_path: str | None = None, + *, + config_dir: Path | None = None, +) -> list[dict[str, Any]]: + """Load one diffusion benchmark JSON, or merge all ``*.json`` under *config_dir*.""" + if config_path is not None: + configs = load_configs(config_path) + source = str(Path(config_path).resolve()) + for cfg in configs: + cfg.setdefault(_DIFFUSION_SOURCE_CONFIG_KEY, source) + return configs + if config_dir is None: + raise ValueError("load_diffusion_benchmark_configs requires config_path or config_dir") + configs: list[dict[str, Any]] = [] + for path in sorted(config_dir.glob("*.json")): + source = str(path.resolve()) + for cfg in load_configs(str(path)): + cfg[_DIFFUSION_SOURCE_CONFIG_KEY] = source + configs.append(cfg) + if not configs: + raise ValueError(f"No benchmark JSON files found under {config_dir}") + return configs -def _append_to_aggregated_file(record: dict[str, Any]) -> None: - """Thread-safe append of *record* to the session-level aggregated JSON file. +if CONFIG_FILE_PATH is None: + _all_configs = load_diffusion_benchmark_configs(config_dir=_PERF_TESTS_DIR) + BENCHMARK_CONFIGS = [cfg for cfg in _all_configs if is_diffusion_perf_config(cfg)] + print( + f"No --test-config-file: loaded {len(BENCHMARK_CONFIGS)} diffusion case(s) from " + f"{_PERF_TESTS_DIR}/*.json (skipped {len(_all_configs) - len(BENCHMARK_CONFIGS)} omni/tts; " + f"use -m to filter, e.g. -m diffusion)" + ) +else: + BENCHMARK_CONFIGS = load_diffusion_benchmark_configs(CONFIG_FILE_PATH) - The file contains a JSON array; each call loads the existing array (or - starts a new one), appends the record, and writes the file back atomically. - """ +_AGGREGATED_RESULT_FILES_BY_SOURCE: dict[str, Path] = {} + + +def _normalized_source_path(source_file: str) -> str: + return str(Path(source_file).resolve()) + + +def _aggregated_result_file_for_source(source_file: str) -> Path: + """One session aggregate per source JSON (same naming as single ``--test-config-file``).""" + key = _normalized_source_path(source_file) + if key not in _AGGREGATED_RESULT_FILES_BY_SOURCE: + stem = Path(key).stem + resource = resource_label_for_filename(get_runtime_resource_label()) + if resource: + result_name = f"diffusion_result_{stem}_{resource}_{_SESSION_TIMESTAMP}.json" + else: + result_name = f"diffusion_result_{stem}_{_SESSION_TIMESTAMP}.json" + _AGGREGATED_RESULT_FILES_BY_SOURCE[key] = BENCHMARK_RESULT_DIR / result_name + return _AGGREGATED_RESULT_FILES_BY_SOURCE[key] + + +def _write_result_record(record: dict[str, Any]) -> Path: + """Append one benchmark record to the aggregate file for its source JSON.""" + source_file = record.get("source_file") or CONFIG_FILE_PATH + if not source_file: + raise ValueError("benchmark record missing source_file") + target = _aggregated_result_file_for_source(str(source_file)) with _RESULT_LOCK: BENCHMARK_RESULT_DIR.mkdir(parents=True, exist_ok=True) - if AGGREGATED_RESULT_FILE.exists(): - with open(AGGREGATED_RESULT_FILE, encoding="utf-8") as f: + if target.exists(): + with open(target, encoding="utf-8") as f: records: list[dict] = json.load(f) else: records = [] records.append(record) - with open(AGGREGATED_RESULT_FILE, "w", encoding="utf-8") as f: + with open(target, "w", encoding="utf-8") as f: json.dump(records, f, indent=2, ensure_ascii=False) + return target + + +def _append_to_aggregated_file(record: dict[str, Any]) -> None: + """Backward-compatible wrapper; prefer :func:`_write_result_record`.""" + _write_result_record(record) _server_lock = threading.Lock() @@ -215,14 +286,6 @@ _server_lock = threading.Lock() # --------------------------------------------------------------------------- -def _get_open_port() -> int: - """Return an available TCP port on localhost.""" - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: - s.bind(("", 0)) - s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - return s.getsockname()[1] - - def _wait_for_port(host: str, port: int, timeout: int = 1200, proc: subprocess.Popen | None = None) -> None: """Block until the given host:port accepts connections or timeout expires. @@ -312,7 +375,7 @@ class DiffusionServer: self.model = server_cfg["model"] self.serve_args = server_cfg["serve_args"] self.host = "127.0.0.1" - self.port = port if port is not None else _get_open_port() + self.port = port if port is not None else get_open_port(self.host) self.proc: subprocess.Popen | None = None self.test_name: str = "" @@ -482,6 +545,7 @@ def _unique_server_params(configs: list[dict[str, Any]]) -> list[dict[str, Any]] "serve_args": _build_serve_args(serve_args_dict), "benchmark_endpoint": cfg.get("benchmark_endpoint", cfg.get("benchmark_backend")), "server_params": cfg["server_params"], + "mark": cfg.get("mark"), } ) return result @@ -496,19 +560,29 @@ def _test_param_mapping(configs: list[dict[str, Any]]) -> dict[str, list[dict]]: return mapping -def _make_server(server_cfg: dict[str, Any]) -> DiffusionServer: - """Factory: return a vLLM-Omni diffusion server instance for the config.""" - return DiffusionServer(server_cfg=server_cfg) +def _marks_by_test_name(configs: list[dict[str, Any]]) -> dict[str, list[pytest.MarkDecorator]]: + return {str(cfg["test_name"]): resolve_pytest_marks(cfg.get("mark")) for cfg in configs} + + +def _paired_diffusion_benchmark_pytest_params(configs: list[dict[str, Any]]) -> list[Any]: + """Paired params for ``run_diffusion_benchmark.py``; same shape as omni runner.""" + test_param_map = _test_param_mapping(configs) + server_entries = [(cfg, cfg["test_name"]) for cfg in _unique_server_params(configs)] + return create_paired_benchmark_pytest_params(server_entries, test_param_map, _marks_by_test_name(configs)) # --------------------------------------------------------------------------- # Parametrize data # --------------------------------------------------------------------------- -server_params = _unique_server_params(BENCHMARK_CONFIGS) test_param_map = _test_param_mapping(BENCHMARK_CONFIGS) +paired_benchmark_params = _paired_diffusion_benchmark_pytest_params(BENCHMARK_CONFIGS) + + +def _make_server(server_cfg: dict[str, Any]) -> DiffusionServer: + """Factory: return a vLLM-Omni diffusion server instance for the config.""" + return DiffusionServer(server_cfg=server_cfg) -benchmark_indices: list[int] = list(range(max(len(v) for v in test_param_map.values()))) # --------------------------------------------------------------------------- # Pytest fixtures @@ -534,16 +608,13 @@ def diffusion_server(request): @pytest.fixture -def benchmark_params(request, diffusion_server): - """Yield the benchmark params dict for the current (server, index) pair.""" - param_index: int = request.param - test_name = diffusion_server.test_name +def benchmark_params(request): + """Benchmark params for the paired server/index parametrization.""" + test_name, param_index = request.param params_list = test_param_map.get(test_name, []) if not params_list: raise ValueError(f"No benchmark params for test: {test_name}") - if param_index >= len(params_list): - pytest.skip(f"Param index {param_index} out of range for {test_name} (has {len(params_list)} params)") current = param_index + 1 total = len(params_list) @@ -575,8 +646,8 @@ def run_benchmark( The raw metrics are written to a temporary file by the subprocess. After the run completes the metrics are merged with full metadata (test_name, endpoint, benchmark_params, timestamp, flat reporting fields) and appended - to the session-wide aggregated JSON file (AGGREGATED_RESULT_FILE). The - temporary file is removed afterwards. Subprocess stdout/stderr are tee'd + to ``diffusion_result_{config_stem}_{hardware}_{timestamp}.json`` for the + source JSON file. The temporary file is removed afterwards. Subprocess stdout/stderr are tee'd to a .log file under BENCHMARK_RESULT_DIR/logs/; its path is stored in the record. """ @@ -586,7 +657,12 @@ def run_benchmark( log_dir = BENCHMARK_RESULT_DIR / "logs" log_dir.mkdir(parents=True, exist_ok=True) endpoint_label = endpoint_filename_token(endpoint) - log_file = log_dir / f"{test_name}_{endpoint_label}_{timestamp}.log" + resource_label = get_runtime_resource_label() + hw_for_filename = resource_label_for_filename(resource_label) + if hw_for_filename: + log_file = log_dir / f"{test_name}_{hw_for_filename}_{endpoint_label}_{timestamp}.log" + else: + log_file = log_dir / f"{test_name}_{endpoint_label}_{timestamp}.log" with tempfile.NamedTemporaryFile(mode="w", suffix=".json", prefix="diffusion_bench_tmp_", delete=False) as tmp: tmp_result_file = Path(tmp.name) @@ -696,7 +772,7 @@ def run_benchmark( "Model": model, "Framework": server_type, "API Endpoint": endpoint, - "Hardware": "", + "Hardware": hardware_json_value(resource_label), "Deployment": "", "Task": params.get("task", "t2i"), "Dataset": params.get("dataset", "random"), @@ -728,8 +804,8 @@ def run_benchmark( "build_url": os.environ.get("BUILDKITE_BUILD_URL", ""), "source_file": source_file, } - _append_to_aggregated_file(record) - print(f"\n Result appended to: {AGGREGATED_RESULT_FILE}") + result_path = _write_result_record(record) + print(f"\n Result saved to: {result_path}") print(f" Log saved to: {log_file}") return metrics @@ -932,12 +1008,10 @@ def _iter_sweep_runs(params: dict[str, Any]) -> list[dict[str, Any]]: # --------------------------------------------------------------------------- @pytest.mark.benchmark @pytest.mark.parametrize( - "diffusion_server", - server_params, - ids=[p["test_name"] for p in server_params], - indirect=True, + "diffusion_server,benchmark_params", + paired_benchmark_params, + indirect=["diffusion_server", "benchmark_params"], ) -@pytest.mark.parametrize("benchmark_params", benchmark_indices, indirect=True) def test_diffusion_performance_benchmark(diffusion_server, benchmark_params, request): """Run the diffusion performance benchmark and verify request completion. @@ -962,7 +1036,10 @@ def test_diffusion_performance_benchmark(diffusion_server, benchmark_params, req test_name=test_name, endpoint=endpoint, server_cfg=server_cfg, - source_file=cast(str, CONFIG_FILE_PATH), + source_file=server_cfg.get( + _DIFFUSION_SOURCE_CONFIG_KEY, + CONFIG_FILE_PATH or f"{_PERF_TESTS_DIR}/*.json", + ), ) print(f"\n{'=' * 60}") @@ -980,7 +1057,9 @@ def test_diffusion_performance_benchmark(diffusion_server, benchmark_params, req if key in result: print(f" {key}: {result[key]:.4f}") - print(f"\n Aggregated results: {AGGREGATED_RESULT_FILE}") + source = server_cfg.get(_DIFFUSION_SOURCE_CONFIG_KEY) or CONFIG_FILE_PATH + if source: + print(f"\n Aggregated results: {_aggregated_result_file_for_source(str(source))}") print("=" * 60) assert_baseline = request.config.getoption("--assert-baseline", default=False) diff --git a/tests/dfx/perf/tests/test_bagel_vllm_omni.json b/tests/dfx/perf/tests/test_bagel_vllm_omni.json index 09fc96030..303839387 100644 --- a/tests/dfx/perf/tests/test_bagel_vllm_omni.json +++ b/tests/dfx/perf/tests/test_bagel_vllm_omni.json @@ -1,6 +1,18 @@ [ { "test_name": "test_bagel_single_device_single_stage_t2i", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H100" + }, + "num_cards": 1 + } + }, + "full_model", + "diffusion" + ], "description": "Single-stage BAGEL (TP=1, CFG=1), t2i Task", "server_type": "vllm-omni", "server_params": { @@ -38,6 +50,18 @@ }, { "test_name": "test_bagel_single_device_single_stage_i2i", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H100" + }, + "num_cards": 1 + } + }, + "full_model", + "diffusion" + ], "description": "Single-stage BAGEL (TP=1, CFG=1), i2i Task", "server_type": "vllm-omni", "server_params": { @@ -76,6 +100,18 @@ }, { "test_name": "test_bagel_single_device_multi_stage_t2i", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H100" + }, + "num_cards": 1 + } + }, + "full_model", + "diffusion" + ], "description": "Two-stage BAGEL (bagel.yaml), TP=1, CFG=1, t2i Task", "server_type": "vllm-omni", "server_params": { @@ -113,6 +149,18 @@ }, { "test_name": "test_bagel_single_device_multi_stage_i2i", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H100" + }, + "num_cards": 1 + } + }, + "full_model", + "diffusion" + ], "description": "Two-stage BAGEL (bagel.yaml), TP=1, CFG=1, i2i Task", "server_type": "vllm-omni", "server_params": { diff --git a/tests/dfx/perf/tests/test_cosmos3_vllm_omni.json b/tests/dfx/perf/tests/test_cosmos3_vllm_omni.json index b0fc9a5a4..9bf31658d 100644 --- a/tests/dfx/perf/tests/test_cosmos3_vllm_omni.json +++ b/tests/dfx/perf/tests/test_cosmos3_vllm_omni.json @@ -1,213 +1,261 @@ [ - { - "test_name": "test_cosmos3_t2i_official_demo_2gpu", - "description": "Cosmos3-Nano text-to-image official demo workload on 2 GPUs", - "server_type": "vllm-omni", - "benchmark_endpoint": "/v1/images/generations", - "server_params": { - "model": "nvidia/Cosmos3-Nano", - "serve_args": { - "cfg-parallel-size": 2, - "use-hsdp": true, - "hsdp-shard-size": 2, - "vae-patch-parallel-size": 2, - "vae-use-tiling": true, - "model-class-name": "Cosmos3OmniDiffusersPipeline", - "no-guardrails": true, - "stage-init-timeout": 900, - "init-timeout": 1200, - "enable-diffusion-pipeline-profiler": true - } - }, - "benchmark_params": [ - { - "name": "1024x1024_steps4", - "dataset": "random", - "task": "t2i", - "width": 1024, - "height": 1024, - "num-inference-steps": 4, - "num-prompts": 3, - "max-concurrency": 1, - "seed": 42, - "extra-body": { - "guidance_scale": 7.0, - "flow_shift": 3.0, - "guardrails": false - }, - "baseline": { - "throughput_qps": 1.107, - "latency_mean": 0.8929, - "peak_memory_mb_mean": 80000.0 - } - } - ] + { + "test_name": "test_cosmos3_t2i_official_demo_2gpu", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H100" + }, + "num_cards": 2 + } + }, + "full_model", + "diffusion" + ], + "description": "Cosmos3-Nano text-to-image official demo workload on 2 GPUs", + "server_type": "vllm-omni", + "benchmark_endpoint": "/v1/images/generations", + "server_params": { + "model": "nvidia/Cosmos3-Nano", + "serve_args": { + "cfg-parallel-size": 2, + "use-hsdp": true, + "hsdp-shard-size": 2, + "vae-patch-parallel-size": 2, + "vae-use-tiling": true, + "model-class-name": "Cosmos3OmniDiffusersPipeline", + "no-guardrails": true, + "stage-init-timeout": 900, + "init-timeout": 1200, + "enable-diffusion-pipeline-profiler": true + } }, - { - "test_name": "test_cosmos3_t2v_official_demo_2gpu", - "description": "Cosmos3-Nano text-to-video official demo workload on 2 GPUs", - "server_type": "vllm-omni", - "benchmark_endpoint": "/v1/videos", - "server_params": { - "model": "nvidia/Cosmos3-Nano", - "serve_args": { - "cfg-parallel-size": 2, - "use-hsdp": true, - "hsdp-shard-size": 2, - "vae-patch-parallel-size": 2, - "vae-use-tiling": true, - "model-class-name": "Cosmos3OmniDiffusersPipeline", - "no-guardrails": true, - "stage-init-timeout": 900, - "init-timeout": 1200, - "enable-diffusion-pipeline-profiler": true - } + "benchmark_params": [ + { + "name": "1024x1024_steps4", + "dataset": "random", + "task": "t2i", + "width": 1024, + "height": 1024, + "num-inference-steps": 4, + "num-prompts": 3, + "max-concurrency": 1, + "seed": 42, + "extra-body": { + "guidance_scale": 7.0, + "flow_shift": 3.0, + "guardrails": false }, - "benchmark_params": [ - { - "name": "1280x720_frames189_steps4", - "dataset": "random", - "task": "t2v", - "num-prompts": 3, - "max-concurrency": 1, - "seed": 42, - "random-request-config": [ - { - "width": 1280, - "height": 720, - "prompt": "A robot arm is cleaning a plate in the kitchen.", - "num_inference_steps": 4, - "num_frames": 189, - "fps": 24, - "weight": 1 - } - ], - "extra-body": { - "guidance_scale": 6.0, - "flow_shift": 10.0, - "max_sequence_length": 4096, - "guardrails": false, - "use_resolution_template": false, - "use_duration_template": false - }, - "baseline": { - "throughput_qps": 0.027, - "latency_mean": 36.6824, - "peak_memory_mb_mean": 30155.4 - } - } - ] + "baseline": { + "throughput_qps": 1.107, + "latency_mean": 0.8929, + "peak_memory_mb_mean": 80000.0 + } + } + ] + }, + { + "test_name": "test_cosmos3_t2v_official_demo_2gpu", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H100" + }, + "num_cards": 2 + } + }, + "full_model", + "diffusion" + ], + "description": "Cosmos3-Nano text-to-video official demo workload on 2 GPUs", + "server_type": "vllm-omni", + "benchmark_endpoint": "/v1/videos", + "server_params": { + "model": "nvidia/Cosmos3-Nano", + "serve_args": { + "cfg-parallel-size": 2, + "use-hsdp": true, + "hsdp-shard-size": 2, + "vae-patch-parallel-size": 2, + "vae-use-tiling": true, + "model-class-name": "Cosmos3OmniDiffusersPipeline", + "no-guardrails": true, + "stage-init-timeout": 900, + "init-timeout": 1200, + "enable-diffusion-pipeline-profiler": true + } }, - { - "test_name": "test_cosmos3_i2v_official_demo_2gpu", - "description": "Cosmos3-Nano image-to-video official demo workload on 2 GPUs", - "server_type": "vllm-omni", - "benchmark_endpoint": "/v1/videos", - "server_params": { - "model": "nvidia/Cosmos3-Nano", - "serve_args": { - "cfg-parallel-size": 2, - "use-hsdp": true, - "hsdp-shard-size": 2, - "vae-patch-parallel-size": 2, - "vae-use-tiling": true, - "model-class-name": "Cosmos3OmniDiffusersPipeline", - "no-guardrails": true, - "stage-init-timeout": 900, - "init-timeout": 1200, - "enable-diffusion-pipeline-profiler": true - } + "benchmark_params": [ + { + "name": "1280x720_frames189_steps4", + "dataset": "random", + "task": "t2v", + "num-prompts": 3, + "max-concurrency": 1, + "seed": 42, + "random-request-config": [ + { + "width": 1280, + "height": 720, + "prompt": "A robot arm is cleaning a plate in the kitchen.", + "num_inference_steps": 4, + "num_frames": 189, + "fps": 24, + "weight": 1 + } + ], + "extra-body": { + "guidance_scale": 6.0, + "flow_shift": 10.0, + "max_sequence_length": 4096, + "guardrails": false, + "use_resolution_template": false, + "use_duration_template": false }, - "benchmark_params": [ - { - "name": "1280x720_frames189_steps4", - "dataset": "random", - "task": "i2v", - "num-prompts": 3, - "max-concurrency": 1, - "num-input-images": 1, - "seed": 42, - "random-request-config": [ - { - "width": 1280, - "height": 720, - "prompt": "The scene comes to life with smooth, natural motion.", - "num_inference_steps": 4, - "num_frames": 189, - "fps": 24, - "weight": 1 - } - ], - "extra-body": { - "guidance_scale": 6.0, - "flow_shift": 10.0, - "max_sequence_length": 4096, - "guardrails": false - }, - "baseline": { - "throughput_qps": 0.027, - "latency_mean": 36.968, - "peak_memory_mb_mean": 30443.6 - } - } - ] + "baseline": { + "throughput_qps": 0.027, + "latency_mean": 36.6824, + "peak_memory_mb_mean": 30155.4 + } + } + ] + }, + { + "test_name": "test_cosmos3_i2v_official_demo_2gpu", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H100" + }, + "num_cards": 2 + } + }, + "full_model", + "diffusion" + ], + "description": "Cosmos3-Nano image-to-video official demo workload on 2 GPUs", + "server_type": "vllm-omni", + "benchmark_endpoint": "/v1/videos", + "server_params": { + "model": "nvidia/Cosmos3-Nano", + "serve_args": { + "cfg-parallel-size": 2, + "use-hsdp": true, + "hsdp-shard-size": 2, + "vae-patch-parallel-size": 2, + "vae-use-tiling": true, + "model-class-name": "Cosmos3OmniDiffusersPipeline", + "no-guardrails": true, + "stage-init-timeout": 900, + "init-timeout": 1200, + "enable-diffusion-pipeline-profiler": true + } }, - { - "test_name": "test_cosmos3_v2v_official_demo_2gpu", - "description": "Cosmos3-Nano video-to-video official demo workload on 2 GPUs", - "server_type": "vllm-omni", - "benchmark_endpoint": "/v1/videos", - "server_params": { - "model": "nvidia/Cosmos3-Nano", - "serve_args": { - "cfg-parallel-size": 2, - "use-hsdp": true, - "hsdp-shard-size": 2, - "vae-patch-parallel-size": 2, - "vae-use-tiling": true, - "model-class-name": "Cosmos3OmniDiffusersPipeline", - "no-guardrails": true, - "stage-init-timeout": 900, - "init-timeout": 1200, - "enable-diffusion-pipeline-profiler": true - } + "benchmark_params": [ + { + "name": "1280x720_frames189_steps4", + "dataset": "random", + "task": "i2v", + "num-prompts": 3, + "max-concurrency": 1, + "num-input-images": 1, + "seed": 42, + "random-request-config": [ + { + "width": 1280, + "height": 720, + "prompt": "The scene comes to life with smooth, natural motion.", + "num_inference_steps": 4, + "num_frames": 189, + "fps": 24, + "weight": 1 + } + ], + "extra-body": { + "guidance_scale": 6.0, + "flow_shift": 10.0, + "max_sequence_length": 4096, + "guardrails": false }, - "benchmark_params": [ - { - "name": "1280x720_frames189_steps4", - "dataset": "random", - "task": "v2v", - "num-prompts": 3, - "max-concurrency": 1, - "seed": 42, - "random-request-config": [ - { - "width": 1280, - "height": 720, - "prompt": "Continue the same scene with smooth natural motion.", - "num_inference_steps": 4, - "num_frames": 189, - "fps": 24, - "weight": 1 - } - ], - "extra-body": { - "guidance_scale": 6.0, - "flow_shift": 10.0, - "max_sequence_length": 4096, - "guardrails": false, - "condition_frame_indexes_vision": [ - 0, - 1 - ], - "condition_video_keep": "first" - }, - "baseline": { - "throughput_qps": 0.027, - "latency_mean": 37.2117, - "peak_memory_mb_mean": 30135.6 - } - } - ] - } + "baseline": { + "throughput_qps": 0.027, + "latency_mean": 36.968, + "peak_memory_mb_mean": 30443.6 + } + } + ] + }, + { + "test_name": "test_cosmos3_v2v_official_demo_2gpu", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H100" + }, + "num_cards": 2 + } + }, + "full_model", + "diffusion" + ], + "description": "Cosmos3-Nano video-to-video official demo workload on 2 GPUs", + "server_type": "vllm-omni", + "benchmark_endpoint": "/v1/videos", + "server_params": { + "model": "nvidia/Cosmos3-Nano", + "serve_args": { + "cfg-parallel-size": 2, + "use-hsdp": true, + "hsdp-shard-size": 2, + "vae-patch-parallel-size": 2, + "vae-use-tiling": true, + "model-class-name": "Cosmos3OmniDiffusersPipeline", + "no-guardrails": true, + "stage-init-timeout": 900, + "init-timeout": 1200, + "enable-diffusion-pipeline-profiler": true + } + }, + "benchmark_params": [ + { + "name": "1280x720_frames189_steps4", + "dataset": "random", + "task": "v2v", + "num-prompts": 3, + "max-concurrency": 1, + "seed": 42, + "random-request-config": [ + { + "width": 1280, + "height": 720, + "prompt": "Continue the same scene with smooth natural motion.", + "num_inference_steps": 4, + "num_frames": 189, + "fps": 24, + "weight": 1 + } + ], + "extra-body": { + "guidance_scale": 6.0, + "flow_shift": 10.0, + "max_sequence_length": 4096, + "guardrails": false, + "condition_frame_indexes_vision": [ + 0, + 1 + ], + "condition_video_keep": "first" + }, + "baseline": { + "throughput_qps": 0.027, + "latency_mean": 37.2117, + "peak_memory_mb_mean": 30135.6 + } + } + ] + } ] diff --git a/tests/dfx/perf/tests/test_higgs_audio_v3.json b/tests/dfx/perf/tests/test_higgs_audio_v3.json index 6fdaec580..9b7b2eb5c 100644 --- a/tests/dfx/perf/tests/test_higgs_audio_v3.json +++ b/tests/dfx/perf/tests/test_higgs_audio_v3.json @@ -1,6 +1,18 @@ [ { "test_name": "test_higgs_audio_v3", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H100" + }, + "num_cards": 1 + } + }, + "full_model", + "tts" + ], "server_params": { "model": "bosonai/higgs-audio-v3-tts-4b", "stage_overrides": { diff --git a/tests/dfx/perf/tests/test_hunyuan_image3_it2i.json b/tests/dfx/perf/tests/test_hunyuan_image3_it2i.json index fa955c763..1b6902cba 100644 --- a/tests/dfx/perf/tests/test_hunyuan_image3_it2i.json +++ b/tests/dfx/perf/tests/test_hunyuan_image3_it2i.json @@ -1,140 +1,152 @@ [ - { - "test_name": "test_hunyuan_image3_ti2i_inline", - "description": "AR (TP=2) + 2 * DiT (TP=2) - using inline dataset and deploy-config", - "server_type": "vllm-omni", - "benchmark_endpoint": "/v1/images/edits", - "server_params": { - "model": "tencent/HunyuanImage-3.0-Instruct", - "serve_args": { - "trust-remote-code": true, - "deploy-config-inline": { - "type": "yaml", - "content": { - "pipeline": "hunyuan_image_3_moe", - "async_chunk": false, - "trust_remote_code": true, - "connectors": { - "rdma_connector": { - "name": "MooncakeTransferEngineConnector", - "extra": { - "host": "auto", - "zmq_port": 50051, - "protocol": "rdma", - "device_name": "", - "memory_pool_size": 17179869184, - "memory_pool_device": "cpu" - } - } - }, - "stages": [ - { - "stage_id": 0, - "is_comprehension": true, - "final_output": true, - "final_output_type": "text", - "max_num_seqs": 16, - "gpu_memory_utilization": 0.95, - "enforce_eager": false, - "enable_prefix_caching": false, - "max_num_batched_tokens": 32768, - "devices": "0,1", - "tensor_parallel_size": 2, - "hf_overrides": { - "rope_parameters": { - "mrope_section": [ - 0, - 32, - 32 - ], - "rope_type": "default" - } - }, - "omni_kv_config": { - "need_send_cache": true - }, - "output_connectors": { - "to_stage_1": "rdma_connector" - }, - "default_sampling_params": { - "temperature": 0.0, - "top_p": 1, - "top_k": -1, - "max_tokens": 8192, - "detokenize": true, - "skip_special_tokens": false, - "stop_token_ids": [ - 128025 - ] - } - }, - { - "stage_id": 1, - "max_num_seqs": 1, - "enforce_eager": true, - "devices": "2,3", - "distributed_executor_backend": "mp", - "omni_kv_config": { - "need_recv_cache": true - }, - "parallel_config": { - "tensor_parallel_size": 2, - "vae_patch_parallel_size": 2, - "enable_expert_parallel": true - }, - "input_connectors": { - "from_stage_0": "rdma_connector" - }, - "default_sampling_params": { - "num_inference_steps": 50, - "guidance_scale": 0 - } - } - ], - "edges": [ - { - "from": 0, - "to": 1, - "window_size": -1, - "max_inflight": 1 - } - ] - } - }, - "enable-diffusion-pipeline-profiler": true - } - }, - "benchmark_params": [ - { - "name": "inline_dataset_5_requests", - "dataset": "custom", - "dataset-path-inline": { - "type": "jsonl", - "content": [ - { - "prompt": "新年宠物海报,Q版圆润的可爱标题“新年快乐汪”,副标题“HAPPY NEW YEAR”。 鱼眼镜头,背景是房间门口,近景,上传的主体歪头笑,围着红色围巾,戴着红色毛线帽,高清,绒毛细节,面部特写。 宝丽莱相纸,超现实主义,写实主义,胶片摄影,打印颗粒感肌理。肌理,超写实,复古感。", - "image_urls": [ - "https://raw.githubusercontent.com/Tencent-Hunyuan/HunyuanImage-3.0/main/assets/demo_instruct_imgs/input_0_0.png" - ], - "num_inference_steps": 8, - "seed": 42, - "guidance_scale": 1.0 - } - ] - }, - "task": "ti2i", - "bot-task": "think_recaption", - "max-concurrency": 16, - "num-prompts": 96, - "skip-performance-assertion": true, - "baseline": { - "throughput_qps": 0.1874, - "latency_p99": 101.852, - "latency_mean": 79.0385, - "peak_memory_mb_max": 70000, - "peak_memory_mb_mean": 70000 + { + "test_name": "test_hunyuan_image3_ti2i_inline", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H100" + }, + "num_cards": 4 + } + }, + "diffusion", + "local_model" + ], + "description": "AR (TP=2) + 2 * DiT (TP=2) - using inline dataset and deploy-config", + "server_type": "vllm-omni", + "benchmark_endpoint": "/v1/images/edits", + "server_params": { + "model": "tencent/HunyuanImage-3.0-Instruct", + "serve_args": { + "trust-remote-code": true, + "deploy-config-inline": { + "type": "yaml", + "content": { + "pipeline": "hunyuan_image_3_moe", + "async_chunk": false, + "trust_remote_code": true, + "connectors": { + "rdma_connector": { + "name": "MooncakeTransferEngineConnector", + "extra": { + "host": "auto", + "zmq_port": 50051, + "protocol": "rdma", + "device_name": "", + "memory_pool_size": 17179869184, + "memory_pool_device": "cpu" } + } + }, + "stages": [ + { + "stage_id": 0, + "is_comprehension": true, + "final_output": true, + "final_output_type": "text", + "max_num_seqs": 16, + "gpu_memory_utilization": 0.95, + "enforce_eager": false, + "enable_prefix_caching": false, + "max_num_batched_tokens": 32768, + "devices": "0,1", + "tensor_parallel_size": 2, + "hf_overrides": { + "rope_parameters": { + "mrope_section": [ + 0, + 32, + 32 + ], + "rope_type": "default" + } + }, + "omni_kv_config": { + "need_send_cache": true + }, + "output_connectors": { + "to_stage_1": "rdma_connector" + }, + "default_sampling_params": { + "temperature": 0.0, + "top_p": 1, + "top_k": -1, + "max_tokens": 8192, + "detokenize": true, + "skip_special_tokens": false, + "stop_token_ids": [ + 128025 + ] + } + }, + { + "stage_id": 1, + "max_num_seqs": 1, + "enforce_eager": true, + "devices": "2,3", + "distributed_executor_backend": "mp", + "omni_kv_config": { + "need_recv_cache": true + }, + "parallel_config": { + "tensor_parallel_size": 2, + "vae_patch_parallel_size": 2, + "enable_expert_parallel": true + }, + "input_connectors": { + "from_stage_0": "rdma_connector" + }, + "default_sampling_params": { + "num_inference_steps": 50, + "guidance_scale": 0 + } + } + ], + "edges": [ + { + "from": 0, + "to": 1, + "window_size": -1, + "max_inflight": 1 + } + ] + } + }, + "enable-diffusion-pipeline-profiler": true + } + }, + "benchmark_params": [ + { + "name": "inline_dataset_5_requests", + "dataset": "custom", + "dataset-path-inline": { + "type": "jsonl", + "content": [ + { + "prompt": "新年宠物海报,Q版圆润的可爱标题“新年快乐汪”,副标题“HAPPY NEW YEAR”。 鱼眼镜头,背景是房间门口,近景,上传的主体歪头笑,围着红色围巾,戴着红色毛线帽,高清,绒毛细节,面部特写。 宝丽莱相纸,超现实主义,写实主义,胶片摄影,打印颗粒感肌理。肌理,超写实,复古感。", + "image_urls": [ + "https://raw.githubusercontent.com/Tencent-Hunyuan/HunyuanImage-3.0/main/assets/demo_instruct_imgs/input_0_0.png" + ], + "num_inference_steps": 8, + "seed": 42, + "guidance_scale": 1.0 } - ] - } + ] + }, + "task": "ti2i", + "bot-task": "think_recaption", + "max-concurrency": 16, + "num-prompts": 96, + "skip-performance-assertion": true, + "baseline": { + "throughput_qps": 0.1874, + "latency_p99": 101.852, + "latency_mean": 79.0385, + "peak_memory_mb_max": 70000, + "peak_memory_mb_mean": 70000 + } + } + ] + } ] diff --git a/tests/dfx/perf/tests/test_hunyuan_image_tp2_cfgp2.json b/tests/dfx/perf/tests/test_hunyuan_image_tp2_cfgp2.json index e6d7ce659..7a699f53a 100644 --- a/tests/dfx/perf/tests/test_hunyuan_image_tp2_cfgp2.json +++ b/tests/dfx/perf/tests/test_hunyuan_image_tp2_cfgp2.json @@ -1,35 +1,47 @@ [ - { - "test_name": "test_hunyuan_image_tp2_cfgp2", - "description": "TP=2 CfgP=2 baseline", - "server_type": "vllm-omni", - "server_params": { - "model": "tencent/HunyuanImage-3.0-Instruct", - "serve_args": { - "deploy_config": "../vllm_omni/deploy/hunyuan_image3_dit.yaml", - "tensor-parallel-size": 2, - "cfg-parallel-size": 2, - "enable-diffusion-pipeline-profiler": true - } - }, - "benchmark_params": [ - { - "name": "1024x1024_steps8", - "dataset": "random", - "task": "t2i", - "width": 1024, - "height": 1024, - "num-inference-steps": 8, - "num-prompts": 10, - "max-concurrency": 1, - "skip-performance-assertion": true, - "baseline": { - "throughput_qps": 0.21, - "latency_p99": 4.7469, - "peak_memory_mb_max": 101100, - "peak_memory_mb_mean": 101100 - } - } - ] - } + { + "test_name": "test_hunyuan_image_tp2_cfgp2", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H100" + }, + "num_cards": 4 + } + }, + "diffusion", + "local_model" + ], + "description": "TP=2 CfgP=2 baseline", + "server_type": "vllm-omni", + "server_params": { + "model": "tencent/HunyuanImage-3.0-Instruct", + "serve_args": { + "deploy_config": "../vllm_omni/deploy/hunyuan_image3_dit.yaml", + "tensor-parallel-size": 2, + "cfg-parallel-size": 2, + "enable-diffusion-pipeline-profiler": true + } + }, + "benchmark_params": [ + { + "name": "1024x1024_steps8", + "dataset": "random", + "task": "t2i", + "width": 1024, + "height": 1024, + "num-inference-steps": 8, + "num-prompts": 10, + "max-concurrency": 1, + "skip-performance-assertion": true, + "baseline": { + "throughput_qps": 0.21, + "latency_p99": 4.7469, + "peak_memory_mb_max": 101100, + "peak_memory_mb_mean": 101100 + } + } + ] + } ] diff --git a/tests/dfx/perf/tests/test_hunyuan_image_tp2_sp2.json b/tests/dfx/perf/tests/test_hunyuan_image_tp2_sp2.json index fbbe1643e..521febd2d 100644 --- a/tests/dfx/perf/tests/test_hunyuan_image_tp2_sp2.json +++ b/tests/dfx/perf/tests/test_hunyuan_image_tp2_sp2.json @@ -1,35 +1,47 @@ [ - { - "test_name": "test_hunyuan_image_tp2_sp2", - "description": "TP=2 SP=2 baseline", - "server_type": "vllm-omni", - "server_params": { - "model": "tencent/HunyuanImage-3.0-Instruct", - "serve_args": { - "deploy-config": "../vllm_omni/deploy/hunyuan_image3_dit.yaml", - "tensor-parallel-size": 2, - "ulysses-degree": 2, - "enable-diffusion-pipeline-profiler": true - } - }, - "benchmark_params": [ - { - "name": "1024x1024_steps8", - "dataset": "random", - "task": "t2i", - "width": 1024, - "height": 1024, - "num-inference-steps": 8, - "num-prompts": 10, - "max-concurrency": 1, - "skip-performance-assertion": true, - "baseline": { - "throughput_qps": 0.20, - "latency_p99": 5.1025, - "peak_memory_mb_max": 97402, - "peak_memory_mb_mean": 97402 - } - } - ] - } + { + "test_name": "test_hunyuan_image_tp2_sp2", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H100" + }, + "num_cards": 4 + } + }, + "diffusion", + "local_model" + ], + "description": "TP=2 SP=2 baseline", + "server_type": "vllm-omni", + "server_params": { + "model": "tencent/HunyuanImage-3.0-Instruct", + "serve_args": { + "deploy-config": "../vllm_omni/deploy/hunyuan_image3_dit.yaml", + "tensor-parallel-size": 2, + "ulysses-degree": 2, + "enable-diffusion-pipeline-profiler": true + } + }, + "benchmark_params": [ + { + "name": "1024x1024_steps8", + "dataset": "random", + "task": "t2i", + "width": 1024, + "height": 1024, + "num-inference-steps": 8, + "num-prompts": 10, + "max-concurrency": 1, + "skip-performance-assertion": true, + "baseline": { + "throughput_qps": 0.20, + "latency_p99": 5.1025, + "peak_memory_mb_max": 97402, + "peak_memory_mb_mean": 97402 + } + } + ] + } ] diff --git a/tests/dfx/perf/tests/test_hunyuan_image_tp4.json b/tests/dfx/perf/tests/test_hunyuan_image_tp4.json index eb37f7a22..f2d05564b 100644 --- a/tests/dfx/perf/tests/test_hunyuan_image_tp4.json +++ b/tests/dfx/perf/tests/test_hunyuan_image_tp4.json @@ -1,35 +1,47 @@ [ - { - "test_name": "test_hunyuan_image_tp4", - "description": "TP=4 baseline", - "server_type": "vllm-omni", - "server_params": { - "model": "tencent/HunyuanImage-3.0-Instruct", - "serve_args": { - "deploy-config": "../vllm_omni/deploy/hunyuan_image3_dit.yaml", - "tensor-parallel-size": 4, - "enable-diffusion-pipeline-profiler": true - } - }, - "benchmark_params": [ - { - "name": "1024x1024_steps8", - "dataset": "random", - "task": "t2i", - "width": 1024, - "height": 1024, - "num-inference-steps": 8, - "num-prompts": 10, - "max-concurrency": 1, - "skip-performance-assertion": true, - "baseline": { - "throughput_qps": 0.213, - "latency_mean": 4.6953, - "latency_p99": 4.8601, - "peak_memory_mb_max": 56912.0, - "peak_memory_mb_mean": 56912.0 - } - } - ] - } + { + "test_name": "test_hunyuan_image_tp4", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H100" + }, + "num_cards": 4 + } + }, + "diffusion", + "local_model" + ], + "description": "TP=4 baseline", + "server_type": "vllm-omni", + "server_params": { + "model": "tencent/HunyuanImage-3.0-Instruct", + "serve_args": { + "deploy-config": "../vllm_omni/deploy/hunyuan_image3_dit.yaml", + "tensor-parallel-size": 4, + "enable-diffusion-pipeline-profiler": true + } + }, + "benchmark_params": [ + { + "name": "1024x1024_steps8", + "dataset": "random", + "task": "t2i", + "width": 1024, + "height": 1024, + "num-inference-steps": 8, + "num-prompts": 10, + "max-concurrency": 1, + "skip-performance-assertion": true, + "baseline": { + "throughput_qps": 0.213, + "latency_mean": 4.6953, + "latency_p99": 4.8601, + "peak_memory_mb_max": 56912.0, + "peak_memory_mb_mean": 56912.0 + } + } + ] + } ] diff --git a/tests/dfx/perf/tests/test_ltx2_vllm_omni.json b/tests/dfx/perf/tests/test_ltx2_vllm_omni.json index 4a6f9e350..1a0f31e98 100644 --- a/tests/dfx/perf/tests/test_ltx2_vllm_omni.json +++ b/tests/dfx/perf/tests/test_ltx2_vllm_omni.json @@ -1,217 +1,273 @@ [ - { - "test_name": "test_ltx2_baseline_eager", - "description": "Single-device baseline with enforce-eager (no torch.compile)", - "server_type": "vllm-omni", - "server_params": { - "model": "Lightricks/LTX-2", - "serve_args": { - "enforce-eager": true, - "enable-diffusion-pipeline-profiler": true - } - }, - "benchmark_params": [ - { - "name": "256x256_145f_steps6", - "dataset": "random", - "task": "t2v", - "backend": "v1/videos", - "width": 256, - "height": 256, - "num-frames": 145, - "fps": 24, - "num-inference-steps": 6, - "num-prompts": 3, - "max-concurrency": 1, - "enable-negative-prompt": true - }, - { - "name": "480x768_41f_steps20", - "dataset": "random", - "task": "t2v", - "backend": "v1/videos", - "width": 768, - "height": 480, - "num-frames": 41, - "fps": 24, - "num-inference-steps": 20, - "num-prompts": 3, - "max-concurrency": 1, - "enable-negative-prompt": true - } - ] + { + "test_name": "test_ltx2_baseline_eager", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H100" + }, + "num_cards": 1 + } + }, + "full_model", + "diffusion" + ], + "description": "Single-device baseline with enforce-eager (no torch.compile)", + "server_type": "vllm-omni", + "server_params": { + "model": "Lightricks/LTX-2", + "serve_args": { + "enforce-eager": true, + "enable-diffusion-pipeline-profiler": true + } }, - - { - "test_name": "test_ltx2_torch_compile", - "description": "Single-device with torch.compile (default, no enforce-eager)", - "server_type": "vllm-omni", - "server_params": { - "model": "Lightricks/LTX-2", - "serve_args": { - "enable-diffusion-pipeline-profiler": true - } - }, - "benchmark_params": [ - { - "name": "256x256_145f_steps6", - "dataset": "random", - "task": "t2v", - "backend": "v1/videos", - "width": 256, - "height": 256, - "num-frames": 145, - "fps": 24, - "num-inference-steps": 6, - "num-prompts": 3, - "max-concurrency": 1, - "enable-negative-prompt": true - }, - { - "name": "480x768_41f_steps20", - "dataset": "random", - "task": "t2v", - "backend": "v1/videos", - "width": 768, - "height": 480, - "num-frames": 41, - "fps": 24, - "num-inference-steps": 20, - "num-prompts": 3, - "max-concurrency": 1, - "enable-negative-prompt": true - } - ] + "benchmark_params": [ + { + "name": "256x256_145f_steps6", + "dataset": "random", + "task": "t2v", + "backend": "v1/videos", + "width": 256, + "height": 256, + "num-frames": 145, + "fps": 24, + "num-inference-steps": 6, + "num-prompts": 3, + "max-concurrency": 1, + "enable-negative-prompt": true + }, + { + "name": "480x768_41f_steps20", + "dataset": "random", + "task": "t2v", + "backend": "v1/videos", + "width": 768, + "height": 480, + "num-frames": 41, + "fps": 24, + "num-inference-steps": 20, + "num-prompts": 3, + "max-concurrency": 1, + "enable-negative-prompt": true + } + ] + }, + { + "test_name": "test_ltx2_torch_compile", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H100" + }, + "num_cards": 1 + } + }, + "full_model", + "diffusion" + ], + "description": "Single-device with torch.compile (default, no enforce-eager)", + "server_type": "vllm-omni", + "server_params": { + "model": "Lightricks/LTX-2", + "serve_args": { + "enable-diffusion-pipeline-profiler": true + } }, - - { - "test_name": "test_ltx2_cfg2_eager", - "description": "CFG-parallel=2 with enforce-eager", - "server_type": "vllm-omni", - "server_params": { - "model": "Lightricks/LTX-2", - "serve_args": { - "cfg-parallel-size": 2, - "enforce-eager": true, - "enable-diffusion-pipeline-profiler": true - } - }, - "benchmark_params": [ - { - "name": "256x256_145f_steps6", - "dataset": "random", - "task": "t2v", - "backend": "v1/videos", - "width": 256, - "height": 256, - "num-frames": 145, - "fps": 24, - "num-inference-steps": 6, - "num-prompts": 3, - "max-concurrency": 1, - "enable-negative-prompt": true - }, - { - "name": "480x768_41f_steps20", - "dataset": "random", - "task": "t2v", - "backend": "v1/videos", - "width": 768, - "height": 480, - "num-frames": 41, - "fps": 24, - "num-inference-steps": 20, - "num-prompts": 3, - "max-concurrency": 1, - "enable-negative-prompt": true - } - ] + "benchmark_params": [ + { + "name": "256x256_145f_steps6", + "dataset": "random", + "task": "t2v", + "backend": "v1/videos", + "width": 256, + "height": 256, + "num-frames": 145, + "fps": 24, + "num-inference-steps": 6, + "num-prompts": 3, + "max-concurrency": 1, + "enable-negative-prompt": true + }, + { + "name": "480x768_41f_steps20", + "dataset": "random", + "task": "t2v", + "backend": "v1/videos", + "width": 768, + "height": 480, + "num-frames": 41, + "fps": 24, + "num-inference-steps": 20, + "num-prompts": 3, + "max-concurrency": 1, + "enable-negative-prompt": true + } + ] + }, + { + "test_name": "test_ltx2_cfg2_eager", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H100" + }, + "num_cards": 2 + } + }, + "full_model", + "diffusion" + ], + "description": "CFG-parallel=2 with enforce-eager", + "server_type": "vllm-omni", + "server_params": { + "model": "Lightricks/LTX-2", + "serve_args": { + "cfg-parallel-size": 2, + "enforce-eager": true, + "enable-diffusion-pipeline-profiler": true + } }, - - { - "test_name": "test_ltx2_cfg2_compile", - "description": "CFG-parallel=2 with torch.compile", - "server_type": "vllm-omni", - "server_params": { - "model": "Lightricks/LTX-2", - "serve_args": { - "cfg-parallel-size": 2, - "enable-diffusion-pipeline-profiler": true - } - }, - "benchmark_params": [ - { - "name": "256x256_145f_steps6", - "dataset": "random", - "task": "t2v", - "backend": "v1/videos", - "width": 256, - "height": 256, - "num-frames": 145, - "fps": 24, - "num-inference-steps": 6, - "num-prompts": 3, - "max-concurrency": 1, - "enable-negative-prompt": true - }, - { - "name": "480x768_41f_steps20", - "dataset": "random", - "task": "t2v", - "backend": "v1/videos", - "width": 768, - "height": 480, - "num-frames": 41, - "fps": 24, - "num-inference-steps": 20, - "num-prompts": 3, - "max-concurrency": 1, - "enable-negative-prompt": true - } - ] + "benchmark_params": [ + { + "name": "256x256_145f_steps6", + "dataset": "random", + "task": "t2v", + "backend": "v1/videos", + "width": 256, + "height": 256, + "num-frames": 145, + "fps": 24, + "num-inference-steps": 6, + "num-prompts": 3, + "max-concurrency": 1, + "enable-negative-prompt": true + }, + { + "name": "480x768_41f_steps20", + "dataset": "random", + "task": "t2v", + "backend": "v1/videos", + "width": 768, + "height": 480, + "num-frames": 41, + "fps": 24, + "num-inference-steps": 20, + "num-prompts": 3, + "max-concurrency": 1, + "enable-negative-prompt": true + } + ] + }, + { + "test_name": "test_ltx2_cfg2_compile", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H100" + }, + "num_cards": 2 + } + }, + "full_model", + "diffusion" + ], + "description": "CFG-parallel=2 with torch.compile", + "server_type": "vllm-omni", + "server_params": { + "model": "Lightricks/LTX-2", + "serve_args": { + "cfg-parallel-size": 2, + "enable-diffusion-pipeline-profiler": true + } }, - - { - "test_name": "test_ltx2_cache_dit_eager", - "description": "CacheDiT with enforce-eager", - "server_type": "vllm-omni", - "server_params": { - "model": "Lightricks/LTX-2", - "serve_args": { - "cache-backend": "cache_dit", - "enforce-eager": true, - "enable-diffusion-pipeline-profiler": true - } - }, - "benchmark_params": [ - { - "name": "256x256_145f_steps6", - "dataset": "random", - "task": "t2v", - "backend": "v1/videos", - "width": 256, - "height": 256, - "num-frames": 145, - "fps": 24, - "num-inference-steps": 6, - "num-prompts": 3, - "max-concurrency": 1, - "enable-negative-prompt": true - }, - { - "name": "480x768_41f_steps20", - "dataset": "random", - "task": "t2v", - "backend": "v1/videos", - "width": 768, - "height": 480, - "num-frames": 41, - "fps": 24, - "num-inference-steps": 20, - "num-prompts": 3, - "max-concurrency": 1, - "enable-negative-prompt": true - } - ] - } + "benchmark_params": [ + { + "name": "256x256_145f_steps6", + "dataset": "random", + "task": "t2v", + "backend": "v1/videos", + "width": 256, + "height": 256, + "num-frames": 145, + "fps": 24, + "num-inference-steps": 6, + "num-prompts": 3, + "max-concurrency": 1, + "enable-negative-prompt": true + }, + { + "name": "480x768_41f_steps20", + "dataset": "random", + "task": "t2v", + "backend": "v1/videos", + "width": 768, + "height": 480, + "num-frames": 41, + "fps": 24, + "num-inference-steps": 20, + "num-prompts": 3, + "max-concurrency": 1, + "enable-negative-prompt": true + } + ] + }, + { + "test_name": "test_ltx2_cache_dit_eager", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H100" + }, + "num_cards": 1 + } + }, + "full_model", + "diffusion" + ], + "description": "CacheDiT with enforce-eager", + "server_type": "vllm-omni", + "server_params": { + "model": "Lightricks/LTX-2", + "serve_args": { + "cache-backend": "cache_dit", + "enforce-eager": true, + "enable-diffusion-pipeline-profiler": true + } + }, + "benchmark_params": [ + { + "name": "256x256_145f_steps6", + "dataset": "random", + "task": "t2v", + "backend": "v1/videos", + "width": 256, + "height": 256, + "num-frames": 145, + "fps": 24, + "num-inference-steps": 6, + "num-prompts": 3, + "max-concurrency": 1, + "enable-negative-prompt": true + }, + { + "name": "480x768_41f_steps20", + "dataset": "random", + "task": "t2v", + "backend": "v1/videos", + "width": 768, + "height": 480, + "num-frames": 41, + "fps": 24, + "num-inference-steps": 20, + "num-prompts": 3, + "max-concurrency": 1, + "enable-negative-prompt": true + } + ] + } ] diff --git a/tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json b/tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json index 459554f63..66ffa231f 100644 --- a/tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json +++ b/tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json @@ -1,331 +1,343 @@ [ - { - "test_name": "test_qwen3_omni_chunk", - "server_params": { - "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct", - "extra_cli_args": [ - "--async-chunk" - ] + { + "test_name": "test_qwen3_omni_chunk", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H100" + }, + "num_cards": 2 + } + }, + "full_model", + "omni" + ], + "server_params": { + "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct", + "extra_cli_args": [ + "--async-chunk" + ] + }, + "benchmark_params": [ + { + "dataset_name": "random", + "backend": "openai-chat-omni", + "endpoint": "/v1/chat/completions", + "num_prompts": [ + 4, + 16, + 32, + 64, + 128 + ], + "max_concurrency": [ + 1, + 4, + 8, + 16, + 32 + ], + "random_input_len": 2500, + "random_output_len": 900, + "ignore_eos": true, + "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", + "baseline": { + "mean_ttft_ms": [ + 135.4379, + 187.0896, + 400, + 435.675, + 724.6836 + ], + "mean_e2el_ms": [ + 17619.9851, + 34811.8335, + 43656.9501, + 56657.1548, + 85097.6016 + ], + "mean_audio_rtf": [ + 0.1279, + 0.2238, + 0.2735, + 0.3712, + 0.5634 + ] + } + }, + { + "dataset_name": "random-mm", + "backend": "openai-chat-omni", + "endpoint": "/v1/chat/completions", + "num_prompts": [ + 10 + ], + "request_rate": [ + 0.1 + ], + "random_input_len": 100, + "random_output_len": 100, + "random_range_ratio": 0.0, + "ignore_eos": true, + "random_mm_base_items_per_request": 1, + "random_mm_num_mm_items_range_ratio": 0.5, + "random_mm_limit_mm_per_prompt": { + "audio": 1 }, - "benchmark_params": [ - { - "dataset_name": "random", - "backend": "openai-chat-omni", - "endpoint": "/v1/chat/completions", - "num_prompts": [ - 4, - 16, - 32, - 64, - 128 - ], - "max_concurrency": [ - 1, - 4, - 8, - 16, - 32 - ], - "random_input_len": 2500, - "random_output_len": 900, - "ignore_eos": true, - "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", - "baseline": { - "mean_ttft_ms": [ - 135.4379, - 187.0896, - 400, - 435.675, - 724.6836 - ], - "mean_e2el_ms": [ - 17619.9851, - 34811.8335, - 43656.9501, - 56657.1548, - 85097.6016 - ], - "mean_audio_rtf": [ - 0.1279, - 0.2238, - 0.2735, - 0.3712, - 0.5634 - ] - } - }, - { - "dataset_name": "random-mm", - "backend": "openai-chat-omni", - "endpoint": "/v1/chat/completions", - "num_prompts": [ - 10 - ], - "request_rate": [ - 0.1 - ], - "random_input_len": 100, - "random_output_len": 100, - "random_range_ratio": 0.0, - "ignore_eos": true, - "random_mm_base_items_per_request": 1, - "random_mm_num_mm_items_range_ratio": 0.5, - "random_mm_limit_mm_per_prompt": { - "audio": 1 - }, - "random_mm_bucket_config": { - "(0, 60, 3)": 1.0 - }, - "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", - "baseline": { - "mean_ttft_ms": [ - 340.329 - ], - "mean_e2el_ms": [ - 4305.477 - ], - "mean_audio_rtf": [ - 0.1305 - ] - } - }, - { - "dataset_name": "random-mm", - "backend": "openai-chat-omni", - "endpoint": "/v1/chat/completions", - "num_prompts": [ - 40 - ], - "request_rate": [ - 0.5 - ], - "random_input_len": 100, - "random_output_len": 100, - "random_range_ratio": 0.0, - "ignore_eos": true, - "random_mm_base_items_per_request": 2, - "random_mm_num_mm_items_range_ratio": 0.5, - "random_mm_limit_mm_per_prompt": { - "image": 1, - "video": 1 - }, - "random_mm_bucket_config": { - "(256, 256, 1)": 0.5, - "(720, 1280, 2)": 0.5 - }, - "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", - "baseline": { - "mean_ttft_ms": [ - 220.0909 - ], - "mean_e2el_ms": [ - 5475.9565 - ], - "mean_audio_rtf": [ - 0.1818 - ] - } - }, - { - "dataset_name": "random-mm", - "backend": "openai-chat-omni", - "endpoint": "/v1/chat/completions", - "num_prompts": [ - 100 - ], - "request_rate": [ - 1.0 - ], - "random_input_len": 100, - "random_output_len": 100, - "random_range_ratio": 0.0, - "ignore_eos": true, - "random_mm_base_items_per_request": 3, - "random_mm_num_mm_items_range_ratio": 0.5, - "random_mm_limit_mm_per_prompt": { - "image": 1, - "video": 1, - "audio": 1 - }, - "random_mm_bucket_config": { - "(256, 256, 1)": 0.34, - "(720, 1280, 2)": 0.33, - "(0, 60, 3)": 0.33 - }, - "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", - "baseline": { - "mean_ttft_ms": [ - 450.0155 - ], - "mean_e2el_ms": [ - 8516.8316 - ], - "mean_audio_rtf": [ - 0.2717 - ] - } - }, - { - "dataset_name": "random", - "backend": "openai-chat-omni", - "endpoint": "/v1/chat/completions", - "num_prompts": [ - 4, - 16, - 32, - 64, - 128 - ], - "max_concurrency": [ - 1, - 4, - 8, - 16, - 32 - ], - "random_input_len": 2500, - "random_output_len": 900, - "ignore_eos": true, - "extra_body": { - "modalities": [ - "text" - ] - }, - "percentile-metrics": "ttft,tpot,itl,e2el", - "baseline": { - "mean_ttft_ms": [ - 78.4278, - 195.8037, - 333.7318, - 561.805, - 1038.043 - ], - "mean_e2el_ms": [ - 4288.3029, - 7429.9429, - 9451.6184, - 12607.5292, - 18153.2547 - ] - } - }, - { - "dataset_name": "random-mm", - "backend": "openai-chat-omni", - "endpoint": "/v1/chat/completions", - "num_prompts": [ - 10 - ], - "request_rate": [ - 0.1 - ], - "random_input_len": 100, - "random_output_len": 100, - "random_range_ratio": 0.0, - "ignore_eos": true, - "extra_body": { - "modalities": [ - "text" - ] - }, - "random_mm_base_items_per_request": 1, - "random_mm_num_mm_items_range_ratio": 0.5, - "random_mm_limit_mm_per_prompt": { - "audio": 1 - }, - "random_mm_bucket_config": { - "(0, 60, 3)": 1.0 - }, - "percentile-metrics": "ttft,tpot,itl,e2el", - "baseline": { - "mean_ttft_ms": [ - 275.3875 - ], - "mean_e2el_ms": [ - 730.6995 - ] - } - }, - { - "dataset_name": "random-mm", - "backend": "openai-chat-omni", - "endpoint": "/v1/chat/completions", - "num_prompts": [ - 40 - ], - "request_rate": [ - 0.5 - ], - "random_input_len": 100, - "random_output_len": 100, - "random_range_ratio": 0.0, - "ignore_eos": true, - "extra_body": { - "modalities": [ - "text" - ] - }, - "random_mm_base_items_per_request": 2, - "random_mm_num_mm_items_range_ratio": 0.5, - "random_mm_limit_mm_per_prompt": { - "image": 1, - "video": 1 - }, - "random_mm_bucket_config": { - "(256, 256, 1)": 0.5, - "(720, 1280, 2)": 0.5 - }, - "percentile-metrics": "ttft,tpot,itl,e2el", - "baseline": { - "mean_ttft_ms": [ - 166.2201 - ], - "mean_e2el_ms": [ - 673.5789 - ] - } - }, - { - "dataset_name": "random-mm", - "backend": "openai-chat-omni", - "endpoint": "/v1/chat/completions", - "num_prompts": [ - 100 - ], - "request_rate": [ - 1.0 - ], - "random_input_len": 100, - "random_output_len": 100, - "random_range_ratio": 0.0, - "ignore_eos": true, - "extra_body": { - "modalities": [ - "text" - ] - }, - "random_mm_base_items_per_request": 3, - "random_mm_num_mm_items_range_ratio": 0.5, - "random_mm_limit_mm_per_prompt": { - "image": 1, - "video": 1, - "audio": 1 - }, - "random_mm_bucket_config": { - "(256, 256, 1)": 0.34, - "(720, 1280, 2)": 0.33, - "(0, 60, 3)": 0.33 - }, - "percentile-metrics": "ttft,tpot,itl,e2el", - "baseline": { - "mean_ttft_ms": [ - 357.7781 - ], - "mean_e2el_ms": [ - 953.3493 - ] - } - } - ] - } + "random_mm_bucket_config": { + "(0, 60, 3)": 1.0 + }, + "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", + "baseline": { + "mean_ttft_ms": [ + 340.329 + ], + "mean_e2el_ms": [ + 4305.477 + ], + "mean_audio_rtf": [ + 0.1305 + ] + } + }, + { + "dataset_name": "random-mm", + "backend": "openai-chat-omni", + "endpoint": "/v1/chat/completions", + "num_prompts": [ + 40 + ], + "request_rate": [ + 0.5 + ], + "random_input_len": 100, + "random_output_len": 100, + "random_range_ratio": 0.0, + "ignore_eos": true, + "random_mm_base_items_per_request": 2, + "random_mm_num_mm_items_range_ratio": 0.5, + "random_mm_limit_mm_per_prompt": { + "image": 1, + "video": 1 + }, + "random_mm_bucket_config": { + "(256, 256, 1)": 0.5, + "(720, 1280, 2)": 0.5 + }, + "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", + "baseline": { + "mean_ttft_ms": [ + 220.0909 + ], + "mean_e2el_ms": [ + 5475.9565 + ], + "mean_audio_rtf": [ + 0.1818 + ] + } + }, + { + "dataset_name": "random-mm", + "backend": "openai-chat-omni", + "endpoint": "/v1/chat/completions", + "num_prompts": [ + 100 + ], + "request_rate": [ + 1.0 + ], + "random_input_len": 100, + "random_output_len": 100, + "random_range_ratio": 0.0, + "ignore_eos": true, + "random_mm_base_items_per_request": 3, + "random_mm_num_mm_items_range_ratio": 0.5, + "random_mm_limit_mm_per_prompt": { + "image": 1, + "video": 1, + "audio": 1 + }, + "random_mm_bucket_config": { + "(256, 256, 1)": 0.34, + "(720, 1280, 2)": 0.33, + "(0, 60, 3)": 0.33 + }, + "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", + "baseline": { + "mean_ttft_ms": [ + 450.0155 + ], + "mean_e2el_ms": [ + 8516.8316 + ], + "mean_audio_rtf": [ + 0.2717 + ] + } + }, + { + "dataset_name": "random", + "backend": "openai-chat-omni", + "endpoint": "/v1/chat/completions", + "num_prompts": [ + 4, + 16, + 32, + 64, + 128 + ], + "max_concurrency": [ + 1, + 4, + 8, + 16, + 32 + ], + "random_input_len": 2500, + "random_output_len": 900, + "ignore_eos": true, + "extra_body": { + "modalities": [ + "text" + ] + }, + "percentile-metrics": "ttft,tpot,itl,e2el", + "baseline": { + "mean_ttft_ms": [ + 78.4278, + 195.8037, + 333.7318, + 561.805, + 1038.043 + ], + "mean_e2el_ms": [ + 4288.3029, + 7429.9429, + 9451.6184, + 12607.5292, + 18153.2547 + ] + } + }, + { + "dataset_name": "random-mm", + "backend": "openai-chat-omni", + "endpoint": "/v1/chat/completions", + "num_prompts": [ + 10 + ], + "request_rate": [ + 0.1 + ], + "random_input_len": 100, + "random_output_len": 100, + "random_range_ratio": 0.0, + "ignore_eos": true, + "extra_body": { + "modalities": [ + "text" + ] + }, + "random_mm_base_items_per_request": 1, + "random_mm_num_mm_items_range_ratio": 0.5, + "random_mm_limit_mm_per_prompt": { + "audio": 1 + }, + "random_mm_bucket_config": { + "(0, 60, 3)": 1.0 + }, + "percentile-metrics": "ttft,tpot,itl,e2el", + "baseline": { + "mean_ttft_ms": [ + 275.3875 + ], + "mean_e2el_ms": [ + 730.6995 + ] + } + }, + { + "dataset_name": "random-mm", + "backend": "openai-chat-omni", + "endpoint": "/v1/chat/completions", + "num_prompts": [ + 40 + ], + "request_rate": [ + 0.5 + ], + "random_input_len": 100, + "random_output_len": 100, + "random_range_ratio": 0.0, + "ignore_eos": true, + "extra_body": { + "modalities": [ + "text" + ] + }, + "random_mm_base_items_per_request": 2, + "random_mm_num_mm_items_range_ratio": 0.5, + "random_mm_limit_mm_per_prompt": { + "image": 1, + "video": 1 + }, + "random_mm_bucket_config": { + "(256, 256, 1)": 0.5, + "(720, 1280, 2)": 0.5 + }, + "percentile-metrics": "ttft,tpot,itl,e2el", + "baseline": { + "mean_ttft_ms": [ + 166.2201 + ], + "mean_e2el_ms": [ + 673.5789 + ] + } + }, + { + "dataset_name": "random-mm", + "backend": "openai-chat-omni", + "endpoint": "/v1/chat/completions", + "num_prompts": [ + 100 + ], + "request_rate": [ + 1.0 + ], + "random_input_len": 100, + "random_output_len": 100, + "random_range_ratio": 0.0, + "ignore_eos": true, + "extra_body": { + "modalities": [ + "text" + ] + }, + "random_mm_base_items_per_request": 3, + "random_mm_num_mm_items_range_ratio": 0.5, + "random_mm_limit_mm_per_prompt": { + "image": 1, + "video": 1, + "audio": 1 + }, + "random_mm_bucket_config": { + "(256, 256, 1)": 0.34, + "(720, 1280, 2)": 0.33, + "(0, 60, 3)": 0.33 + }, + "percentile-metrics": "ttft,tpot,itl,e2el", + "baseline": { + "mean_ttft_ms": [ + 357.7781 + ], + "mean_e2el_ms": [ + 953.3493 + ] + } + } + ] + } ] diff --git a/tests/dfx/perf/tests/test_qwen3_omni_multi_replicas.json b/tests/dfx/perf/tests/test_qwen3_omni_multi_replicas.json index bee799a77..ac6f679c1 100644 --- a/tests/dfx/perf/tests/test_qwen3_omni_multi_replicas.json +++ b/tests/dfx/perf/tests/test_qwen3_omni_multi_replicas.json @@ -1,115 +1,127 @@ [ - { - "test_name": "test_qwen3_omni_3gpu_replica2", - "server_params": { - "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct", - "stage_overrides": { - "1": {"num_replicas": 2, "devices": "1,2"}, - "2": {"num_replicas": 2, "devices": "1,2"} - }, - "extra_cli_args": [ - "--async-chunk" - ] + { + "test_name": "test_qwen3_omni_3gpu_replica2", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H100" + }, + "num_cards": 3 + } + }, + "full_model", + "omni" + ], + "server_params": { + "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct", + "stage_overrides": { + "1": {"num_replicas": 2, "devices": "1,2"}, + "2": {"num_replicas": 2, "devices": "1,2"} + }, + "extra_cli_args": [ + "--async-chunk" + ] + }, + "benchmark_params": [ + { + "dataset_name": "random", + "backend": "openai-chat-omni", + "endpoint": "/v1/chat/completions", + "num_prompts": [128, 128, 128, 128], + "max_concurrency": [8, 16, 24, 32], + "random_input_len": 2500, + "random_output_len": 900, + "ignore_eos": true, + "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", + "baseline": { + "mean_ttft_ms": [203, 283, 470, 869], + "mean_tpot_ms": [28, 36, 41, 45], + "mean_audio_ttfp_ms": [861, 1200, 1528, 2100], + "mean_audio_rtf": [0.24, 0.30, 0.35, 0.39] + } + }, + { + "dataset_name": "random-mm", + "backend": "openai-chat-omni", + "endpoint": "/v1/chat/completions", + "num_prompts": [10], + "request_rate": [0.1], + "random_input_len": 100, + "random_output_len": 100, + "random_range_ratio": 0.0, + "ignore_eos": true, + "random_mm_base_items_per_request": 1, + "random_mm_num_mm_items_range_ratio": 0.5, + "random_mm_limit_mm_per_prompt": { + "audio": 1 }, - "benchmark_params": [ - { - "dataset_name": "random", - "backend": "openai-chat-omni", - "endpoint": "/v1/chat/completions", - "num_prompts": [128, 128, 128, 128], - "max_concurrency": [8, 16, 24, 32], - "random_input_len": 2500, - "random_output_len": 900, - "ignore_eos": true, - "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", - "baseline": { - "mean_ttft_ms": [203, 283, 470, 869], - "mean_tpot_ms": [28, 36, 41, 45], - "mean_audio_ttfp_ms": [861, 1200, 1528, 2100], - "mean_audio_rtf": [0.24, 0.30, 0.35, 0.39] - } - }, - { - "dataset_name": "random-mm", - "backend": "openai-chat-omni", - "endpoint": "/v1/chat/completions", - "num_prompts": [10], - "request_rate": [0.1], - "random_input_len": 100, - "random_output_len": 100, - "random_range_ratio": 0.0, - "ignore_eos": true, - "random_mm_base_items_per_request": 1, - "random_mm_num_mm_items_range_ratio": 0.5, - "random_mm_limit_mm_per_prompt": { - "audio": 1 - }, - "random_mm_bucket_config": { - "(0, 60, 3)": 1.0 - }, - "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", - "baseline": { - "mean_ttft_ms": [377], - "mean_audio_ttfp_ms": [684], - "mean_audio_rtf": [0.14] - } - }, - { - "dataset_name": "random-mm", - "backend": "openai-chat-omni", - "endpoint": "/v1/chat/completions", - "num_prompts": [40], - "request_rate": [0.5], - "random_input_len": 100, - "random_output_len": 100, - "random_range_ratio": 0.0, - "ignore_eos": true, - "random_mm_base_items_per_request": 2, - "random_mm_num_mm_items_range_ratio": 0.5, - "random_mm_limit_mm_per_prompt": { - "image": 1, - "video": 1 - }, - "random_mm_bucket_config": { - "(256, 256, 1)": 0.5, - "(720, 1280, 2)": 0.5 - }, - "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", - "baseline": { - "mean_ttft_ms": [228], - "mean_audio_ttfp_ms": [686], - "mean_audio_rtf": [0.16] - } - }, - { - "dataset_name": "random-mm", - "backend": "openai-chat-omni", - "endpoint": "/v1/chat/completions", - "num_prompts": [100], - "request_rate": [1.0], - "random_input_len": 100, - "random_output_len": 100, - "random_range_ratio": 0.0, - "ignore_eos": true, - "random_mm_base_items_per_request": 3, - "random_mm_num_mm_items_range_ratio": 0.5, - "random_mm_limit_mm_per_prompt": { - "image": 1, - "video": 1, - "audio": 1 - }, - "random_mm_bucket_config": { - "(256, 256, 1)": 0.34, - "(720, 1280, 2)": 0.33, - "(0, 60, 3)": 0.33 - }, - "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", - "baseline": { - "mean_ttft_ms": [522], - "mean_audio_ttfp_ms": [1119], - "mean_audio_rtf": [0.21] - } - } - ] - } + "random_mm_bucket_config": { + "(0, 60, 3)": 1.0 + }, + "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", + "baseline": { + "mean_ttft_ms": [377], + "mean_audio_ttfp_ms": [684], + "mean_audio_rtf": [0.14] + } + }, + { + "dataset_name": "random-mm", + "backend": "openai-chat-omni", + "endpoint": "/v1/chat/completions", + "num_prompts": [40], + "request_rate": [0.5], + "random_input_len": 100, + "random_output_len": 100, + "random_range_ratio": 0.0, + "ignore_eos": true, + "random_mm_base_items_per_request": 2, + "random_mm_num_mm_items_range_ratio": 0.5, + "random_mm_limit_mm_per_prompt": { + "image": 1, + "video": 1 + }, + "random_mm_bucket_config": { + "(256, 256, 1)": 0.5, + "(720, 1280, 2)": 0.5 + }, + "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", + "baseline": { + "mean_ttft_ms": [228], + "mean_audio_ttfp_ms": [686], + "mean_audio_rtf": [0.16] + } + }, + { + "dataset_name": "random-mm", + "backend": "openai-chat-omni", + "endpoint": "/v1/chat/completions", + "num_prompts": [100], + "request_rate": [1.0], + "random_input_len": 100, + "random_output_len": 100, + "random_range_ratio": 0.0, + "ignore_eos": true, + "random_mm_base_items_per_request": 3, + "random_mm_num_mm_items_range_ratio": 0.5, + "random_mm_limit_mm_per_prompt": { + "image": 1, + "video": 1, + "audio": 1 + }, + "random_mm_bucket_config": { + "(256, 256, 1)": 0.34, + "(720, 1280, 2)": 0.33, + "(0, 60, 3)": 0.33 + }, + "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", + "baseline": { + "mean_ttft_ms": [522], + "mean_audio_ttfp_ms": [1119], + "mean_audio_rtf": [0.21] + } + } + ] + } ] diff --git a/tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json b/tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json index d01ca06b4..78daf691b 100644 --- a/tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json +++ b/tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json @@ -1,170 +1,182 @@ [ - { - "test_name": "test_qwen3_omni", - "server_params": { - "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct", - "extra_cli_args": [ - "--no-async-chunk" - ] + { + "test_name": "test_qwen3_omni", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H100" + }, + "num_cards": 2 + } + }, + "full_model", + "omni" + ], + "server_params": { + "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct", + "extra_cli_args": [ + "--no-async-chunk" + ] + }, + "benchmark_params": [ + { + "dataset_name": "random", + "backend": "openai-chat-omni", + "endpoint": "/v1/chat/completions", + "num_prompts": [ + 4, + 16, + 32, + 64, + 128 + ], + "max_concurrency": [ + 1, + 4, + 8, + 16, + 32 + ], + "random_input_len": 2500, + "random_output_len": 900, + "ignore_eos": true, + "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", + "baseline": { + "mean_ttft_ms": [ + 109.1218, + 247.4411, + 354.9384, + 447.7885, + 694.7487 + ], + "mean_e2el_ms": [ + 22413.667, + 30808.9715, + 35677.844, + 46345.8978, + 68350.4853 + ], + "mean_audio_rtf": [ + 0.1695, + 0.2033, + 0.2467, + 0.3234, + 0.4438 + ] + } + }, + { + "dataset_name": "random-mm", + "backend": "openai-chat-omni", + "endpoint": "/v1/chat/completions", + "num_prompts": [ + 10 + ], + "request_rate": [ + 0.1 + ], + "random_input_len": 100, + "random_output_len": 100, + "random_range_ratio": 0.0, + "ignore_eos": true, + "random_mm_base_items_per_request": 1, + "random_mm_num_mm_items_range_ratio": 0.5, + "random_mm_limit_mm_per_prompt": { + "audio": 1 }, - "benchmark_params": [ - { - "dataset_name": "random", - "backend": "openai-chat-omni", - "endpoint": "/v1/chat/completions", - "num_prompts": [ - 4, - 16, - 32, - 64, - 128 - ], - "max_concurrency": [ - 1, - 4, - 8, - 16, - 32 - ], - "random_input_len": 2500, - "random_output_len": 900, - "ignore_eos": true, - "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", - "baseline": { - "mean_ttft_ms": [ - 109.1218, - 247.4411, - 354.9384, - 447.7885, - 694.7487 - ], - "mean_e2el_ms": [ - 22413.667, - 30808.9715, - 35677.844, - 46345.8978, - 68350.4853 - ], - "mean_audio_rtf": [ - 0.1695, - 0.2033, - 0.2467, - 0.3234, - 0.4438 - ] - } - }, - { - "dataset_name": "random-mm", - "backend": "openai-chat-omni", - "endpoint": "/v1/chat/completions", - "num_prompts": [ - 10 - ], - "request_rate": [ - 0.1 - ], - "random_input_len": 100, - "random_output_len": 100, - "random_range_ratio": 0.0, - "ignore_eos": true, - "random_mm_base_items_per_request": 1, - "random_mm_num_mm_items_range_ratio": 0.5, - "random_mm_limit_mm_per_prompt": { - "audio": 1 - }, - "random_mm_bucket_config": { - "(0, 60, 3)": 1.0 - }, - "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", - "baseline": { - "mean_ttft_ms": [ - 323.3573 - ], - "mean_e2el_ms": [ - 4623.4173 - ], - "mean_audio_rtf": [ - 0.1406 - ] - } - }, - { - "dataset_name": "random-mm", - "backend": "openai-chat-omni", - "endpoint": "/v1/chat/completions", - "num_prompts": [ - 40 - ], - "request_rate": [ - 0.5 - ], - "random_input_len": 100, - "random_output_len": 100, - "random_range_ratio": 0.0, - "ignore_eos": true, - "random_mm_base_items_per_request": 2, - "random_mm_num_mm_items_range_ratio": 0.5, - "random_mm_limit_mm_per_prompt": { - "image": 1, - "video": 1 - }, - "random_mm_bucket_config": { - "(256, 256, 1)": 0.5, - "(720, 1280, 2)": 0.5 - }, - "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", - "baseline": { - "mean_ttft_ms": [ - 215.4502 - ], - "mean_e2el_ms": [ - 4924.149 - ], - "mean_audio_rtf": [ - 0.1637 - ] - } - }, - { - "dataset_name": "random-mm", - "backend": "openai-chat-omni", - "endpoint": "/v1/chat/completions", - "num_prompts": [ - 100 - ], - "request_rate": [ - 1.0 - ], - "random_input_len": 100, - "random_output_len": 100, - "random_range_ratio": 0.0, - "ignore_eos": true, - "random_mm_base_items_per_request": 3, - "random_mm_num_mm_items_range_ratio": 0.5, - "random_mm_limit_mm_per_prompt": { - "image": 1, - "video": 1, - "audio": 1 - }, - "random_mm_bucket_config": { - "(256, 256, 1)": 0.34, - "(720, 1280, 2)": 0.33, - "(0, 60, 3)": 0.33 - }, - "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", - "baseline": { - "mean_ttft_ms": [ - 442.5815 - ], - "mean_e2el_ms": [ - 6441.0986 - ], - "mean_audio_rtf": [ - 0.2091 - ] - } - } - ] - } + "random_mm_bucket_config": { + "(0, 60, 3)": 1.0 + }, + "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", + "baseline": { + "mean_ttft_ms": [ + 323.3573 + ], + "mean_e2el_ms": [ + 4623.4173 + ], + "mean_audio_rtf": [ + 0.1406 + ] + } + }, + { + "dataset_name": "random-mm", + "backend": "openai-chat-omni", + "endpoint": "/v1/chat/completions", + "num_prompts": [ + 40 + ], + "request_rate": [ + 0.5 + ], + "random_input_len": 100, + "random_output_len": 100, + "random_range_ratio": 0.0, + "ignore_eos": true, + "random_mm_base_items_per_request": 2, + "random_mm_num_mm_items_range_ratio": 0.5, + "random_mm_limit_mm_per_prompt": { + "image": 1, + "video": 1 + }, + "random_mm_bucket_config": { + "(256, 256, 1)": 0.5, + "(720, 1280, 2)": 0.5 + }, + "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", + "baseline": { + "mean_ttft_ms": [ + 215.4502 + ], + "mean_e2el_ms": [ + 4924.149 + ], + "mean_audio_rtf": [ + 0.1637 + ] + } + }, + { + "dataset_name": "random-mm", + "backend": "openai-chat-omni", + "endpoint": "/v1/chat/completions", + "num_prompts": [ + 100 + ], + "request_rate": [ + 1.0 + ], + "random_input_len": 100, + "random_output_len": 100, + "random_range_ratio": 0.0, + "ignore_eos": true, + "random_mm_base_items_per_request": 3, + "random_mm_num_mm_items_range_ratio": 0.5, + "random_mm_limit_mm_per_prompt": { + "image": 1, + "video": 1, + "audio": 1 + }, + "random_mm_bucket_config": { + "(256, 256, 1)": 0.34, + "(720, 1280, 2)": 0.33, + "(0, 60, 3)": 0.33 + }, + "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", + "baseline": { + "mean_ttft_ms": [ + 442.5815 + ], + "mean_e2el_ms": [ + 6441.0986 + ], + "mean_audio_rtf": [ + 0.2091 + ] + } + } + ] + } ] diff --git a/tests/dfx/perf/tests/test_qwen3_omni_vllm_text.json b/tests/dfx/perf/tests/test_qwen3_omni_vllm_text.json index f608eca0e..b9d553098 100644 --- a/tests/dfx/perf/tests/test_qwen3_omni_vllm_text.json +++ b/tests/dfx/perf/tests/test_qwen3_omni_vllm_text.json @@ -1,180 +1,192 @@ [ - { - "test_name": "test_qwen3_omni_vllm_text", - "server_params": { - "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct", - "use_omni": false, - "extra_cli_args": [ - "--no-enable-prefix-caching", - "--skip-mm-profiling" - ] + { + "test_name": "test_qwen3_omni_vllm_text", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H100" + }, + "num_cards": 2 + } + }, + "full_model", + "omni" + ], + "server_params": { + "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct", + "use_omni": false, + "extra_cli_args": [ + "--no-enable-prefix-caching", + "--skip-mm-profiling" + ] + }, + "benchmark_params": [ + { + "dataset_name": "random", + "backend": "vllm", + "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct", + "endpoint": "/v1/completions", + "num_prompts": [ + 4, + 16, + 32, + 64, + 128 + ], + "max_concurrency": [ + 1, + 4, + 8, + 16, + 32 + ], + "random_input_len": 2500, + "random_output_len": 900, + "ignore_eos": true, + "extra_body": { + "modalities": [ + "text" + ] }, - "benchmark_params": [ - { - "dataset_name": "random", - "backend": "vllm", - "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct", - "endpoint": "/v1/completions", - "num_prompts": [ - 4, - 16, - 32, - 64, - 128 - ], - "max_concurrency": [ - 1, - 4, - 8, - 16, - 32 - ], - "random_input_len": 2500, - "random_output_len": 900, - "ignore_eos": true, - "extra_body": { - "modalities": [ - "text" - ] - }, - "percentile-metrics": "ttft,tpot,itl,e2el", - "baseline": { - "mean_ttft_ms": [ - 69.1276, - 255.1882, - 310.639, - 422.0337, - 532.0582 - ], - "mean_e2el_ms": [ - 4382.7211, - 7510.3815, - 9440.4854, - 12553.0186, - 16932.6065 - ] - } - }, - { - "dataset_name": "random-mm", - "backend": "vllm", - "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct", - "endpoint": "/v1/completions", - "num_prompts": [ - 10 - ], - "request_rate": [ - 0.1 - ], - "random_input_len": 100, - "random_output_len": 100, - "random_range_ratio": 0.0, - "ignore_eos": true, - "extra_body": { - "modalities": [ - "text" - ] - }, - "random_mm_base_items_per_request": 1, - "random_mm_num_mm_items_range_ratio": 0.5, - "random_mm_limit_mm_per_prompt": { - "audio": 1 - }, - "random_mm_bucket_config": { - "(0, 60, 3)": 1.0 - }, - "percentile-metrics": "ttft,tpot,itl,e2el", - "baseline": { - "mean_ttft_ms": [ - 26.3381 - ], - "mean_e2el_ms": [ - 485.2436 - ] - } - }, - { - "dataset_name": "random-mm", - "backend": "vllm", - "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct", - "endpoint": "/v1/completions", - "num_prompts": [ - 40 - ], - "request_rate": [ - 0.5 - ], - "random_input_len": 100, - "random_output_len": 100, - "random_range_ratio": 0.0, - "ignore_eos": true, - "extra_body": { - "modalities": [ - "text" - ] - }, - "random_mm_base_items_per_request": 2, - "random_mm_num_mm_items_range_ratio": 0.5, - "random_mm_limit_mm_per_prompt": { - "image": 1, - "video": 1 - }, - "random_mm_bucket_config": { - "(256, 256, 1)": 0.5, - "(720, 1280, 2)": 0.5 - }, - "percentile-metrics": "ttft,tpot,itl,e2el", - "baseline": { - "mean_ttft_ms": [ - 26.4517 - ], - "mean_e2el_ms": [ - 517.2017 - ] - } - }, - { - "dataset_name": "random-mm", - "backend": "vllm", - "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct", - "endpoint": "/v1/completions", - "num_prompts": [ - 100 - ], - "request_rate": [ - 1.0 - ], - "random_input_len": 100, - "random_output_len": 100, - "random_range_ratio": 0.0, - "ignore_eos": true, - "extra_body": { - "modalities": [ - "text" - ] - }, - "random_mm_base_items_per_request": 3, - "random_mm_num_mm_items_range_ratio": 0.5, - "random_mm_limit_mm_per_prompt": { - "image": 1, - "video": 1, - "audio": 1 - }, - "random_mm_bucket_config": { - "(256, 256, 1)": 0.34, - "(720, 1280, 2)": 0.33, - "(0, 60, 3)": 0.33 - }, - "percentile-metrics": "ttft,tpot,itl,e2el", - "baseline": { - "mean_ttft_ms": [ - 27.6972 - ], - "mean_e2el_ms": [ - 561.3684 - ] - } - } - ] - } + "percentile-metrics": "ttft,tpot,itl,e2el", + "baseline": { + "mean_ttft_ms": [ + 69.1276, + 255.1882, + 310.639, + 422.0337, + 532.0582 + ], + "mean_e2el_ms": [ + 4382.7211, + 7510.3815, + 9440.4854, + 12553.0186, + 16932.6065 + ] + } + }, + { + "dataset_name": "random-mm", + "backend": "vllm", + "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct", + "endpoint": "/v1/completions", + "num_prompts": [ + 10 + ], + "request_rate": [ + 0.1 + ], + "random_input_len": 100, + "random_output_len": 100, + "random_range_ratio": 0.0, + "ignore_eos": true, + "extra_body": { + "modalities": [ + "text" + ] + }, + "random_mm_base_items_per_request": 1, + "random_mm_num_mm_items_range_ratio": 0.5, + "random_mm_limit_mm_per_prompt": { + "audio": 1 + }, + "random_mm_bucket_config": { + "(0, 60, 3)": 1.0 + }, + "percentile-metrics": "ttft,tpot,itl,e2el", + "baseline": { + "mean_ttft_ms": [ + 26.3381 + ], + "mean_e2el_ms": [ + 485.2436 + ] + } + }, + { + "dataset_name": "random-mm", + "backend": "vllm", + "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct", + "endpoint": "/v1/completions", + "num_prompts": [ + 40 + ], + "request_rate": [ + 0.5 + ], + "random_input_len": 100, + "random_output_len": 100, + "random_range_ratio": 0.0, + "ignore_eos": true, + "extra_body": { + "modalities": [ + "text" + ] + }, + "random_mm_base_items_per_request": 2, + "random_mm_num_mm_items_range_ratio": 0.5, + "random_mm_limit_mm_per_prompt": { + "image": 1, + "video": 1 + }, + "random_mm_bucket_config": { + "(256, 256, 1)": 0.5, + "(720, 1280, 2)": 0.5 + }, + "percentile-metrics": "ttft,tpot,itl,e2el", + "baseline": { + "mean_ttft_ms": [ + 26.4517 + ], + "mean_e2el_ms": [ + 517.2017 + ] + } + }, + { + "dataset_name": "random-mm", + "backend": "vllm", + "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct", + "endpoint": "/v1/completions", + "num_prompts": [ + 100 + ], + "request_rate": [ + 1.0 + ], + "random_input_len": 100, + "random_output_len": 100, + "random_range_ratio": 0.0, + "ignore_eos": true, + "extra_body": { + "modalities": [ + "text" + ] + }, + "random_mm_base_items_per_request": 3, + "random_mm_num_mm_items_range_ratio": 0.5, + "random_mm_limit_mm_per_prompt": { + "image": 1, + "video": 1, + "audio": 1 + }, + "random_mm_bucket_config": { + "(256, 256, 1)": 0.34, + "(720, 1280, 2)": 0.33, + "(0, 60, 3)": 0.33 + }, + "percentile-metrics": "ttft,tpot,itl,e2el", + "baseline": { + "mean_ttft_ms": [ + 27.6972 + ], + "mean_e2el_ms": [ + 561.3684 + ] + } + } + ] + } ] diff --git a/tests/dfx/perf/tests/test_qwen_image_edit_2511_vllm_omni.json b/tests/dfx/perf/tests/test_qwen_image_edit_2511_vllm_omni.json index 118fa6733..53e13daed 100644 --- a/tests/dfx/perf/tests/test_qwen_image_edit_2511_vllm_omni.json +++ b/tests/dfx/perf/tests/test_qwen_image_edit_2511_vllm_omni.json @@ -1,149 +1,185 @@ [ - { - "test_name": "test_qwen_image_edit_2511_single_device", - "description": "Single-device baseline (two input images)", - "server_type": "vllm-omni", - "server_params": { - "model": "Qwen/Qwen-Image-Edit-2511", - "serve_args": { - "enable-diffusion-pipeline-profiler": true - } - }, - "benchmark_params": [ - { - "name": "512x512_steps20_i2i_2img", - "dataset": "random", - "task": "i2i", - "width": 512, - "height": 512, - "num-inference-steps": 20, - "num-prompts": 10, - "max-concurrency": 1, - "num-input-images": 2, - "enable-negative-prompt": true, - "baseline": { - "throughput_qps": 0.0721, - "latency_mean": 13.8398, - "peak_memory_mb_max": 54731.4, - "peak_memory_mb_mean": 54731.4 - } - }, - { - "name": "1536x1536_steps35_i2i_2img", - "dataset": "random", - "task": "i2i", - "width": 1536, - "height": 1536, - "num-inference-steps": 35, - "num-prompts": 10, - "max-concurrency": 1, - "num-input-images": 2, - "enable-negative-prompt": true, - "baseline": { - "throughput_qps": 0.0186, - "latency_mean": 53.6942, - "peak_memory_mb_max": 64402.4, - "peak_memory_mb_mean": 64402.4 - } - } - ] + { + "test_name": "test_qwen_image_edit_2511_single_device", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H100" + }, + "num_cards": 1 + } + }, + "full_model", + "diffusion" + ], + "description": "Single-device baseline (two input images)", + "server_type": "vllm-omni", + "server_params": { + "model": "Qwen/Qwen-Image-Edit-2511", + "serve_args": { + "enable-diffusion-pipeline-profiler": true + } }, - { - "test_name": "test_qwen_image_edit_2511_ulysses2_cfg2_vae_patch4", - "description": "Ulysses SP=2 + CFG=2 + VAE patch parallel=4", - "server_type": "vllm-omni", - "server_params": { - "model": "Qwen/Qwen-Image-Edit-2511", - "serve_args": { - "ulysses-degree": 2, - "cfg-parallel-size": 2, - "vae-patch-parallel-size": 4, - "vae-use-tiling": true, - "enable-diffusion-pipeline-profiler": true - } - }, - "benchmark_params": [ - { - "name": "1536x1536_steps35_i2i_2img", - "dataset": "random", - "task": "i2i", - "width": 1536, - "height": 1536, - "num-inference-steps": 35, - "num-prompts": 10, - "max-concurrency": 1, - "num-input-images": 2, - "enable-negative-prompt": true, - "baseline": { - "throughput_qps": 0.0538, - "latency_mean": 18.5591, - "peak_memory_mb_max": 53485.0, - "peak_memory_mb_mean": 53485.0 - } - } - ] + "benchmark_params": [ + { + "name": "512x512_steps20_i2i_2img", + "dataset": "random", + "task": "i2i", + "width": 512, + "height": 512, + "num-inference-steps": 20, + "num-prompts": 10, + "max-concurrency": 1, + "num-input-images": 2, + "enable-negative-prompt": true, + "baseline": { + "throughput_qps": 0.0721, + "latency_mean": 13.8398, + "peak_memory_mb_max": 54731.4, + "peak_memory_mb_mean": 54731.4 + } + }, + { + "name": "1536x1536_steps35_i2i_2img", + "dataset": "random", + "task": "i2i", + "width": 1536, + "height": 1536, + "num-inference-steps": 35, + "num-prompts": 10, + "max-concurrency": 1, + "num-input-images": 2, + "enable-negative-prompt": true, + "baseline": { + "throughput_qps": 0.0186, + "latency_mean": 53.6942, + "peak_memory_mb_max": 64402.4, + "peak_memory_mb_mean": 64402.4 + } + } + ] + }, + { + "test_name": "test_qwen_image_edit_2511_ulysses2_cfg2_vae_patch4", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H100" + }, + "num_cards": 4 + } + }, + "full_model", + "diffusion" + ], + "description": "Ulysses SP=2 + CFG=2 + VAE patch parallel=4", + "server_type": "vllm-omni", + "server_params": { + "model": "Qwen/Qwen-Image-Edit-2511", + "serve_args": { + "ulysses-degree": 2, + "cfg-parallel-size": 2, + "vae-patch-parallel-size": 4, + "vae-use-tiling": true, + "enable-diffusion-pipeline-profiler": true + } }, - { - "test_name": "test_qwen_image_edit_2511_ulysses2_cfg2_cache_dit", - "description": "Ulysses SP=2 + CFG=2 + CacheDiT", - "server_type": "vllm-omni", - "server_params": { - "model": "Qwen/Qwen-Image-Edit-2511", - "serve_args": { - "ulysses-degree": 2, - "cfg-parallel-size": 2, - "cache-backend": "cache_dit", - "cache-config": { - "Fn_compute_blocks": 1, - "Bn_compute_blocks": 0, - "max_warmup_steps": 4, - "residual_diff_threshold": 0.24, - "max_continuous_cached_steps": 3, - "enable_taylorseer": false, - "taylorseer_order": 1, - "scm_steps_mask_policy": null, - "scm_steps_policy": "dynamic" - }, - "enable-diffusion-pipeline-profiler": true - } + "benchmark_params": [ + { + "name": "1536x1536_steps35_i2i_2img", + "dataset": "random", + "task": "i2i", + "width": 1536, + "height": 1536, + "num-inference-steps": 35, + "num-prompts": 10, + "max-concurrency": 1, + "num-input-images": 2, + "enable-negative-prompt": true, + "baseline": { + "throughput_qps": 0.0538, + "latency_mean": 18.5591, + "peak_memory_mb_max": 53485.0, + "peak_memory_mb_mean": 53485.0 + } + } + ] + }, + { + "test_name": "test_qwen_image_edit_2511_ulysses2_cfg2_cache_dit", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H100" + }, + "num_cards": 4 + } + }, + "full_model", + "diffusion" + ], + "description": "Ulysses SP=2 + CFG=2 + CacheDiT", + "server_type": "vllm-omni", + "server_params": { + "model": "Qwen/Qwen-Image-Edit-2511", + "serve_args": { + "ulysses-degree": 2, + "cfg-parallel-size": 2, + "cache-backend": "cache_dit", + "cache-config": { + "Fn_compute_blocks": 1, + "Bn_compute_blocks": 0, + "max_warmup_steps": 4, + "residual_diff_threshold": 0.24, + "max_continuous_cached_steps": 3, + "enable_taylorseer": false, + "taylorseer_order": 1, + "scm_steps_mask_policy": null, + "scm_steps_policy": "dynamic" }, - "benchmark_params": [ - { - "name": "512x512_steps20_i2i_2img", - "dataset": "random", - "task": "i2i", - "width": 512, - "height": 512, - "num-inference-steps": 20, - "num-prompts": 10, - "max-concurrency": 1, - "num-input-images": 2, - "enable-negative-prompt": true, - "baseline": { - "throughput_qps": 0.2776, - "latency_mean": 3.5938, - "peak_memory_mb_max": 55214.0, - "peak_memory_mb_mean": 55214.0 - } - }, - { - "name": "1536x1536_steps35_i2i_2img", - "dataset": "random", - "task": "i2i", - "width": 1536, - "height": 1536, - "num-inference-steps": 35, - "num-prompts": 10, - "max-concurrency": 1, - "num-input-images": 2, - "enable-negative-prompt": true, - "baseline": { - "throughput_qps": 0.1051, - "latency_mean": 9.4918, - "peak_memory_mb_max": 64265.6, - "peak_memory_mb_mean": 64265.6 - } - } - ] - } + "enable-diffusion-pipeline-profiler": true + } + }, + "benchmark_params": [ + { + "name": "512x512_steps20_i2i_2img", + "dataset": "random", + "task": "i2i", + "width": 512, + "height": 512, + "num-inference-steps": 20, + "num-prompts": 10, + "max-concurrency": 1, + "num-input-images": 2, + "enable-negative-prompt": true, + "baseline": { + "throughput_qps": 0.2776, + "latency_mean": 3.5938, + "peak_memory_mb_max": 55214.0, + "peak_memory_mb_mean": 55214.0 + } + }, + { + "name": "1536x1536_steps35_i2i_2img", + "dataset": "random", + "task": "i2i", + "width": 1536, + "height": 1536, + "num-inference-steps": 35, + "num-prompts": 10, + "max-concurrency": 1, + "num-input-images": 2, + "enable-negative-prompt": true, + "baseline": { + "throughput_qps": 0.1051, + "latency_mean": 9.4918, + "peak_memory_mb_max": 64265.6, + "peak_memory_mb_mean": 64265.6 + } + } + ] + } ] diff --git a/tests/dfx/perf/tests/test_qwen_image_edit_vllm_omni.json b/tests/dfx/perf/tests/test_qwen_image_edit_vllm_omni.json index da3288182..bfda30731 100644 --- a/tests/dfx/perf/tests/test_qwen_image_edit_vllm_omni.json +++ b/tests/dfx/perf/tests/test_qwen_image_edit_vllm_omni.json @@ -1,144 +1,180 @@ [ - { - "test_name": "test_qwen_image_edit_single_device", - "description": "Single-device baseline", - "server_type": "vllm-omni", - "server_params": { - "model": "Qwen/Qwen-Image-Edit", - "serve_args": { - "enable-diffusion-pipeline-profiler": true - } - }, - "benchmark_params": [ - { - "name": "512x512_steps20_i2i", - "dataset": "random", - "task": "i2i", - "width": 512, - "height": 512, - "num-inference-steps": 20, - "num-prompts": 10, - "max-concurrency": 1, - "enable-negative-prompt": true, - "baseline": { - "throughput_qps": 0.1129, - "latency_mean": 8.8352, - "peak_memory_mb_max": 55356.5, - "peak_memory_mb_mean": 55356.5 - } - }, - { - "name": "1536x1536_steps35_i2i", - "dataset": "random", - "task": "i2i", - "width": 1536, - "height": 1536, - "num-inference-steps": 35, - "num-prompts": 10, - "max-concurrency": 1, - "enable-negative-prompt": true, - "baseline": { - "throughput_qps": 0.0240, - "latency_mean": 41.5686, - "peak_memory_mb_max": 62762.7, - "peak_memory_mb_mean": 62762.7 - } - } - ] + { + "test_name": "test_qwen_image_edit_single_device", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H100" + }, + "num_cards": 1 + } + }, + "full_model", + "diffusion" + ], + "description": "Single-device baseline", + "server_type": "vllm-omni", + "server_params": { + "model": "Qwen/Qwen-Image-Edit", + "serve_args": { + "enable-diffusion-pipeline-profiler": true + } }, - { - "test_name": "test_qwen_image_edit_ulysses2_cfg2_vae_patch4", - "description": "Ulysses SP=2 + CFG=2 + VAE patch parallel=4", - "server_type": "vllm-omni", - "server_params": { - "model": "Qwen/Qwen-Image-Edit", - "serve_args": { - "ulysses-degree": 2, - "cfg-parallel-size": 2, - "vae-patch-parallel-size": 4, - "vae-use-tiling": true, - "enable-diffusion-pipeline-profiler": true - } - }, - "benchmark_params": [ - { - "name": "1536x1536_steps35_i2i", - "dataset": "random", - "task": "i2i", - "width": 1536, - "height": 1536, - "num-inference-steps": 35, - "num-prompts": 10, - "max-concurrency": 1, - "enable-negative-prompt": true, - "baseline": { - "throughput_qps": 0.0672, - "latency_mean": 14.8517, - "peak_memory_mb_max": 53931.5, - "peak_memory_mb_mean": 53931.5 - } - } - ] + "benchmark_params": [ + { + "name": "512x512_steps20_i2i", + "dataset": "random", + "task": "i2i", + "width": 512, + "height": 512, + "num-inference-steps": 20, + "num-prompts": 10, + "max-concurrency": 1, + "enable-negative-prompt": true, + "baseline": { + "throughput_qps": 0.1129, + "latency_mean": 8.8352, + "peak_memory_mb_max": 55356.5, + "peak_memory_mb_mean": 55356.5 + } + }, + { + "name": "1536x1536_steps35_i2i", + "dataset": "random", + "task": "i2i", + "width": 1536, + "height": 1536, + "num-inference-steps": 35, + "num-prompts": 10, + "max-concurrency": 1, + "enable-negative-prompt": true, + "baseline": { + "throughput_qps": 0.0240, + "latency_mean": 41.5686, + "peak_memory_mb_max": 62762.7, + "peak_memory_mb_mean": 62762.7 + } + } + ] + }, + { + "test_name": "test_qwen_image_edit_ulysses2_cfg2_vae_patch4", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H100" + }, + "num_cards": 4 + } + }, + "full_model", + "diffusion" + ], + "description": "Ulysses SP=2 + CFG=2 + VAE patch parallel=4", + "server_type": "vllm-omni", + "server_params": { + "model": "Qwen/Qwen-Image-Edit", + "serve_args": { + "ulysses-degree": 2, + "cfg-parallel-size": 2, + "vae-patch-parallel-size": 4, + "vae-use-tiling": true, + "enable-diffusion-pipeline-profiler": true + } }, - { - "test_name": "test_qwen_image_edit_ulysses2_cfg2_cache_dit", - "description": "Ulysses SP=2 + CFG=2 + CacheDiT", - "server_type": "vllm-omni", - "server_params": { - "model": "Qwen/Qwen-Image-Edit", - "serve_args": { - "ulysses-degree": 2, - "cfg-parallel-size": 2, - "cache-backend": "cache_dit", - "cache-config": { - "Fn_compute_blocks": 1, - "Bn_compute_blocks": 0, - "max_warmup_steps": 4, - "residual_diff_threshold": 0.24, - "max_continuous_cached_steps": 3, - "enable_taylorseer": false, - "taylorseer_order": 1, - "scm_steps_mask_policy": null, - "scm_steps_policy": "dynamic" - }, - "enable-diffusion-pipeline-profiler": true - } + "benchmark_params": [ + { + "name": "1536x1536_steps35_i2i", + "dataset": "random", + "task": "i2i", + "width": 1536, + "height": 1536, + "num-inference-steps": 35, + "num-prompts": 10, + "max-concurrency": 1, + "enable-negative-prompt": true, + "baseline": { + "throughput_qps": 0.0672, + "latency_mean": 14.8517, + "peak_memory_mb_max": 53931.5, + "peak_memory_mb_mean": 53931.5 + } + } + ] + }, + { + "test_name": "test_qwen_image_edit_ulysses2_cfg2_cache_dit", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H100" + }, + "num_cards": 4 + } + }, + "full_model", + "diffusion" + ], + "description": "Ulysses SP=2 + CFG=2 + CacheDiT", + "server_type": "vllm-omni", + "server_params": { + "model": "Qwen/Qwen-Image-Edit", + "serve_args": { + "ulysses-degree": 2, + "cfg-parallel-size": 2, + "cache-backend": "cache_dit", + "cache-config": { + "Fn_compute_blocks": 1, + "Bn_compute_blocks": 0, + "max_warmup_steps": 4, + "residual_diff_threshold": 0.24, + "max_continuous_cached_steps": 3, + "enable_taylorseer": false, + "taylorseer_order": 1, + "scm_steps_mask_policy": null, + "scm_steps_policy": "dynamic" }, - "benchmark_params": [ - { - "name": "512x512_steps20_i2i", - "dataset": "random", - "task": "i2i", - "width": 512, - "height": 512, - "num-inference-steps": 20, - "num-prompts": 10, - "max-concurrency": 1, - "enable-negative-prompt": true, - "baseline": { - "throughput_qps": 0.2879, - "latency_mean": 3.4701, - "peak_memory_mb_max": 55525.6, - "peak_memory_mb_mean": 55525.6 - } - }, - { - "name": "1536x1536_steps35_i2i", - "dataset": "random", - "task": "i2i", - "width": 1536, - "height": 1536, - "num-inference-steps": 35, - "num-prompts": 10, - "max-concurrency": 1, - "enable-negative-prompt": true, - "baseline": { - "throughput_qps": 0.1113, - "latency_mean": 8.9656, - "peak_memory_mb_max": 63758.3, - "peak_memory_mb_mean": 63758.3 - } - } - ] - } + "enable-diffusion-pipeline-profiler": true + } + }, + "benchmark_params": [ + { + "name": "512x512_steps20_i2i", + "dataset": "random", + "task": "i2i", + "width": 512, + "height": 512, + "num-inference-steps": 20, + "num-prompts": 10, + "max-concurrency": 1, + "enable-negative-prompt": true, + "baseline": { + "throughput_qps": 0.2879, + "latency_mean": 3.4701, + "peak_memory_mb_max": 55525.6, + "peak_memory_mb_mean": 55525.6 + } + }, + { + "name": "1536x1536_steps35_i2i", + "dataset": "random", + "task": "i2i", + "width": 1536, + "height": 1536, + "num-inference-steps": 35, + "num-prompts": 10, + "max-concurrency": 1, + "enable-negative-prompt": true, + "baseline": { + "throughput_qps": 0.1113, + "latency_mean": 8.9656, + "peak_memory_mb_max": 63758.3, + "peak_memory_mb_mean": 63758.3 + } + } + ] + } ] diff --git a/tests/dfx/perf/tests/test_qwen_image_layered_vllm_omni.json b/tests/dfx/perf/tests/test_qwen_image_layered_vllm_omni.json index a94db56f5..3445dd94c 100644 --- a/tests/dfx/perf/tests/test_qwen_image_layered_vllm_omni.json +++ b/tests/dfx/perf/tests/test_qwen_image_layered_vllm_omni.json @@ -1,49 +1,61 @@ [ - { - "test_name": "test_qwen_image_layered_single_device", - "description": "Single-device baseline", - "server_type": "vllm-omni", - "server_params": { - "model": "Qwen/Qwen-Image-Layered", - "serve_args": { - "enable-diffusion-pipeline-profiler": true - } - }, - "benchmark_params": [ - { - "name": "640x640_steps20_i2i", - "dataset": "random", - "task": "i2i", - "width": 640, - "height": 640, - "num-inference-steps": 20, - "num-prompts": 10, - "max-concurrency": 1, - "enable-negative-prompt": true, - "baseline": { - "throughput_qps": 0.0703, - "latency_mean": 14.208, - "peak_memory_mb_max": 60543.5, - "peak_memory_mb_mean": 60543.5 - } - }, - { - "name": "1024x1024_steps35_i2i", - "dataset": "random", - "task": "i2i", - "width": 1024, - "height": 1024, - "num-inference-steps": 35, - "num-prompts": 10, - "max-concurrency": 1, - "enable-negative-prompt": true, - "baseline": { - "throughput_qps": 0.0417, - "latency_mean": 23.914, - "peak_memory_mb_max": 60543.5, - "peak_memory_mb_mean": 60543.5 - } - } - ] - } + { + "test_name": "test_qwen_image_layered_single_device", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H100" + }, + "num_cards": 1 + } + }, + "full_model", + "diffusion" + ], + "description": "Single-device baseline", + "server_type": "vllm-omni", + "server_params": { + "model": "Qwen/Qwen-Image-Layered", + "serve_args": { + "enable-diffusion-pipeline-profiler": true + } + }, + "benchmark_params": [ + { + "name": "640x640_steps20_i2i", + "dataset": "random", + "task": "i2i", + "width": 640, + "height": 640, + "num-inference-steps": 20, + "num-prompts": 10, + "max-concurrency": 1, + "enable-negative-prompt": true, + "baseline": { + "throughput_qps": 0.0703, + "latency_mean": 14.208, + "peak_memory_mb_max": 60543.5, + "peak_memory_mb_mean": 60543.5 + } + }, + { + "name": "1024x1024_steps35_i2i", + "dataset": "random", + "task": "i2i", + "width": 1024, + "height": 1024, + "num-inference-steps": 35, + "num-prompts": 10, + "max-concurrency": 1, + "enable-negative-prompt": true, + "baseline": { + "throughput_qps": 0.0417, + "latency_mean": 23.914, + "peak_memory_mb_max": 60543.5, + "peak_memory_mb_mean": 60543.5 + } + } + ] + } ] diff --git a/tests/dfx/perf/tests/test_qwen_image_vllm_omni.json b/tests/dfx/perf/tests/test_qwen_image_vllm_omni.json index 5ca6ed61a..b3daf7587 100644 --- a/tests/dfx/perf/tests/test_qwen_image_vllm_omni.json +++ b/tests/dfx/perf/tests/test_qwen_image_vllm_omni.json @@ -1,219 +1,279 @@ [ - { - "test_name": "test_qwen_image_single_device", - "description": "Single-device baseline (no parallelism)", - "server_type": "vllm-omni", - "server_params": { - "model": "Qwen/Qwen-Image", - "serve_args": { - "enable-diffusion-pipeline-profiler": true - } - }, - "benchmark_params": [ - { - "name": "512x512_steps20", - "dataset": "random", - "task": "t2i", - "width": 512, - "height": 512, - "num-inference-steps": 20, - "num-prompts": 10, - "max-concurrency": 1, - "enable-negative-prompt": true, - "baseline": { - "throughput_qps": 0.4153, - "latency_mean": 2.4065, - "peak_memory_mb_mean": 52289.9 - } - }, - { - "name": "1536x1536_steps35", - "dataset": "random", - "task": "t2i", - "width": 1536, - "height": 1536, - "num-inference-steps": 35, - "num-prompts": 10, - "max-concurrency": 1, - "enable-negative-prompt": true, - "baseline": { - "throughput_qps": 0.0423, - "latency_mean": 23.5991, - "peak_memory_mb_mean": 61360.5 - } - } - ] + { + "test_name": "test_qwen_image_single_device", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H100" + }, + "num_cards": 1 + } + }, + "full_model", + "diffusion" + ], + "description": "Single-device baseline (no parallelism)", + "server_type": "vllm-omni", + "server_params": { + "model": "Qwen/Qwen-Image", + "serve_args": { + "enable-diffusion-pipeline-profiler": true + } }, - { - "test_name": "test_qwen_image_single_device_step_execution", - "description": "Single-device baseline (no parallelism) with step execution", - "server_type": "vllm-omni", - "server_params": { - "model": "Qwen/Qwen-Image", - "serve_args": { - "enable-diffusion-pipeline-profiler": true, - "step-execution": true - } - }, - "benchmark_params": [ - { - "name": "512x512_steps20", - "dataset": "random", - "task": "t2i", - "width": 512, - "height": 512, - "num-inference-steps": 20, - "num-prompts": 10, - "max-concurrency": 1, - "enable-negative-prompt": true, - "baseline": { - "throughput_qps": 0.4175, - "latency_mean": 2.3953, - "peak_memory_mb_mean": 56000 - } - }, - { - "name": "1536x1536_steps35", - "dataset": "random", - "task": "t2i", - "width": 1536, - "height": 1536, - "num-inference-steps": 35, - "num-prompts": 10, - "max-concurrency": 1, - "enable-negative-prompt": true, - "baseline": { - "throughput_qps": 0.0423, - "latency_mean": 23.5768, - "peak_memory_mb_mean": 65000 - } - } - ] + "benchmark_params": [ + { + "name": "512x512_steps20", + "dataset": "random", + "task": "t2i", + "width": 512, + "height": 512, + "num-inference-steps": 20, + "num-prompts": 10, + "max-concurrency": 1, + "enable-negative-prompt": true, + "baseline": { + "throughput_qps": 0.4153, + "latency_mean": 2.4065, + "peak_memory_mb_mean": 52289.9 + } + }, + { + "name": "1536x1536_steps35", + "dataset": "random", + "task": "t2i", + "width": 1536, + "height": 1536, + "num-inference-steps": 35, + "num-prompts": 10, + "max-concurrency": 1, + "enable-negative-prompt": true, + "baseline": { + "throughput_qps": 0.0423, + "latency_mean": 23.5991, + "peak_memory_mb_mean": 61360.5 + } + } + ] + }, + { + "test_name": "test_qwen_image_single_device_step_execution", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H100" + }, + "num_cards": 1 + } + }, + "full_model", + "diffusion" + ], + "description": "Single-device baseline (no parallelism) with step execution", + "server_type": "vllm-omni", + "server_params": { + "model": "Qwen/Qwen-Image", + "serve_args": { + "enable-diffusion-pipeline-profiler": true, + "step-execution": true + } }, - { - "test_name": "test_qwen_image_ulysses2_cfg2_vae_patch4", - "description": "Ulysses SP=2 + CFG-parallel=2 + VAE Patch Parallel=4", - "server_type": "vllm-omni", - "server_params": { - "model": "Qwen/Qwen-Image", - "serve_args": { - "ulysses-degree": 2, - "cfg-parallel-size": 2, - "vae-patch-parallel-size": 4, - "vae-use-tiling": true, - "enable-diffusion-pipeline-profiler": true - } - }, - "benchmark_params": [ - { - "name": "1536x1536_steps35", - "dataset": "random", - "task": "t2i", - "width": 1536, - "height": 1536, - "num-inference-steps": 35, - "num-prompts": 10, - "max-concurrency": 1, - "enable-negative-prompt": true, - "baseline": { - "throughput_qps": 0.1214, - "latency_mean": 8.2211, - "peak_memory_mb_mean": 51704.7 - } - } - ] + "benchmark_params": [ + { + "name": "512x512_steps20", + "dataset": "random", + "task": "t2i", + "width": 512, + "height": 512, + "num-inference-steps": 20, + "num-prompts": 10, + "max-concurrency": 1, + "enable-negative-prompt": true, + "baseline": { + "throughput_qps": 0.4175, + "latency_mean": 2.3953, + "peak_memory_mb_mean": 56000 + } + }, + { + "name": "1536x1536_steps35", + "dataset": "random", + "task": "t2i", + "width": 1536, + "height": 1536, + "num-inference-steps": 35, + "num-prompts": 10, + "max-concurrency": 1, + "enable-negative-prompt": true, + "baseline": { + "throughput_qps": 0.0423, + "latency_mean": 23.5768, + "peak_memory_mb_mean": 65000 + } + } + ] + }, + { + "test_name": "test_qwen_image_ulysses2_cfg2_vae_patch4", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H100" + }, + "num_cards": 4 + } + }, + "full_model", + "diffusion" + ], + "description": "Ulysses SP=2 + CFG-parallel=2 + VAE Patch Parallel=4", + "server_type": "vllm-omni", + "server_params": { + "model": "Qwen/Qwen-Image", + "serve_args": { + "ulysses-degree": 2, + "cfg-parallel-size": 2, + "vae-patch-parallel-size": 4, + "vae-use-tiling": true, + "enable-diffusion-pipeline-profiler": true + } }, - { - "test_name": "test_qwen_image_ulysses2_cfg2_cache_dit", - "description": "Ulysses SP=2 + CFG-parallel=2 + CacheDiT acceleration", - "server_type": "vllm-omni", - "server_params": { - "model": "Qwen/Qwen-Image", - "serve_args": { - "ulysses-degree": 2, - "cfg-parallel-size": 2, - "cache-backend": "cache_dit", - "cache-config": { - "Fn_compute_blocks": 1, - "Bn_compute_blocks": 0, - "max_warmup_steps": 4, - "residual_diff_threshold": 0.24, - "max_continuous_cached_steps": 3, - "enable_taylorseer": false, - "taylorseer_order": 1, - "scm_steps_mask_policy": null, - "scm_steps_policy": "dynamic" - }, - "enable-diffusion-pipeline-profiler": true - } + "benchmark_params": [ + { + "name": "1536x1536_steps35", + "dataset": "random", + "task": "t2i", + "width": 1536, + "height": 1536, + "num-inference-steps": 35, + "num-prompts": 10, + "max-concurrency": 1, + "enable-negative-prompt": true, + "baseline": { + "throughput_qps": 0.1214, + "latency_mean": 8.2211, + "peak_memory_mb_mean": 51704.7 + } + } + ] + }, + { + "test_name": "test_qwen_image_ulysses2_cfg2_cache_dit", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H100" + }, + "num_cards": 4 + } + }, + "full_model", + "diffusion" + ], + "description": "Ulysses SP=2 + CFG-parallel=2 + CacheDiT acceleration", + "server_type": "vllm-omni", + "server_params": { + "model": "Qwen/Qwen-Image", + "serve_args": { + "ulysses-degree": 2, + "cfg-parallel-size": 2, + "cache-backend": "cache_dit", + "cache-config": { + "Fn_compute_blocks": 1, + "Bn_compute_blocks": 0, + "max_warmup_steps": 4, + "residual_diff_threshold": 0.24, + "max_continuous_cached_steps": 3, + "enable_taylorseer": false, + "taylorseer_order": 1, + "scm_steps_mask_policy": null, + "scm_steps_policy": "dynamic" }, - "benchmark_params": [ - { - "name": "512x512_steps20", - "dataset": "random", - "task": "t2i", - "width": 512, - "height": 512, - "num-inference-steps": 20, - "num-prompts": 10, - "max-concurrency": 1, - "enable-negative-prompt": true, - "baseline": { - "throughput_qps": 0.559, - "latency_mean": 1.794, - "peak_memory_mb_mean": 52397.6775 - } - }, - { - "name": "1536x1536_steps35", - "dataset": "random", - "task": "t2i", - "width": 1536, - "height": 1536, - "num-inference-steps": 35, - "num-prompts": 10, - "max-concurrency": 1, - "enable-negative-prompt": true, - "baseline": { - "throughput_qps": 0.1918, - "latency_mean": 5.2022, - "peak_memory_mb_mean": 61468.8 - } - } - ] + "enable-diffusion-pipeline-profiler": true + } }, - { - "test_name": "test_qwen_image_single_device_step_execution_high_concurrency", - "description": "Single-device Qwen-Image step-level execution benchmark at 512x512, 20 steps, concurrency sweep", - "server_type": "vllm-omni", - "server_params": { - "model": "Qwen/Qwen-Image", - "serve_args": { - "enable-diffusion-pipeline-profiler": true, - "step-execution": true, - "max-num-seqs": 8 - } - }, - "benchmark_params": [ - { - "name": "512x512_steps20_high_concurrency", - "dataset": "random", - "task": "t2i", - "width": 512, - "height": 512, - "num-inference-steps": 20, - "num-prompts": [20, 40, 80, 160], - "max-concurrency": [1, 2, 4, 8], - "warmup-requests": 8, - "warmup-concurrency": 8, - "warmup-num-inference-steps": 20, - "enable-negative-prompt": true, - "baseline": { - "throughput_qps": [0.4125, 0.6714, 0.6674, 0.8159], - "latency_mean": [2.4215, 2.992, 5.9293, 9.656], - "peak_memory_mb_mean": [67000, 67000, 67000, 67000] - } - } - ] - } + "benchmark_params": [ + { + "name": "512x512_steps20", + "dataset": "random", + "task": "t2i", + "width": 512, + "height": 512, + "num-inference-steps": 20, + "num-prompts": 10, + "max-concurrency": 1, + "enable-negative-prompt": true, + "baseline": { + "throughput_qps": 0.559, + "latency_mean": 1.794, + "peak_memory_mb_mean": 52397.6775 + } + }, + { + "name": "1536x1536_steps35", + "dataset": "random", + "task": "t2i", + "width": 1536, + "height": 1536, + "num-inference-steps": 35, + "num-prompts": 10, + "max-concurrency": 1, + "enable-negative-prompt": true, + "baseline": { + "throughput_qps": 0.1918, + "latency_mean": 5.2022, + "peak_memory_mb_mean": 61468.8 + } + } + ] + }, + { + "test_name": "test_qwen_image_single_device_step_execution_high_concurrency", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H100" + }, + "num_cards": 1 + } + }, + "full_model", + "diffusion" + ], + "description": "Single-device Qwen-Image step-level execution benchmark at 512x512, 20 steps, concurrency sweep", + "server_type": "vllm-omni", + "server_params": { + "model": "Qwen/Qwen-Image", + "serve_args": { + "enable-diffusion-pipeline-profiler": true, + "step-execution": true, + "max-num-seqs": 8 + } + }, + "benchmark_params": [ + { + "name": "512x512_steps20_high_concurrency", + "dataset": "random", + "task": "t2i", + "width": 512, + "height": 512, + "num-inference-steps": 20, + "num-prompts": [20, 40, 80, 160], + "max-concurrency": [1, 2, 4, 8], + "warmup-requests": 8, + "warmup-concurrency": 8, + "warmup-num-inference-steps": 20, + "enable-negative-prompt": true, + "baseline": { + "throughput_qps": [0.4125, 0.6714, 0.6674, 0.8159], + "latency_mean": [2.4215, 2.992, 5.9293, 9.656], + "peak_memory_mb_mean": [67000, 67000, 67000, 67000] + } + } + ] + } ] diff --git a/tests/dfx/perf/tests/test_runner_metadata.py b/tests/dfx/perf/tests/test_runner_metadata.py index a195c4546..71d4aa95b 100644 --- a/tests/dfx/perf/tests/test_runner_metadata.py +++ b/tests/dfx/perf/tests/test_runner_metadata.py @@ -86,3 +86,202 @@ def test_enabled_false_entry_is_skipped(): # Only the enabled=True entry should appear assert len(params) == 1 assert params[0].get("task") == "voice_clone" + + +def test_resolve_pytest_marks_hardware_dict_with_extra(): + from tests.dfx.conftest import resolve_pytest_marks + + marks = resolve_pytest_marks( + [ + {"hardware_marks": {"res": {"cuda": "H100"}, "num_cards": 2}}, + "full_model", + "diffusion", + "local_model", + ] + ) + names = {m.name for m in marks} + assert "H100" in names + assert "cuda" in names + assert "gpu" in names + assert "distributed_cuda" in names + assert "full_model" in names + assert "diffusion" in names + assert "local_model" in names + + +def test_resolve_pytest_marks_rejects_legacy_object_format(): + from tests.dfx.conftest import resolve_pytest_marks + + with pytest.raises(ValueError, match="mark must be a list"): + resolve_pytest_marks( + { + "hardware_marks": {"res": {"cuda": "H100"}, "num_cards": 1}, + "marks": ["full_model"], + } + ) + + +def test_extract_mark_resource_label(): + from tests.dfx.conftest import extract_mark_resource_label + + assert extract_mark_resource_label(None) == "na" + assert ( + extract_mark_resource_label( + [ + {"hardware_marks": {"res": {"cuda": "H100"}, "num_cards": 1}}, + "full_model", + ] + ) + == "H100" + ) + assert ( + extract_mark_resource_label( + [ + { + "hardware_marks": { + "res": {"cuda": "H100", "rocm": "MI325"}, + "num_cards": 2, + } + }, + "diffusion", + ] + ) + == "H100-MI325" + ) + + +def test_resource_label_for_filename(): + from tests.dfx.conftest import resource_label_for_filename + + assert resource_label_for_filename("H100") == "" + assert resource_label_for_filename("L4") == "L4" + assert resource_label_for_filename("910B") == "910B" + assert resource_label_for_filename("na") == "na" + + +def test_hardware_json_value(): + from tests.dfx.conftest import hardware_json_value + + assert hardware_json_value("H100") == "H100" + assert hardware_json_value("na") == "" + assert hardware_json_value(None) == "" + + +def test_extract_configs_resource_label(monkeypatch): + from tests.dfx.conftest import extract_configs_resource_label, get_runtime_resource_label + + monkeypatch.setattr( + "tests.dfx.conftest._read_runtime_device_name", + lambda *, device_id=0: "NVIDIA H100 80GB HBM3", + ) + get_runtime_resource_label(refresh=True) + assert extract_configs_resource_label([]) == "H100" + get_runtime_resource_label(refresh=True) + monkeypatch.setattr( + "tests.dfx.conftest._read_runtime_device_name", + lambda *, device_id=0: "Ascend910B2", + ) + assert get_runtime_resource_label(refresh=True) == "910B" + + +def test_load_benchmark_configs_from_dir(tmp_path): + from tests.dfx.conftest import load_benchmark_configs + + (tmp_path / "a.json").write_text( + json.dumps([{"test_name": "test_a", "server_params": {"model": "m/a"}, "benchmark_params": []}]), + encoding="utf-8", + ) + (tmp_path / "b.json").write_text( + json.dumps([{"test_name": "test_b", "server_params": {"model": "m/b"}, "benchmark_params": []}]), + encoding="utf-8", + ) + configs = load_benchmark_configs(config_dir=tmp_path) + assert [c["test_name"] for c in configs] == ["test_a", "test_b"] + + +def test_create_unique_server_pytest_params_applies_marks(tmp_path): + from tests.dfx.conftest import create_unique_server_pytest_params + + configs = [ + { + "test_name": "test_with_mark", + "mark": [ + {"hardware_marks": {"res": {"cuda": "H100"}, "num_cards": 1}}, + "full_model", + ], + "server_params": {"model": "some/model"}, + "benchmark_params": [{"name": "p0", "num_prompts": 1}], + }, + { + "test_name": "test_without_mark", + "server_params": {"model": "other/model"}, + "benchmark_params": [{"name": "p0", "num_prompts": 1}], + }, + ] + params = create_unique_server_pytest_params(configs, tmp_path) + by_id = {p.id: p for p in params} + assert len(by_id["test_with_mark"].values) == 1 + assert isinstance(by_id["test_with_mark"].values[0], tuple) + assert any(m.name == "H100" for m in by_id["test_with_mark"].marks) + assert not any(m.name == "H100" for m in by_id["test_without_mark"].marks) + + +def test_is_diffusion_perf_config(): + from tests.dfx.conftest import is_diffusion_perf_config + + assert not is_diffusion_perf_config( + {"test_name": "omni_a", "mark": [{"hardware_marks": {"res": {"cuda": "H100"}}}, "omni"]} + ) + assert is_diffusion_perf_config( + { + "test_name": "diff_a", + "server_type": "vllm-omni", + "mark": [{"hardware_marks": {"res": {"cuda": "H100"}}}, "diffusion"], + } + ) + + +def test_benchmark_param_id_suffix_from_task_eval_phase(): + from tests.dfx.conftest import _unique_benchmark_param_id_suffixes + + params = [ + {"task": "default_voice", "eval_phase": "latency"}, + {"task": "default_voice", "eval_phase": "throughput"}, + ] + assert _unique_benchmark_param_id_suffixes(params) == [ + "default_voice_latency", + "default_voice_throughput", + ] + + +def test_create_paired_omni_benchmark_pytest_params(tmp_path): + from tests.dfx.conftest import create_paired_omni_benchmark_pytest_params + + configs = [ + { + "test_name": "test_omni", + "mark": [ + {"hardware_marks": {"res": {"cuda": "H100"}, "num_cards": 1}}, + "omni", + ], + "server_params": {"model": "m/omni"}, + "benchmark_params": [{"name": "p0", "num_prompts": 1}, {"name": "p1", "num_prompts": 2}], + }, + { + "test_name": "test_tts", + "mark": [ + {"hardware_marks": {"res": {"cuda": "H100"}, "num_cards": 1}}, + "tts", + ], + "server_params": {"model": "m/tts"}, + "benchmark_params": [{"name": "p0", "num_prompts": 1}], + }, + ] + params = create_paired_omni_benchmark_pytest_params(configs, tmp_path) + by_id = {p.id: p for p in params} + assert set(by_id) == {"test_omni-p0", "test_omni-p1", "test_tts-p0"} + omni_row, bench_row = by_id["test_tts-p0"].values + assert omni_row[0] == "test_tts" + assert bench_row == ("test_tts", 0) + assert any(m.name == "tts" for m in by_id["test_tts-p0"].marks) + assert not any(m.name == "tts" for m in by_id["test_omni-p0"].marks) diff --git a/tests/dfx/perf/tests/test_tts.json b/tests/dfx/perf/tests/test_tts.json index 1ba273a2c..3bbf60935 100644 --- a/tests/dfx/perf/tests/test_tts.json +++ b/tests/dfx/perf/tests/test_tts.json @@ -1,6 +1,18 @@ [ { "test_name": "test_qwen3_tts_base", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H100" + }, + "num_cards": 2 + } + }, + "full_model", + "tts" + ], "server_params": { "model": "Qwen/Qwen3-TTS-12Hz-1.7B-Base", "stage_overrides": { @@ -98,6 +110,18 @@ }, { "test_name": "test_qwen3_tts_customvoice", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H100" + }, + "num_cards": 2 + } + }, + "full_model", + "tts" + ], "server_params": { "model": "Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice", "stage_overrides": { @@ -342,6 +366,18 @@ }, { "test_name": "test_voxtral_tts", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H100" + }, + "num_cards": 1 + } + }, + "full_model", + "tts" + ], "server_params": { "model": "mistralai/Voxtral-4B-TTS-2603" }, diff --git a/tests/dfx/perf/tests/test_voxcpm2.json b/tests/dfx/perf/tests/test_voxcpm2.json index 5c99f2596..ef72b5d88 100644 --- a/tests/dfx/perf/tests/test_voxcpm2.json +++ b/tests/dfx/perf/tests/test_voxcpm2.json @@ -1,6 +1,18 @@ [ { "test_name": "test_voxcpm2", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H100" + }, + "num_cards": 1 + } + }, + "full_model", + "tts" + ], "server_params": { "model": "openbmb/VoxCPM2", "trust_remote_code": true, diff --git a/tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json b/tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json index 07873a64b..78f720a6a 100644 --- a/tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json +++ b/tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json @@ -1,107 +1,131 @@ [ - { - "test_name": "test_wan22_i2v_single_device", - "description": "Single-device baseline", - "server_type": "vllm-omni", - "server_params": { - "model": "Wan-AI/Wan2.2-I2V-A14B-Diffusers", - "serve_args": { - "enable-diffusion-pipeline-profiler": true - } - }, - "benchmark_params": [ - { - "name": "832x480_frames81_steps4", - "dataset": "random", - "task": "i2v", - "num-prompts": 10, - "max-concurrency": 1, - "num-input-images": 1, - "seed": 42, - "enable-negative-prompt": true, - "random-request-config": [ - { - "width": 832, - "height": 480, - "num_inference_steps": 4, - "num_frames": 81, - "fps": 16, - "weight": 1 - } - ], - "baseline": { - "throughput_qps": 0.0465, - "latency_mean": 21.4354, - "peak_memory_mb_mean": 72557.58 - } - } - ] + { + "test_name": "test_wan22_i2v_single_device", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H100" + }, + "num_cards": 1 + } + }, + "full_model", + "diffusion" + ], + "description": "Single-device baseline", + "server_type": "vllm-omni", + "server_params": { + "model": "Wan-AI/Wan2.2-I2V-A14B-Diffusers", + "serve_args": { + "enable-diffusion-pipeline-profiler": true + } }, - { - "test_name": "test_wan22_i2v_usp2_vae_patch2_hsdp_slicing", - "description": "USP=2 + VAE patch parallel=2 + HSDP + VAE slicing", - "server_type": "vllm-omni", - "server_params": { - "model": "Wan-AI/Wan2.2-I2V-A14B-Diffusers", - "serve_args": { - "usp": 2, - "vae-patch-parallel-size": 2, - "use-hsdp": true, - "vae-use-slicing": true, - "enable-diffusion-pipeline-profiler": true - } - }, - "benchmark_params": [ - { - "name": "832x480_frames81_steps4", - "dataset": "random", - "task": "i2v", - "num-prompts": 10, - "max-concurrency": 1, - "num-input-images": 1, - "seed": 42, - "enable-negative-prompt": true, - "random-request-config": [ - { - "width": 832, - "height": 480, - "num_inference_steps": 4, - "num_frames": 81, - "fps": 16, - "weight": 1 - } - ], - "baseline": { - "throughput_qps": 0.0642, - "latency_mean": 15.5589, - "peak_memory_mb_mean": 45742.88 - } - }, - { - "name": "1280x720_frames121_steps4", - "dataset": "random", - "task": "i2v", - "num-prompts": 10, - "max-concurrency": 1, - "num-input-images": 1, - "seed": 42, - "enable-negative-prompt": true, - "random-request-config": [ - { - "width": 1280, - "height": 720, - "num_inference_steps": 4, - "num_frames": 121, - "fps": 16, - "weight": 1 - } - ], - "baseline": { - "throughput_qps": 0.0128, - "latency_mean": 77.8565, - "peak_memory_mb_mean": 53947.84 - } - } - ] - } + "benchmark_params": [ + { + "name": "832x480_frames81_steps4", + "dataset": "random", + "task": "i2v", + "num-prompts": 10, + "max-concurrency": 1, + "num-input-images": 1, + "seed": 42, + "enable-negative-prompt": true, + "random-request-config": [ + { + "width": 832, + "height": 480, + "num_inference_steps": 4, + "num_frames": 81, + "fps": 16, + "weight": 1 + } + ], + "baseline": { + "throughput_qps": 0.0465, + "latency_mean": 21.4354, + "peak_memory_mb_mean": 72557.58 + } + } + ] + }, + { + "test_name": "test_wan22_i2v_usp2_vae_patch2_hsdp_slicing", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H100" + }, + "num_cards": 2 + } + }, + "full_model", + "diffusion" + ], + "description": "USP=2 + VAE patch parallel=2 + HSDP + VAE slicing", + "server_type": "vllm-omni", + "server_params": { + "model": "Wan-AI/Wan2.2-I2V-A14B-Diffusers", + "serve_args": { + "usp": 2, + "vae-patch-parallel-size": 2, + "use-hsdp": true, + "vae-use-slicing": true, + "enable-diffusion-pipeline-profiler": true + } + }, + "benchmark_params": [ + { + "name": "832x480_frames81_steps4", + "dataset": "random", + "task": "i2v", + "num-prompts": 10, + "max-concurrency": 1, + "num-input-images": 1, + "seed": 42, + "enable-negative-prompt": true, + "random-request-config": [ + { + "width": 832, + "height": 480, + "num_inference_steps": 4, + "num_frames": 81, + "fps": 16, + "weight": 1 + } + ], + "baseline": { + "throughput_qps": 0.0642, + "latency_mean": 15.5589, + "peak_memory_mb_mean": 45742.88 + } + }, + { + "name": "1280x720_frames121_steps4", + "dataset": "random", + "task": "i2v", + "num-prompts": 10, + "max-concurrency": 1, + "num-input-images": 1, + "seed": 42, + "enable-negative-prompt": true, + "random-request-config": [ + { + "width": 1280, + "height": 720, + "num_inference_steps": 4, + "num_frames": 121, + "fps": 16, + "weight": 1 + } + ], + "baseline": { + "throughput_qps": 0.0128, + "latency_mean": 77.8565, + "peak_memory_mb_mean": 53947.84 + } + } + ] + } ] From 0a9a5c231e39f6523ae404fcd17033c2d6134408 Mon Sep 17 00:00:00 2001 From: zzh <67720593+BLANKETusers@users.noreply.github.com> Date: Thu, 16 Jul 2026 22:49:13 +0800 Subject: [PATCH 004/240] Fix hunyuan ci (#5143) Signed-off-by: zzh <943967662@qq.com> --- tests/e2e/accuracy/test_hunyuan_image3_pixel_accuracy.py | 1 + vllm_omni/config/omni_config.py | 1 + vllm_omni/diffusion/data.py | 3 +++ vllm_omni/diffusion/worker/diffusion_worker.py | 6 ++++++ 4 files changed, 11 insertions(+) diff --git a/tests/e2e/accuracy/test_hunyuan_image3_pixel_accuracy.py b/tests/e2e/accuracy/test_hunyuan_image3_pixel_accuracy.py index 6ac930f46..3f799c038 100644 --- a/tests/e2e/accuracy/test_hunyuan_image3_pixel_accuracy.py +++ b/tests/e2e/accuracy/test_hunyuan_image3_pixel_accuracy.py @@ -48,6 +48,7 @@ _DEPLOY_CONFIG = { "trust_remote_code": True, "devices": "0,1,2,3", # set dynamically by _write_deploy_config "vae_use_slicing": False, + "moe_backend": "flashinfer_cutlass", "vae_use_tiling": False, "parallel_config": { "pipeline_parallel_size": 1, diff --git a/vllm_omni/config/omni_config.py b/vllm_omni/config/omni_config.py index dc5f4ef9c..256dcd5c7 100644 --- a/vllm_omni/config/omni_config.py +++ b/vllm_omni/config/omni_config.py @@ -469,6 +469,7 @@ class _DiffusionConfigProjection: diffusion_kv_cache_skip_layers: str | list[int] | tuple[int, ...] | set[int] | None = None diffusion_kv_cache_skip_step_indices: set[int] | None = None diffusion_kv_cache_skip_layer_indices: set[int] | None = None + moe_backend: str = "auto" force_cutlass_fp8: bool = False enable_diffusion_pipeline_profiler: bool = False step_execution: bool = False diff --git a/vllm_omni/diffusion/data.py b/vllm_omni/diffusion/data.py index 0feab5b28..e131b833b 100644 --- a/vllm_omni/diffusion/data.py +++ b/vllm_omni/diffusion/data.py @@ -664,6 +664,9 @@ class OmniDiffusionConfig: # STA_mode: STA_Mode = STA_Mode.STA_INFERENCE skip_time_steps: int = 15 + # MoE kernel backend selection + moe_backend: str = "auto" + # Compilation enforce_eager: bool = False diff --git a/vllm_omni/diffusion/worker/diffusion_worker.py b/vllm_omni/diffusion/worker/diffusion_worker.py index 8cce47897..006bbae48 100644 --- a/vllm_omni/diffusion/worker/diffusion_worker.py +++ b/vllm_omni/diffusion/worker/diffusion_worker.py @@ -288,6 +288,12 @@ class DiffusionWorker: # Since vLLM v0.20.0, IR wraps GPU ops. Set IR op priority preference to enforce GPU op fusion during wrapping. # Also need to log, because vLLM internally logs another line in VllmConfig.__post_init__. Avoid confusion. vllm_config.kernel_config.ir_op_priority = _resolve_ir_op_priority(self.od_config, vllm_config) + if self.od_config.moe_backend != "auto": + logger.warning( + "Overriding MoE backend from default 'auto' to '%s' per deploy config.", + self.od_config.moe_backend, + ) + vllm_config.kernel_config.moe_backend = self.od_config.moe_backend logger.info( "Final IR op priority after setting vLLM-Omni overrides: %s", vllm_config.kernel_config.ir_op_priority ) From 85f7a5b374cbed6953df96b61dd8db1ff7e86429 Mon Sep 17 00:00:00 2001 From: bastefaniak Date: Thu, 16 Jul 2026 18:39:40 +0200 Subject: [PATCH 005/240] Add Cosmos3 Edge and Distilled checkpoints support (#5001) Signed-off-by: Bartosz Stefaniak --- .../models/cosmos3/test_cosmos3_pipeline.py | 809 ++++++++++++++++-- .../cosmos3/test_cosmos3_transformer.py | 309 ++++++- tests/e2e/accuracy/test_cosmos3_similarity.py | 2 - tests/model_extras/test_model_extras.py | 16 +- tests/model_tests/diffusion/test_alignment.py | 1 + .../diffusion/cache/cache_dit_backend.py | 1 + .../diffusion/models/cosmos3/__init__.py | 2 + .../models/cosmos3/pipeline_cosmos3.py | 397 +++++++-- .../models/cosmos3/transformer_cosmos3.py | 59 +- .../cosmos3/transformer_cosmos3_edge.py | 396 +++++++++ vllm_omni/diffusion/registry.py | 8 + vllm_omni/model_extras/registry.py | 5 + 12 files changed, 1789 insertions(+), 216 deletions(-) create mode 100644 vllm_omni/diffusion/models/cosmos3/transformer_cosmos3_edge.py diff --git a/tests/diffusion/models/cosmos3/test_cosmos3_pipeline.py b/tests/diffusion/models/cosmos3/test_cosmos3_pipeline.py index ba482e793..1085e3878 100644 --- a/tests/diffusion/models/cosmos3/test_cosmos3_pipeline.py +++ b/tests/diffusion/models/cosmos3/test_cosmos3_pipeline.py @@ -36,20 +36,36 @@ class StubScheduler: timesteps: list[int] | None = None, *, flow_shift: float = 1.0, - use_karras_sigmas: bool = True, ) -> None: self.timesteps = torch.tensor(timesteps or [9, 3], dtype=torch.int64) self.config = SimpleNamespace( num_train_timesteps=1000, flow_shift=flow_shift, - use_karras_sigmas=use_karras_sigmas, ) - self.set_timesteps_calls: list[tuple[int, torch.device]] = [] + self.set_timesteps_calls: list[dict[str, Any]] = [] self.step_calls: list[tuple[torch.Tensor, torch.Tensor, torch.Tensor]] = [] - def set_timesteps(self, num_steps: int, device: torch.device) -> None: - self.set_timesteps_calls.append((num_steps, device)) - self.timesteps = torch.arange(num_steps, 0, -1, dtype=torch.int64, device=device) + def set_timesteps( + self, + num_inference_steps: int | None = None, + device: str | torch.device | None = None, + *, + shift: float | None = None, + sigmas: list[float] | None = None, + ) -> None: + self.set_timesteps_calls.append( + { + "num_inference_steps": num_inference_steps, + "device": device, + "shift": shift, + "sigmas": sigmas, + } + ) + if sigmas is not None: + self.timesteps = torch.tensor(sigmas, device=device) + else: + assert num_inference_steps is not None + self.timesteps = torch.arange(num_inference_steps, 0, -1, dtype=torch.int64, device=device) def step(self, noise_pred: torch.Tensor, timestep: torch.Tensor, latents: torch.Tensor, **kwargs): del kwargs @@ -200,6 +216,7 @@ def fake_cosmos3_guardrails(monkeypatch: pytest.MonkeyPatch): def make_cosmos3_pipeline(): def _make(): from vllm_omni.diffusion.models.cosmos3.pipeline_cosmos3 import ( + COSMOS3_VIDEO_DEFAULT_FLOW_SHIFT, Cosmos3OmniDiffusersPipeline, ) @@ -213,11 +230,10 @@ def make_cosmos3_pipeline(): pipeline.vae_scale_factor_temporal = 4 pipeline.vae_scale_factor_spatial = 8 pipeline.scheduler = StubScheduler([9, 3], flow_shift=1.0) - pipeline._base_scheduler_config = pipeline.scheduler.config - pipeline._engine_init_flow_shift = 1.0 - pipeline._current_flow_shift = 1.0 - pipeline._base_scheduler_use_karras_sigmas = True - pipeline._current_scheduler_use_karras_sigmas = True + pipeline._engine_init_flow_shift = COSMOS3_VIDEO_DEFAULT_FLOW_SHIFT + pipeline._current_flow_shift = COSMOS3_VIDEO_DEFAULT_FLOW_SHIFT + pipeline.is_distilled_model = False + pipeline.is_edge_model = False pipeline._guidance_scale = None pipeline._num_timesteps = None pipeline._cosmos3_branch_caches = None @@ -243,6 +259,7 @@ def make_sampling_params(**overrides: Any) -> SimpleNamespace: "num_frames": None, "num_inference_steps": None, "guidance_scale": None, + "guidance_scale_provided": False, "generator": None, "seed": 123, "num_outputs_per_prompt": 1, @@ -288,6 +305,165 @@ def _mask() -> torch.Tensor: return torch.ones(1, 1, dtype=torch.long) +def _capture_tokenize_calls(pipeline: Any) -> list[dict[str, Any]]: + calls: list[dict[str, Any]] = [] + + def _tokenize( + text: str, + max_sequence_length: int, + use_system_prompt: bool = False, + system_prompt: str | None = None, + ) -> tuple[torch.Tensor, torch.Tensor]: + index = len(calls) + 1 + calls.append( + { + "text": text, + "max_sequence_length": max_sequence_length, + "use_system_prompt": use_system_prompt, + "system_prompt": system_prompt, + } + ) + return _ids(index), torch.full((1, 1), index, dtype=torch.long) + + pipeline._tokenize_prompt = _tokenize + return calls + + +@pytest.mark.parametrize( + ("provided", "value", "default", "is_distilled", "expected"), + [ + (False, 1.0, 7.0, False, 7.0), + (True, 1.0, 7.0, False, 1.0), + (True, 4.5, 7.0, False, 4.5), + (False, 1.0, 7.0, True, 1.0), + (True, 4.5, 7.0, True, 1.0), + ], +) +def test_resolve_guidance_scale( + make_cosmos3_pipeline, + provided: bool, + value: float, + default: float, + is_distilled: bool, + expected: float, +) -> None: + pipeline = make_cosmos3_pipeline() + pipeline.is_distilled_model = is_distilled + sp = make_sampling_params( + guidance_scale=value, + guidance_scale_provided=provided, + ) + + assert pipeline._resolve_guidance_scale(sp, default) == expected + + +def test_distilled_generation_accepts_t2i_and_i2v(make_cosmos3_pipeline) -> None: + pipeline = make_cosmos3_pipeline() + pipeline.is_distilled_model = True + common = { + "action_enabled": False, + "transfer_config": None, + "is_v2v": False, + "sound_enabled": False, + } + + pipeline._validate_distilled_generation_mode( + is_t2i=True, + image_tensor=None, + **common, + ) + pipeline._validate_distilled_generation_mode( + is_t2i=False, + image_tensor=torch.zeros(1), + **common, + ) + + +@pytest.mark.parametrize( + ("overrides", "message"), + [ + ({"action_enabled": True}, "action requests are unsupported"), + ({"transfer_config": SimpleNamespace()}, "transfer requests are unsupported"), + ({"is_v2v": True}, "video-to-video requests are unsupported"), + ({"sound_enabled": True}, "sound generation is unsupported"), + ( + {"is_t2i": False, "image_tensor": None}, + "text-to-video requests are unsupported", + ), + ( + {"is_t2i": True, "image_tensor": torch.zeros(1)}, + "image-conditioned image generation is unsupported", + ), + ], +) +def test_distilled_generation_rejects_unsupported_modes( + make_cosmos3_pipeline, + overrides: dict[str, Any], + message: str, +) -> None: + pipeline = make_cosmos3_pipeline() + pipeline.is_distilled_model = True + kwargs = { + "is_t2i": True, + "image_tensor": None, + "action_enabled": False, + "transfer_config": None, + "is_v2v": False, + "sound_enabled": False, + } + kwargs.update(overrides) + + with pytest.raises(ValueError, match=message): + pipeline._validate_distilled_generation_mode(**kwargs) + + +def test_distilled_generation_rejects_robolab_policy(make_cosmos3_pipeline) -> None: + pipeline = make_cosmos3_pipeline() + pipeline.is_distilled_model = True + + with pytest.raises(ValueError, match="RoboLab/action policy requests are unsupported"): + pipeline._forward_robolab_policy(make_sampling_params(), None, 0.0) + + +@pytest.mark.parametrize( + ("prompt", "sampling_params", "message"), + [ + ( + {"prompt": "x", "modalities": ["video"], "generate_sound": True}, + make_sampling_params(), + "do not support sound generation", + ), + ( + {"prompt": "x", "modalities": ["video"]}, + make_sampling_params(extra_args={"edge": {"control_path": "/tmp/control.mp4"}}), + "do not support transfer inference", + ), + ( + { + "prompt": "x", + "modalities": ["video"], + "additional_information": { + "preprocessed_video": torch.zeros(1, 3, 5, 16, 16), + }, + }, + make_sampling_params(height=16, width=16, num_frames=5), + "do not support video-to-video generation", + ), + ], +) +def test_edge_forward_rejects_unsupported_modes( + make_cosmos3_pipeline, + prompt: dict[str, Any], + sampling_params: SimpleNamespace, + message: str, +) -> None: + pipeline = make_cosmos3_pipeline() + pipeline.is_edge_model = True + + with pytest.raises(ValueError, match=message): + pipeline.forward(SimpleNamespace(prompts=[prompt], sampling_params=sampling_params)) + + def test_pipeline_registered_and_exported() -> None: from vllm_omni.diffusion.cache.cache_dit_backend import CUSTOM_DIT_ENABLERS from vllm_omni.diffusion.models import cosmos3 @@ -303,17 +479,45 @@ def test_pipeline_registered_and_exported() -> None: assert issubclass(Cosmos3OmniDiffusersPipeline, nn.Module) assert issubclass(Cosmos3OmniDiffusersPipeline, ProgressBarMixin) assert Cosmos3OmniDiffusersPipeline.support_image_input is True - assert _DIFFUSION_MODELS["Cosmos3OmniDiffusersPipeline"] == ( - "cosmos3", - "pipeline_cosmos3", - "Cosmos3OmniDiffusersPipeline", - ) - assert _DIFFUSION_PRE_PROCESS_FUNCS["Cosmos3OmniDiffusersPipeline"] == "get_cosmos3_pre_process_func" - assert _DIFFUSION_POST_PROCESS_FUNCS["Cosmos3OmniDiffusersPipeline"] == "get_cosmos3_post_process_func" - assert _DIFFUSION_IR_OP_PRIORITY_FUNCS["Cosmos3OmniDiffusersPipeline"] == "get_cosmos3_ir_op_priority_func" - assert "Cosmos3OmniDiffusersPipeline" in CUSTOM_DIT_ENABLERS assert "Cosmos3OmniDiffusersPipeline" in cosmos3.__all__ + for pipeline_name in ("Cosmos3OmniDiffusersPipeline", "Cosmos3OmniPipeline"): + assert _DIFFUSION_MODELS[pipeline_name] == ( + "cosmos3", + "pipeline_cosmos3", + "Cosmos3OmniDiffusersPipeline", + ) + assert _DIFFUSION_PRE_PROCESS_FUNCS[pipeline_name] == "get_cosmos3_pre_process_func" + assert _DIFFUSION_POST_PROCESS_FUNCS[pipeline_name] == "get_cosmos3_post_process_func" + assert _DIFFUSION_IR_OP_PRIORITY_FUNCS[pipeline_name] == "get_cosmos3_ir_op_priority_func" + assert pipeline_name in CUSTOM_DIT_ENABLERS + + +@pytest.mark.parametrize( + "pipeline_name", + ["Cosmos3OmniDiffusersPipeline", "Cosmos3OmniPipeline"], +) +def test_cosmos3_model_index_resolves_pipeline( + tmp_path, + pipeline_name: str, +) -> None: + import json + + from vllm_omni.diffusion.data import OmniDiffusionConfig + from vllm_omni.diffusion.models.cosmos3.pipeline_cosmos3 import ( + Cosmos3OmniDiffusersPipeline, + ) + from vllm_omni.diffusion.registry import DiffusionModelRegistry + + (tmp_path / "model_index.json").write_text(json.dumps({"_class_name": pipeline_name})) + + config = OmniDiffusionConfig(model=str(tmp_path)) + config.enrich_config() + + assert config.model_class_name == pipeline_name + resolved_pipeline_cls = DiffusionModelRegistry._try_load_model_cls(config.model_class_name) + assert resolved_pipeline_cls is Cosmos3OmniDiffusersPipeline + @pytest.fixture def stub_real_pipeline_init(monkeypatch: pytest.MonkeyPatch): @@ -335,25 +539,24 @@ def stub_real_pipeline_init(monkeypatch: pytest.MonkeyPatch): return self class _StubDiffusersScheduler: + load_config_calls: list[dict[str, Any]] = [] from_config_calls: list[dict[str, Any]] = [] - def __init__(self, *, flow_shift: float = 1.0, use_karras_sigmas: bool = True) -> None: - self.config = SimpleNamespace(flow_shift=flow_shift, use_karras_sigmas=use_karras_sigmas) + def __init__(self, *, flow_shift: float = 1.0) -> None: + self.config = SimpleNamespace(flow_shift=flow_shift) @classmethod - def from_pretrained(cls, *args, **kwargs): - return cls() + def load_config(cls, *args, **kwargs): + cls.load_config_calls.append({"args": args, "kwargs": dict(kwargs)}) + return { + "_class_name": "UniPCMultistepScheduler", + "flow_shift": 1.0, + } @classmethod def from_config(cls, config, **kwargs): cls.from_config_calls.append({"config": config, "kwargs": dict(kwargs)}) - return cls( - flow_shift=kwargs.get("flow_shift", getattr(config, "flow_shift", 1.0)), - use_karras_sigmas=kwargs.get( - "use_karras_sigmas", - getattr(config, "use_karras_sigmas", True), - ), - ) + return cls(flow_shift=kwargs.get("shift", config.get("flow_shift", 1.0))) class _StubVideoProcessor: def __init__(self, *args, **kwargs) -> None: @@ -361,13 +564,18 @@ def stub_real_pipeline_init(monkeypatch: pytest.MonkeyPatch): monkeypatch.setattr(pipeline_cosmos3, "AutoTokenizer", _StubAutoTokenizer) monkeypatch.setattr(pipeline_cosmos3, "DistributedAutoencoderKLWan", _StubDiffusersVAE) - monkeypatch.setattr(pipeline_cosmos3, "UniPCMultistepScheduler", _StubDiffusersScheduler) + monkeypatch.setattr(pipeline_cosmos3, "FlowUniPCMultistepScheduler", _StubDiffusersScheduler) monkeypatch.setattr(pipeline_cosmos3, "VideoProcessor", _StubVideoProcessor) monkeypatch.setattr(pipeline_cosmos3, "get_local_device", lambda: torch.device("cpu")) return _StubDiffusersScheduler -def _make_od_config(*, sound_gen: bool) -> SimpleNamespace: +def _make_od_config( + *, + sound_gen: bool, + tf_model_config_overrides: dict[str, Any] | None = None, + model_config: dict[str, Any] | None = None, +) -> SimpleNamespace: tf_model_config = { "hidden_size": 8, "num_hidden_layers": 0, @@ -382,6 +590,8 @@ def _make_od_config(*, sound_gen: bool) -> SimpleNamespace: } if sound_gen: tf_model_config["sound_gen"] = True + if tf_model_config_overrides: + tf_model_config.update(tf_model_config_overrides) return SimpleNamespace( enable_cpu_offload=False, enable_diffusion_pipeline_profiler=False, @@ -390,7 +600,7 @@ def _make_od_config(*, sound_gen: bool) -> SimpleNamespace: flow_shift=None, quantization_config=None, custom_pipeline_args={}, - model_config={}, + model_config=model_config or {}, tf_model_config=tf_model_config, ) @@ -400,16 +610,13 @@ def test_pipeline_init_skips_tokenizer_when_sound_disabled(stub_real_pipeline_in pipeline = Cosmos3OmniDiffusersPipeline(od_config=_make_od_config(sound_gen=False)) - assert stub_real_pipeline_init.from_config_calls == [] assert pipeline._sound_tokenizer is None assert pipeline.transformer.sound_gen is False assert not hasattr(pipeline.transformer, "audio_proj_in") assert not hasattr(pipeline.transformer, "audio_proj_out") - assert pipeline._base_scheduler_use_karras_sigmas is True - assert pipeline._current_scheduler_use_karras_sigmas is True -def test_pipeline_init_rebuilds_scheduler_with_cosmos3_defaults(stub_real_pipeline_init) -> None: +def test_pipeline_init_uses_flow_unipc_with_cosmos3_defaults(stub_real_pipeline_init) -> None: from vllm_omni.diffusion.models.cosmos3.pipeline_cosmos3 import Cosmos3OmniDiffusersPipeline od_config = _make_od_config(sound_gen=False) @@ -417,55 +624,302 @@ def test_pipeline_init_rebuilds_scheduler_with_cosmos3_defaults(stub_real_pipeli pipeline = Cosmos3OmniDiffusersPipeline(od_config=od_config) - assert stub_real_pipeline_init.from_config_calls - call = stub_real_pipeline_init.from_config_calls[-1] - assert getattr(call["config"], "flow_shift") == 1.0 - assert call["kwargs"] == {"flow_shift": 2.5} + assert len(stub_real_pipeline_init.load_config_calls) == 1 + assert len(stub_real_pipeline_init.from_config_calls) == 1 + call = stub_real_pipeline_init.from_config_calls[0] + assert call["config"]["_class_name"] == "UniPCMultistepScheduler" + assert call["kwargs"] == { + "shift": 1.0, + "use_dynamic_shifting": False, + "prediction_type": "flow_prediction", + } + assert pipeline.is_distilled_model is False assert pipeline._engine_init_flow_shift == 2.5 - assert pipeline._base_scheduler_use_karras_sigmas is True -def test_set_flow_shift_restores_checkpoint_scheduler_mode( - make_cosmos3_pipeline, +@pytest.mark.parametrize( + ("scheduler_class_name", "expected_distilled"), + [ + ("FlowMatchEulerDiscreteScheduler", True), + ("UniPCMultistepScheduler", False), + ], +) +def test_pipeline_resolves_scheduler_class_from_checkpoint_file( + tmp_path, + stub_real_pipeline_init, monkeypatch: pytest.MonkeyPatch, + scheduler_class_name: str, + expected_distilled: bool, ) -> None: + import json + from vllm_omni.diffusion.models.cosmos3 import pipeline_cosmos3 + from vllm_omni.diffusion.models.cosmos3.pipeline_cosmos3 import Cosmos3OmniDiffusersPipeline + from vllm_omni.diffusion.models.schedulers.scheduling_flow_unipc_multistep import ( + FlowUniPCMultistepScheduler as RealFlowUniPCMultistepScheduler, + ) - class RecordingScheduler: - from_config_calls: list[dict[str, Any]] = [] + t_list = [1.0, 0.75, 0.5, 0.25] + scheduler_dir = tmp_path / "scheduler" + scheduler_dir.mkdir() + scheduler_config = {"_class_name": scheduler_class_name} + if expected_distilled: + scheduler_config["fixed_step_sampler_config"] = {"sample_type": "sde", "t_list": t_list} + (scheduler_dir / "scheduler_config.json").write_text(json.dumps(scheduler_config)) - def __init__(self, *, flow_shift: float, use_karras_sigmas: bool) -> None: - self.config = SimpleNamespace(flow_shift=flow_shift, use_karras_sigmas=use_karras_sigmas) + class StubFlowUniPCScheduler(StubScheduler): + from_config_calls: list[tuple[Any, dict[str, Any]]] = [] + + @classmethod + def load_config(cls, *args, **kwargs): + return RealFlowUniPCMultistepScheduler.load_config(*args, **kwargs) @classmethod def from_config(cls, config, **kwargs): - cls.from_config_calls.append({"config": config, "kwargs": dict(kwargs)}) - return cls( - flow_shift=kwargs.get("flow_shift", getattr(config, "flow_shift", 1.0)), - use_karras_sigmas=kwargs.get( - "use_karras_sigmas", - getattr(config, "use_karras_sigmas", True), - ), - ) + cls.from_config_calls.append((config, dict(kwargs))) + return cls() - monkeypatch.setattr(pipeline_cosmos3, "UniPCMultistepScheduler", RecordingScheduler) - pipeline = make_cosmos3_pipeline() + class StubFlowMatchScheduler(StubScheduler): + from_config_calls: list[tuple[Any, dict[str, Any]]] = [] - pipeline._set_flow_shift(10.0, use_karras_sigmas=False) - assert RecordingScheduler.from_config_calls[-1]["kwargs"] == { - "flow_shift": 10.0, - "use_karras_sigmas": False, + @classmethod + def from_config(cls, config, **kwargs): + cls.from_config_calls.append((config, dict(kwargs))) + scheduler = cls() + scheduler.config.fixed_step_sampler_config = config["fixed_step_sampler_config"] + return scheduler + + monkeypatch.setattr(pipeline_cosmos3, "FlowUniPCMultistepScheduler", StubFlowUniPCScheduler) + monkeypatch.setattr( + pipeline_cosmos3, + "FlowMatchEulerDiscreteScheduler", + StubFlowMatchScheduler, + ) + + od_config = _make_od_config(sound_gen=False) + od_config.model = str(tmp_path) + pipeline = Cosmos3OmniDiffusersPipeline(od_config=od_config) + + assert pipeline.is_distilled_model is expected_distilled + if expected_distilled: + assert len(StubFlowMatchScheduler.from_config_calls) == 1 + assert StubFlowMatchScheduler.from_config_calls[0][1] == {"stochastic_sampling": True} + assert StubFlowUniPCScheduler.from_config_calls == [] + assert pipeline._scheduler_init_t_list == t_list + else: + assert len(StubFlowUniPCScheduler.from_config_calls) == 1 + assert StubFlowMatchScheduler.from_config_calls == [] + assert not hasattr(pipeline, "_scheduler_init_t_list") + + +def test_distilled_pipeline_initializes_sde_scheduler( + stub_real_pipeline_init, + monkeypatch: pytest.MonkeyPatch, +) -> None: + from diffusers import FlowMatchEulerDiscreteScheduler + + from vllm_omni.diffusion.models.cosmos3.pipeline_cosmos3 import ( + Cosmos3OmniDiffusersPipeline, + ) + + t_list = [1.0, 0.9375, 0.8333333333333334, 0.625] + scheduler_config = { + "_class_name": "FlowMatchEulerDiscreteScheduler", + "shift": 1.0, + "stochastic_sampling": False, + "fixed_step_sampler_config": { + "sample_type": "sde", + "t_list": t_list, + }, } - assert pipeline._current_flow_shift == 10.0 - assert pipeline._current_scheduler_use_karras_sigmas is False + monkeypatch.setattr( + stub_real_pipeline_init, + "load_config", + classmethod(lambda cls, *args, **kwargs: scheduler_config), + ) - pipeline._set_flow_shift(10.0) - assert RecordingScheduler.from_config_calls[-1]["kwargs"] == {"flow_shift": 10.0} - assert pipeline._current_flow_shift == 10.0 - assert pipeline._current_scheduler_use_karras_sigmas is True + pipeline = Cosmos3OmniDiffusersPipeline(od_config=_make_od_config(sound_gen=False)) + assert pipeline.is_distilled_model is True + assert isinstance(pipeline.scheduler, FlowMatchEulerDiscreteScheduler) + assert pipeline.scheduler.config.stochastic_sampling is True + assert pipeline._scheduler_init_t_list == t_list - pipeline._set_flow_shift(10.0) - assert len(RecordingScheduler.from_config_calls) == 2 + +@pytest.mark.parametrize( + ("fixed_step_config", "error_pattern"), + [ + pytest.param( + {"t_list": [1.0, 0.5]}, + r"fixed_step_sampler_config\.sample_type=sde", + id="missing-sample-type", + ), + pytest.param( + {"sample_type": "ode", "t_list": [1.0, 0.5]}, + r"fixed_step_sampler_config\.sample_type=sde", + id="unsupported-sample-type", + ), + pytest.param( + {"sample_type": "sde"}, + r"non-empty fixed_step_sampler_config\.t_list", + id="missing-t-list", + ), + pytest.param( + {"sample_type": "sde", "t_list": []}, + r"non-empty fixed_step_sampler_config\.t_list", + id="empty-t-list", + ), + ], +) +def test_distilled_scheduler_validates_fixed_step_config( + stub_real_pipeline_init, + monkeypatch: pytest.MonkeyPatch, + fixed_step_config: dict[str, Any], + error_pattern: str, +) -> None: + from vllm_omni.diffusion.models.cosmos3.pipeline_cosmos3 import ( + Cosmos3OmniDiffusersPipeline, + ) + + scheduler_config = { + "_class_name": "FlowMatchEulerDiscreteScheduler", + "fixed_step_sampler_config": fixed_step_config, + } + monkeypatch.setattr( + stub_real_pipeline_init, + "load_config", + classmethod(lambda cls, *args, **kwargs: scheduler_config), + ) + + with pytest.raises(ValueError, match=error_pattern): + Cosmos3OmniDiffusersPipeline(od_config=_make_od_config(sound_gen=False)) + + +def test_pipeline_init_selects_edge_transformer_from_backbone_type(stub_real_pipeline_init) -> None: + from vllm_omni.diffusion.models.cosmos3.pipeline_cosmos3 import ( + COSMOS3_EDGE_VIDEO_DEFAULT_FLOW_SHIFT, + Cosmos3OmniDiffusersPipeline, + ) + from vllm_omni.diffusion.models.cosmos3.transformer_cosmos3_edge import ( + COSMOS3_EDGE_BACKBONE_TYPE, + Cosmos3EdgeVFMTransformer, + ) + + pipeline = Cosmos3OmniDiffusersPipeline( + od_config=_make_od_config( + sound_gen=False, + tf_model_config_overrides={ + "backbone_type": COSMOS3_EDGE_BACKBONE_TYPE, + "qk_norm_for_text": False, + "latent_channel": 48, + "latent_patch_size": 2, + "temporal_compression_factor": 4, + "layer_norm_epsilon": 1e-5, + }, + ) + ) + + assert isinstance(pipeline.transformer, Cosmos3EdgeVFMTransformer) + assert pipeline.is_edge_model is True + assert pipeline._engine_init_flow_shift == COSMOS3_EDGE_VIDEO_DEFAULT_FLOW_SHIFT + + +def test_pipeline_init_does_not_select_edge_from_model_index_class_name(stub_real_pipeline_init) -> None: + from vllm_omni.diffusion.models.cosmos3.pipeline_cosmos3 import Cosmos3OmniDiffusersPipeline + from vllm_omni.diffusion.models.cosmos3.transformer_cosmos3 import Cosmos3VFMTransformer + from vllm_omni.diffusion.models.cosmos3.transformer_cosmos3_edge import Cosmos3EdgeVFMTransformer + + pipeline = Cosmos3OmniDiffusersPipeline( + od_config=_make_od_config( + sound_gen=False, + model_config={"_class_name": "Cosmos3EdgeOmniDiffusersPipeline"}, + ) + ) + + assert isinstance(pipeline.transformer, Cosmos3VFMTransformer) + assert not isinstance(pipeline.transformer, Cosmos3EdgeVFMTransformer) + + +def test_flow_unipc_reuses_scheduler_and_forwards_each_request_shift(make_cosmos3_pipeline) -> None: + pipeline = make_cosmos3_pipeline() + scheduler = pipeline.scheduler + + assert pipeline._engine_init_flow_shift == 10.0 + assert pipeline._current_flow_shift == 10.0 + assert scheduler.config.flow_shift == 1.0 + + for shift in (3.0, 10.0): + pipeline._set_flow_shift(shift) + pipeline._set_timesteps(4, torch.device("cpu"), shift=pipeline._current_flow_shift) + + assert pipeline.scheduler is scheduler + assert pipeline._current_flow_shift == 10.0 + assert [call["shift"] for call in scheduler.set_timesteps_calls] == [3.0, 10.0] + assert all(call["num_inference_steps"] == 4 for call in scheduler.set_timesteps_calls) + assert all(call["sigmas"] is None for call in scheduler.set_timesteps_calls) + + +def test_flow_unipc_reproducible_with_same_seed(make_cosmos3_pipeline) -> None: + from vllm_omni.diffusion.models.schedulers.scheduling_flow_unipc_multistep import ( + FlowUniPCMultistepScheduler, + ) + + pipeline = make_cosmos3_pipeline() + pipeline.scheduler = FlowUniPCMultistepScheduler( + shift=1.0, + use_dynamic_shifting=False, + prediction_type="flow_prediction", + ) + pipeline._format_and_tokenize_prompts = lambda *args, **kwargs: ( + _ids(1), + _mask(), + _ids(0), + _mask(), + ) + + def run(seed: int) -> torch.Tensor: + request = make_request_batch( + {"prompt": "A test video.", "modalities": ["video"]}, + make_sampling_params( + seed=seed, + guidance_scale=1.0, + guidance_scale_provided=True, + num_inference_steps=4, + num_frames=5, + height=16, + width=16, + extra_args={"flow_shift": 3.0}, + ), + ) + output = pipeline.forward(request) + return output.output["video"] + + first = run(123) + second = run(123) + different_seed = run(456) + + torch.testing.assert_close(first, second) + assert not torch.equal(first, different_seed) + + +def test_distilled_set_timesteps_uses_fixed_sigma_schedule(make_cosmos3_pipeline) -> None: + pipeline = make_cosmos3_pipeline() + pipeline.is_distilled_model = True + pipeline._scheduler_init_t_list = [1.0, 0.75, 0.5, 0.25] + + pipeline._set_timesteps( + num_inference_steps=99, + device=torch.device("cpu"), + shift=7.0, + ) + + assert pipeline.scheduler.set_timesteps_calls == [ + { + "num_inference_steps": None, + "device": torch.device("cpu"), + "shift": None, + "sigmas": pipeline._scheduler_init_t_list, + } + ] def test_pipeline_init_passes_tokenizer_attrs_into_transformer( @@ -806,12 +1260,35 @@ def test_ir_op_priority_hook_preserves_platform_fields(monkeypatch: pytest.Monke assert merged.custom_op == ["platform_kernel", "native"] -def test_prompt_formatting_and_checkpoint_key_remap(make_cosmos3_pipeline) -> None: - from vllm_omni.diffusion.models.cosmos3.pipeline_cosmos3 import Cosmos3OmniDiffusersPipeline +def test_format_and_tokenize_prompts_leaves_plain_prompts_unchanged_by_default(make_cosmos3_pipeline) -> None: + from vllm_omni.diffusion.models.cosmos3.pipeline_cosmos3 import COSMOS3_SYSTEM_PROMPT pipeline = make_cosmos3_pipeline() - captured: list[str] = [] - pipeline._tokenize_prompt = lambda text, *args, **kwargs: (captured.append(text) or _ids(len(captured)), _mask()) + calls = _capture_tokenize_calls(pipeline) + + result = pipeline._format_and_tokenize_prompts( + " A robot. ", + " bad. ", + num_frames=48, + frame_rate=24, + height=720, + width=1280, + max_sequence_length=32, + sp=SimpleNamespace(extra_args={}), + use_system_prompt=False, + is_t2i=False, + ) + + assert [call["text"] for call in calls] == ["A robot.", "bad."] + assert all(call["max_sequence_length"] == 32 for call in calls) + assert all(call["use_system_prompt"] is False for call in calls) + assert all(call["system_prompt"] == COSMOS3_SYSTEM_PROMPT for call in calls) + assert [tensor.item() for tensor in result] == [1, 1, 2, 2] + + +def test_format_and_tokenize_prompts_applies_video_templates_and_system_override(make_cosmos3_pipeline) -> None: + pipeline = make_cosmos3_pipeline() + calls = _capture_tokenize_calls(pipeline) pipeline._format_and_tokenize_prompts( "A robot", @@ -821,21 +1298,167 @@ def test_prompt_formatting_and_checkpoint_key_remap(make_cosmos3_pipeline) -> No height=720, width=1280, max_sequence_length=32, + sp=SimpleNamespace( + system_prompt="direct system prompt", + extra_args={ + "use_duration_template": True, + "use_resolution_template": True, + "system_prompt": "API system prompt", + }, + ), + use_system_prompt=True, + is_t2i=False, + ) + + assert calls == [ + { + "text": ("A robot. The video is 2.0 seconds long and is of 24 FPS. This video is of 720x1280 resolution."), + "max_sequence_length": 32, + "use_system_prompt": True, + "system_prompt": "API system prompt", + }, + { + "text": ( + "bad. The video is not 2.0 seconds long and is not of 24 FPS. This video is not of 720x1280 resolution." + ), + "max_sequence_length": 32, + "use_system_prompt": True, + "system_prompt": "API system prompt", + }, + ] + + +def test_format_and_tokenize_prompts_uses_image_templates_for_t2i(make_cosmos3_pipeline) -> None: + from vllm_omni.diffusion.models.cosmos3.pipeline_cosmos3 import COSMOS3_T2I_SYSTEM_PROMPT + + pipeline = make_cosmos3_pipeline() + calls = _capture_tokenize_calls(pipeline) + + pipeline._format_and_tokenize_prompts( + "A robot", + "bad", + num_frames=1, + frame_rate=24, + height=1024, + width=768, + max_sequence_length=64, sp=SimpleNamespace( extra_args={ - "negative_metadata_mode": "inverse", - # Duration/resolution metadata templates are off by default - # (commit "Change resolution and duration templates to off by - # default"); enable them explicitly to exercise the formatting. "use_duration_template": True, "use_resolution_template": True, } ), use_system_prompt=True, + is_t2i=True, + ) + + assert [call["text"] for call in calls] == [ + "A robot. This image is of 1024x768 resolution.", + "bad. This image is not of 1024x768 resolution.", + ] + assert all("seconds" not in call["text"] and "FPS" not in call["text"] for call in calls) + assert all(call["system_prompt"] == COSMOS3_T2I_SYSTEM_PROMPT for call in calls) + + +def test_format_and_tokenize_prompts_rewrites_json_object_metadata(make_cosmos3_pipeline) -> None: + import json + + pipeline = make_cosmos3_pipeline() + calls = _capture_tokenize_calls(pipeline) + + pipeline._format_and_tokenize_prompts( + ('{"caption": "A robot", "duration": "old", "fps": 1, "resolution": {"H": 1, "W": 2}, "aspect_ratio": "1:1"}'), + "bad", + num_frames=48, + frame_rate=24, + height=720, + width=1280, + max_sequence_length=32, + sp=SimpleNamespace( + extra_args={ + "aspect_ratio": "16:9", + "use_duration_template": True, + "use_resolution_template": True, + } + ), is_t2i=False, ) - assert "The video is 2.0 seconds long" in captured[0] - assert "The video is not 2.0 seconds long" in captured[1] + + assert json.loads(calls[0]["text"]) == { + "caption": "A robot", + "duration": "2s", + "fps": 24.0, + "resolution": {"H": 720, "W": 1280}, + "aspect_ratio": "16:9", + } + assert "The video is 2.0 seconds long" not in calls[0]["text"] + assert calls[1]["text"] == ( + "bad. The video is not 2.0 seconds long and is not of 24 FPS. This video is not of 720x1280 resolution." + ) + + +def test_format_and_tokenize_prompts_removes_video_metadata_from_t2i_json(make_cosmos3_pipeline) -> None: + import json + + pipeline = make_cosmos3_pipeline() + calls = _capture_tokenize_calls(pipeline) + + pipeline._format_and_tokenize_prompts( + '{"caption": "A robot", "duration": "2s", "fps": 24}', + "", + num_frames=1, + frame_rate=24, + height=1024, + width=768, + max_sequence_length=32, + sp=SimpleNamespace(extra_args={}), + is_t2i=True, + ) + + assert json.loads(calls[0]["text"]) == { + "caption": "A robot", + "resolution": {"H": 1024, "W": 768}, + } + + +@pytest.mark.parametrize( + "prompt", + [ + "{malformed", + '["A robot"]', + ], +) +def test_format_and_tokenize_prompts_falls_back_for_non_object_json( + make_cosmos3_pipeline, + prompt: str, +) -> None: + pipeline = make_cosmos3_pipeline() + calls = _capture_tokenize_calls(pipeline) + + pipeline._format_and_tokenize_prompts( + prompt, + "", + num_frames=48, + frame_rate=24, + height=720, + width=1280, + max_sequence_length=32, + sp=SimpleNamespace( + extra_args={ + "use_duration_template": True, + "use_resolution_template": True, + } + ), + is_t2i=False, + ) + + assert calls[0]["text"] == ( + f"{prompt}. The video is 2.0 seconds long and is of 24 FPS. This video is of 720x1280 resolution." + ) + + +def test_checkpoint_key_remap() -> None: + from vllm_omni.diffusion.models.cosmos3.pipeline_cosmos3 import Cosmos3OmniDiffusersPipeline remaps = { "embed_tokens.weight": "transformer.language_model.embed_tokens.weight", @@ -847,9 +1470,14 @@ def test_prompt_formatting_and_checkpoint_key_remap(make_cosmos3_pipeline) -> No "layers.3.self_attn.to_q.weight": "transformer.language_model.layers.3.self_attn.to_q.weight", "layers.3.self_attn.to_out.weight": "transformer.language_model.layers.3.self_attn.to_out.weight", "layers.3.self_attn.norm_q.weight": "transformer.language_model.layers.3.self_attn.norm_q.weight", + "layers.3.self_attn.k_norm_und_for_gen.weight": ( + "transformer.language_model.layers.3.self_attn.k_norm_und_for_gen.weight" + ), "layers.3.self_attn.add_q_proj.weight": "transformer.gen_layers.3.cross_attention.to_q.weight", "layers.3.self_attn.to_add_out.weight": "transformer.gen_layers.3.cross_attention.to_out.weight", "layers.3.self_attn.norm_added_q.weight": "transformer.gen_layers.3.cross_attention.norm_q.weight", + "layers.3.mlp_moe_gen.up_proj.weight": "transformer.gen_layers.3.mlp.up_proj.weight", + "layers.3.mlp_moe_gen.down_proj.weight": "transformer.gen_layers.3.mlp.down_proj.weight", "transformer.model.layers.3.self_attn.add_k_proj.weight": ( "transformer.gen_layers.3.cross_attention.to_k.weight" ), @@ -1193,9 +1821,9 @@ def test_forward_transfer_uses_source_fps_except_wsm(make_cosmos3_pipeline, hint pipeline._prepare_transfer_latents = fake_prepare pipeline.diffuse_transfer = fake_diffuse_transfer - def fake_set_flow_shift(target, *, use_karras_sigmas=None): + def fake_set_flow_shift(target): captured.setdefault("flow_shifts", []).append(target) - captured.setdefault("scheduler_use_karras_sigmas", []).append(use_karras_sigmas) + pipeline._current_flow_shift = float(target) pipeline._set_flow_shift = fake_set_flow_shift pipeline._decode_latents = lambda latents: torch.zeros(1, 3, 5, 16, 16, device="meta") @@ -1230,8 +1858,8 @@ def test_forward_transfer_uses_source_fps_except_wsm(make_cosmos3_pipeline, hint assert captured["format_frame_rate"] == expected_fps assert captured["shared_kwargs"]["fps"] == expected_fps assert captured["flow_shifts"] == [10.0] - # Transfer uses the V2V flow-sigma schedule (karras off) so flow_shift applies. - assert captured["scheduler_use_karras_sigmas"] == [False] + # Transfer applies the V2V flow shift when building its timestep schedule. + assert [call["shift"] for call in pipeline.scheduler.set_timesteps_calls] == [10.0] assert output.output["metadata"]["video"]["fps"] == expected_fps assert output.output["payload"]["video"].device.type == "meta" @@ -1391,9 +2019,9 @@ class TestForwardRouting: pipeline._format_and_tokenize_prompts = fake_format pipeline._prepare_latents = fake_prepare - def fake_set_flow_shift(target, *, use_karras_sigmas=None): + def fake_set_flow_shift(target): captured.setdefault("flow_shifts", []).append(target) - captured.setdefault("scheduler_use_karras_sigmas", []).append(use_karras_sigmas) + pipeline._current_flow_shift = float(target) pipeline._set_flow_shift = fake_set_flow_shift pipeline.diffuse = fake_diffuse @@ -1420,7 +2048,7 @@ class TestForwardRouting: { "key": "video", "is_t2i": False, - "flow": [1.0], + "flow": [10.0], "steps": [35], "frames": 189, }, @@ -1443,8 +2071,8 @@ class TestForwardRouting: assert captured["format"]["is_t2i"] is expected["is_t2i"] assert captured["format"]["num_frames"] == expected["frames"] assert captured["flow_shifts"] == expected["flow"] - assert captured["scheduler_use_karras_sigmas"] == [None] - assert [call[0] for call in pipeline.scheduler.set_timesteps_calls] == expected["steps"] + assert [call["num_inference_steps"] for call in pipeline.scheduler.set_timesteps_calls] == expected["steps"] + assert all(call["shift"] == expected["flow"][0] for call in pipeline.scheduler.set_timesteps_calls) def test_forward_i2v_sound_and_action_routes(self, make_cosmos3_pipeline) -> None: pipeline = make_cosmos3_pipeline() @@ -1491,7 +2119,6 @@ class TestForwardRouting: ) ) assert captured["flow_shifts"][-1] == 10.0 - assert captured["scheduler_use_karras_sigmas"][-1] is False assert captured["format"]["negative_prompt"] == "" assert captured["diffuse_calls"][-1]["shared_kwargs"]["noisy_frame_mask"] is v2v_mask assert captured["diffuse_calls"][-1]["condition_latents"] is v2v_condition diff --git a/tests/diffusion/models/cosmos3/test_cosmos3_transformer.py b/tests/diffusion/models/cosmos3/test_cosmos3_transformer.py index 13d1c399b..cb59dffe5 100644 --- a/tests/diffusion/models/cosmos3/test_cosmos3_transformer.py +++ b/tests/diffusion/models/cosmos3/test_cosmos3_transformer.py @@ -15,6 +15,25 @@ from vllm_omni.platforms import current_omni_platform pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion] +@pytest.fixture(autouse=True) +def _single_rank_tensor_parallel(monkeypatch: pytest.MonkeyPatch) -> None: + """Provide the TP metadata required by vLLM parallel linear layers.""" + from vllm.model_executor import parameter + from vllm.model_executor.layers import linear + + from vllm_omni.diffusion.models.cosmos3 import ( + transformer_cosmos3, + transformer_cosmos3_edge, + ) + + monkeypatch.setattr(linear, "get_tensor_model_parallel_rank", lambda: 0) + monkeypatch.setattr(linear, "get_tensor_model_parallel_world_size", lambda: 1) + monkeypatch.setattr(parameter, "get_tensor_model_parallel_rank", lambda: 0) + monkeypatch.setattr(parameter, "get_tensor_model_parallel_world_size", lambda: 1) + monkeypatch.setattr(transformer_cosmos3, "get_tensor_model_parallel_world_size", lambda: 1) + monkeypatch.setattr(transformer_cosmos3_edge, "get_tensor_model_parallel_world_size", lambda: 1) + + @pytest.fixture def accelerator_device() -> torch.device: """Provide an accelerator device, skipping when none is available.""" @@ -40,6 +59,22 @@ def _tiny_cosmos3_config(**overrides): return config +def _tiny_cosmos3_edge_config(**overrides): + from vllm_omni.diffusion.models.cosmos3.transformer_cosmos3_edge import ( + COSMOS3_EDGE_BACKBONE_TYPE, + ) + + config = _tiny_cosmos3_config( + latent_patch_size=2, + latent_channel=48, + temporal_compression_factor=4, + backbone_type=COSMOS3_EDGE_BACKBONE_TYPE, + qk_norm_for_text=False, + ) + config.update(overrides) + return config + + def test_mrope_position_ids_cover_text_video_sound_and_action() -> None: from vllm_omni.diffusion.models.cosmos3.transformer_cosmos3 import ( compute_mrope_position_ids_action, @@ -78,6 +113,16 @@ def test_mrope_position_ids_cover_text_video_sound_and_action() -> None: assert action_offset == 14 +def test_timestep_embedder_stores_frequencies_in_fp32() -> None: + from vllm_omni.diffusion.models.cosmos3.transformer_cosmos3 import ( + TimestepEmbedder, + ) + + embedder = TimestepEmbedder(hidden_size=8, frequency_embedding_size=16) + + assert embedder.freqs.dtype == torch.float32 + + @pytest.mark.parametrize( ("key", "value"), [ @@ -97,30 +142,245 @@ def test_validate_supported_config_rejects_unsupported_flags(key: str, value) -> Cosmos3VFMTransformer._validate_supported_config(None) +def test_edge_config_resolves_nemotron_defaults() -> None: + from vllm_omni.diffusion.models.cosmos3.transformer_cosmos3_edge import ( + Cosmos3EdgeVFMTransformer, + ) + + model = Cosmos3EdgeVFMTransformer( + SimpleNamespace( + tf_model_config=_tiny_cosmos3_edge_config( + layer_norm_epsilon=1e-5, + rope_scaling={}, + rope_parameters={"mrope_section": [24, 20, 20]}, + ), + dtype=torch.float32, + ) + ) + + assert model.rms_norm_eps == 1e-5 + assert model.rope_theta == 100_000_000 + assert model.mrope_section == [24, 20, 20] + assert model.latent_channel_size == 48 + assert model.latent_patch_size == 2 + assert model.temporal_compression_factor == 4 + + +def test_edge_config_requires_backbone_type() -> None: + from vllm_omni.diffusion.models.cosmos3.transformer_cosmos3_edge import ( + Cosmos3EdgeVFMTransformer, + ) + + config = _tiny_cosmos3_edge_config() + config.pop("backbone_type") + + with pytest.raises(ValueError, match="must declare backbone_type"): + Cosmos3EdgeVFMTransformer( + SimpleNamespace( + tf_model_config=config, + dtype=torch.float32, + ) + ) + + +def test_edge_und_and_gen_use_relu2_and_no_und_qk_norm() -> None: + from vllm_omni.diffusion.models.cosmos3.transformer_cosmos3_edge import ( + Cosmos3EdgeVFMTransformer, + Cosmos3Relu2MLP, + ) + + model = Cosmos3EdgeVFMTransformer( + SimpleNamespace( + tf_model_config=_tiny_cosmos3_edge_config( + num_hidden_layers=1, + use_k_norm_und_for_gen=False, + ), + dtype=torch.float32, + ) + ) + + layer = model.language_model.layers[0] + gen_layer = model.gen_layers[0] + assert isinstance(layer.mlp, Cosmos3Relu2MLP) + assert not hasattr(layer.mlp, "gate_proj") + assert isinstance(gen_layer.mlp, Cosmos3Relu2MLP) + assert not hasattr(gen_layer.mlp, "gate_proj") + assert not hasattr(layer.self_attn, "norm_q") + assert not hasattr(layer.self_attn, "norm_k") + assert layer.self_attn.k_norm_und_for_gen is None + + +def test_edge_creates_gen_k_norm_when_enabled() -> None: + from vllm_omni.diffusion.models.cosmos3.transformer_cosmos3 import RMSNorm + from vllm_omni.diffusion.models.cosmos3.transformer_cosmos3_edge import ( + Cosmos3EdgeVFMTransformer, + ) + + model = Cosmos3EdgeVFMTransformer( + SimpleNamespace( + tf_model_config=_tiny_cosmos3_edge_config( + num_hidden_layers=1, + qk_norm_for_diffusion=True, + use_k_norm_und_for_gen=True, + ), + dtype=torch.float32, + ) + ) + + k_norm = model.language_model.layers[0].self_attn.k_norm_und_for_gen + assert model.use_k_norm_und_for_gen is True + assert isinstance(k_norm, RMSNorm) + assert isinstance(model.gen_layers[0].cross_attention.norm_q, RMSNorm) + assert isinstance(model.gen_layers[0].cross_attention.norm_k, RMSNorm) + + +def test_edge_supports_no_gen_qk_norm_variant() -> None: + from vllm_omni.diffusion.models.cosmos3.transformer_cosmos3_edge import ( + Cosmos3EdgeVFMTransformer, + ) + + model = Cosmos3EdgeVFMTransformer( + SimpleNamespace( + tf_model_config=_tiny_cosmos3_edge_config( + num_hidden_layers=1, + qk_norm_for_diffusion=False, + use_k_norm_und_for_gen=False, + ), + dtype=torch.float32, + ) + ) + + assert model.qk_norm_for_diffusion is False + assert model.language_model.layers[0].self_attn.k_norm_und_for_gen is None + assert model.gen_layers[0].cross_attention.qk_norm is False + assert not hasattr(model.gen_layers[0].cross_attention, "norm_q") + assert not hasattr(model.gen_layers[0].cross_attention, "norm_k") + + +def test_edge_validates_required_relu2_weights() -> None: + from vllm_omni.diffusion.models.cosmos3.transformer_cosmos3_edge import ( + Cosmos3EdgeVFMTransformer, + ) + + model = object.__new__(Cosmos3EdgeVFMTransformer) + nn.Module.__init__(model) + model.num_hidden_layers = 1 + model.qk_norm_for_diffusion = True + model.use_k_norm_und_for_gen = True + + complete = { + "transformer.language_model.layers.0.mlp.up_proj.weight", + "transformer.language_model.layers.0.mlp.down_proj.weight", + "transformer.language_model.layers.0.self_attn.k_norm_und_for_gen.weight", + "transformer.gen_layers.0.mlp.up_proj.weight", + "transformer.gen_layers.0.mlp.down_proj.weight", + } + model.validate_loaded_weights(complete) + + mlp_weights = complete - { + "transformer.language_model.layers.0.self_attn.k_norm_und_for_gen.weight", + } + for missing_weight in mlp_weights: + with pytest.raises(ValueError, match="missing required weights"): + model.validate_loaded_weights(complete - {missing_weight}) + + missing_k_norm = complete - {"transformer.language_model.layers.0.self_attn.k_norm_und_for_gen.weight"} + with pytest.raises(ValueError, match=r"self_attn\.k_norm_und_for_gen"): + model.validate_loaded_weights(missing_k_norm) + + model.use_k_norm_und_for_gen = False + model.validate_loaded_weights(missing_k_norm) + + +def test_edge_gen_cached_k_is_normalized_but_reasoner_uses_raw_k() -> None: + from vllm_omni.diffusion.models.cosmos3.transformer_cosmos3 import RMSNorm + from vllm_omni.diffusion.models.cosmos3.transformer_cosmos3_edge import ( + Cosmos3EdgeCausalAttention, + ) + + class ScaleProjection(nn.Module): + def __init__(self, scale: float) -> None: + super().__init__() + self.scale = scale + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return x * self.scale + + class IdentityOutput(nn.Module): + def forward(self, x: torch.Tensor) -> torch.Tensor: + return x + + class RawKAttention(nn.Module): + def forward(self, q: torch.Tensor, k: torch.Tensor, v: torch.Tensor) -> torch.Tensor: + del v + return q + k + + attn = object.__new__(Cosmos3EdgeCausalAttention) + nn.Module.__init__(attn) + attn.hidden_size = 4 + attn.num_heads = 2 + attn.num_kv_heads = 2 + attn.head_dim = 2 + attn.num_heads_local = 2 + attn.num_kv_heads_local = 2 + attn.to_q = ScaleProjection(1.0) + attn.to_k = ScaleProjection(1.0) + attn.to_v = ScaleProjection(1.0) + attn.to_out = IdentityOutput() + attn.k_norm_und_for_gen = RMSNorm(2, eps=1e-5) + attn.attn = RawKAttention() + + hidden_states = torch.tensor( + [[[1.0, 2.0, 3.0, 4.0], [2.0, 1.0, 4.0, 3.0]]], + dtype=torch.float32, + ) + freqs_cos = torch.ones(1, 2, 1, 2) + freqs_sin = torch.zeros(1, 2, 1, 2) + + reasoner_out, gen_k, gen_v = attn(hidden_states, freqs_cos, freqs_sin) + attn.to_k.scale = 3.0 + scaled_reasoner_out, scaled_gen_k, scaled_gen_v = attn(hidden_states, freqs_cos, freqs_sin) + + torch.testing.assert_close(gen_k, scaled_gen_k) + torch.testing.assert_close(gen_v, scaled_gen_v) + assert not torch.allclose(reasoner_out, scaled_reasoner_out) + + def test_transformer_sharding_offload_and_patch_round_trip_contracts() -> None: from vllm_omni.diffusion.models.cosmos3.transformer_cosmos3 import Cosmos3LanguageModel, Cosmos3VFMTransformer + from vllm_omni.diffusion.models.cosmos3.transformer_cosmos3_edge import ( + Cosmos3EdgeLanguageModel, + Cosmos3EdgeVFMTransformer, + ) - model = object.__new__(Cosmos3VFMTransformer) - nn.Module.__init__(model) - model.language_model = nn.Module() - model.language_model.layers = nn.ModuleList([nn.Linear(2, 2) for _ in range(2)]) - model.gen_layers = nn.ModuleList([nn.Linear(2, 2)]) - model.norm_moe_gen = nn.LayerNorm(2) + for transformer_cls, language_model_cls in ( + (Cosmos3VFMTransformer, Cosmos3LanguageModel), + (Cosmos3EdgeVFMTransformer, Cosmos3EdgeLanguageModel), + ): + model = object.__new__(transformer_cls) + nn.Module.__init__(model) + model.language_model = nn.Module() + model.language_model.layers = nn.ModuleList([nn.Linear(2, 2) for _ in range(2)]) + model.gen_layers = nn.ModuleList([nn.Linear(2, 2)]) + model.norm_moe_gen = nn.LayerNorm(2) - matched = [ - name - for name, module in model.named_modules() - if any(condition(name, module) for condition in model._hsdp_shard_conditions) - ] - assert matched == ["language_model.layers.0", "language_model.layers.1", "gen_layers.0"] - assert Cosmos3VFMTransformer._layerwise_offload_blocks_attrs == ["gen_layers"] - assert Cosmos3LanguageModel._layerwise_offload_blocks_attrs == ["layers"] - assert Cosmos3VFMTransformer._repeated_blocks == ["Cosmos3GenDecoderLayer"] + matched = [ + name + for name, module in model.named_modules() + if any(condition(name, module) for condition in model._hsdp_shard_conditions) + ] + assert matched == ["language_model.layers.0", "language_model.layers.1", "gen_layers.0"] + assert transformer_cls._layerwise_offload_blocks_attrs == ["gen_layers"] + assert language_model_cls._layerwise_offload_blocks_attrs == ["layers"] + assert transformer_cls._repeated_blocks == ["Cosmos3GenDecoderLayer"] - model.latent_patch_size = 2 - model.latent_channel_size = 3 - latents = torch.arange(1 * 3 * 1 * 3 * 5, dtype=torch.float32).reshape(1, 3, 1, 3, 5) - torch.testing.assert_close(model.unpatchify(model.patchify(latents, t=1, h=3, w=5), t=1, h=3, w=5), latents) + model.latent_patch_size = 2 + model.latent_channel_size = 3 + latents = torch.arange(1 * 3 * 1 * 3 * 5, dtype=torch.float32).reshape(1, 3, 1, 3, 5) + torch.testing.assert_close( + model.unpatchify(model.patchify(latents, t=1, h=3, w=5), t=1, h=3, w=5), + latents, + ) def test_forward_returns_video_prediction(monkeypatch: pytest.MonkeyPatch) -> None: @@ -142,6 +402,17 @@ def test_forward_returns_video_prediction(monkeypatch: pytest.MonkeyPatch) -> No assert tuple(output.shape) == (1, 2, 1, 2, 2) +def test_qwen_gen_mlp_remains_gated() -> None: + from vllm_omni.diffusion.models.cosmos3.transformer_cosmos3 import Cosmos3GatedMLP, Cosmos3VFMTransformer + + model = Cosmos3VFMTransformer( + SimpleNamespace(tf_model_config=_tiny_cosmos3_config(num_hidden_layers=1), dtype=torch.float32) + ) + + assert isinstance(model.gen_layers[0].mlp, Cosmos3GatedMLP) + assert hasattr(model.gen_layers[0].mlp, "gate_proj") + + def test_model_cpu_offload_swaps_back_to_generator(monkeypatch: pytest.MonkeyPatch) -> None: from vllm_omni.diffusion.models.cosmos3 import transformer_cosmos3 diff --git a/tests/e2e/accuracy/test_cosmos3_similarity.py b/tests/e2e/accuracy/test_cosmos3_similarity.py index ff2350096..efa095fb1 100644 --- a/tests/e2e/accuracy/test_cosmos3_similarity.py +++ b/tests/e2e/accuracy/test_cosmos3_similarity.py @@ -42,8 +42,6 @@ def _server_args() -> list[str]: return [ "--num-gpus", "1", - "--model-class-name", - "Cosmos3OmniDiffusersPipeline", "--stage-init-timeout", "900", "--init-timeout", diff --git a/tests/model_extras/test_model_extras.py b/tests/model_extras/test_model_extras.py index f2db9ded0..76ef84e7e 100644 --- a/tests/model_extras/test_model_extras.py +++ b/tests/model_extras/test_model_extras.py @@ -60,8 +60,9 @@ def test_sensenova_extra_registry_declares_request_and_response_params() -> None @pytest.mark.core_model @pytest.mark.cpu -def test_cosmos3_extra_registry_declares_request_and_response_params() -> None: - assert get_extra_body_params("Cosmos3OmniDiffusersPipeline") == frozenset( +@pytest.mark.parametrize("pipeline_name", ["Cosmos3OmniDiffusersPipeline", "Cosmos3OmniPipeline"]) +def test_cosmos3_extra_registry_declares_request_and_response_params(pipeline_name: str) -> None: + assert get_extra_body_params(pipeline_name) == frozenset( { "flow_shift", "max_sequence_length", @@ -98,7 +99,7 @@ def test_cosmos3_extra_registry_declares_request_and_response_params() -> None: "session_id", } ) - assert get_extra_output_params("Cosmos3OmniDiffusersPipeline") == frozenset( + assert get_extra_output_params(pipeline_name) == frozenset( { "action", "raw_action_dim", @@ -106,7 +107,7 @@ def test_cosmos3_extra_registry_declares_request_and_response_params() -> None: "action_mode", } ) - assert should_init_extra_args_for_non_diffusion_stages("Cosmos3OmniDiffusersPipeline") is False + assert should_init_extra_args_for_non_diffusion_stages(pipeline_name) is False @pytest.mark.core_model @@ -224,9 +225,10 @@ def test_ming_flash_omni_image_to_image_prompt_builder() -> None: @pytest.mark.core_model @pytest.mark.cpu -def test_cosmos3_text_to_image_prompt_builder_selects_image_modality() -> None: +@pytest.mark.parametrize("pipeline_name", ["Cosmos3OmniDiffusersPipeline", "Cosmos3OmniPipeline"]) +def test_cosmos3_text_to_image_prompt_builder_selects_image_modality(pipeline_name: str) -> None: assert build_text_to_image_prompt( - "Cosmos3OmniDiffusersPipeline", + pipeline_name, prompt="a red sports car at golden hour", negative_prompt="blurry, distorted", height=1024, @@ -237,7 +239,7 @@ def test_cosmos3_text_to_image_prompt_builder_selects_image_modality() -> None: "negative_prompt": "blurry, distorted", } assert build_text_to_image_prompt( - "Cosmos3OmniDiffusersPipeline", + pipeline_name, prompt="a red sports car", negative_prompt=None, ) == {"prompt": "a red sports car", "modalities": ["image"]} diff --git a/tests/model_tests/diffusion/test_alignment.py b/tests/model_tests/diffusion/test_alignment.py index ae01334f6..c95644143 100644 --- a/tests/model_tests/diffusion/test_alignment.py +++ b/tests/model_tests/diffusion/test_alignment.py @@ -71,6 +71,7 @@ EXCLUDED_MODELS = [ "OmniVoicePipeline", "OmniVoice", "Cosmos3OmniDiffusersPipeline", + "Cosmos3OmniPipeline", "DiffusersAdapterPipeline", "HiDreamImagePipeline", "DreamZeroPipeline", diff --git a/vllm_omni/diffusion/cache/cache_dit_backend.py b/vllm_omni/diffusion/cache/cache_dit_backend.py index 48e64de0a..5d63ce3c0 100644 --- a/vllm_omni/diffusion/cache/cache_dit_backend.py +++ b/vllm_omni/diffusion/cache/cache_dit_backend.py @@ -1016,6 +1016,7 @@ CUSTOM_DIT_ENABLERS.update( "Wan22VACEPipeline": enable_cache_for_wan22, "Wan22S2VPipeline": enable_cache_for_wan22_s2v, "Cosmos3OmniDiffusersPipeline": enable_cache_for_cosmos3, + "Cosmos3OmniPipeline": enable_cache_for_cosmos3, "Krea2Pipeline": enable_cache_for_krea2, } ) diff --git a/vllm_omni/diffusion/models/cosmos3/__init__.py b/vllm_omni/diffusion/models/cosmos3/__init__.py index 6df062b5c..af4eeaacf 100644 --- a/vllm_omni/diffusion/models/cosmos3/__init__.py +++ b/vllm_omni/diffusion/models/cosmos3/__init__.py @@ -7,10 +7,12 @@ from .pipeline_cosmos3 import ( get_cosmos3_pre_process_func, ) from .transformer_cosmos3 import Cosmos3VFMTransformer +from .transformer_cosmos3_edge import Cosmos3EdgeVFMTransformer __all__ = [ "Cosmos3OmniDiffusersPipeline", "get_cosmos3_post_process_func", "get_cosmos3_pre_process_func", "Cosmos3VFMTransformer", + "Cosmos3EdgeVFMTransformer", ] diff --git a/vllm_omni/diffusion/models/cosmos3/pipeline_cosmos3.py b/vllm_omni/diffusion/models/cosmos3/pipeline_cosmos3.py index 5d0e24789..58e74ccf4 100644 --- a/vllm_omni/diffusion/models/cosmos3/pipeline_cosmos3.py +++ b/vllm_omni/diffusion/models/cosmos3/pipeline_cosmos3.py @@ -29,6 +29,7 @@ is produced from sound latents rather than from ``multi_modal_data["audio"]``. from __future__ import annotations +import json import math import os import time @@ -39,7 +40,7 @@ from typing import Any, ClassVar import numpy as np import PIL.Image import torch -from diffusers import UniPCMultistepScheduler +from diffusers import FlowMatchEulerDiscreteScheduler from diffusers.utils.torch_utils import randn_tensor from diffusers.video_processor import VideoProcessor from torch import nn @@ -61,6 +62,9 @@ from vllm_omni.diffusion.models.interface import ( SupportsComponentDiscovery, ) from vllm_omni.diffusion.models.progress_bar import ProgressBarMixin, _is_rank_zero +from vllm_omni.diffusion.models.schedulers.scheduling_flow_unipc_multistep import ( + FlowUniPCMultistepScheduler, +) from vllm_omni.diffusion.profiler.diffusion_pipeline_profiler import DiffusionPipelineProfilerMixin from vllm_omni.diffusion.request import OmniDiffusionRequest from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch @@ -94,7 +98,8 @@ from .transfer import ( transfer_max_frames_from_extra_args, uint8_cthw_to_normalized_5d, ) -from .transformer_cosmos3 import Cosmos3VFMTransformer, resolve_sound_gen +from .transformer_cosmos3 import Cosmos3VFMTransformer, _tf_config_get, resolve_sound_gen +from .transformer_cosmos3_edge import COSMOS3_EDGE_BACKBONE_TYPE, Cosmos3EdgeVFMTransformer from .utils import ( COSMOS3_DEFAULT_CONDITION_FRAME_INDEXES_VISION, COSMOS3_VAE_TEMPORAL_COMPRESSION, @@ -154,6 +159,7 @@ COSMOS3_T2V_DEFAULT_WIDTH = 1280 COSMOS3_T2V_DEFAULT_NUM_FRAMES = 189 COSMOS3_T2V_DEFAULT_NUM_INFERENCE_STEPS = 35 COSMOS3_T2V_DEFAULT_GUIDANCE_SCALE = 6.0 +COSMOS3_VIDEO_DEFAULT_FLOW_SHIFT = 10.0 COSMOS3_T2I_DEFAULT_HEIGHT = 1024 COSMOS3_T2I_DEFAULT_WIDTH = 1024 @@ -162,12 +168,73 @@ COSMOS3_T2I_DEFAULT_GUIDANCE_SCALE = 7.0 COSMOS3_T2I_DEFAULT_FLOW_SHIFT = 3.0 COSMOS3_T2I_DEFAULT_GUIDANCE_INTERVAL: tuple[float, float] = (400.0, 1000.0) +COSMOS3_EDGE_T2V_DEFAULT_HEIGHT = 480 +COSMOS3_EDGE_T2V_DEFAULT_WIDTH = 832 +COSMOS3_EDGE_T2V_DEFAULT_GUIDANCE_SCALE = 5.0 +COSMOS3_EDGE_VIDEO_DEFAULT_FLOW_SHIFT = 3.0 +COSMOS3_EDGE_T2I_DEFAULT_HEIGHT = 640 +COSMOS3_EDGE_T2I_DEFAULT_WIDTH = 640 + +COSMOS3_DISTILLED_CHECKPOINT_SCHEDULER_CLASS = "FlowMatchEulerDiscreteScheduler" + # Truncation cap on the prompt token count (shared by T2I and T2V). Prompts # are tokenized to their natural length (no padding); this only bounds the # UND pathway / GEN cross-attention cost for pathologically long prompts. COSMOS3_DEFAULT_MAX_SEQUENCE_LENGTH = 4096 +def _ceil_video_num_frames(num_frames: int, temporal_compression_factor: int) -> int: + """Round a video length up to the causal VAE's ``factor * k + 1`` grid.""" + if temporal_compression_factor <= 0: + raise ValueError(f"Cosmos3 temporal_compression_factor must be positive, got {temporal_compression_factor}.") + if num_frames <= 1: + return num_frames + latent_frames = math.ceil((num_frames - 1) / temporal_compression_factor) + return latent_frames * temporal_compression_factor + 1 + + +def _format_json_object_prompt( + prompt: str, + *, + num_frames: int, + frame_rate: float, + height: int, + width: int, + aspect_ratio: str | None, +) -> str | None: + """Match Imaginaire4 metadata injection for JSON-object prompts.""" + try: + prompt_obj = json.loads(prompt) + except (json.JSONDecodeError, TypeError, ValueError): + return None + if not isinstance(prompt_obj, dict): + return None + + metadata: dict[str, Any] = {} + if num_frames > 1: + duration_seconds = int(num_frames / frame_rate) if frame_rate > 0 else 0 + metadata.update({"duration": f"{duration_seconds}s", "fps": float(frame_rate)}) + else: + prompt_obj.pop("duration", None) + prompt_obj.pop("fps", None) + metadata["resolution"] = {"H": int(height), "W": int(width)} + if aspect_ratio is not None: + metadata["aspect_ratio"] = aspect_ratio + + prompt_obj.update(metadata) + return json.dumps(prompt_obj) + + +def resolve_cosmos3_transformer_cls(model_config: Any) -> type[Cosmos3VFMTransformer]: + """Select the Cosmos3 transformer implementation from transformer/config.json.""" + backbone_type = _tf_config_get(model_config, "backbone_type", None) + if backbone_type is None: + return Cosmos3VFMTransformer + if backbone_type == COSMOS3_EDGE_BACKBONE_TYPE: + return Cosmos3EdgeVFMTransformer + raise ValueError(f"Unsupported Cosmos3 transformer backbone_type={backbone_type!r}.") + + # --------------------------------------------------------------------------- # Post-process function (registered in registry.py) # --------------------------------------------------------------------------- @@ -190,6 +257,9 @@ def get_cosmos3_pre_process_func(od_config: OmniDiffusionConfig): from .guardrails import check_text_safety, ensure_initialized, is_guardrails_enabled video_processor = VideoProcessor(vae_scale_factor=16) + is_edge_model = ( + _tf_config_get(getattr(od_config, "tf_model_config", None), "backbone_type", None) == COSMOS3_EDGE_BACKBONE_TYPE + ) # Eager-load guardrail models at pipeline build time when the server-level # gate is on. Per-request overrides only decide whether the loaded models # are *invoked* — they cannot turn checks on without a server-side preload. @@ -410,12 +480,17 @@ def get_cosmos3_pre_process_func(od_config: OmniDiffusionConfig): extra = _extra_args(request) transfer_requested = action_mode is None and has_transfer_hints(extra) - # Auto-calculate H/W from aspect ratio (720p max area) + # Resolve missing H/W. if transfer_requested: _set_transfer_size_from_image(request, image) elif request.sampling_params.height is None or request.sampling_params.width is None: if action_mode is not None: _set_action_size_from_image(request, image) + elif is_edge_model: + if request.sampling_params.height is None: + request.sampling_params.height = COSMOS3_EDGE_T2V_DEFAULT_HEIGHT + if request.sampling_params.width is None: + request.sampling_params.width = COSMOS3_EDGE_T2V_DEFAULT_WIDTH else: max_area = 720 * 1280 aspect_ratio = image.height / image.width @@ -823,27 +898,52 @@ class Cosmos3OmniDiffusersPipeline( sound_latent_fps = self._sound_tokenizer.latent_fps # --- Transformer (weights loaded later via weights_sources) --- - self.transformer = Cosmos3VFMTransformer( + transformer_cls = resolve_cosmos3_transformer_cls(od_config.tf_model_config) + self.transformer = transformer_cls( od_config=od_config, temporal_compression_factor=self.vae_scale_factor_temporal, sound_gen=sound_gen, sound_dim=sound_dim, sound_latent_fps=sound_latent_fps, ) + self.is_edge_model = transformer_cls is Cosmos3EdgeVFMTransformer # --- Scheduler --- - # Load from checkpoint to preserve solver_order, timestep_spacing, - # beta_schedule, sigma bounds, flow_shift, etc. Only override - # flow_shift when explicitly requested by the user. - self.scheduler = UniPCMultistepScheduler.from_pretrained( + # Distilled model differs from regular one only by scheduler, + # distilled one uses FlowMatchEulerDiscreteScheduler, while + # regular should use FlowUniPCMultistepScheduler + + scheduler_config = FlowUniPCMultistepScheduler.load_config( model_path, subfolder="scheduler", local_files_only=local_files_only, ) - if od_config.flow_shift is not None: - self.scheduler = UniPCMultistepScheduler.from_config( - self.scheduler.config, - flow_shift=od_config.flow_shift, + + scheduler_class_name = scheduler_config.get("_class_name") + + self.is_distilled_model = False + if scheduler_class_name == COSMOS3_DISTILLED_CHECKPOINT_SCHEDULER_CLASS: + fixed_step_config = scheduler_config.get("fixed_step_sampler_config") + if not isinstance(fixed_step_config, dict) or fixed_step_config.get("sample_type") != "sde": + raise ValueError("Cosmos3 distilled scheduler requires fixed_step_sampler_config.sample_type=sde.") + t_list = fixed_step_config.get("t_list") + if not isinstance(t_list, list) or not t_list: + raise ValueError("Cosmos3 distilled scheduler requires a non-empty fixed_step_sampler_config.t_list.") + self.scheduler = FlowMatchEulerDiscreteScheduler.from_config( + scheduler_config, + stochastic_sampling=True, + ) + self._scheduler_init_t_list = list(t_list) + self.is_distilled_model = True + else: + # Preserve compatible solver settings from the checkpoint, but keep + # the base shift neutral. The concrete request shift is applied when + # FlowUniPC builds its timesteps. + self.scheduler = FlowUniPCMultistepScheduler.from_config( + scheduler_config, + shift=1.0, + use_dynamic_shifting=False, + prediction_type="flow_prediction", ) self._cpu_scheduler_state() @@ -862,14 +962,16 @@ class Cosmos3OmniDiffusersPipeline( ), ] - # Snapshot the loaded scheduler config so we can rebuild the - # scheduler at request time when a per-request flow_shift override - # is supplied (T2I uses shift=3.0; T2V/I2V use the engine default). - self._base_scheduler_config = self.scheduler.config - self._engine_init_flow_shift = float(getattr(self.scheduler.config, "flow_shift", 1.0) or 1.0) + # An engine-level override becomes the video default; otherwise use + # the matching regular/Edge default. Per-request values still take + # precedence. + default_video_flow_shift = ( + COSMOS3_EDGE_VIDEO_DEFAULT_FLOW_SHIFT if self.is_edge_model else COSMOS3_VIDEO_DEFAULT_FLOW_SHIFT + ) + self._engine_init_flow_shift = float( + od_config.flow_shift if od_config.flow_shift is not None else default_video_flow_shift + ) self._current_flow_shift = self._engine_init_flow_shift - self._base_scheduler_use_karras_sigmas = self._scheduler_use_karras_sigmas(self.scheduler.config) - self._current_scheduler_use_karras_sigmas = self._base_scheduler_use_karras_sigmas self._guidance_scale = None self._num_timesteps = None @@ -989,6 +1091,7 @@ class Cosmos3OmniDiffusersPipeline( "self_attn.to_out.": f"{und_lp}.self_attn.to_out.", "self_attn.norm_q.": f"{und_lp}.self_attn.norm_q.", "self_attn.norm_k.": f"{und_lp}.self_attn.norm_k.", + "self_attn.k_norm_und_for_gen.": f"{und_lp}.self_attn.k_norm_und_for_gen.", # GEN attention "self_attn.add_q_proj.": f"{gen_lp}.cross_attention.to_q.", "self_attn.add_k_proj.": f"{gen_lp}.cross_attention.to_k.", @@ -1054,6 +1157,7 @@ class Cosmos3OmniDiffusersPipeline( loaded = loader.load_weights(_remapped_weights()) self.transformer.post_load_weights() self.transformer.eval() + self.transformer.validate_loaded_weights(loaded) if getattr(self.transformer, "sound_gen", False): sound_markers = ("audio_proj_in.", "audio_proj_out.", "audio_modality_embed") missing = [marker.rstrip(".") for marker in sound_markers if not any(marker in name for name in loaded)] @@ -1359,6 +1463,8 @@ class Cosmos3OmniDiffusersPipeline( inputs: RoboLabPolicyInputs, pipeline_start: float, ) -> DiffusionOutput: + if self.is_distilled_model: + raise self._distilled_unsupported_error("RoboLab/action policy requests are unsupported.") if not getattr(self.transformer, "action_gen", False): raise ValueError( "Cosmos3 RoboLab policy serving was requested, but the transformer " @@ -1469,6 +1575,7 @@ class Cosmos3OmniDiffusersPipeline( guidance_interval=None, raw_action_dim=raw_action_dim, scheduler=scheduler, + generator=generator, ) if _is_rank_zero(): @@ -1560,38 +1667,34 @@ class Cosmos3OmniDiffusersPipeline( raise ValueError(f"Incorrect modality value in {modalities}, expected one of {accepted_modalities}.") return "image" in modalities + def _set_timesteps(self, num_inference_steps: int, device: str | torch.device, shift: float) -> None: + if self.is_distilled_model: + self.scheduler.set_timesteps(sigmas=self._scheduler_init_t_list, device=device) + else: + self.scheduler.set_timesteps( + num_inference_steps, + device=device, + shift=shift, + ) + + def _set_flow_shift(self, target_shift: float) -> None: + """Select the shift applied by FlowUniPC when timesteps are built.""" + self._current_flow_shift = float(target_shift) + @staticmethod - def _scheduler_use_karras_sigmas(config: Any) -> bool | None: - value = getattr(config, "use_karras_sigmas", None) - return None if value is None else bool(value) + def _resolve_seed(sp: OmniDiffusionSamplingParams, generator: torch.Generator | None, default: int = 42) -> int: + if sp.seed is not None: + return int(sp.seed) + if isinstance(generator, torch.Generator): + return int(generator.initial_seed()) + return int(default) - def _set_flow_shift(self, target_shift: float, *, use_karras_sigmas: bool | None = None) -> None: - """Set UniPC scheduler mode for a concrete request. - - The scheduler is rebuilt from the saved base config if - the target differs from the current shift or Karras-sigma mode. - Tracking explicit scheduler state is required because the previous - mode may have rebuilt the scheduler - we cannot rely on - ``self.scheduler.config`` reflecting the last requested target if a - rebuild was skipped via the equality check. - """ - target = float(target_shift) - target_use_karras_sigmas = ( - self._base_scheduler_use_karras_sigmas if use_karras_sigmas is None else bool(use_karras_sigmas) - ) - if ( - target == float(self._current_flow_shift) - and target_use_karras_sigmas == self._current_scheduler_use_karras_sigmas - ): - return - - scheduler_kwargs: dict[str, Any] = {"flow_shift": target} - if use_karras_sigmas is not None: - scheduler_kwargs["use_karras_sigmas"] = bool(use_karras_sigmas) - self.scheduler = UniPCMultistepScheduler.from_config(self._base_scheduler_config, **scheduler_kwargs) - self._cpu_scheduler_state() - self._current_flow_shift = target - self._current_scheduler_use_karras_sigmas = self._scheduler_use_karras_sigmas(self.scheduler.config) + def _resolve_guidance_scale(self, sp: OmniDiffusionSamplingParams, default: float) -> float: + if self.is_distilled_model: + return 1.0 + if sp.guidance_scale_provided: + return float(sp.guidance_scale) + return float(default) def _cpu_scheduler_state(self) -> None: # We need to move scheduler tensors to CPU, as unipc from diffusers assumes they are on CPU. @@ -1612,6 +1715,53 @@ class Cosmos3OmniDiffusersPipeline( def num_timesteps(self): return self._num_timesteps + @staticmethod + def _distilled_unsupported_error(detail: str) -> ValueError: + return ValueError( + f"Cosmos3 distilled checkpoints support only text-to-image and image-to-video generation; {detail}" + ) + + def _validate_distilled_generation_mode( + self, + *, + is_t2i: bool, + image_tensor: torch.Tensor | None, + action_enabled: bool, + transfer_config: Cosmos3TransferConfig | None, + is_v2v: bool, + sound_enabled: bool, + ) -> None: + if not self.is_distilled_model: + return + if action_enabled: + raise self._distilled_unsupported_error("action requests are unsupported.") + if transfer_config is not None: + raise self._distilled_unsupported_error("transfer requests are unsupported.") + if is_v2v: + raise self._distilled_unsupported_error("video-to-video requests are unsupported.") + if sound_enabled: + raise self._distilled_unsupported_error("sound generation is unsupported.") + if not is_t2i and image_tensor is None: + raise self._distilled_unsupported_error("text-to-video requests are unsupported.") + if is_t2i and image_tensor is not None: + raise self._distilled_unsupported_error("image-conditioned image generation is unsupported.") + + def _validate_edge_generation_mode( + self, + *, + transfer_config: Cosmos3TransferConfig | None, + is_v2v: bool, + sound_enabled: bool, + ) -> None: + if not self.is_edge_model: + return + if sound_enabled: + raise ValueError("Cosmos3 Edge checkpoints do not support sound generation.") + if transfer_config is not None: + raise ValueError("Cosmos3 Edge checkpoints do not support transfer inference.") + if is_v2v: + raise ValueError("Cosmos3 Edge checkpoints do not support video-to-video generation.") + # -- Prompt formatting ----------------------------------------------------- @staticmethod @@ -1628,6 +1778,10 @@ class Cosmos3OmniDiffusersPipeline( """ Append duration and resolution metadata to a prompt. """ + prompt = prompt.strip() + if duration_template is None and resolution_template is None: + return prompt + parts: list[str] = [] head = prompt.rstrip(".").strip() if head: @@ -1903,15 +2057,26 @@ class Cosmos3OmniDiffusersPipeline( res_tmpl = COSMOS3_IMAGE_RESOLUTION_TEMPLATE if is_t2i else COSMOS3_RESOLUTION_TEMPLATE else: res_tmpl = None - prompt = self._apply_metadata_templates( + json_prompt = _format_json_object_prompt( prompt, - num_frames, - frame_rate, - height, - width, - duration_template=dur_tmpl, - resolution_template=res_tmpl, + num_frames=num_frames, + frame_rate=frame_rate, + height=height, + width=width, + aspect_ratio=self._get_sp_param(sp, "aspect_ratio", None), ) + if json_prompt is not None: + prompt = json_prompt + else: + prompt = self._apply_metadata_templates( + prompt, + num_frames, + frame_rate, + height, + width, + duration_template=dur_tmpl, + resolution_template=res_tmpl, + ) if _is_rank_zero(): logger.info("Final prompt: '%s'", prompt) @@ -2272,6 +2437,7 @@ class Cosmos3OmniDiffusersPipeline( guidance_interval: tuple[float, float] | None = None, raw_action_dim: int | None = None, scheduler: Any | None = None, + generator: torch.Generator | None = None, ) -> torch.Tensor | tuple[torch.Tensor, ...]: """Denoising loop with 3-mode CFG support (parallel, sequential, none). @@ -2383,11 +2549,23 @@ class Cosmos3OmniDiffusersPipeline( if raw_action_dim is not None and 0 < raw_action_dim < action_pred.shape[-1]: action_pred[..., raw_action_dim:] = 0 if action_latents is None and sound_latents is None: - latents = step_scheduler.step(video_pred, t, latents, return_dict=False)[0] + latents = step_scheduler.step( + video_pred, + t, + latents, + generator=generator, + return_dict=False, + )[0] else: packed_noise, shapes, numels = _pack_joint(video_pred, action_pred, sound_pred) packed_latents, _, _ = _pack_joint(latents, action_latents, sound_latents) - packed_next = step_scheduler.step(packed_noise, t, packed_latents, return_dict=False)[0] + packed_next = step_scheduler.step( + packed_noise, + t, + packed_latents, + generator=generator, + return_dict=False, + )[0] unpacked = _unpack_joint(packed_next, shapes, numels) latents = unpacked[0] idx = 1 @@ -2613,6 +2791,7 @@ class Cosmos3OmniDiffusersPipeline( velocity_mask: torch.Tensor, condition_latents: torch.Tensor, guidance_interval: tuple[float, float] | None = None, + generator: torch.Generator | None = None, ) -> torch.Tensor: def _active_at(t: torch.Tensor, interval: tuple[float, float] | None) -> bool: if interval is None: @@ -2724,7 +2903,13 @@ class Cosmos3OmniDiffusersPipeline( if isinstance(noise_pred, tuple): raise ValueError("Cosmos3 transfer diffusion expects video-only tensor predictions.") noise_pred = noise_pred * velocity_mask - latents = self.scheduler.step(noise_pred, t, latents, return_dict=False)[0] + latents = self.scheduler.step( + noise_pred, + t, + latents, + generator=generator, + return_dict=False, + )[0] latents = velocity_mask * latents + (1.0 - velocity_mask) * condition_latents finally: self._cosmos3_branch_caches = None @@ -2741,6 +2926,8 @@ class Cosmos3OmniDiffusersPipeline( transfer_video_tensor: torch.Tensor | None, transfer_input_fps: float | None, ) -> DiffusionOutput: + if self.is_distilled_model: + raise self._distilled_unsupported_error("transfer requests are unsupported.") input_frames = None if transfer_video_tensor is not None: input_frames = normalized_video_to_uint8_cthw(transfer_video_tensor) @@ -2806,7 +2993,7 @@ class Cosmos3OmniDiffusersPipeline( guidance_scale = ( float(transfer_config.guidance_scale) if transfer_config.guidance_scale is not None - else float(sp.guidance_scale or COSMOS3_T2V_DEFAULT_GUIDANCE_SCALE) + else self._resolve_guidance_scale(sp, COSMOS3_T2V_DEFAULT_GUIDANCE_SCALE) ) flow_shift_target = float( transfer_config.flow_shift @@ -2821,11 +3008,11 @@ class Cosmos3OmniDiffusersPipeline( self._guidance_scale = guidance_scale self._num_timesteps = num_inference_steps - self._set_flow_shift(flow_shift_target, use_karras_sigmas=False) + self._set_flow_shift(flow_shift_target) generator = sp.generator + seed = self._resolve_seed(sp, generator) if generator is None: - seed = sp.seed if sp.seed is not None else 42 generator = torch.Generator(device=self.device).manual_seed(seed) cond_ids, cond_mask, uncond_ids, uncond_mask = self._format_and_tokenize_prompts( @@ -2915,7 +3102,11 @@ class Cosmos3OmniDiffusersPipeline( transfer_share_vision_temporal_positions=transfer_config.share_vision_temporal_positions, ) - self.scheduler.set_timesteps(num_inference_steps, device=self.device) + self._set_timesteps( + num_inference_steps, + device=self.device, + shift=self._current_flow_shift, + ) latents = self.diffuse_transfer( latents=latents, timesteps=self.scheduler.timesteps, @@ -2930,6 +3121,7 @@ class Cosmos3OmniDiffusersPipeline( shared_kwargs=shared_kwargs, velocity_mask=velocity_mask, condition_latents=condition_latents, + generator=generator, ) output_video = self._decode_latents(latents).clamp(-1, 1) previous_output = output_video @@ -3009,6 +3201,20 @@ class Cosmos3OmniDiffusersPipeline( action_enabled = action_mode is not None transfer_config = resolve_transfer_config(sp, prompt_data) action_video_tensor = video_tensor if action_enabled else None + is_v2v = video_tensor is not None and not is_t2i and not action_enabled + self._validate_distilled_generation_mode( + is_t2i=is_t2i, + image_tensor=image_tensor, + action_enabled=action_enabled, + transfer_config=transfer_config, + is_v2v=is_v2v, + sound_enabled=sound_enabled, + ) + self._validate_edge_generation_mode( + transfer_config=transfer_config, + is_v2v=is_v2v, + sound_enabled=sound_enabled, + ) if transfer_config is not None: if is_t2i: raise ValueError("Cosmos3 transfer inference is supported only for video outputs.") @@ -3056,31 +3262,36 @@ class Cosmos3OmniDiffusersPipeline( raise ValueError("Cosmos3 non-action generation accepts either image or video input, not both.") if video_tensor is not None and is_t2i: raise ValueError("Cosmos3 video-to-video generation is supported only for video outputs.") - is_v2v = video_tensor is not None and not is_t2i and not action_enabled - # T2I and T2V share the same model + forward path; only defaults - # differ: - # T2I: 1024x1024, 50 steps, shift=3.0, guidance_interval=[400, 1000] - # T2V: 720x1280, 35 steps, shift=engine-init, no interval + # T2I and T2V share the same model and forward path; their defaults are: + # T2I: regular 1024x1024, Edge 640x640; 50 steps, shift=3.0, + # guidance_interval=[400, 1000] + # T2V: 189 frames and 35 steps; regular guidance=6, shift=10; + # Edge guidance=5, shift=3; + # no guidance interval if is_t2i: - height = sp.height or COSMOS3_T2I_DEFAULT_HEIGHT - width = sp.width or COSMOS3_T2I_DEFAULT_WIDTH + height = sp.height or ( + COSMOS3_EDGE_T2I_DEFAULT_HEIGHT if self.is_edge_model else COSMOS3_T2I_DEFAULT_HEIGHT + ) + width = sp.width or (COSMOS3_EDGE_T2I_DEFAULT_WIDTH if self.is_edge_model else COSMOS3_T2I_DEFAULT_WIDTH) num_frames = 1 num_inference_steps = sp.num_inference_steps or COSMOS3_T2I_DEFAULT_NUM_INFERENCE_STEPS - guidance_scale = sp.guidance_scale if sp.guidance_scale else COSMOS3_T2I_DEFAULT_GUIDANCE_SCALE + guidance_scale = self._resolve_guidance_scale(sp, COSMOS3_T2I_DEFAULT_GUIDANCE_SCALE) default_flow_shift = COSMOS3_T2I_DEFAULT_FLOW_SHIFT default_guidance_interval: tuple[float, float] | None = COSMOS3_T2I_DEFAULT_GUIDANCE_INTERVAL batch_size = max(1, int(sp.num_outputs_per_prompt or 1)) else: - height = sp.height or COSMOS3_T2V_DEFAULT_HEIGHT - width = sp.width or COSMOS3_T2V_DEFAULT_WIDTH + height = sp.height or ( + COSMOS3_EDGE_T2V_DEFAULT_HEIGHT if self.is_edge_model else COSMOS3_T2V_DEFAULT_HEIGHT + ) + width = sp.width or (COSMOS3_EDGE_T2V_DEFAULT_WIDTH if self.is_edge_model else COSMOS3_T2V_DEFAULT_WIDTH) + default_guidance_scale = ( + COSMOS3_EDGE_T2V_DEFAULT_GUIDANCE_SCALE if self.is_edge_model else COSMOS3_T2V_DEFAULT_GUIDANCE_SCALE + ) num_frames = sp.num_frames or COSMOS3_T2V_DEFAULT_NUM_FRAMES num_inference_steps = sp.num_inference_steps or COSMOS3_T2V_DEFAULT_NUM_INFERENCE_STEPS - guidance_scale = sp.guidance_scale if sp.guidance_scale else COSMOS3_T2V_DEFAULT_GUIDANCE_SCALE - # Fall back to the engine-init shift, NOT None: passing None - # to ``_set_flow_shift`` would leak a prior T2I rebuild - # (shift=3.0) into a subsequent video request. - default_flow_shift = COSMOS3_V2V_DEFAULT_FLOW_SHIFT if is_v2v else self._engine_init_flow_shift + guidance_scale = self._resolve_guidance_scale(sp, default_guidance_scale) + default_flow_shift = self._engine_init_flow_shift default_guidance_interval = None batch_size = 1 # Existing video pipeline assumes B=1. @@ -3103,9 +3314,23 @@ class Cosmos3OmniDiffusersPipeline( f"or action_chunk_size + 1; got num_frames={num_frames}, action_chunk_size={action_chunk_size}." ) num_inference_steps = sp.num_inference_steps or 30 - guidance_scale = sp.guidance_scale if sp.guidance_scale is not None else 1.0 + guidance_scale = self._resolve_guidance_scale(sp, 1.0) default_flow_shift = 5.0 + if not is_t2i and not action_enabled: + requested_num_frames = int(num_frames) + num_frames = _ceil_video_num_frames( + requested_num_frames, + self.vae_scale_factor_temporal, + ) + if num_frames != requested_num_frames and _is_rank_zero(): + logger.info( + "Rounded Cosmos3 num_frames from %d to %d for temporal compression factor %d.", + requested_num_frames, + num_frames, + self.vae_scale_factor_temporal, + ) + domain_id = None if action_enabled: domain_id = resolve_domain_id( @@ -3146,13 +3371,12 @@ class Cosmos3OmniDiffusersPipeline( self._guidance_scale = guidance_scale self._num_timesteps = num_inference_steps - # Always resolve to a concrete target shift for this request, then - # update the shared Diffusers scheduler. - self._set_flow_shift(flow_shift_target, use_karras_sigmas=False if is_v2v else None) + # Always resolve to a concrete target shift for this request. + self._set_flow_shift(flow_shift_target) generator = sp.generator + seed = self._resolve_seed(sp, generator) if generator is None: - seed = sp.seed if sp.seed is not None else 42 generator = torch.Generator(device=self.device).manual_seed(seed) # --- Format prompts & tokenize (B=1; reused across loop iterations @@ -3293,7 +3517,11 @@ class Cosmos3OmniDiffusersPipeline( ) def _run_diffusion(start_latents): - self.scheduler.set_timesteps(num_inference_steps, device=self.device) + self._set_timesteps( + num_inference_steps, + device=self.device, + shift=self._current_flow_shift, + ) scheduler = self.scheduler return self.diffuse( latents=start_latents, @@ -3314,6 +3542,7 @@ class Cosmos3OmniDiffusersPipeline( guidance_interval=guidance_interval, raw_action_dim=raw_action_dim, scheduler=scheduler, + generator=generator, ) if is_t2i and batch_size > 1: diff --git a/vllm_omni/diffusion/models/cosmos3/transformer_cosmos3.py b/vllm_omni/diffusion/models/cosmos3/transformer_cosmos3.py index 21e723ced..de9767d6f 100644 --- a/vllm_omni/diffusion/models/cosmos3/transformer_cosmos3.py +++ b/vllm_omni/diffusion/models/cosmos3/transformer_cosmos3.py @@ -396,7 +396,6 @@ class TimestepEmbedder(nn.Module): hidden_size: int, frequency_embedding_size: int = 256, max_period: int = 10000, - target_dtype: torch.dtype = torch.bfloat16, ) -> None: super().__init__() # Following diffusers naming pattern here for checkpoint compatibility. @@ -407,7 +406,7 @@ class TimestepEmbedder(nn.Module): self.hidden_size = hidden_size half = frequency_embedding_size // 2 - freqs = torch.exp(-math.log(max_period) * torch.arange(start=0, end=half, dtype=target_dtype) / half) + freqs = torch.exp(-math.log(max_period) * torch.arange(start=0, end=half, dtype=torch.float32) / half) self.register_buffer("freqs", freqs, persistent=False) def forward(self, t: torch.Tensor) -> torch.Tensor: @@ -587,6 +586,7 @@ class Cosmos3CrossAttention(nn.Module): head_dim: int, rms_norm_eps: float, quant_config: QuantizationConfig | None = None, + qk_norm: bool = True, prefix: str = "", ) -> None: super().__init__() @@ -636,8 +636,10 @@ class Cosmos3CrossAttention(nn.Module): prefix=f"{prefix}.to_out", ) - self.norm_q = RMSNorm(self.head_dim, eps=rms_norm_eps) - self.norm_k = RMSNorm(self.head_dim, eps=rms_norm_eps) + self.qk_norm = qk_norm + if self.qk_norm: + self.norm_q = RMSNorm(self.head_dim, eps=rms_norm_eps) + self.norm_k = RMSNorm(self.head_dim, eps=rms_norm_eps) self.attn = FrameworkAttention( num_heads=self.num_heads, @@ -721,8 +723,9 @@ class Cosmos3CrossAttention(nn.Module): v = self.to_v(hidden_states).view(B, S_gen, self.num_kv_heads_local, self.head_dim) # Per-head QK norm - q = F.rms_norm(q, (self.head_dim,), self.norm_q.weight, eps=self.norm_q.variance_epsilon) - k = F.rms_norm(k, (self.head_dim,), self.norm_k.weight, eps=self.norm_k.variance_epsilon) + if self.qk_norm: + q = F.rms_norm(q, (self.head_dim,), self.norm_q.weight, eps=self.norm_q.variance_epsilon) + k = F.rms_norm(k, (self.head_dim,), self.norm_k.weight, eps=self.norm_k.variance_epsilon) # Qwen3-style RoPE q, k = _apply_rotary_pos_emb(q, k, freqs_cos, freqs_sin) @@ -806,6 +809,8 @@ class Cosmos3GenDecoderLayer(nn.Module): head_dim: int, rms_norm_eps: float, quant_config: QuantizationConfig | None = None, + mlp_cls: type[nn.Module] = Cosmos3GatedMLP, + qk_norm: bool = True, prefix: str = "", ) -> None: super().__init__() @@ -817,11 +822,12 @@ class Cosmos3GenDecoderLayer(nn.Module): head_dim=head_dim, rms_norm_eps=rms_norm_eps, quant_config=quant_config, + qk_norm=qk_norm, prefix=f"{prefix}.cross_attention", ) self.input_layernorm = RMSNorm(hidden_size, eps=rms_norm_eps) self.post_attention_layernorm = RMSNorm(hidden_size, eps=rms_norm_eps) - self.mlp = Cosmos3GatedMLP( + self.mlp = mlp_cls( hidden_size=hidden_size, intermediate_size=intermediate_size, quant_config=quant_config, @@ -996,6 +1002,9 @@ class Cosmos3VFMTransformer(nn.Module): packed_modules_mapping = {} + _language_model_cls = Cosmos3LanguageModel + _gen_mlp_cls = Cosmos3GatedMLP + @staticmethod def _is_transformer_block(name: str, module) -> bool: return ("gen_layers" in name or "language_model.layers" in name) and name.split(".")[-1].isdigit() @@ -1032,6 +1041,25 @@ class Cosmos3VFMTransformer(nn.Module): if actual != expected: raise ValueError(f"Unsupported Cosmos3 transformer config: {key}={actual!r}; expected {expected!r}.") + @classmethod + def _resolve_rms_norm_eps(cls, model_config: Any) -> float: + return float(_tf_config_get(model_config, "rms_norm_eps", 1e-6)) + + @classmethod + def _resolve_rope_theta(cls, model_config: Any) -> float: + return float(_tf_config_get(model_config, "rope_theta", 5_000_000)) + + @classmethod + def _resolve_mrope_section(cls, model_config: Any) -> list[int]: + rope_scaling = _tf_config_get(model_config, "rope_scaling", {}) or {} + return list(rope_scaling.get("mrope_section", [24, 20, 20])) + + def _language_model_kwargs(self) -> dict[str, Any]: + return {} + + def validate_loaded_weights(self, loaded: set[str]) -> None: + del loaded + def __init__( self, od_config: OmniDiffusionConfig, @@ -1044,7 +1072,6 @@ class Cosmos3VFMTransformer(nn.Module): super().__init__() model_config = od_config.tf_model_config self._validate_supported_config(model_config) - rope_scaling = _tf_config_get(model_config, "rope_scaling", {}) or {} self.hidden_size = int(_tf_config_get(model_config, "hidden_size", 4096)) self.num_hidden_layers = int(_tf_config_get(model_config, "num_hidden_layers", 36)) @@ -1053,9 +1080,10 @@ class Cosmos3VFMTransformer(nn.Module): self.head_dim = int(_tf_config_get(model_config, "head_dim", 128)) self.intermediate_size = int(_tf_config_get(model_config, "intermediate_size", 12288)) self.vocab_size = int(_tf_config_get(model_config, "vocab_size", 151936)) - self.rms_norm_eps = float(_tf_config_get(model_config, "rms_norm_eps", 1e-6)) - self.rope_theta = float(_tf_config_get(model_config, "rope_theta", 5_000_000)) - self.mrope_section = list(rope_scaling.get("mrope_section", [24, 20, 20])) + self.rms_norm_eps = self._resolve_rms_norm_eps(model_config) + self.rope_theta = self._resolve_rope_theta(model_config) + self.mrope_section = self._resolve_mrope_section(model_config) + self.qk_norm_for_diffusion = bool(_tf_config_get(model_config, "qk_norm_for_diffusion", True)) self.latent_patch_size = int(_tf_config_get(model_config, "latent_patch_size", 2)) self.latent_channel_size = int(_tf_config_get(model_config, "latent_channel", 48)) self.timestep_scale = float(_tf_config_get(model_config, "timestep_scale", 0.001)) @@ -1095,10 +1123,12 @@ class Cosmos3VFMTransformer(nn.Module): ) self.patch_latent_dim = (self.latent_patch_size**2) * self.latent_channel_size + self.use_k_norm_und_for_gen = _tf_config_get(model_config, "use_k_norm_und_for_gen", None) + dtype = od_config.dtype quant_config = getattr(od_config, "quantization_config", None) if od_config else None - self.language_model = Cosmos3LanguageModel( + self.language_model = self._language_model_cls( hidden_size=self.hidden_size, intermediate_size=self.intermediate_size, num_hidden_layers=self.num_hidden_layers, @@ -1111,12 +1141,13 @@ class Cosmos3VFMTransformer(nn.Module): mrope_section=self.mrope_section, quant_config=quant_config, prefix="language_model", + **self._language_model_kwargs(), ) # Video projection layers are small; not worth quantizing. self.proj_in = nn.Linear(self.patch_latent_dim, self.hidden_size) self.proj_out = nn.Linear(self.hidden_size, self.patch_latent_dim) - self.time_embedder = TimestepEmbedder(self.hidden_size, target_dtype=dtype) + self.time_embedder = TimestepEmbedder(self.hidden_size) if self.action_gen: self.action_proj_in = DomainAwareLinear( self.action_dim, @@ -1147,6 +1178,8 @@ class Cosmos3VFMTransformer(nn.Module): head_dim=self.head_dim, rms_norm_eps=self.rms_norm_eps, quant_config=quant_config, + mlp_cls=self._gen_mlp_cls, + qk_norm=self.qk_norm_for_diffusion, prefix=f"gen_layers.{i}", ) for i in range(self.num_hidden_layers) diff --git a/vllm_omni/diffusion/models/cosmos3/transformer_cosmos3_edge.py b/vllm_omni/diffusion/models/cosmos3/transformer_cosmos3_edge.py new file mode 100644 index 000000000..6a1b4313b --- /dev/null +++ b/vllm_omni/diffusion/models/cosmos3/transformer_cosmos3_edge.py @@ -0,0 +1,396 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Cosmos3 Edge transformer variant with a Nemotron dense UND backbone.""" + +from __future__ import annotations + +from typing import Any + +import torch +import torch.nn as nn +import torch.nn.functional as F +from vllm.distributed import get_tensor_model_parallel_world_size +from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear +from vllm.model_executor.layers.quantization.base_config import QuantizationConfig + +from vllm_omni.diffusion.attention.layer import Attention as FrameworkAttention + +from .transformer_cosmos3 import ( + Cosmos3VFMTransformer, + Qwen3VLTextRotaryEmbedding, + RMSNorm, + _apply_rotary_pos_emb, + _nested_get, + _tf_config_get, +) + +COSMOS3_EDGE_BACKBONE_TYPE = "cosmos3_edge_nemotron_dense" + + +def _tf_config_nested_get(config: Any, key: str, default: Any = None) -> Any: + if config is None: + return default + if isinstance(config, dict): + value = _nested_get(config, key) + elif hasattr(config, "to_dict"): + value = _nested_get(config.to_dict(), key) + elif hasattr(config, "params"): + value = _nested_get(config.params, key) + else: + value = getattr(config, key, None) + return default if value is None else value + + +class Cosmos3Relu2MLP(nn.Module): + """Nemotron dense MLP: down_proj(relu(up_proj(x)) ** 2).""" + + def __init__( + self, + hidden_size: int, + intermediate_size: int, + quant_config: QuantizationConfig | None = None, + prefix: str = "", + ) -> None: + super().__init__() + self.up_proj = ColumnParallelLinear( + hidden_size, + intermediate_size, + bias=False, + gather_output=False, + return_bias=False, + quant_config=quant_config, + prefix=f"{prefix}.up_proj", + ) + self.down_proj = RowParallelLinear( + intermediate_size, + hidden_size, + bias=False, + input_is_parallel=True, + return_bias=False, + quant_config=quant_config, + prefix=f"{prefix}.down_proj", + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + hidden = F.relu(self.up_proj(x)) + return self.down_proj(hidden * hidden) + + +class Cosmos3EdgeCausalAttention(nn.Module): + """Edge UND causal attention. + + Reasoner self-attention uses raw UND Q/K with RoPE because + ``qk_norm_for_text=false``. The GEN-facing UND key cache is a separate + normalized-then-RoPE'd view of the raw K tensor when GEN QK norm is enabled; + no-GEN-QK-norm checkpoints reuse the raw RoPE'd UND K. + """ + + def __init__( + self, + *, + hidden_size: int, + num_attention_heads: int, + num_key_value_heads: int, + head_dim: int, + rms_norm_eps: float, + use_k_norm_und_for_gen: bool, + quant_config: QuantizationConfig | None = None, + prefix: str = "", + ) -> None: + super().__init__() + self.hidden_size = hidden_size + self.num_heads = num_attention_heads + self.num_kv_heads = num_key_value_heads + self.head_dim = head_dim + + tp_size = get_tensor_model_parallel_world_size() + self.num_heads_local = self.num_heads // tp_size + self.num_kv_heads_local = self.num_kv_heads // tp_size + + self.to_q = ColumnParallelLinear( + hidden_size, + self.num_heads * self.head_dim, + bias=False, + gather_output=False, + return_bias=False, + quant_config=quant_config, + prefix=f"{prefix}.to_q", + ) + self.to_k = ColumnParallelLinear( + hidden_size, + self.num_kv_heads * self.head_dim, + bias=False, + gather_output=False, + return_bias=False, + quant_config=quant_config, + prefix=f"{prefix}.to_k", + ) + self.to_v = ColumnParallelLinear( + hidden_size, + self.num_kv_heads * self.head_dim, + bias=False, + gather_output=False, + return_bias=False, + quant_config=quant_config, + prefix=f"{prefix}.to_v", + ) + self.to_out = RowParallelLinear( + self.num_heads * self.head_dim, + hidden_size, + bias=False, + input_is_parallel=True, + return_bias=False, + quant_config=quant_config, + prefix=f"{prefix}.to_out", + ) + + self.k_norm_und_for_gen = RMSNorm(self.head_dim, eps=rms_norm_eps) if use_k_norm_und_for_gen else None + self.attn = FrameworkAttention( + num_heads=self.num_heads, + head_size=self.head_dim, + causal=True, + softmax_scale=1.0 / (self.head_dim**0.5), + num_kv_heads=self.num_kv_heads, + skip_sequence_parallel=True, + ) + + def forward( + self, + hidden_states: torch.Tensor, + freqs_cos: torch.Tensor, + freqs_sin: torch.Tensor, + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + B, S, _ = hidden_states.shape + + q = self.to_q(hidden_states).view(B, S, self.num_heads_local, self.head_dim) + k = self.to_k(hidden_states).view(B, S, self.num_kv_heads_local, self.head_dim) + v = self.to_v(hidden_states).view(B, S, self.num_kv_heads_local, self.head_dim) + + q_reasoner, k_reasoner = _apply_rotary_pos_emb(q, k, freqs_cos, freqs_sin) + + if self.k_norm_und_for_gen is not None: + k_gen = F.rms_norm( + k, + (self.head_dim,), + self.k_norm_und_for_gen.weight, + eps=self.k_norm_und_for_gen.variance_epsilon, + ) + _, k_gen = _apply_rotary_pos_emb(q, k_gen, freqs_cos, freqs_sin) + else: + k_gen = k_reasoner + + out = self.attn(q_reasoner, k_reasoner, v).reshape(B, S, -1) + return self.to_out(out), k_gen, v + + +class Cosmos3EdgeUndDecoderLayer(nn.Module): + """Edge UND decoder layer: causal self-attention + ReLU2 MLP.""" + + def __init__( + self, + *, + hidden_size: int, + intermediate_size: int, + num_attention_heads: int, + num_key_value_heads: int, + head_dim: int, + rms_norm_eps: float, + use_k_norm_und_for_gen: bool, + quant_config: QuantizationConfig | None = None, + prefix: str = "", + ) -> None: + super().__init__() + self.self_attn = Cosmos3EdgeCausalAttention( + hidden_size=hidden_size, + num_attention_heads=num_attention_heads, + num_key_value_heads=num_key_value_heads, + head_dim=head_dim, + rms_norm_eps=rms_norm_eps, + use_k_norm_und_for_gen=use_k_norm_und_for_gen, + quant_config=quant_config, + prefix=f"{prefix}.self_attn", + ) + self.input_layernorm = RMSNorm(hidden_size, eps=rms_norm_eps) + self.post_attention_layernorm = RMSNorm(hidden_size, eps=rms_norm_eps) + self.mlp = Cosmos3Relu2MLP( + hidden_size=hidden_size, + intermediate_size=intermediate_size, + quant_config=quant_config, + prefix=f"{prefix}.mlp", + ) + + def forward( + self, + hidden_states: torch.Tensor, + freqs: tuple[torch.Tensor, torch.Tensor], + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + residual = hidden_states + hidden_states = self.input_layernorm(hidden_states) + + cos, sin = freqs + attn_out, k, v = self.self_attn(hidden_states, cos, sin) + hidden_states = residual + attn_out + + residual = hidden_states + hidden_states = self.post_attention_layernorm(hidden_states) + hidden_states = residual + self.mlp(hidden_states) + + return hidden_states, k, v + + +class Cosmos3EdgeLanguageModel(nn.Module): + """Nemotron dense UND tower that returns GEN-facing K plus raw V per layer.""" + + _layerwise_offload_blocks_attrs = ["layers"] + + def __init__( + self, + *, + hidden_size: int, + intermediate_size: int, + num_hidden_layers: int, + num_attention_heads: int, + num_key_value_heads: int, + head_dim: int, + vocab_size: int, + rms_norm_eps: float, + rope_theta: float, + mrope_section: list[int], + use_k_norm_und_for_gen: bool, + quant_config: QuantizationConfig | None = None, + prefix: str = "", + ) -> None: + super().__init__() + self.embed_tokens = nn.Embedding(vocab_size, hidden_size) + self.rotary_emb = Qwen3VLTextRotaryEmbedding( + head_dim=head_dim, + rope_theta=rope_theta, + mrope_section=mrope_section, + ) + self.layers = nn.ModuleList( + [ + Cosmos3EdgeUndDecoderLayer( + hidden_size=hidden_size, + intermediate_size=intermediate_size, + num_attention_heads=num_attention_heads, + num_key_value_heads=num_key_value_heads, + head_dim=head_dim, + rms_norm_eps=rms_norm_eps, + use_k_norm_und_for_gen=use_k_norm_und_for_gen, + quant_config=quant_config, + prefix=f"{prefix}.layers.{i}", + ) + for i in range(num_hidden_layers) + ] + ) + self.norm = RMSNorm(hidden_size, eps=rms_norm_eps) + + def forward( + self, + text_ids: torch.Tensor, + freqs: tuple[torch.Tensor, torch.Tensor], + ) -> list[tuple[torch.Tensor, torch.Tensor]]: + hidden = self.embed_tokens(text_ids) + + cached_kv: list[tuple[torch.Tensor, torch.Tensor]] = [] + for layer in self.layers: + hidden, k, v = layer(hidden, freqs) + cached_kv.append((k, v)) + + return cached_kv + + +class Cosmos3EdgeVFMTransformer(Cosmos3VFMTransformer): + """Cosmos3 Edge variant with Nemotron dense UND and shared GEN diffusion.""" + + _language_model_cls = Cosmos3EdgeLanguageModel + _gen_mlp_cls = Cosmos3Relu2MLP + + @staticmethod + def _validate_supported_config(model_config: Any) -> None: + expected_values = { + "qk_norm_for_text": False, + "position_embedding_type": "unified_3d_mrope", + "unified_3d_mrope_reset_spatial_ids": True, + "joint_attn_implementation": "two_way", + } + for key, expected in expected_values.items(): + actual = _tf_config_get(model_config, key, expected) + if actual != expected: + raise ValueError( + f"Unsupported Cosmos3 Edge transformer config: {key}={actual!r}; expected {expected!r}." + ) + + backbone_type = _tf_config_get(model_config, "backbone_type", None) + if backbone_type is None: + raise ValueError( + f"Cosmos3 Edge transformer config must declare backbone_type={COSMOS3_EDGE_BACKBONE_TYPE!r}." + ) + if backbone_type != COSMOS3_EDGE_BACKBONE_TYPE: + raise ValueError( + "Unsupported Cosmos3 Edge transformer config: " + f"backbone_type={backbone_type!r}; expected {COSMOS3_EDGE_BACKBONE_TYPE!r}." + ) + + required_values = { + "latent_channel": 48, + "latent_patch_size": 2, + "temporal_compression_factor": 4, + } + for key, expected in required_values.items(): + actual = _tf_config_get(model_config, key, None) + if actual is None: + raise ValueError(f"Cosmos3 Edge transformer config must declare {key}={expected}.") + if int(actual) != expected: + raise ValueError( + f"Unsupported Cosmos3 Edge transformer config: {key}={actual!r}; expected {expected!r}." + ) + + @classmethod + def _resolve_rms_norm_eps(cls, model_config: Any) -> float: + return float( + _tf_config_get( + model_config, + "rms_norm_eps", + _tf_config_get(model_config, "layer_norm_epsilon", 1e-5), + ) + ) + + @classmethod + def _resolve_rope_theta(cls, model_config: Any) -> float: + return float(_tf_config_get(model_config, "rope_theta", 100_000_000)) + + @classmethod + def _resolve_mrope_section(cls, model_config: Any) -> list[int]: + mrope_section = _tf_config_get(model_config, "mrope_section", None) + if mrope_section is None: + mrope_section = _tf_config_nested_get(model_config, "mrope_section", [24, 20, 20]) + return list(mrope_section) + + def _language_model_kwargs(self) -> dict[str, Any]: + return {"use_k_norm_und_for_gen": self.use_k_norm_und_for_gen} + + def validate_loaded_weights(self, loaded: set[str]) -> None: + missing: list[str] = [] + for layer_idx in range(self.num_hidden_layers): + required_markers = ( + f"language_model.layers.{layer_idx}.mlp.up_proj.", + f"language_model.layers.{layer_idx}.mlp.down_proj.", + f"gen_layers.{layer_idx}.mlp.up_proj.", + f"gen_layers.{layer_idx}.mlp.down_proj.", + ) + if self.use_k_norm_und_for_gen: + required_markers = ( + *required_markers, + f"language_model.layers.{layer_idx}.self_attn.k_norm_und_for_gen.", + ) + missing.extend( + marker.rstrip(".") for marker in required_markers if not any(marker in name for name in loaded) + ) + if missing: + preview = ", ".join(missing[:10]) + suffix = " ..." if len(missing) > 10 else "" + raise ValueError( + "Cosmos3 Edge transformer checkpoint is missing required weights for " + f"{preview}{suffix}. Use a converted Edge checkpoint with ReLU2 UND/GEN MLP weights." + ) diff --git a/vllm_omni/diffusion/registry.py b/vllm_omni/diffusion/registry.py index 1454a37b1..5b8b2fba1 100644 --- a/vllm_omni/diffusion/registry.py +++ b/vllm_omni/diffusion/registry.py @@ -276,6 +276,11 @@ _DIFFUSION_MODELS = { "pipeline_cosmos3", "Cosmos3OmniDiffusersPipeline", ), + "Cosmos3OmniPipeline": ( + "cosmos3", + "pipeline_cosmos3", + "Cosmos3OmniDiffusersPipeline", + ), "SoulXSingerPipeline": ( "soulx_singer", "pipeline_soulx_singer_svs", @@ -536,6 +541,7 @@ _DIFFUSION_POST_PROCESS_FUNCS = { "DreamIDOmniPipeline": "get_dreamid_omni_post_process_func", "SenseNovaU1Pipeline": "get_sensenova_u1_post_process_func", "Cosmos3OmniDiffusersPipeline": "get_cosmos3_post_process_func", + "Cosmos3OmniPipeline": "get_cosmos3_post_process_func", "HiDreamImagePipeline": "get_hidream_image_post_process_func", "StableDiffusionXLPipeline": "get_sdxl_image_post_process_func", "Krea2Pipeline": "get_krea2_post_process_func", @@ -546,6 +552,7 @@ _DIFFUSION_IR_OP_PRIORITY_FUNCS = { # `ir_op_priority_func` function must be placed in {mod_folder}/{mod_relname}.py, # where mod_folder and mod_relname are defined and mapped using `_DIFFUSION_MODELS` via the `arch` key. "Cosmos3OmniDiffusersPipeline": "get_cosmos3_ir_op_priority_func", + "Cosmos3OmniPipeline": "get_cosmos3_ir_op_priority_func", } _DIFFUSION_PRE_PROCESS_FUNCS = { @@ -570,6 +577,7 @@ _DIFFUSION_PRE_PROCESS_FUNCS = { "HunyuanImage3ForCausalMM": "get_hunyuan_image_3_pre_process_func", "MagiHumanPipeline": "get_magi_human_pre_process_func", "Cosmos3OmniDiffusersPipeline": "get_cosmos3_pre_process_func", + "Cosmos3OmniPipeline": "get_cosmos3_pre_process_func", "SoulXSingerPipeline": "get_soulxsinger_pre_process_func", "SoulXSingerSVCPipeline": "get_soulxsinger_svc_pre_process_func", } diff --git a/vllm_omni/model_extras/registry.py b/vllm_omni/model_extras/registry.py index 7103991c7..79f8b4360 100644 --- a/vllm_omni/model_extras/registry.py +++ b/vllm_omni/model_extras/registry.py @@ -153,6 +153,11 @@ _EXTRA_SPECS: dict[str, dict[str, Any]] = { "extra_output_params": COSMOS3_EXTRA_OUTPUT_PARAMS, "text_to_image_prompt_builder": build_cosmos3_text_to_image_prompt, }, + "Cosmos3OmniPipeline": { + "extra_body_params": COSMOS3_EXTRA_BODY_PARAMS, + "extra_output_params": COSMOS3_EXTRA_OUTPUT_PARAMS, + "text_to_image_prompt_builder": build_cosmos3_text_to_image_prompt, + }, "MagiHumanPipeline": { "extra_body_params": MAGI_HUMAN_EXTRA_BODY_PARAMS, "extra_output_params": MAGI_HUMAN_EXTRA_OUTPUT_PARAMS, From ae949d129f4d6dbf58640397b926dda252f30122 Mon Sep 17 00:00:00 2001 From: Alex Steiner Date: Thu, 16 Jul 2026 12:33:57 -0700 Subject: [PATCH 006/240] [Bugfix][Quantization] Remap ModelOpt NVFP4 scale tensors (#5087) Signed-off-by: Alex Steiner --- .../model_loader/test_modelopt_fp8_adapter.py | 49 +++++++++++++++++-- .../checkpoint_adapters/modelopt.py | 31 ++++++++++-- 2 files changed, 73 insertions(+), 7 deletions(-) diff --git a/tests/diffusion/model_loader/test_modelopt_fp8_adapter.py b/tests/diffusion/model_loader/test_modelopt_fp8_adapter.py index 66fa87a3c..8c429a00f 100644 --- a/tests/diffusion/model_loader/test_modelopt_fp8_adapter.py +++ b/tests/diffusion/model_loader/test_modelopt_fp8_adapter.py @@ -9,6 +9,7 @@ import torch.nn as nn from vllm_omni.diffusion.model_loader.checkpoint_adapters import ( ModelOptFp8CheckpointAdapter, + ModelOptNvFp4CheckpointAdapter, ) pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu] @@ -89,6 +90,23 @@ class _QuantizedRemappedModelOptModel(nn.Module): return name +class _RemappedNvFp4Model(nn.Module): + def __init__(self) -> None: + super().__init__() + self.runtime = nn.Module() + self.runtime.proj = nn.Module() + self.runtime.proj.register_parameter( + "weight", + nn.Parameter(torch.empty(1, 1, dtype=torch.uint8), requires_grad=False), + ) + for name in ("input_scale", "weight_scale", "weight_scale_2"): + self.runtime.proj.register_parameter(name, nn.Parameter(torch.empty(1), requires_grad=False)) + + @staticmethod + def remap_checkpoint_key(name: str) -> str: + return name.replace("transformer.orig.proj.", "runtime.proj.") + + def _make_source() -> SimpleNamespace: return SimpleNamespace( subfolder="transformer", @@ -188,13 +206,36 @@ def test_modelopt_adapter_remaps_and_keeps_scales_for_quantized_target(): emitted = dict(adapted) assert len(adapted) == len(emitted) == 3 - # Both scales are retained (the dropped-scale regression). They keep their - # source-checkpoint names; the model's load_weights remaps them downstream. - assert torch.equal(emitted["transformer.orig.proj.weight_scale"], weight_scale) - assert torch.equal(emitted["transformer.orig.proj.input_scale"], input_scale) + # Both scales are retained and emitted under the remapped runtime names. + assert torch.equal(emitted["runtime.proj.weight_scale"], weight_scale) + assert torch.equal(emitted["runtime.proj.input_scale"], input_scale) # The FP8 weight is passed through unchanged under the remapped target name # (the quantized param is FP8, so it must NOT be dequantized). Compare via # float32 since torch.equal has no FP8 CPU kernel. assert emitted["runtime.proj.weight"].dtype == torch.float8_e4m3fn assert torch.equal(emitted["runtime.proj.weight"].to(torch.float32), fp8_weight.to(torch.float32)) + + +def test_modelopt_nvfp4_adapter_remaps_quantized_weights_and_scales(): + adapter = ModelOptNvFp4CheckpointAdapter(_RemappedNvFp4Model(), _make_source()) + prefix = "transformer.orig.proj" + checkpoint_tensors = [ + (f"{prefix}.input_scale", torch.tensor([1.0])), + (f"{prefix}.weight_scale", torch.tensor([0.5])), + (f"{prefix}.weight_scale_2", torch.tensor([0.25])), + (f"{prefix}.weight", torch.tensor([[1]], dtype=torch.uint8)), + ] + + adapted = list(adapter.adapt(iter(checkpoint_tensors))) + + assert [name for name, _ in adapted] == [ + name.replace(f"{prefix}.", "runtime.proj.") for name, _ in checkpoint_tensors + ] + + +def test_modelopt_nvfp4_adapter_rejects_unconsumed_pre_quant_scale(): + adapter = ModelOptNvFp4CheckpointAdapter(nn.Module(), _make_source()) + + with pytest.raises(ValueError, match="does not consume pre_quant_scale"): + list(adapter.adapt(iter([("transformer.proj.pre_quant_scale", torch.tensor([1.0]))]))) diff --git a/vllm_omni/diffusion/model_loader/checkpoint_adapters/modelopt.py b/vllm_omni/diffusion/model_loader/checkpoint_adapters/modelopt.py index d6f7b3e0c..68ec98d5a 100644 --- a/vllm_omni/diffusion/model_loader/checkpoint_adapters/modelopt.py +++ b/vllm_omni/diffusion/model_loader/checkpoint_adapters/modelopt.py @@ -11,7 +11,12 @@ from vllm.model_executor.utils import get_packed_modules_mapping logger = init_logger(__name__) -MODEL_OPT_SCALE_SUFFIXES = (".input_scale", ".weight_scale", ".weight_scale_2", ".weight_scale_inv") +MODEL_OPT_SCALE_SUFFIXES = ( + ".input_scale", + ".weight_scale", + ".weight_scale_2", + ".weight_scale_inv", +) DEFAULT_PACKED_MODULES_MAPPING = { "to_qkv": ("to_q", "to_k", "to_v"), "add_kv_proj": ("add_q_proj", "add_k_proj", "add_v_proj"), @@ -122,6 +127,7 @@ class ModelOptFp8CheckpointAdapter: for candidate in self._weights_mapper.apply_list([name]): if candidate != name and candidate in self._loadable_tensors: + # Preserve shard names so downstream packed-weight loaders can route them. return candidate, name return None, name @@ -178,6 +184,7 @@ class ModelOptFp8CheckpointAdapter: def _handle_scale_tensor( self, name: str, + output_name: str, tensor: torch.Tensor, target_name: str | None, state: _AdaptState, @@ -186,7 +193,7 @@ class ModelOptFp8CheckpointAdapter: if target_name is None: state.skipped_scales += 1 else: - yield name, tensor + yield output_name, tensor yield from self._flush_pending_weights(name, state) def _target_dtype_for_dequantization( @@ -249,7 +256,7 @@ class ModelOptFp8CheckpointAdapter: for name, tensor in weights: target_name, output_name = self._resolve_target_and_output_names(name) if self._is_scale(name): - yield from self._handle_scale_tensor(name, tensor, target_name, state) + yield from self._handle_scale_tensor(name, output_name, tensor, target_name, state) continue target_dtype = self._target_dtype_for_dequantization(tensor, target_name) @@ -264,6 +271,8 @@ class ModelOptFp8CheckpointAdapter: class ModelOptNvFp4CheckpointAdapter(ModelOptFp8CheckpointAdapter): + _PRE_QUANT_SCALE_SUFFIX = ".pre_quant_scale" + @staticmethod def _is_checkpoint_quant_config(quant_config: object | None) -> bool: return ( @@ -273,6 +282,22 @@ class ModelOptNvFp4CheckpointAdapter(ModelOptFp8CheckpointAdapter): and bool(getattr(quant_config, "is_checkpoint_nvfp4_serialized", False)) ) + def adapt( + self, + weights: Iterable[tuple[str, torch.Tensor]], + ) -> Generator[tuple[str, torch.Tensor], None, None]: + def validated_weights() -> Generator[tuple[str, torch.Tensor], None, None]: + for name, tensor in weights: + if name.endswith(self._PRE_QUANT_SCALE_SUFFIX): + raise ValueError( + f"ModelOpt NVFP4 checkpoint tensor {name!r} is unsupported: " + "vLLM 0.25.0 does not consume pre_quant_scale. Export the checkpoint " + "with pre-quant scales folded into the weights." + ) + yield name, tensor + + yield from super().adapt(validated_weights()) + class ModelOptMixedPrecisionCheckpointAdapter(ModelOptFp8CheckpointAdapter): @staticmethod From 07957e12f45611363970f717dd6be080f5c7b71c Mon Sep 17 00:00:00 2001 From: bastefaniak Date: Fri, 17 Jul 2026 00:01:21 +0200 Subject: [PATCH 007/240] Fix FP8 Quack GEMM call when inference_mode() and torch.compile() is used. (#5153) Signed-off-by: Bartosz Stefaniak Co-authored-by: Roger Wang --- vllm_omni/quantization/quack_fp8.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vllm_omni/quantization/quack_fp8.py b/vllm_omni/quantization/quack_fp8.py index 8c7251e43..865958386 100644 --- a/vllm_omni/quantization/quack_fp8.py +++ b/vllm_omni/quantization/quack_fp8.py @@ -94,7 +94,7 @@ def quack_scaled_fp8_mm( return None out = torch.empty(a.shape[0], b.shape[1], device=a.device, dtype=out_dtype) alpha = scale_a.reshape(1).float() * scale_b.reshape(1).float() - gemm.gemm_tuned(a, b, out, bias=bias, alpha=alpha) + gemm.gemm(a, b, out=out, bias=bias, alpha=alpha, tuned=True) return out From be517db18a68fb8983ce72a4a5f5312f5324b26e Mon Sep 17 00:00:00 2001 From: linzhenpl07 <60651557+linzhenpl07@users.noreply.github.com> Date: Fri, 17 Jul 2026 11:26:35 +0800 Subject: [PATCH 008/240] examples: replica data-parallel recipe + benchmark for video DiT (#4707) (#5052) Signed-off-by: linzhenpl07 --- .../replica_data_parallel/README.md | 91 +++++++ .../replica_data_parallel/bench_replica_dp.py | 241 ++++++++++++++++++ .../replica_data_parallel/run_server.sh | 33 +++ .../replica_data_parallel/wan2_2_ti2v_dp.yaml | 43 ++++ vllm_omni/entrypoints/cli/serve.py | 3 +- 5 files changed, 410 insertions(+), 1 deletion(-) create mode 100644 examples/online_serving/replica_data_parallel/README.md create mode 100755 examples/online_serving/replica_data_parallel/bench_replica_dp.py create mode 100755 examples/online_serving/replica_data_parallel/run_server.sh create mode 100644 examples/online_serving/replica_data_parallel/wan2_2_ti2v_dp.yaml diff --git a/examples/online_serving/replica_data_parallel/README.md b/examples/online_serving/replica_data_parallel/README.md new file mode 100644 index 000000000..4bc98d6f7 --- /dev/null +++ b/examples/online_serving/replica_data_parallel/README.md @@ -0,0 +1,91 @@ +# Replica Data Parallelism for Video Generation + +A runnable recipe + benchmark for **replica data parallelism** (replica-DP) on a +video DiT. Replica-DP runs `N` independent diffusion engine replicas, one per GPU, +and routes each request to a single replica. It scales **request throughput** +near-linearly; it does **not** change single-request latency (for that, use an +intra-request axis such as tensor/sequence/CFG parallelism). + +This example uses `Wan-AI/Wan2.2-TI2V-5B-Diffusers` with one GPU per replica. + +## Files + +| File | Purpose | +|------|---------| +| `wan2_2_ti2v_dp.yaml` | Diffusion stage config; `runtime.num_replicas` + `runtime.devices` drive the fan-out. | +| `run_server.sh` | Substitutes `NUM_REPLICAS` / `DEVICES` into the config and serves. | +| `bench_replica_dp.py` | Replica-agnostic load driver; reports throughput, latency, and a per-input isolation check. | + +## Run + +Start the server with the replica count you want (one GPU per replica): + +```bash +# baseline +NUM_REPLICAS=1 DEVICES=0 ./run_server.sh +# 2 replicas +NUM_REPLICAS=2 DEVICES=0,1 ./run_server.sh +# 4 replicas +NUM_REPLICAS=4 DEVICES=0,1,2,3 ./run_server.sh +``` + +In another shell, drive a fixed workload and read the throughput: + +```bash +python bench_replica_dp.py --url http://127.0.0.1:8098 \ + --num-requests 28 --concurrency 8 --label "replicas=4" +``` + +Re-run the server at `NUM_REPLICAS=1,2,4` (same client workload each time) and +compare `videos/min`. Pass `--baseline-tpm ` to print a scaling factor. + +### Isolation check + +Identical outputs under a *shared* prompt+seed prove nothing -- every request +would be identical regardless of isolation. Instead each request gets a distinct +input (a prompt from the pool + a unique seed `base_seed+i`), and we check that a +request's output is unchanged whether it runs alone or concurrently amid other +distinct requests. Capture a per-input baseline on `N=1`, then verify it under +`N>=2` concurrency: + +```bash +# 1) against NUM_REPLICAS=1 (serial), record each input's output hash +python bench_replica_dp.py --num-requests 8 --concurrency 1 \ + --write-baseline baseline.json + +# 2) against NUM_REPLICAS>=2, replay the SAME inputs concurrently and verify +python bench_replica_dp.py --num-requests 8 --concurrency 8 \ + --check-baseline baseline.json +``` + +Step 2 exits non-zero if any request's output differs from its own baseline (a +mismatch means cross-request state bleed or misrouting). Keep `--num-requests`, +`--base-seed`, and the prompt source identical between the two runs so the input +sets match. + +## Measured scaling + +Wan2.2-TI2V-5B (832×480, 33 frames, 30 steps), 4× A800-80GB (NVLink), one GPU per replica: + +| Replicas | Throughput (videos/min) | Scaling | Efficiency | +|----------|-------------------------|---------|------------| +| 1 | 4.71 | 1.00× | — | +| 2 | 9.20 | 1.95× | 98% | +| 4 | 18.02 | 3.83× | 96% | + +Per-request latency stayed flat (~13–14 s) across all replica counts. Isolation +is checked per-input via the baseline/verify protocol above: each request's +output under `N>=2` concurrency must match its own single-replica baseline. + +## Notes on the config surface + +- `runtime.num_replicas: N` fans out `N` replicas; `runtime.devices` assigns their + GPUs. With `tensor_parallel_size = 1`, list one GPU per replica + (`num_replicas * tensor_parallel_size` entries, pool mode). +- Replica fan-out here comes from the stage config, not a CLI flag. (The headless / + multi-runtime launch path uses the process-local `--omni-dp-size-local`, which + requires `--stage-id`.) +- This recipe serves via `--stage-configs-path`. Expressing replica-DP under the + newer `--deploy-config` schema is part of the ongoing serving-config alignment + (see the diffusion parallelism guide and the `#4707` discussion); `num_replicas` + is accepted by both paths. diff --git a/examples/online_serving/replica_data_parallel/bench_replica_dp.py b/examples/online_serving/replica_data_parallel/bench_replica_dp.py new file mode 100755 index 000000000..977f475ac --- /dev/null +++ b/examples/online_serving/replica_data_parallel/bench_replica_dp.py @@ -0,0 +1,241 @@ +#!/usr/bin/env python3 +"""Throughput-scaling + isolation benchmark for diffusion replica data parallelism. + +Sends `--num-requests` text-to-video requests (up to `--concurrency` in flight) +to a running vLLM-Omni video server via `POST /v1/videos/sync` (blocks until the +clip is produced) and reports: + + completed / failed, makespan (wall clock), throughput (videos/min), + per-request latency p50/p90/max (nearest-rank), and -- when a baseline + manifest is supplied -- a per-request isolation check. + +Each request gets a *distinct* input (a prompt from the pool + a unique seed), +so an output is a deterministic function of its own input and nothing else. + +Measuring replica scaling +------------------------- +This client is replica-agnostic. Vary the *server's* replica count +(`runtime.num_replicas` in the stage config; see run_server.sh) and keep the +client workload fixed, then compare throughput: + + N=1 -> baseline + N=2 -> expect ~2x + N=4 -> expect ~4x (near-linear is the goal) + +Pass the single-replica videos/min via `--baseline-tpm` to print a scaling +factor for the current run. + +Isolation / correctness +----------------------- +Byte-identical outputs under a *shared* prompt+seed prove nothing -- every +request would be identical regardless of isolation. Instead, give each request a +distinct input and check that its output is unchanged whether it runs alone or +concurrently amid other distinct requests: + + 1. Against an N=1 server, capture a per-input baseline: + bench_replica_dp.py --num-requests 8 --concurrency 1 \ + --write-baseline baseline.json + 2. Against an N>=2 server, replay the *same* inputs concurrently and verify + each output matches its own baseline: + bench_replica_dp.py --num-requests 8 --concurrency 8 \ + --check-baseline baseline.json + +A mismatch means a request's output changed under concurrency -- cross-request +state bleed or misrouting -- and the run exits non-zero. + +Requires: requests (`pip install requests`). +""" + +import argparse +import concurrent.futures as cf +import hashlib +import json +import math +import sys +import time +from pathlib import Path + +import requests + +# Distinct prompts, cycled across requests; combined with a per-request unique +# seed this makes every request a distinct input (see input_key()). +DEFAULT_PROMPTS = [ + "A cat playing piano, cinematic, high detail", + "A drone shot over a snowy mountain range at sunrise", + "A red sports car driving through a neon-lit city at night", + "Ocean waves crashing on a rocky shore, slow motion", + "A hot air balloon floating over green rolling hills", + "Timelapse of clouds moving over a desert canyon at dusk", + "A steam train crossing a stone viaduct in autumn", + "A jellyfish drifting through deep blue water, backlit", +] + + +def input_key(prompt, seed, args): + """Stable identity of a request's input; matches baseline<->verify by content, not order.""" + return "\x1f".join([prompt, str(seed), args.size, str(args.num_frames), str(args.steps)]) + + +def nearest_rank(xs, q): + """Nearest-rank percentile; q in [0, 1]. Correct for tiny samples (unlike int(q*n)-1).""" + if not xs: + return float("nan") + s = sorted(xs) + rank = max(1, math.ceil(q * len(s))) + return s[rank - 1] + + +def one_request(args, spec, idx, save): + """Fire one /v1/videos/sync request; return (ok, latency, nbytes, md5, key, err).""" + prompt, seed = spec["prompt"], spec["seed"] + key = input_key(prompt, seed, args) + # multipart form fields match POST /v1/videos/sync in the omni API server + form = { + "prompt": (None, prompt), + "size": (None, args.size), # e.g. "832x480" + "num_frames": (None, str(args.num_frames)), + "num_inference_steps": (None, str(args.steps)), + "seed": (None, str(seed)), # per-request unique -> distinct, reproducible input + } + if args.model: + form["model"] = (None, args.model) + if args.negative_prompt: + form["negative_prompt"] = (None, args.negative_prompt) + + t0 = time.perf_counter() + try: + r = requests.post(f"{args.url}/v1/videos/sync", files=form, timeout=args.timeout) + dt = time.perf_counter() - t0 + if r.status_code != 200: + return (False, dt, 0, "", key, f"HTTP {r.status_code}: {r.text[:160]}") + body = r.content + md5 = hashlib.md5(body).hexdigest() + if save and args.save_dir: + Path(args.save_dir).mkdir(parents=True, exist_ok=True) + (Path(args.save_dir) / f"req{idx:03d}_seed{seed}.mp4").write_bytes(body) + return (True, dt, len(body), md5, key, "") + except Exception as e: # noqa: BLE001 - report any client-side failure + return (False, time.perf_counter() - t0, 0, "", key, repr(e)) + + +def build_specs(args): + """Deterministic list of distinct (prompt, seed) inputs; identical across baseline/verify runs.""" + prompts = DEFAULT_PROMPTS + if args.prompts_file: + prompts = [ln.strip() for ln in Path(args.prompts_file).read_text().splitlines() if ln.strip()] + return [{"prompt": prompts[i % len(prompts)], "seed": args.base_seed + i} for i in range(args.num_requests)] + + +def run_batch(args, specs, save): + """Fire specs concurrently; return (results, makespan). results = list of one_request tuples.""" + results = [] + wall0 = time.perf_counter() + with cf.ThreadPoolExecutor(max_workers=args.concurrency) as ex: + futs = {ex.submit(one_request, args, spec, i, save): i for i, spec in enumerate(specs)} + for done, f in enumerate(cf.as_completed(futs), 1): + res = f.result() + results.append(res) + good, dt, nbytes, _md5, _key, err = res + print(f" [{done}/{len(specs)}] {'ok' if good else 'FAIL'} {dt:6.1f}s {(nbytes / 1e6):5.1f}MB {err}") + return results, time.perf_counter() - wall0 + + +def main(): + ap = argparse.ArgumentParser(description="Diffusion replica-DP throughput + isolation benchmark") + ap.add_argument("--url", default="http://127.0.0.1:8098", help="server base URL (no path)") + ap.add_argument("--num-requests", type=int, default=16, help="total requests to send") + ap.add_argument("--concurrency", type=int, default=4, help="max in-flight (set >= replica count)") + ap.add_argument("--warmup", type=int, default=-1, help="warmup requests before timing (default: =concurrency)") + ap.add_argument("--base-seed", type=int, default=42, help="request i uses seed base_seed+i (distinct inputs)") + ap.add_argument("--prompts-file", default="", help="optional file, one prompt per line (else a built-in pool)") + ap.add_argument("--negative-prompt", default="") + ap.add_argument("--size", default="832x480", help="WIDTHxHEIGHT") + ap.add_argument("--num-frames", type=int, default=33) + ap.add_argument("--steps", type=int, default=30, help="num_inference_steps") + ap.add_argument("--model", default="", help="optional; defaults to the served model") + ap.add_argument("--timeout", type=float, default=1800, help="per-request timeout (s)") + ap.add_argument("--save-dir", default="", help="save returned clips") + ap.add_argument("--label", default="", help="run label, e.g. 'replicas=2'") + ap.add_argument( + "--baseline-tpm", type=float, default=0.0, help="single-replica videos/min, to print a scaling factor" + ) + ap.add_argument("--write-baseline", default="", help="run against N=1: write per-input md5 manifest to this path") + ap.add_argument("--check-baseline", default="", help="run against N>=2: verify each output matches this manifest") + args = ap.parse_args() + + if args.write_baseline and args.check_baseline: + ap.error("--write-baseline and --check-baseline are mutually exclusive") + + specs = build_specs(args) + warmup = args.concurrency if args.warmup < 0 else args.warmup + + print( + f"-> {args.url}/v1/videos/sync | {args.num_requests} reqs " + f"| concurrency {args.concurrency} | warmup {warmup} | {args.size} {args.num_frames}f {args.steps}steps " + f"| {args.label}" + ) + + # Warmup: absorb each replica's first-call compile/init cost so it does not + # skew the measured makespan. Results are discarded and never saved. + if warmup > 0: + print(f"warming up ({warmup} reqs, not timed)...") + run_batch(args, [specs[i % len(specs)] for i in range(warmup)], save=False) + + results, makespan = run_batch(args, specs, save=True) + + ok = sum(1 for r in results if r[0]) + fail = len(results) - ok + lat = [r[1] for r in results if r[0]] + errs = [r[5] for r in results if not r[0]] + got = {r[4]: r[3] for r in results if r[0]} # input_key -> md5 + + print(f"\n===== result {args.label} =====") + print(f"completed {ok} / failed {fail} | makespan {makespan:.1f}s") + if ok: + tpm = ok / makespan * 60.0 + print(f"throughput: {tpm:.2f} videos/min ({ok / makespan:.4f} videos/s)") + print( + f"latency (s): p50={nearest_rank(lat, 0.50):.1f} " + f"p90={nearest_rank(lat, 0.90):.1f} " + f"min={min(lat):.1f} max={max(lat):.1f}" + ) + if args.baseline_tpm > 0: + print(f"scaling vs baseline ({args.baseline_tpm:.2f}/min): {tpm / args.baseline_tpm:.2f}x") + if errs: + print("sample failures:", errs[:3]) + + # Isolation check: compare each request's output against its own single-replica baseline. + isolation_failed = False + if args.write_baseline: + Path(args.write_baseline).write_text(json.dumps(got, indent=2)) + print(f"baseline: wrote {len(got)} per-input hashes -> {args.write_baseline}") + elif args.check_baseline: + baseline = json.loads(Path(args.check_baseline).read_text()) + matched = mismatched = unknown = 0 + for key, md5 in got.items(): + if key not in baseline: + unknown += 1 + print(f" isolation: NO BASELINE for input {key!r} (not in the baseline manifest)") + elif baseline[key] == md5: + matched += 1 + else: + mismatched += 1 + print(f" isolation: MISMATCH for input {key!r} (output changed under concurrency)") + # PASS requires the run to cover the *whole* baseline: a subset (fewer requests, or + # some failed) must not pass just because the inputs it did run happened to match. + unverified = sorted(set(baseline) - set(got)) + for key in unverified: + print(f" isolation: NOT VERIFIED for baseline input {key!r} (absent from this run)") + isolation_failed = mismatched > 0 or unknown > 0 or len(unverified) > 0 + verdict = "PASS (isolated)" if not isolation_failed else "FAIL" + print( + f"isolation: {matched} matched / {mismatched} mismatched / {unknown} unknown / " + f"{len(unverified)} unverified vs baseline({len(baseline)}) -> {verdict}" + ) + + if fail > 0 or ok == 0 or isolation_failed: + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/examples/online_serving/replica_data_parallel/run_server.sh b/examples/online_serving/replica_data_parallel/run_server.sh new file mode 100755 index 000000000..915732eee --- /dev/null +++ b/examples/online_serving/replica_data_parallel/run_server.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +# Serve N independent Wan2.2 TI2V-5B replicas (replica data parallelism). +# +# Usage: +# NUM_REPLICAS=1 DEVICES=0 ./run_server.sh # baseline +# NUM_REPLICAS=2 DEVICES=0,1 ./run_server.sh +# NUM_REPLICAS=4 DEVICES=0,1,2,3 ./run_server.sh +# +# Then drive load with bench_replica_dp.py (see README.md) and compare +# throughput across replica counts. +set -euo pipefail + +MODEL="${MODEL:-Wan-AI/Wan2.2-TI2V-5B-Diffusers}" +HOST="${HOST:-127.0.0.1}" +PORT="${PORT:-8098}" +NUM_REPLICAS="${NUM_REPLICAS:-1}" +DEVICES="${DEVICES:-0}" # one GPU per replica for tensor_parallel_size=1 + +HERE="$(cd "$(dirname "$0")" && pwd)" +TEMPLATE="$HERE/wan2_2_ti2v_dp.yaml" +CFG="$(mktemp --suffix=_wan2_2_dp.yaml)" +trap 'rm -f "$CFG"' EXIT + +sed -e "s/__NUM_REPLICAS__/${NUM_REPLICAS}/" \ + -e "s/__DEVICES__/${DEVICES}/" \ + "$TEMPLATE" > "$CFG" + +echo "Serving ${MODEL}: num_replicas=${NUM_REPLICAS} devices=${DEVICES} (config ${CFG})" + +vllm serve "$MODEL" --omni \ + --host "$HOST" \ + --port "$PORT" \ + --stage-configs-path "$CFG" diff --git a/examples/online_serving/replica_data_parallel/wan2_2_ti2v_dp.yaml b/examples/online_serving/replica_data_parallel/wan2_2_ti2v_dp.yaml new file mode 100644 index 000000000..b472d7e2d --- /dev/null +++ b/examples/online_serving/replica_data_parallel/wan2_2_ti2v_dp.yaml @@ -0,0 +1,43 @@ +# Replica data-parallel recipe for Wan2.2 TI2V-5B (video DiT). +# +# Fans out `num_replicas` independent DiffusionEngine replicas, one per GPU, +# and lets the serving layer route each request to a single replica. This +# scales request throughput near-linearly; it does not change single-request +# latency. See ./README.md for measured numbers and the `run_server.sh` driver. +# +# run_server.sh substitutes __NUM_REPLICAS__ / __DEVICES__ from env before serving. +# For tensor_parallel_size = 1, `devices` lists one GPU per replica (pool mode): +# num_replicas=4 -> devices "0,1,2,3". + +stage_args: + - stage_id: 0 + stage_type: diffusion + runtime: + num_replicas: __NUM_REPLICAS__ + devices: "__DEVICES__" + max_batch_size: 1 + engine_args: + model_stage: dit + # `model_class_name` takes the diffusion registry KEY (WanPipeline), which + # resolves to the Wan22Pipeline class. Wan2.2 TI2V-5B is served by the base + # WanPipeline, which auto-detects TI2V-5B mode from model_index.json. + model_class_name: WanPipeline + max_num_seqs: 1 + enforce_eager: true + trust_remote_code: true + distributed_executor_backend: "mp" + vae_use_tiling: true + parallel_config: + tensor_parallel_size: 1 + + final_output: true + final_output_type: video + is_comprehension: false + default_sampling_params: + seed: 42 + +runtime: + enabled: true + defaults: + window_size: -1 + max_inflight: 1 diff --git a/vllm_omni/entrypoints/cli/serve.py b/vllm_omni/entrypoints/cli/serve.py index 5d435bc2f..c1ca73e44 100644 --- a/vllm_omni/entrypoints/cli/serve.py +++ b/vllm_omni/entrypoints/cli/serve.py @@ -148,7 +148,8 @@ class OmniServeCommand(CLISubcommand): "The following CLI args are not supported under --omni: " f"{', '.join(offenders)}. Configure parallelism through the " "per-stage YAML (`--deploy-config` / `--stage-configs-path`) " - "and replica count via `--omni-dp-size-local`." + "and replica count via the per-stage `num_replicas` config field " + "(single-runtime) or `--omni-dp-size-local` (headless / multi-runtime)." ) # --omni-lb-policy is validated against the LoadBalancingPolicy enum. From 7aa5c9a0901b7b9254052c4d342d0c3fa447eb95 Mon Sep 17 00:00:00 2001 From: Weiming Liao Date: Fri, 17 Jul 2026 17:32:48 +0800 Subject: [PATCH 009/240] [CI][NPU]: Add nightly performance test for NPU - Qwen3-TTS on A3 (#5158) Signed-off-by: Weiming Liao Co-authored-by: wangyu <53896905+yenuo26@users.noreply.github.com> --- .buildkite/pipeline-npu.yaml | 87 +++++++++++++++++++++++++++++- .buildkite/test-npu-nightly.yaml | 28 ++++++++++ docker/Dockerfile.npu.ci | 2 + docker/Dockerfile.npu.ci.a3 | 2 + tests/dfx/perf/tests/test_tts.json | 3 +- 5 files changed, 119 insertions(+), 3 deletions(-) diff --git a/.buildkite/pipeline-npu.yaml b/.buildkite/pipeline-npu.yaml index 161b0e944..d035ffc94 100644 --- a/.buildkite/pipeline-npu.yaml +++ b/.buildkite/pipeline-npu.yaml @@ -78,8 +78,89 @@ steps: echo "--- Image pushed successfully" echo "$${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG}" + - label: ":buildkit: Build and Push NPU Test Image - A3" + key: image-build-a3 + agents: + queue: "ascend-a2b3" + resource_class: "npu-1" + plugins: + - kubernetes: + metadata: + annotations: + vault.hashicorp.com/agent-init-first: "true" + vault.hashicorp.com/agent-inject: "true" + vault.hashicorp.com/agent-inject-perms-ca.pem: "0400" + vault.hashicorp.com/agent-inject-perms-cert.pem: "0400" + vault.hashicorp.com/agent-inject-perms-config.json: "0400" + vault.hashicorp.com/agent-inject-perms-key.pem: "0400" + vault.hashicorp.com/agent-inject-secret-ca.pem: internal/data/ascend/buildkitd + vault.hashicorp.com/agent-inject-secret-cert.pem: internal/data/ascend/buildkitd + vault.hashicorp.com/agent-inject-secret-config.json: internal/data/ascend/buildkitd + vault.hashicorp.com/agent-inject-secret-key.pem: internal/data/ascend/buildkitd + vault.hashicorp.com/agent-inject-template-ca.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.RootCA }}\n{{- end }}" + vault.hashicorp.com/agent-inject-template-cert.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.ClientCaCert }}\n{{- end }}" + vault.hashicorp.com/agent-inject-template-config.json: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.dockerConfig }}\n{{- end }}" + vault.hashicorp.com/agent-inject-template-key.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.ClientCaKey }}\n{{- end }}" + vault.hashicorp.com/agent-pre-populate-only: "true" + vault.hashicorp.com/agent-run-as-group: "1000" + vault.hashicorp.com/agent-run-as-user: "1000" + vault.hashicorp.com/agent-service-account-token-volume-name: token-vol + vault.hashicorp.com/role: ascend-gha-runners + vault.hashicorp.com/secret-volume-path: /home/user/.docker/ + vault.hashicorp.com/tls-skip-verify: "true" + podSpecPatch: + volumes: + - name: token-vol + projected: + defaultMode: 420 + sources: + - serviceAccountToken: + audience: api + expirationSeconds: 600 + path: token + env: + VLLM_IMAGE_TAG: "${BUILDKITE_COMMIT}-a3" + IMAGE_NAME: "vllm-omni-ci-npu" + IMAGE_REGISTRY: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry" + BUILDKITD_ADDR: "tcp://buildkitd-service.buildkitd:1234" + command: | + set -ex + + echo "--- Building and pushing NPU Test Image" + echo "Image: $${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG}" + echo "buildkitd address: $${BUILDKITD_ADDR}" + + if ! command -v buildctl &> /dev/null; then + echo "Installing buildctl..." + mkdir -p /tmp/buildkit + BUILDKIT_VERSION="v0.29.0" + wget -q "https://github.com/moby/buildkit/releases/download/$${BUILDKIT_VERSION}/buildkit-$${BUILDKIT_VERSION}.linux-arm64.tar.gz" -O /tmp/buildkit.tar.gz + tar -xzf /tmp/buildkit.tar.gz -C /tmp/buildkit + cp /tmp/buildkit/bin/buildctl /usr/local/bin/ + fi + + export DOCKER_CONFIG=/home/user/.docker + buildctl \ + --addr="$${BUILDKITD_ADDR}" \ + --tlscacert=/home/user/.docker/ca.pem \ + --tlscert=/home/user/.docker/cert.pem \ + --tlskey=/home/user/.docker/key.pem \ + build \ + --frontend dockerfile.v0 \ + --local context=. \ + --local dockerfile=./docker \ + --opt filename=Dockerfile.npu.ci.a3 \ + --secret id=dockerconfig,src=/home/user/.docker/config.json \ + --output type=image,name=$${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG},push=true \ + --progress=plain + + echo "--- Image pushed successfully" + echo "$${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG}" + - label: "Upload Ready Pipeline" - depends_on: image-build + depends_on: + - image-build + - image-build-a3 key: upload-ready-pipeline if: >- build.branch != "main" && ( @@ -94,7 +175,9 @@ steps: # L4 Test — main+NIGHTLY=1 (scheduled), or PR with specific label (e.g. add label then Rebuild) - label: "Upload Nightly Pipeline" - depends_on: image-build + depends_on: + - image-build + - image-build-a3 key: upload-nightly-pipeline if: >- (build.branch == "main" && build.env("NIGHTLY") == "1") || diff --git a/.buildkite/test-npu-nightly.yaml b/.buildkite/test-npu-nightly.yaml index 2c2372f4c..4dbda17f9 100644 --- a/.buildkite/test-npu-nightly.yaml +++ b/.buildkite/test-npu-nightly.yaml @@ -22,3 +22,31 @@ steps: HF_TOKEN: "${HF_TOKEN}" commands: - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k npu_i2v --run-level "full_model" + - group: ":card_index_dividers: TTS Model Test" + key: nightly-tts-test-group + depends_on: upload-nightly-pipeline + if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" || build.pull_request.labels includes "tts-test" + steps: + - label: ":full_moon: TTS · Perf Test" + agents: + queue: "ascend-a3" + resource_class: "npu-2" + image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}-a3" + plugins: + - kubernetes: + podSpecPatch: + imagePullSecrets: + - name: swr-secret + env: + VLLM_WORKER_MULTIPROC_METHOD: spawn + VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1" + HF_TOKEN: "${HF_TOKEN}" + commands: + - export BENCHMARK_DIR=tests/dfx/perf/results + - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1" + - | + set +e + pytest -s -v tests/dfx/perf/scripts/run_benchmark.py -m npu --test-config-file tests/dfx/perf/tests/test_tts.json + EXIT=$$? + buildkite-agent artifact upload "tests/dfx/perf/results/*.json" + exit $$EXIT diff --git a/docker/Dockerfile.npu.ci b/docker/Dockerfile.npu.ci index f5c3cdfaf..a3cede3a7 100644 --- a/docker/Dockerfile.npu.ci +++ b/docker/Dockerfile.npu.ci @@ -6,6 +6,8 @@ ARG APP_DIR=/vllm-workspace/vllm-omni WORKDIR ${APP_DIR} COPY . . +RUN sed -i "s@^\(deb.*\)https\?://[a-z0-9.-]*\.ubuntu\.com@\1http://cache-service.nginx-pypi-cache.svc.cluster.local:8081@g" /etc/apt/sources.list + RUN apt-get update && \ apt-get install -y --no-install-recommends \ ffmpeg \ diff --git a/docker/Dockerfile.npu.ci.a3 b/docker/Dockerfile.npu.ci.a3 index 6a8ac4678..1369d8c85 100644 --- a/docker/Dockerfile.npu.ci.a3 +++ b/docker/Dockerfile.npu.ci.a3 @@ -6,6 +6,8 @@ ARG APP_DIR=/vllm-workspace/vllm-omni WORKDIR ${APP_DIR} COPY . . +RUN sed -i "s@^\(deb.*\)https\?://[a-z0-9.-]*\.ubuntu\.com@\1http://cache-service.nginx-pypi-cache.svc.cluster.local:8081@g" /etc/apt/sources.list + RUN apt-get update && \ apt-get install -y --no-install-recommends \ ffmpeg \ diff --git a/tests/dfx/perf/tests/test_tts.json b/tests/dfx/perf/tests/test_tts.json index 3bbf60935..85236eab7 100644 --- a/tests/dfx/perf/tests/test_tts.json +++ b/tests/dfx/perf/tests/test_tts.json @@ -5,7 +5,8 @@ { "hardware_marks": { "res": { - "cuda": "H100" + "cuda": "H100", + "npu": "A3" }, "num_cards": 2 } From d09f549e862c58ca87c195d4050e77b20a126a38 Mon Sep 17 00:00:00 2001 From: bastefaniak Date: Fri, 17 Jul 2026 20:38:04 +0200 Subject: [PATCH 010/240] [bugfix] Use FlowMatchEulerDiscreteScheduler from diffusers 0.39.0 with correct RNG handling for upcoming Cosmos3 distilled models (#5176) Signed-off-by: Bartosz Stefaniak --- .../models/cosmos3/pipeline_cosmos3.py | 4 +- .../diffusion/models/schedulers/__init__.py | 4 + .../scheduling_flow_match_euler_discrete.py | 660 ++++++++++++++++++ 3 files changed, 667 insertions(+), 1 deletion(-) create mode 100644 vllm_omni/diffusion/models/schedulers/scheduling_flow_match_euler_discrete.py diff --git a/vllm_omni/diffusion/models/cosmos3/pipeline_cosmos3.py b/vllm_omni/diffusion/models/cosmos3/pipeline_cosmos3.py index 58e74ccf4..41a3e80fc 100644 --- a/vllm_omni/diffusion/models/cosmos3/pipeline_cosmos3.py +++ b/vllm_omni/diffusion/models/cosmos3/pipeline_cosmos3.py @@ -40,7 +40,6 @@ from typing import Any, ClassVar import numpy as np import PIL.Image import torch -from diffusers import FlowMatchEulerDiscreteScheduler from diffusers.utils.torch_utils import randn_tensor from diffusers.video_processor import VideoProcessor from torch import nn @@ -62,6 +61,9 @@ from vllm_omni.diffusion.models.interface import ( SupportsComponentDiscovery, ) from vllm_omni.diffusion.models.progress_bar import ProgressBarMixin, _is_rank_zero +from vllm_omni.diffusion.models.schedulers.scheduling_flow_match_euler_discrete import ( + FlowMatchEulerDiscreteScheduler, +) from vllm_omni.diffusion.models.schedulers.scheduling_flow_unipc_multistep import ( FlowUniPCMultistepScheduler, ) diff --git a/vllm_omni/diffusion/models/schedulers/__init__.py b/vllm_omni/diffusion/models/schedulers/__init__.py index e683ed272..11aedcb1d 100644 --- a/vllm_omni/diffusion/models/schedulers/__init__.py +++ b/vllm_omni/diffusion/models/schedulers/__init__.py @@ -2,11 +2,15 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from vllm_omni.diffusion.models.schedulers.scheduling_dmd2_euler import DMD2EulerScheduler +from vllm_omni.diffusion.models.schedulers.scheduling_flow_match_euler_discrete import ( + FlowMatchEulerDiscreteScheduler, +) from vllm_omni.diffusion.models.schedulers.scheduling_flow_unipc_multistep import ( FlowUniPCMultistepScheduler, ) __all__ = [ "DMD2EulerScheduler", + "FlowMatchEulerDiscreteScheduler", "FlowUniPCMultistepScheduler", ] diff --git a/vllm_omni/diffusion/models/schedulers/scheduling_flow_match_euler_discrete.py b/vllm_omni/diffusion/models/schedulers/scheduling_flow_match_euler_discrete.py new file mode 100644 index 000000000..8aa7d3a00 --- /dev/null +++ b/vllm_omni/diffusion/models/schedulers/scheduling_flow_match_euler_discrete.py @@ -0,0 +1,660 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project +# Originally from https://github.com/huggingface/diffusers/blob/v0.39.0/src/diffusers/schedulers/scheduling_flow_match_euler_discrete.py +# Copyright 2025 Stability AI, Katherine Crowson and The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Flow Match Euler scheduler copied from Diffusers v0.39.0. + +Modified by vLLM-Omni contributors for local integration and RNG compatibility. +""" + +import math +from dataclasses import dataclass +from typing import Literal + +import numpy as np +import torch +from diffusers.configuration_utils import ConfigMixin, register_to_config +from diffusers.schedulers.scheduling_utils import SchedulerMixin +from diffusers.utils import BaseOutput, is_scipy_available, logging +from diffusers.utils.torch_utils import randn_tensor + +if is_scipy_available(): + import scipy.stats + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +@dataclass +class FlowMatchEulerDiscreteSchedulerOutput(BaseOutput): + """ + Output class for the scheduler's `step` function output. + + Args: + prev_sample (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)` for images): + Computed sample `(x_{t-1})` of previous timestep. `prev_sample` should be used as next model input in the + denoising loop. + """ + + prev_sample: torch.FloatTensor + + +class FlowMatchEulerDiscreteScheduler(SchedulerMixin, ConfigMixin): + """ + Euler scheduler. + + This model inherits from [`SchedulerMixin`] and [`ConfigMixin`]. Check the superclass documentation for the generic + methods the library implements for all schedulers such as loading and saving. + + Args: + num_train_timesteps (`int`, defaults to 1000): + The number of diffusion steps to train the model. + shift (`float`, defaults to 1.0): + The shift value for the timestep schedule. + use_dynamic_shifting (`bool`, defaults to False): + Whether to apply timestep shifting on-the-fly based on the image resolution. + base_shift (`float`, defaults to 0.5): + Value to stabilize image generation. Increasing `base_shift` reduces variation and image is more consistent + with desired output. + max_shift (`float`, defaults to 1.15): + Value change allowed to latent vectors. Increasing `max_shift` encourages more variation and image may be + more exaggerated or stylized. + base_image_seq_len (`int`, defaults to 256): + The base image sequence length. + max_image_seq_len (`int`, defaults to 4096): + The maximum image sequence length. + invert_sigmas (`bool`, defaults to False): + Whether to invert the sigmas. + shift_terminal (`float`, defaults to None): + The end value of the shifted timestep schedule. + use_karras_sigmas (`bool`, defaults to False): + Whether to use Karras sigmas for step sizes in the noise schedule during sampling. + use_exponential_sigmas (`bool`, defaults to False): + Whether to use exponential sigmas for step sizes in the noise schedule during sampling. + use_beta_sigmas (`bool`, defaults to False): + Whether to use beta sigmas for step sizes in the noise schedule during sampling. + time_shift_type (`str`, defaults to "exponential"): + The type of dynamic resolution-dependent timestep shifting to apply. Either "exponential" or "linear". + stochastic_sampling (`bool`, defaults to False): + Whether to use stochastic sampling. + """ + + _compatibles = [] + order = 1 + + @register_to_config + def __init__( + self, + num_train_timesteps: int = 1000, + shift: float = 1.0, + use_dynamic_shifting: bool = False, + base_shift: float | None = 0.5, + max_shift: float | None = 1.15, + base_image_seq_len: int = 256, + max_image_seq_len: int = 4096, + invert_sigmas: bool = False, + shift_terminal: float = None, + use_karras_sigmas: bool = False, + use_exponential_sigmas: bool = False, + use_beta_sigmas: bool = False, + time_shift_type: Literal["exponential", "linear"] = "exponential", + stochastic_sampling: bool = False, + ): + if self.config.use_beta_sigmas and not is_scipy_available(): + raise ImportError("Make sure to install scipy if you want to use beta sigmas.") + if ( + sum( + [ + self.config.use_beta_sigmas, + self.config.use_exponential_sigmas, + self.config.use_karras_sigmas, + ] + ) + > 1 + ): + raise ValueError( + "Only one of `config.use_beta_sigmas`, `config.use_exponential_sigmas`, " + "`config.use_karras_sigmas` can be used." + ) + if time_shift_type not in {"exponential", "linear"}: + raise ValueError("`time_shift_type` must either be 'exponential' or 'linear'.") + + timesteps = np.linspace(1, num_train_timesteps, num_train_timesteps, dtype=np.float32)[::-1].copy() + timesteps = torch.from_numpy(timesteps).to(dtype=torch.float32) + + sigmas = timesteps / num_train_timesteps + if not use_dynamic_shifting: + # when use_dynamic_shifting is True, we apply the timestep shifting on the fly based on the image resolution + sigmas = shift * sigmas / (1 + (shift - 1) * sigmas) + + self.timesteps = sigmas * num_train_timesteps + + self._step_index = None + self._begin_index = None + + self._shift = shift + + self.sigmas = sigmas.to("cpu") # to avoid too much CPU/GPU communication + self.sigma_min = self.sigmas[-1].item() + self.sigma_max = self.sigmas[0].item() + + @property + def shift(self): + """ + The value used for shifting. + """ + return self._shift + + @property + def step_index(self): + """ + The index counter for current timestep. It will increase 1 after each scheduler step. + """ + return self._step_index + + @property + def begin_index(self): + """ + The index for the first timestep. It should be set from pipeline with `set_begin_index` method. + """ + return self._begin_index + + # Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler.set_begin_index + def set_begin_index(self, begin_index: int = 0): + """ + Sets the begin index for the scheduler. This function should be run from pipeline before the inference. + + Args: + begin_index (`int`, defaults to `0`): + The begin index for the scheduler. + """ + self._begin_index = begin_index + + def set_shift(self, shift: float): + """ + Sets the shift value for the scheduler. + + Args: + shift (`float`): + The shift value to be set. + """ + self._shift = shift + + def scale_noise( + self, + sample: torch.FloatTensor, + timestep: float | torch.FloatTensor, + noise: torch.FloatTensor | None = None, + ) -> torch.FloatTensor: + """ + Forward process in flow-matching + + Args: + sample (`torch.FloatTensor`): + The input sample. + timestep (`torch.FloatTensor`): + The current timestep in the diffusion chain. + noise (`torch.FloatTensor`): + The noise tensor. + + Returns: + `torch.FloatTensor`: + A scaled input sample. + """ + # Make sure sigmas and timesteps have the same device and dtype as original_samples + sigmas = self.sigmas.to(device=sample.device, dtype=sample.dtype) + + if sample.device.type == "mps" and torch.is_floating_point(timestep): + # mps does not support float64 + schedule_timesteps = self.timesteps.to(sample.device, dtype=torch.float32) + timestep = timestep.to(sample.device, dtype=torch.float32) + else: + schedule_timesteps = self.timesteps.to(sample.device) + timestep = timestep.to(sample.device) + + # self.begin_index is None when scheduler is used for training, or pipeline does not implement set_begin_index + if self.begin_index is None: + step_indices = [self.index_for_timestep(t, schedule_timesteps) for t in timestep] + elif self.step_index is not None: + # add_noise is called after first denoising step (for inpainting) + step_indices = [self.step_index] * timestep.shape[0] + else: + # add noise is called before first denoising step to create initial latent(img2img) + step_indices = [self.begin_index] * timestep.shape[0] + + sigma = sigmas[step_indices].flatten() + while len(sigma.shape) < len(sample.shape): + sigma = sigma.unsqueeze(-1) + + sample = sigma * noise + (1.0 - sigma) * sample + + return sample + + def _sigma_to_t(self, sigma) -> float: + return sigma * self.config.num_train_timesteps + + def time_shift(self, mu: float, sigma: float, t: torch.Tensor) -> torch.Tensor: + """ + Apply time shifting to the sigmas. + + Args: + mu (`float`): + The mu parameter for the time shift. + sigma (`float`): + The sigma parameter for the time shift. + t (`torch.Tensor`): + The input timesteps. + + Returns: + `torch.Tensor`: + The time-shifted timesteps. + """ + if self.config.time_shift_type == "exponential": + return self._time_shift_exponential(mu, sigma, t) + elif self.config.time_shift_type == "linear": + return self._time_shift_linear(mu, sigma, t) + + def stretch_shift_to_terminal(self, t: torch.Tensor) -> torch.Tensor: + r""" + Stretches and shifts the timestep schedule to ensure it terminates at the configured `shift_terminal` config + value. + + Reference: + https://github.com/Lightricks/LTX-Video/blob/a01a171f8fe3d99dce2728d60a73fecf4d4238ae/ltx_video/schedulers/rf.py#L51 + + Args: + t (`torch.Tensor`): + A tensor of timesteps to be stretched and shifted. + + Returns: + `torch.Tensor`: + A tensor of adjusted timesteps such that the final value equals `self.config.shift_terminal`. + """ + one_minus_z = 1 - t + scale_factor = one_minus_z[-1] / (1 - self.config.shift_terminal) + stretched_t = 1 - (one_minus_z / scale_factor) + return stretched_t + + def set_timesteps( + self, + num_inference_steps: int | None = None, + device: str | torch.device = None, + sigmas: list[float] | None = None, + mu: float | None = None, + timesteps: list[float] | None = None, + ): + """ + Sets the discrete timesteps used for the diffusion chain (to be run before inference). + + Args: + num_inference_steps (`int`, *optional*): + The number of diffusion steps used when generating samples with a pre-trained model. + device (`str` or `torch.device`, *optional*): + The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. + sigmas (`list[float]`, *optional*): + Custom values for sigmas to be used for each diffusion step. If `None`, the sigmas are computed + automatically. + mu (`float`, *optional*): + Determines the amount of shifting applied to sigmas when performing resolution-dependent timestep + shifting. + timesteps (`list[float]`, *optional*): + Custom values for timesteps to be used for each diffusion step. If `None`, the timesteps are computed + automatically. + """ + if self.config.use_dynamic_shifting and mu is None: + raise ValueError("`mu` must be passed when `use_dynamic_shifting` is set to be `True`") + + if sigmas is not None and timesteps is not None: + if len(sigmas) != len(timesteps): + raise ValueError("`sigmas` and `timesteps` should have the same length") + + if num_inference_steps is not None: + if (sigmas is not None and len(sigmas) != num_inference_steps) or ( + timesteps is not None and len(timesteps) != num_inference_steps + ): + raise ValueError( + "`sigmas` and `timesteps` should have the same length as num_inference_steps, " + "if `num_inference_steps` is provided" + ) + else: + num_inference_steps = len(sigmas) if sigmas is not None else len(timesteps) + + self.num_inference_steps = num_inference_steps + + # 1. Prepare default sigmas + is_timesteps_provided = timesteps is not None + + if is_timesteps_provided: + timesteps = np.array(timesteps).astype(np.float32) + + if sigmas is None: + if timesteps is None: + timesteps = np.linspace( + self._sigma_to_t(self.sigma_max), + self._sigma_to_t(self.sigma_min), + num_inference_steps, + ) + sigmas = timesteps / self.config.num_train_timesteps + else: + sigmas = np.array(sigmas).astype(np.float32) + num_inference_steps = len(sigmas) + + # 2. Perform timestep shifting. Either no shifting is applied, or resolution-dependent shifting of + # "exponential" or "linear" type is applied + if self.config.use_dynamic_shifting: + sigmas = self.time_shift(mu, 1.0, sigmas) + else: + sigmas = self.shift * sigmas / (1 + (self.shift - 1) * sigmas) + + # 3. If required, stretch the sigmas schedule to terminate at the configured `shift_terminal` value + if self.config.shift_terminal: + sigmas = self.stretch_shift_to_terminal(sigmas) + + # 4. If required, convert sigmas to one of karras, exponential, or beta sigma schedules + if self.config.use_karras_sigmas: + sigmas = self._convert_to_karras(in_sigmas=sigmas, num_inference_steps=num_inference_steps) + elif self.config.use_exponential_sigmas: + sigmas = self._convert_to_exponential(in_sigmas=sigmas, num_inference_steps=num_inference_steps) + elif self.config.use_beta_sigmas: + sigmas = self._convert_to_beta(in_sigmas=sigmas, num_inference_steps=num_inference_steps) + + # 5. Convert sigmas and timesteps to tensors and move to specified device + sigmas = torch.from_numpy(sigmas).to(dtype=torch.float32, device=device) + if not is_timesteps_provided: + timesteps = sigmas * self.config.num_train_timesteps + else: + timesteps = torch.from_numpy(timesteps).to(dtype=torch.float32, device=device) + + # 6. Append the terminal sigma value. + # If a model requires inverted sigma schedule for denoising but timesteps without inversion, the + # `invert_sigmas` flag can be set to `True`. This case is only required in Mochi + if self.config.invert_sigmas: + sigmas = 1.0 - sigmas + timesteps = sigmas * self.config.num_train_timesteps + sigmas = torch.cat([sigmas, torch.ones(1, device=sigmas.device)]) + else: + sigmas = torch.cat([sigmas, torch.zeros(1, device=sigmas.device)]) + + self.timesteps = timesteps + self.sigmas = sigmas + self._step_index = None + self._begin_index = None + + def index_for_timestep( + self, + timestep: float | torch.FloatTensor, + schedule_timesteps: torch.FloatTensor | None = None, + ) -> int: + """ + Get the index for the given timestep. + + Args: + timestep (`float` or `torch.FloatTensor`): + The timestep to find the index for. + schedule_timesteps (`torch.FloatTensor`, *optional*): + The schedule timesteps to validate against. If `None`, the scheduler's timesteps are used. + + Returns: + `int`: + The index of the timestep. + """ + if schedule_timesteps is None: + schedule_timesteps = self.timesteps + + indices = (schedule_timesteps == timestep).nonzero() + + # The sigma index that is taken for the **very** first `step` + # is always the second index (or the last index if there is only 1) + # This way we can ensure we don't accidentally skip a sigma in + # case we start in the middle of the denoising schedule (e.g. for image-to-image) + pos = 1 if len(indices) > 1 else 0 + + return indices[pos].item() + + def _init_step_index(self, timestep: float | torch.FloatTensor) -> None: + if self.begin_index is None: + if isinstance(timestep, torch.Tensor): + timestep = timestep.to(self.timesteps.device) + self._step_index = self.index_for_timestep(timestep) + else: + self._step_index = self._begin_index + + def step( + self, + model_output: torch.FloatTensor, + timestep: float | torch.FloatTensor, + sample: torch.FloatTensor, + s_churn: float = 0.0, + s_tmin: float = 0.0, + s_tmax: float = float("inf"), + s_noise: float = 1.0, + generator: torch.Generator | None = None, + per_token_timesteps: torch.Tensor | None = None, + return_dict: bool = True, + ) -> FlowMatchEulerDiscreteSchedulerOutput | tuple: + """ + Predict the sample from the previous timestep by reversing the SDE. This function propagates the diffusion + process from the learned model outputs (most often the predicted noise). + + Args: + model_output (`torch.FloatTensor`): + The direct output from learned diffusion model. + timestep (`float`): + The current discrete timestep in the diffusion chain. + sample (`torch.FloatTensor`): + A current instance of a sample created by the diffusion process. + s_churn (`float`): + s_tmin (`float`): + s_tmax (`float`): + s_noise (`float`, defaults to 1.0): + Scaling factor for noise added to the sample. + generator (`torch.Generator`, *optional*): + A random number generator. + per_token_timesteps (`torch.Tensor`, *optional*): + The timesteps for each token in the sample. + return_dict (`bool`, defaults to `True`): + Whether or not to return a + [`~schedulers.scheduling_flow_match_euler_discrete.FlowMatchEulerDiscreteSchedulerOutput`] or tuple. + + Returns: + [`~schedulers.scheduling_flow_match_euler_discrete.FlowMatchEulerDiscreteSchedulerOutput`] or `tuple`: + If return_dict is `True`, + [`~schedulers.scheduling_flow_match_euler_discrete.FlowMatchEulerDiscreteSchedulerOutput`] is returned, + otherwise a tuple is returned where the first element is the sample tensor. + """ + + if isinstance(timestep, int) or isinstance(timestep, torch.IntTensor) or isinstance(timestep, torch.LongTensor): + raise ValueError( + ( + "Passing integer indices (e.g. from `enumerate(timesteps)`) as timesteps to" + " `FlowMatchEulerDiscreteScheduler.step()` is not supported. Make sure to pass" + " one of the `scheduler.timesteps` as a timestep." + ), + ) + + if self.step_index is None: + self._init_step_index(timestep) + + # Upcast to avoid precision issues when computing prev_sample + sample = sample.to(torch.float32) + + if per_token_timesteps is not None: + per_token_sigmas = per_token_timesteps / self.config.num_train_timesteps + + sigmas = self.sigmas[:, None, None] + lower_mask = sigmas < per_token_sigmas[None] - 1e-6 + lower_sigmas = lower_mask * sigmas + lower_sigmas, _ = lower_sigmas.max(dim=0) + + current_sigma = per_token_sigmas[..., None] + next_sigma = lower_sigmas[..., None] + dt = current_sigma - next_sigma + else: + sigma_idx = self.step_index + sigma = self.sigmas[sigma_idx] + sigma_next = self.sigmas[sigma_idx + 1] + + current_sigma = sigma + next_sigma = sigma_next + dt = sigma_next - sigma + + if self.config.stochastic_sampling: + x0 = sample - current_sigma * model_output + noise = randn_tensor(sample.shape, generator=generator, device=sample.device, dtype=sample.dtype) + prev_sample = (1.0 - next_sigma) * x0 + next_sigma * noise + else: + prev_sample = sample + dt * model_output + + # upon completion increase step index by one + self._step_index += 1 + if per_token_timesteps is None: + # Cast sample back to model compatible dtype + prev_sample = prev_sample.to(model_output.dtype) + + if not return_dict: + return (prev_sample,) + + return FlowMatchEulerDiscreteSchedulerOutput(prev_sample=prev_sample) + + # Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler._convert_to_karras + def _convert_to_karras(self, in_sigmas: torch.Tensor, num_inference_steps: int) -> torch.Tensor: + """ + Construct the noise schedule as proposed in [Elucidating the Design Space of Diffusion-Based Generative + Models](https://huggingface.co/papers/2206.00364). + + Args: + in_sigmas (`torch.Tensor`): + The input sigma values to be converted. + num_inference_steps (`int`): + The number of inference steps to generate the noise schedule for. + + Returns: + `torch.Tensor`: + The converted sigma values following the Karras noise schedule. + """ + + # Hack to make sure that other schedulers which copy this function don't break + # TODO: Add this logic to the other schedulers + if hasattr(self.config, "sigma_min"): + sigma_min = self.config.sigma_min + else: + sigma_min = None + + if hasattr(self.config, "sigma_max"): + sigma_max = self.config.sigma_max + else: + sigma_max = None + + sigma_min = sigma_min if sigma_min is not None else in_sigmas[-1].item() + sigma_max = sigma_max if sigma_max is not None else in_sigmas[0].item() + + rho = 7.0 # 7.0 is the value used in the paper + ramp = np.linspace(0, 1, num_inference_steps) + min_inv_rho = sigma_min ** (1 / rho) + max_inv_rho = sigma_max ** (1 / rho) + sigmas = (max_inv_rho + ramp * (min_inv_rho - max_inv_rho)) ** rho + return sigmas + + # Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler._convert_to_exponential + def _convert_to_exponential(self, in_sigmas: torch.Tensor, num_inference_steps: int) -> torch.Tensor: + """ + Construct an exponential noise schedule. + + Args: + in_sigmas (`torch.Tensor`): + The input sigma values to be converted. + num_inference_steps (`int`): + The number of inference steps to generate the noise schedule for. + + Returns: + `torch.Tensor`: + The converted sigma values following an exponential schedule. + """ + + # Hack to make sure that other schedulers which copy this function don't break + # TODO: Add this logic to the other schedulers + if hasattr(self.config, "sigma_min"): + sigma_min = self.config.sigma_min + else: + sigma_min = None + + if hasattr(self.config, "sigma_max"): + sigma_max = self.config.sigma_max + else: + sigma_max = None + + sigma_min = sigma_min if sigma_min is not None else in_sigmas[-1].item() + sigma_max = sigma_max if sigma_max is not None else in_sigmas[0].item() + + sigmas = np.exp(np.linspace(math.log(sigma_max), math.log(sigma_min), num_inference_steps)) + return sigmas + + # Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler._convert_to_beta + def _convert_to_beta( + self, in_sigmas: torch.Tensor, num_inference_steps: int, alpha: float = 0.6, beta: float = 0.6 + ) -> torch.Tensor: + """ + Construct a beta noise schedule as proposed in [Beta Sampling is All You + Need](https://huggingface.co/papers/2407.12173). + + Args: + in_sigmas (`torch.Tensor`): + The input sigma values to be converted. + num_inference_steps (`int`): + The number of inference steps to generate the noise schedule for. + alpha (`float`, *optional*, defaults to `0.6`): + The alpha parameter for the beta distribution. + beta (`float`, *optional*, defaults to `0.6`): + The beta parameter for the beta distribution. + + Returns: + `torch.Tensor`: + The converted sigma values following a beta distribution schedule. + """ + + # Hack to make sure that other schedulers which copy this function don't break + # TODO: Add this logic to the other schedulers + if hasattr(self.config, "sigma_min"): + sigma_min = self.config.sigma_min + else: + sigma_min = None + + if hasattr(self.config, "sigma_max"): + sigma_max = self.config.sigma_max + else: + sigma_max = None + + sigma_min = sigma_min if sigma_min is not None else in_sigmas[-1].item() + sigma_max = sigma_max if sigma_max is not None else in_sigmas[0].item() + + sigmas = np.array( + [ + sigma_min + (ppf * (sigma_max - sigma_min)) + for ppf in [ + scipy.stats.beta.ppf(timestep, alpha, beta) + for timestep in 1 - np.linspace(0, 1, num_inference_steps) + ] + ] + ) + return sigmas + + def _time_shift_exponential(self, mu: float, sigma: float, t: torch.Tensor) -> torch.Tensor: + return math.exp(mu) / (math.exp(mu) + (1 / t - 1) ** sigma) + + def _time_shift_linear(self, mu: float, sigma: float, t: torch.Tensor) -> torch.Tensor: + return mu / (mu + (1 / t - 1) ** sigma) + + def __len__(self) -> int: + return self.config.num_train_timesteps From 6853d7204d724aa5b5ca7a23c835f902f7a1e546 Mon Sep 17 00:00:00 2001 From: Inesh Reddy Chappidi Date: Sat, 18 Jul 2026 01:49:10 -0400 Subject: [PATCH 011/240] [Bugfix] Add single-80GB deploy config for MOSS-TTS (resolves #4643 OOM) (#4761) Signed-off-by: Inesh Reddy Signed-off-by: Inesh Reddy Chappidi Co-authored-by: SYLAR <125541396+lishunyang12@users.noreply.github.com> Co-authored-by: Hongsheng Liu --- vllm_omni/deploy/moss_tts_80gb.yaml | 64 +++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 vllm_omni/deploy/moss_tts_80gb.yaml diff --git a/vllm_omni/deploy/moss_tts_80gb.yaml b/vllm_omni/deploy/moss_tts_80gb.yaml new file mode 100644 index 000000000..e61d4cf5d --- /dev/null +++ b/vllm_omni/deploy/moss_tts_80gb.yaml @@ -0,0 +1,64 @@ +# Single-80GB variant of moss_tts.yaml (see #4643): lowers stage-0 +# gpu_memory_utilization and disables codec FlashInfer autotune so both +# stages initialize on one H100/H800 80GB at full max_model_len. +# MOSS-TTS (MossTTSDelayModel, 8B, n_vq=32, 24 kHz) +# HF: OpenMOSS-Team/MOSS-TTS +# General-purpose TTS with voice cloning. Production-recommended variant. +async_chunk: true +trust_remote_code: true # OpenMOSS-Team/MOSS-TTS ships custom modeling code + +connectors: + shm: + name: SharedMemoryConnector + extra: + shm_threshold_bytes: 65536 + codec_streaming: true + connector_get_sleep_s: 0.01 + connector_get_max_wait_first_chunk: 3000 + connector_get_max_wait: 300 + codec_chunk_frames: 25 + codec_left_context_frames: 0 + decode_cudagraph_capture_sizes: [4, 8, 16, 25, 32, 50, 64, 100, 128, 200, 256] + +stages: + - stage_id: 0 + max_num_seqs: 4 + gpu_memory_utilization: 0.60 + enable_prefix_caching: false + async_scheduling: true + max_num_batched_tokens: 4096 + max_model_len: 4096 + devices: "0" + output_connectors: + to_stage_1: shm + default_sampling_params: + temperature: 1.7 + top_p: 0.8 + top_k: 25 + max_tokens: 4096 + seed: 42 + repetition_penalty: 1.0 + stop_token_ids: [151645] # <|im_end|> — terminates the talker AR loop + + - stage_id: 1 + max_num_seqs: 1 + gpu_memory_utilization: 0.12 + enforce_eager: true + enable_flashinfer_autotune: false + enable_prefix_caching: false + async_scheduling: true + # Talker can emit up to max_tokens=4096 frames × n_vq=32 = 131072 codec + # tokens per request; cap codec stage above that so a single long output + # doesn't trigger "could not broadcast (>65536,) into (65536,)". + max_num_batched_tokens: 196608 + max_model_len: 196608 + devices: "0" + input_connectors: + from_stage_0: shm + default_sampling_params: + temperature: 0.0 + top_p: 1.0 + top_k: -1 + max_tokens: 65536 + seed: 42 + repetition_penalty: 1.0 From 1f0ed96fcd2a3d25385a7fa2e6e3f2d03c606d12 Mon Sep 17 00:00:00 2001 From: Nick Cao Date: Sat, 18 Jul 2026 01:52:07 -0400 Subject: [PATCH 012/240] [Bugfix] Fix broken FrameSimilarityFilter threshold test (#4786) Signed-off-by: Nick Cao --- tests/entrypoints/openai_api/test_video_frame_filter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/entrypoints/openai_api/test_video_frame_filter.py b/tests/entrypoints/openai_api/test_video_frame_filter.py index a734aec75..70df6e34c 100644 --- a/tests/entrypoints/openai_api/test_video_frame_filter.py +++ b/tests/entrypoints/openai_api/test_video_frame_filter.py @@ -29,8 +29,8 @@ class TestFrameSimilarityFilter: assert f.should_retain(make_jpeg(255, 255, 255)) is True assert f.should_retain(make_jpeg(0, 0, 0)) is True - def test_low_threshold_keeps_slightly_different(self): - f = FrameSimilarityFilter(threshold=0.50) + def test_high_threshold_keeps_slightly_different(self): + f = FrameSimilarityFilter(threshold=0.999) assert f.should_retain(make_jpeg(100, 100, 100)) is True assert f.should_retain(make_jpeg(110, 110, 110)) is True From 907f932b72e50b726436c1aa42a475eb962a14c9 Mon Sep 17 00:00:00 2001 From: wangyu <53896905+yenuo26@users.noreply.github.com> Date: Sat, 18 Jul 2026 17:15:17 +0800 Subject: [PATCH 013/240] [CI][Test] Tier diffusion offloader CI and reduce gpu_4_queue merge usage and fix #5182 (#5185) Signed-off-by: wangyu <410167048@qq.com> --- .buildkite/test-merge.yml | 55 ++++++------------- .buildkite/test-nightly.yml | 23 +++++++- .buildkite/test-ready.yml | 6 ++ .../models/cosmos3/test_cosmos3_pipeline.py | 3 +- .../offloader/test_diffusion_cpu_offload.py | 13 ++++- ...dextts2.py => test_indextts2_expansion.py} | 2 +- ...dextts2.py => test_indextts2_expansion.py} | 2 +- .../openai_api/test_qwen3_tts_websocket.py | 2 +- .../openai_api/test_serving_speech_batch.py | 2 +- 9 files changed, 60 insertions(+), 48 deletions(-) rename tests/e2e/offline_inference/{test_indextts2.py => test_indextts2_expansion.py} (98%) rename tests/e2e/online_serving/{test_indextts2.py => test_indextts2_expansion.py} (97%) diff --git a/.buildkite/test-merge.yml b/.buildkite/test-merge.yml index 51c4223f2..7a78eda4a 100644 --- a/.buildkite/test-merge.yml +++ b/.buildkite/test-merge.yml @@ -60,6 +60,23 @@ steps: volumes: - "/fsx/hf_cache:/fsx/hf_cache" + - label: "Diffusion · Offloader Test" + timeout_in_minutes: 40 + commands: + - timeout 40m pytest -s -v tests/diffusion/offloader -m "advanced_model and cuda" + agents: + queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU + plugins: + - docker#v5.2.0: + image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT + always-pull: true + propagate-environment: true + environment: + - "HF_HOME=/fsx/hf_cache" + - "HF_TOKEN" + volumes: + - "/fsx/hf_cache:/fsx/hf_cache" + - label: "Distributed Test" depends_on: upload-merge-pipeline timeout_in_minutes: 60 @@ -150,7 +167,7 @@ steps: commands: - pytest -s -v tests/entrypoints/ -m "advanced_model and cuda and L4" --run-level "advanced_model" agents: - queue: "gpu_4_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU + queue: "gpu_1_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU plugins: - docker#v5.2.0: image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT @@ -247,42 +264,6 @@ steps: volumes: - "/fsx/hf_cache:/fsx/hf_cache" - - label: "TTS · IndexTTS2 Test" - source_file_dependencies: - - tests/e2e/online_serving/test_indextts2.py - - tests/e2e/offline_inference/test_indextts2.py - - tests/model_executor/models/indextts2/ - - tests/model_executor/stage_input_processors/test_indextts2.py - - tests/assets/indextts2/ - - vllm_omni/model_executor/models/indextts2/ - - vllm_omni/model_executor/stage_input_processors/indextts2.py - - vllm_omni/deploy/indextts2.yaml - - vllm_omni/deploy/indextts2_low_latency.yaml - - vllm_omni/entrypoints/openai/serving_speech.py - - vllm_omni/entrypoints/openai/tts_adapters/indextts2.py - - vllm_omni/config/pipeline_registry.py - - vllm_omni/model_executor/models/registry.py - timeout_in_minutes: 60 - commands: - - | - timeout 45m bash -c " - export VLLM_LOGGING_LEVEL=DEBUG - pytest -s -v tests/e2e/online_serving/test_indextts2.py tests/e2e/offline_inference/test_indextts2.py -m 'advanced_model and cuda' --run-level 'advanced_model' - " - agents: - queue: "gpu_4_queue" - plugins: - - docker#v5.2.0: - image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - always-pull: true - propagate-environment: true - environment: - - "HF_HOME=/fsx/hf_cache" - - "HF_TOKEN" - gpus: all - volumes: - - "/fsx/hf_cache:/fsx/hf_cache" - - label: "Omni · Qwen3-Omni Test" source_file_dependencies: - tests/e2e/offline_inference/test_qwen3_omni.py diff --git a/.buildkite/test-nightly.yml b/.buildkite/test-nightly.yml index 783d0ab1f..0a23e5825 100644 --- a/.buildkite/test-nightly.yml +++ b/.buildkite/test-nightly.yml @@ -1529,11 +1529,28 @@ steps: path: /mnt/hf-cache type: DirectoryOrCreate - - group: ":card_index_dividers: Diffusion · Quantization Test" + - group: ":card_index_dividers: Diffusion Test" depends_on: upload-nightly-pipeline if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" || build.pull_request.labels includes "diffusion-x2iat-test" steps: - - label: ":full_moon: Quantization Test with H100" + - label: ":full_moon: Diffusion · Offloader Test" + timeout_in_minutes: 40 + commands: + - timeout 40m pytest -s -v tests/diffusion/offloader -m "full_model and cuda" + agents: + queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU + plugins: + - docker#v5.2.0: + image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT + always-pull: true + propagate-environment: true + environment: + - "HF_HOME=/fsx/hf_cache" + - "HF_TOKEN" + volumes: + - "/fsx/hf_cache:/fsx/hf_cache" + + - label: ":full_moon: Diffusion · Quantization Test with H100" commands: - timeout 60m pytest -sv tests/diffusion/quantization -m 'full_model and cuda and H100' --run-level "full_model" agents: @@ -1570,7 +1587,7 @@ steps: path: /mnt/hf-cache type: DirectoryOrCreate - - label: ":full_moon: Quantization Test with L4" + - label: ":full_moon: Diffusion · Quantization Test with L4" timeout_in_minutes: 60 commands: - pytest -sv tests/diffusion/quantization -m 'full_model and cuda and L4' --run-level "full_model" diff --git a/.buildkite/test-ready.yml b/.buildkite/test-ready.yml index c9b60fe2c..75e834be9 100644 --- a/.buildkite/test-ready.yml +++ b/.buildkite/test-ready.yml @@ -108,6 +108,12 @@ steps: - "/fsx/hf_cache:/fsx/hf_cache" - label: "Diffusion · Distributed Test" + source_file_dependencies: + - tests/diffusion/distributed/ + - vllm_omni/diffusion/distributed/ + - vllm_omni/diffusion/attention/parallel/ + - vllm_omni/diffusion/data.py + - vllm_omni/diffusion/models/ commands: - timeout 20m pytest -sv tests/diffusion/distributed -m 'core_model and cuda' --run-level "core_model" agents: diff --git a/tests/diffusion/models/cosmos3/test_cosmos3_pipeline.py b/tests/diffusion/models/cosmos3/test_cosmos3_pipeline.py index 1085e3878..404979a78 100644 --- a/tests/diffusion/models/cosmos3/test_cosmos3_pipeline.py +++ b/tests/diffusion/models/cosmos3/test_cosmos3_pipeline.py @@ -716,10 +716,9 @@ def test_distilled_pipeline_initializes_sde_scheduler( stub_real_pipeline_init, monkeypatch: pytest.MonkeyPatch, ) -> None: - from diffusers import FlowMatchEulerDiscreteScheduler - from vllm_omni.diffusion.models.cosmos3.pipeline_cosmos3 import ( Cosmos3OmniDiffusersPipeline, + FlowMatchEulerDiscreteScheduler, ) t_list = [1.0, 0.9375, 0.8333333333333334, 0.625] diff --git a/tests/diffusion/offloader/test_diffusion_cpu_offload.py b/tests/diffusion/offloader/test_diffusion_cpu_offload.py index 22f6839fd..555a62ca0 100644 --- a/tests/diffusion/offloader/test_diffusion_cpu_offload.py +++ b/tests/diffusion/offloader/test_diffusion_cpu_offload.py @@ -24,6 +24,13 @@ IMAGE_VIDEO_MODELS = { MODELS = {**AUDIO_MODEL, **IMAGE_VIDEO_MODELS} +MODEL_MARKS = { + "riverclouds/qwen_image_random": pytest.mark.core_model, + "Tongyi-MAI/Z-Image-Turbo": pytest.mark.advanced_model, + "stabilityai/stable-audio-open-1.0": pytest.mark.full_model, + "OmniGen2/OmniGen2": pytest.mark.full_model, +} + _GATED_MODELS = {"stabilityai/stable-audio-open-1.0"} # Aliased for backward compatibility (imported by test_diffusion_layerwise_offload.py). @@ -97,10 +104,12 @@ def check_audio_determinism(audio1, audio2, atol=1e-2): return True -@pytest.mark.core_model @pytest.mark.diffusion @hardware_test(res={"cuda": "L4", "rocm": "MI325"}) -@pytest.mark.parametrize("model_name", list(MODELS.keys())) +@pytest.mark.parametrize( + "model_name", + [pytest.param(name, marks=MODEL_MARKS[name]) for name in MODELS], +) def test_cpu_offload_diffusion_model(model_name: str): if model_name == "OmniGen2/OmniGen2": pytest.skip("issue #4537") diff --git a/tests/e2e/offline_inference/test_indextts2.py b/tests/e2e/offline_inference/test_indextts2_expansion.py similarity index 98% rename from tests/e2e/offline_inference/test_indextts2.py rename to tests/e2e/offline_inference/test_indextts2_expansion.py index 6c2786efa..ba5b984f9 100644 --- a/tests/e2e/offline_inference/test_indextts2.py +++ b/tests/e2e/offline_inference/test_indextts2_expansion.py @@ -33,7 +33,7 @@ ASSETS_DIR = Path(__file__).resolve().parents[2] / "assets" / "indextts2" REF_AUDIO = str(ASSETS_DIR / "ref_audio.wav") pytestmark = [ - pytest.mark.advanced_model, + pytest.mark.slow, pytest.mark.tts, pytest.mark.parametrize("omni_runner", [_OMNI_RUNNER_PARAM], indirect=True), ] diff --git a/tests/e2e/online_serving/test_indextts2.py b/tests/e2e/online_serving/test_indextts2_expansion.py similarity index 97% rename from tests/e2e/online_serving/test_indextts2.py rename to tests/e2e/online_serving/test_indextts2_expansion.py index 5ab0d1f8d..d971ef166 100644 --- a/tests/e2e/online_serving/test_indextts2.py +++ b/tests/e2e/online_serving/test_indextts2_expansion.py @@ -19,7 +19,7 @@ from tests.helpers.media import load_test_audio_data_url from tests.helpers.runtime import OmniServerParams from tests.helpers.stage_config import get_deploy_config_path -pytestmark = [pytest.mark.advanced_model, pytest.mark.tts] +pytestmark = [pytest.mark.slow, pytest.mark.tts] MODEL = "IndexTeam/IndexTTS-2" diff --git a/tests/entrypoints/openai_api/test_qwen3_tts_websocket.py b/tests/entrypoints/openai_api/test_qwen3_tts_websocket.py index 058b20331..ecae21158 100644 --- a/tests/entrypoints/openai_api/test_qwen3_tts_websocket.py +++ b/tests/entrypoints/openai_api/test_qwen3_tts_websocket.py @@ -102,7 +102,7 @@ async def _run_ws_session(host: str, port: int, model: str) -> dict: class TestQwen3TTSWebSocket: @pytest.mark.advanced_model @pytest.mark.tts - @hardware_test(res={"cuda": "L4"}, num_cards=4) + @hardware_test(res={"cuda": "L4"}, num_cards=1) @pytest.mark.parametrize("omni_server", tts_ws_server_params, indirect=True) def test_streaming_pcm_output(self, omni_server) -> None: result = asyncio.run(_run_ws_session(omni_server.host, omni_server.port, omni_server.model)) diff --git a/tests/entrypoints/openai_api/test_serving_speech_batch.py b/tests/entrypoints/openai_api/test_serving_speech_batch.py index ce0a5e21f..686dba7d0 100644 --- a/tests/entrypoints/openai_api/test_serving_speech_batch.py +++ b/tests/entrypoints/openai_api/test_serving_speech_batch.py @@ -88,7 +88,7 @@ class TestSpeechBatchE2E: @pytest.mark.advanced_model @pytest.mark.tts - @hardware_test(res={"cuda": "H100"}, num_cards=1) + @hardware_test(res={"cuda": "L4"}, num_cards=1) @pytest.mark.parametrize("omni_server", default_server_params, indirect=True) def test_batch_basic_two_items(self, omni_server) -> None: """Batch with two items returns two successful base64-encoded results.""" From 793d3e9ab7c4939eb9148be359375015f2e590a6 Mon Sep 17 00:00:00 2001 From: Yukim1 <121286183+zwhzzz0821@users.noreply.github.com> Date: Sat, 18 Jul 2026 17:21:07 +0800 Subject: [PATCH 014/240] [Config] Return VllmOmniConfig from create_from_model (#4818) Signed-off-by: zwhzzz0821 <2831474076@qq.com> Co-authored-by: Hongsheng Liu --- .../test_factory_integration.py | 6 +- tests/config/test_omni_config.py | 201 ++++++-- tests/e2e/accuracy/test_hunyuan_image3.py | 9 +- .../test_covo_audio_expansion.py | 2 +- .../test_dynin_omni_expansion.py | 2 +- .../test_glm_tts_expansion.py | 1 + .../offline_inference/test_hunyuanimage3.py | 1 + .../test_hunyuanimage3_i2t_expansion.py | 1 + .../test_moss_tts_expansion.py | 6 +- .../test_moss_tts_nano_expansion.py | 1 + .../test_moss_tts_realtime.py | 6 +- .../test_moss_tts_realtime_expansion.py | 6 +- .../test_moss_tts_v1_5_expansion.py | 6 +- .../e2e/offline_inference/test_voxcpm2_tts.py | 4 +- .../test_async_omni_engine_stage_init.py | 3 + .../test_async_omni_diffusion_config.py | 2 + .../test_resolve_dreamzero_config.py | 11 +- tests/entrypoints/test_utils.py | 8 +- tests/test_config_factory.py | 479 +++++++++++++++--- tests/utils/test_tracking_parser.py | 26 +- vllm_omni/config/config_factory.py | 184 +++---- vllm_omni/config/omni_config.py | 91 ++-- vllm_omni/config/pipeline_registry.py | 12 + vllm_omni/config/stage_config.py | 3 + vllm_omni/engine/async_omni_engine.py | 15 +- vllm_omni/entrypoints/cli/serve.py | 1 + vllm_omni/entrypoints/omni_base.py | 2 +- vllm_omni/entrypoints/utils.py | 19 +- .../models/aura_omni/pipeline.py | 1 + .../model_executor/models/bagel/pipeline.py | 3 + .../models/cosyvoice3/pipeline.py | 1 + .../models/covo_audio/pipeline.py | 1 + .../models/dreamzero/pipeline.py | 1 + .../models/dynin_omni/pipeline.py | 1 + .../models/fish_speech/pipeline.py | 1 + .../models/glm_image/pipeline.py | 1 + .../model_executor/models/glm_tts/pipeline.py | 1 + .../model_executor/models/gr00t/pipeline.py | 1 + .../models/higgs_audio_v2/pipeline.py | 1 + .../models/higgs_audio_v3/pipeline.py | 1 + .../models/hunyuan_image3/pipeline.py | 3 + .../models/indextts2/pipeline.py | 1 + .../model_executor/models/lance/pipeline.py | 1 + .../models/mammoth_moda2/pipeline.py | 2 + .../models/mimo_audio/pipeline.py | 1 + .../models/ming_flash_omni/pipeline.py | 4 + .../models/ming_tts/pipeline.py | 2 + .../models/minicpmo_4_5/pipeline.py | 1 + .../models/moss_tts/pipeline.py | 3 + .../models/moss_tts_nano/pipeline.py | 1 + .../models/omnivoice/pipeline.py | 1 + .../models/qwen2_5_omni/pipeline.py | 1 + .../models/qwen3_omni/pipeline.py | 1 + .../models/qwen3_tts/pipeline.py | 1 + .../model_executor/models/voxcpm2/pipeline.py | 1 + .../models/voxtral_tts/pipeline.py | 1 + 56 files changed, 865 insertions(+), 282 deletions(-) diff --git a/tests/config/composable_parallel/test_factory_integration.py b/tests/config/composable_parallel/test_factory_integration.py index ea685541c..468b3db2c 100644 --- a/tests/config/composable_parallel/test_factory_integration.py +++ b/tests/config/composable_parallel/test_factory_integration.py @@ -100,8 +100,7 @@ def test_device_check_survives_cli_override(): # guard: effective world=1, replicas=2 admits only 1 or 2 device ids. pipeline_cfg = OMNI_PIPELINES["qwen2_5_omni"] with pytest.raises(StrategyApplyError): - StageConfigFactory._create_from_registry( - "qwen2_5_omni", + StageConfigFactory._create_legacy_from_registry( pipeline_cfg, cli_overrides={"stage_1_devices": "0,1,2"}, strategy_specs={"talker": [_stage_replica(2, "round_robin")]}, @@ -126,8 +125,7 @@ def test_cli_overrides_strategy_with_warning(): log.addHandler(handler) try: pipeline_cfg = OMNI_PIPELINES["qwen2_5_omni"] - stages, _ = StageConfigFactory._create_from_registry( - "qwen2_5_omni", + stages, _ = StageConfigFactory._create_legacy_from_registry( pipeline_cfg, cli_overrides={"stage_1_num_replicas": 3}, strategy_specs={"talker": [_stage_replica(2, "round_robin")]}, diff --git a/tests/config/test_omni_config.py b/tests/config/test_omni_config.py index bf2dba701..23e53ea3d 100644 --- a/tests/config/test_omni_config.py +++ b/tests/config/test_omni_config.py @@ -9,7 +9,9 @@ from pathlib import Path import pytest from pydantic import ValidationError +from transformers import Qwen3OmniMoeConfig +from tests.helpers.stage_config import get_deploy_config_path from vllm_omni.config import omni_config as omni_config_module from vllm_omni.config.omni_config import ( BaseVllmOmniStageConfig, @@ -26,7 +28,7 @@ from vllm_omni.config.omni_config import ( VllmOmniDiffusionStageConfig, VllmOmniGenerationStageConfig, ) -from vllm_omni.config.pipeline_registry import OMNI_PIPELINES +from vllm_omni.config.pipeline_registry import OMNI_PIPELINES, resolve_pipeline_config from vllm_omni.config.stage_config import ( _STAGE_DEPLOY_FIELDS, PIPELINE_WIDE_ENGINE_FIELDS, @@ -53,28 +55,39 @@ def _stable_test_platform(monkeypatch): monkeypatch.setattr(platform, "device_type", "cpu", raising=False) -def _load_default_deploy(model_type: str) -> DeployConfig: - deploy_path = _DEPLOY_DIR / f"{model_type}.yaml" - if deploy_path.exists(): - return load_deploy_config(deploy_path) +def _load_default_deploy(pipeline: PipelineConfig) -> DeployConfig: + if pipeline.default_deploy_config_name is not None: + return load_deploy_config(_DEPLOY_DIR / pipeline.default_deploy_config_name) return DeployConfig() -def _resolve_pipeline_or_skip(model_type: str) -> PipelineConfig: - registered = OMNI_PIPELINES[model_type] - pipeline = registered(None) if callable(registered) else registered +def _resolve_pipeline_or_skip(model_type: str, hf_config=None) -> PipelineConfig: + pipeline = resolve_pipeline_config(model_type, hf_config) if pipeline is None: pytest.skip(f"Pipeline {model_type!r} requires an HF config to resolve") return pipeline +def _from_pipeline_key( + model_type: str, + hf_config=None, + deploy_config_path: str | None = None, + cli_overrides: dict | None = None, +) -> VllmOmniConfig: + return VllmOmniConfig.from_pipeline_config( + _resolve_pipeline_or_skip(model_type, hf_config), + deploy_config_path=deploy_config_path, + cli_overrides=cli_overrides, + ) + + @pytest.mark.parametrize("model_type", sorted(OMNI_PIPELINES)) -def test_vllm_omni_config_from_registry_matches_merge_pipeline_deploy(model_type: str): +def test_vllm_omni_config_from_pipeline_config_matches_merge_pipeline_deploy(model_type: str): pipeline = _resolve_pipeline_or_skip(model_type) - legacy_deploy = _load_default_deploy(model_type) + legacy_deploy = _load_default_deploy(pipeline) legacy_stages = merge_pipeline_deploy(pipeline, legacy_deploy) - omni_config = VllmOmniConfig.from_registry(model_type) + omni_config = VllmOmniConfig.from_pipeline_config(pipeline) assert omni_config.pipeline_config is pipeline assert len(omni_config.stage_configs) == len(legacy_stages) @@ -119,7 +132,7 @@ def test_vllm_omni_config_from_registry_matches_merge_pipeline_deploy(model_type def test_stage_by_id_raises_for_unknown_stage(): - omni_config = VllmOmniConfig.from_registry("qwen3_tts") + omni_config = _from_pipeline_key("qwen3_tts") with pytest.raises(KeyError, match="no stage 99"): omni_config.stage_by_id(99) @@ -130,8 +143,8 @@ def test_resolve_execution_mode_rejects_unknown_execution_type(): omni_config_module._resolve_execution_mode("unknown_execution_type") -def test_from_registry_preserves_current_pipeline_config_object(): - omni_config = VllmOmniConfig.from_registry("minicpmo_4_5") +def test_from_pipeline_config_preserves_current_pipeline_config_object(): + omni_config = _from_pipeline_key("minicpmo_4_5") pipeline = _resolve_pipeline_or_skip("minicpmo_4_5") assert omni_config.pipeline_config is pipeline @@ -140,11 +153,11 @@ def test_from_registry_preserves_current_pipeline_config_object(): assert omni_config.pipeline_config.hf_config_predicate is pipeline.hf_config_predicate -def test_from_registry_normalizes_stage_engine_extras_without_expanding_stage_deploy_config(): +def test_from_pipeline_config_normalizes_stage_engine_extras_without_expanding_stage_deploy_config(): assert not hasattr(StageDeployConfig, "model_config") assert not hasattr(StageDeployConfig, "parallel_config") - stage = VllmOmniConfig.from_registry("dreamzero", deploy_config_path="dreamzero_tp1_cfg2").stage_by_id(0) + stage = _from_pipeline_key("dreamzero", deploy_config_path="dreamzero_tp1_cfg2").stage_by_id(0) assert isinstance(stage, VllmOmniDiffusionStageConfig) assert stage.parallel_config.tensor_parallel_size == 1 @@ -152,8 +165,8 @@ def test_from_registry_normalizes_stage_engine_extras_without_expanding_stage_de assert stage.diffusion_config.model_config["default_robot_embodiment"] == "roboarena" -def test_from_registry_applies_cli_overrides_without_stage_config_runtime_bridge(): - omni_config = VllmOmniConfig.from_registry( +def test_from_pipeline_config_applies_cli_overrides_without_stage_config_runtime_bridge(): + omni_config = _from_pipeline_key( "qwen3_tts", cli_overrides={ "stage_0_max_num_seqs": 7, @@ -170,7 +183,7 @@ def test_from_registry_applies_cli_overrides_without_stage_config_runtime_bridge def test_runtime_num_gpus_is_derived_from_parallel_world_size(): - omni_config = VllmOmniConfig.from_registry("hunyuan_image3_dit") + omni_config = _from_pipeline_key("hunyuan_image3_dit") stage = omni_config.stage_by_id(0) assert stage.parallel_config.tensor_parallel_size == 4 @@ -179,7 +192,7 @@ def test_runtime_num_gpus_is_derived_from_parallel_world_size(): def test_runtime_num_gpus_ignores_stale_runtime_override(): - omni_config = VllmOmniConfig.from_registry( + omni_config = _from_pipeline_key( "hunyuan_image3_dit", cli_overrides={ "stage_0_num_gpus": 1, @@ -191,8 +204,8 @@ def test_runtime_num_gpus_ignores_stale_runtime_override(): assert stage.runtime_config.num_gpus == 4 -def test_from_registry_does_not_route_server_cli_keys_to_diffusion_stage(): - omni_config = VllmOmniConfig.from_registry( +def test_from_pipeline_config_does_not_route_server_cli_keys_to_diffusion_stage(): + omni_config = _from_pipeline_key( "dreamzero", deploy_config_path="dreamzero_tp1_cfg2", cli_overrides={ @@ -220,7 +233,7 @@ def test_pipeline_deploy_cli_fields_reuse_legacy_pipeline_wide_engine_fields(): def test_pipeline_wide_model_fields_are_retained_on_structured_stage_configs(tmp_path): custom_voice_dir = tmp_path / "voices" - omni_config = VllmOmniConfig.from_registry( + omni_config = _from_pipeline_key( "qwen3_tts", cli_overrides={ "active_stream_window": 2, @@ -267,8 +280,8 @@ def test_public_config_exports_use_stage_specific_sub_config_names(): }.issubset(config_pkg.__all__) -def test_from_registry_keeps_worker_backend_separate_from_distributed_executor_backend(): - omni_config = VllmOmniConfig.from_registry("dreamzero", deploy_config_path="dreamzero_tp1_cfg2") +def test_from_pipeline_config_keeps_worker_backend_separate_from_distributed_executor_backend(): + omni_config = _from_pipeline_key("dreamzero", deploy_config_path="dreamzero_tp1_cfg2") stage = omni_config.stage_by_id(0) assert isinstance(stage, VllmOmniDiffusionStageConfig) @@ -276,8 +289,8 @@ def test_from_registry_keeps_worker_backend_separate_from_distributed_executor_b assert omni_config.orchestrator_config.worker_backend == "multi_process" -def test_from_registry_maps_orchestrator_cli_overrides(): - omni_config = VllmOmniConfig.from_registry( +def test_from_pipeline_config_maps_orchestrator_cli_overrides(): + omni_config = _from_pipeline_key( "qwen3_tts", cli_overrides={ "stage_init_timeout": 1200, @@ -308,21 +321,21 @@ def test_from_registry_maps_orchestrator_cli_overrides(): assert orchestrator_config.batch_timeout == 3 -def test_from_registry_records_loaded_deploy_path_on_orchestrator_config(): - omni_config = VllmOmniConfig.from_registry("dreamzero", deploy_config_path="dreamzero_tp1_cfg2") +def test_from_pipeline_config_records_loaded_deploy_path_on_orchestrator_config(): + omni_config = _from_pipeline_key("dreamzero", deploy_config_path="dreamzero_tp1_cfg2") assert omni_config.pipeline_config.model_type == "dreamzero" assert omni_config.orchestrator_config.deploy_config_path == str(_DEPLOY_DIR / "dreamzero_tp1_cfg2.yaml") -def test_from_registry_dispatches_async_chunk_processors_without_mutating_topology(): +def test_from_pipeline_config_dispatches_async_chunk_processors_without_mutating_topology(): pipeline = _resolve_pipeline_or_skip("qwen3_tts") - async_config = VllmOmniConfig.from_registry("qwen3_tts") + async_config = _from_pipeline_key("qwen3_tts") assert async_config.stage_by_id(0).custom_process_next_stage_input_func.endswith("talker2code2wav_async_chunk") assert async_config.stage_by_id(1).custom_process_input_func is None - sync_config = VllmOmniConfig.from_registry("qwen3_tts", cli_overrides={"async_chunk": False}) + sync_config = _from_pipeline_key("qwen3_tts", cli_overrides={"async_chunk": False}) assert sync_config.stage_by_id(0).custom_process_next_stage_input_func.endswith("talker2code2wav_full_payload") assert sync_config.stage_by_id(1).custom_process_input_func.endswith("talker2code2wav_token_only") @@ -503,8 +516,8 @@ def test_diffusion_parallel_config_rejects_hsdp_with_tp_or_dp(): OmniStageDiffusionParallelConfig(data_parallel_size=2, use_hsdp=True, hsdp_shard_size=2) -def test_from_registry_preserves_legacy_pp_dp_for_world_size(): - cfg = VllmOmniConfig.from_registry("hunyuan_image3_dit").stage_by_id(0).parallel_config +def test_from_pipeline_config_preserves_legacy_pp_dp_for_world_size(): + cfg = _from_pipeline_key("hunyuan_image3_dit").stage_by_id(0).parallel_config assert cfg.pipeline_parallel_size == 1 assert cfg.data_parallel_size == 1 @@ -512,7 +525,7 @@ def test_from_registry_preserves_legacy_pp_dp_for_world_size(): assert cfg.world_size == 4 -def test_from_registry_derives_sequence_parallel_size_from_degrees(tmp_path): +def test_from_pipeline_config_derives_sequence_parallel_size_from_degrees(tmp_path): deploy_path = tmp_path / "dreamzero_derived_parallel.yaml" deploy_path.write_text( "\n".join( @@ -529,7 +542,7 @@ def test_from_registry_derives_sequence_parallel_size_from_degrees(tmp_path): ) ) - stage = VllmOmniConfig.from_registry("dreamzero", deploy_config_path=str(deploy_path)).stage_by_id(0) + stage = _from_pipeline_key("dreamzero", deploy_config_path=str(deploy_path)).stage_by_id(0) assert isinstance(stage, VllmOmniDiffusionStageConfig) assert stage.parallel_config.sequence_parallel_size == 6 @@ -542,10 +555,10 @@ def test_diffusion_parallel_config_rejects_cfg_parallel_size_outside_current_bou def test_stage_realizations_use_stage_specific_parallel_config_types(): - qwen_config = VllmOmniConfig.from_registry("qwen3_tts") + qwen_config = _from_pipeline_key("qwen3_tts") ar_stage = qwen_config.stage_by_id(0) generation_stage = qwen_config.stage_by_id(1) - diffusion_stage = VllmOmniConfig.from_registry("hunyuan_image3_dit").stage_by_id(0) + diffusion_stage = _from_pipeline_key("hunyuan_image3_dit").stage_by_id(0) assert isinstance(ar_stage, VllmOmniARStageConfig) assert type(ar_stage.parallel_config) is OmniStageParallelConfig @@ -566,7 +579,7 @@ def test_stage_realizations_use_stage_specific_parallel_config_types(): assert diffusion_stage.parallel_config.ulysses_degree == 1 -def test_from_registry_preserves_diffusion_parallel_mask_sp_padding(tmp_path): +def test_from_pipeline_config_preserves_diffusion_parallel_mask_sp_padding(tmp_path): deploy_path = tmp_path / "dreamzero_mask_sp_padding.yaml" deploy_path.write_text( "\n".join( @@ -581,17 +594,17 @@ def test_from_registry_preserves_diffusion_parallel_mask_sp_padding(tmp_path): ) ) - stage = VllmOmniConfig.from_registry("dreamzero", deploy_config_path=str(deploy_path)).stage_by_id(0) + stage = _from_pipeline_key("dreamzero", deploy_config_path=str(deploy_path)).stage_by_id(0) assert isinstance(stage, VllmOmniDiffusionStageConfig) assert stage.parallel_config.mask_sp_padding is True -def test_from_registry_matches_stage_config_to_omegaconf_behavior_for_representative_stage(): +def test_from_pipeline_config_matches_stage_config_to_omegaconf_behavior_for_representative_stage(): pipeline = _resolve_pipeline_or_skip("qwen3_tts") - legacy_stage = merge_pipeline_deploy(pipeline, _load_default_deploy("qwen3_tts"))[0] + legacy_stage = merge_pipeline_deploy(pipeline, _load_default_deploy(pipeline))[0] omega_stage = legacy_stage.to_omegaconf() - omni_stage = VllmOmniConfig.from_registry("qwen3_tts").stage_by_id(legacy_stage.stage_id) + omni_stage = _from_pipeline_key("qwen3_tts").stage_by_id(legacy_stage.stage_id) assert omega_stage.stage_id == omni_stage.stage_id assert omega_stage.stage_type == omni_stage.stage_type.value @@ -606,11 +619,93 @@ def test_from_registry_matches_stage_config_to_omegaconf_behavior_for_representa assert omega_stage.runtime.requires_multimodal_data == omni_stage.requires_multimodal_data -def test_from_registry_matches_to_omegaconf_diffusion_parallel_config(): +def test_from_pipeline_config_uses_hf_config_for_callable_resolver(): + hf_config = Qwen3OmniMoeConfig() + hf_config.enable_audio_output = False + + omni_config = _from_pipeline_key("qwen3_omni_moe", hf_config=hf_config) + + assert omni_config.pipeline_config.model_type == "qwen3_omni_moe_thinker_only" + assert len(omni_config.stage_configs) == 1 + assert omni_config.orchestrator_config.deploy_config_path is None + + thinker = omni_config.stage_configs[0] + assert thinker.model_stage == "thinker" + assert thinker.model_config.default_sampling_params == {"detokenize": True} + + +def test_from_pipeline_config_accepts_pre_resolved_pipeline(): + resolved_pipeline = PipelineConfig(model_type="callable_resolved_variant") + + omni_config = VllmOmniConfig.from_pipeline_config(resolved_pipeline) + + assert omni_config.pipeline_config is resolved_pipeline + + +def test_from_pipeline_config_prefers_loaded_user_deploy_config(monkeypatch): + pipeline = _resolve_pipeline_or_skip("qwen3_tts") + user_deploy_config = DeployConfig( + stages=[StageDeployConfig(stage_id=0, max_num_seqs=7)], + ) + monkeypatch.setattr( + omni_config_module, + "load_deploy_config", + lambda _path: pytest.fail("default deploy config should not be loaded"), + ) + + omni_config = VllmOmniConfig.from_pipeline_config( + pipeline, + user_deploy_config=user_deploy_config, + ) + + assert omni_config.stage_by_id(0).scheduler_config.max_num_seqs == 7 + + +def test_from_pipeline_config_default_deploy_name_ignores_cwd(monkeypatch, tmp_path): + default_name = "pipeline_default.yaml" + (tmp_path / default_name).write_text("stages: []\n", encoding="utf-8") + monkeypatch.chdir(tmp_path) + pipeline = PipelineConfig( + model_type="pipeline_with_default", + default_deploy_config_name=default_name, + ) + loaded_paths = [] + + def _load_deploy_config(path): + loaded_paths.append(Path(path)) + return DeployConfig() + + monkeypatch.setattr(omni_config_module, "load_deploy_config", _load_deploy_config) + + omni_config = VllmOmniConfig.from_pipeline_config(pipeline) + + assert omni_config.orchestrator_config.deploy_config_path == str(_DEPLOY_DIR / default_name) + assert loaded_paths == [_DEPLOY_DIR / default_name] + + +def test_from_pipeline_config_uses_resolved_deploy_pipeline(): + deploy_path = get_deploy_config_path("aura_omni.yaml") + pipeline = _resolve_pipeline_or_skip("aura_omni") + + omni_config = VllmOmniConfig.from_pipeline_config( + pipeline, + deploy_config_path=str(deploy_path), + ) + + assert omni_config.pipeline_config.model_type == "aura_omni" + assert [stage.model_stage for stage in omni_config.stage_configs] == [ + "asr", + "aura", + "qwen3_tts", + "code2wav", + ] + + +def test_from_pipeline_config_matches_to_omegaconf_diffusion_parallel_config(): pipeline = _resolve_pipeline_or_skip("hunyuan_image3_dit") - legacy_stage = merge_pipeline_deploy(pipeline, _load_default_deploy("hunyuan_image3_dit"))[0] + legacy_stage = merge_pipeline_deploy(pipeline, _load_default_deploy(pipeline))[0] omega_stage = legacy_stage.to_omegaconf() - omni_stage = VllmOmniConfig.from_registry("hunyuan_image3_dit").stage_by_id(legacy_stage.stage_id) + omni_stage = _from_pipeline_key("hunyuan_image3_dit").stage_by_id(legacy_stage.stage_id) assert ( omega_stage.engine_args.parallel_config.pipeline_parallel_size @@ -631,19 +726,19 @@ def test_from_registry_matches_to_omegaconf_diffusion_parallel_config(): ) -def test_from_registry_matches_build_engine_args_dict_behavior_for_representative_stage(monkeypatch): +def test_from_pipeline_config_matches_build_engine_args_dict_behavior_for_representative_stage(monkeypatch): from vllm_omni.engine import stage_init_utils monkeypatch.setattr(stage_init_utils, "resolve_worker_cls", lambda engine_args: None) pipeline = _resolve_pipeline_or_skip("qwen3_tts") - legacy_stage = merge_pipeline_deploy(pipeline, _load_default_deploy("qwen3_tts"))[0] + legacy_stage = merge_pipeline_deploy(pipeline, _load_default_deploy(pipeline))[0] omega_stage = legacy_stage.to_omegaconf() legacy_engine_args = build_engine_args_dict( omega_stage, model="/tmp/qwen3-tts", stage_connector_spec={"name": "SharedMemoryConnector", "extra": {}}, ) - omni_stage = VllmOmniConfig.from_registry("qwen3_tts").stage_by_id(legacy_stage.stage_id) + omni_stage = _from_pipeline_key("qwen3_tts").stage_by_id(legacy_stage.stage_id) assert legacy_engine_args["model"] == "/tmp/qwen3-tts" assert legacy_engine_args["stage_id"] == omni_stage.stage_id @@ -657,8 +752,8 @@ def test_from_registry_matches_build_engine_args_dict_behavior_for_representativ assert omni_stage.model_config.has_sampling_extra_args == legacy_engine_args["has_sampling_extra_args"] -def test_from_registry_derives_has_sampling_extra_args_from_stage_defaults(): - stage = VllmOmniConfig.from_registry("voxtral_tts").stage_by_id(0) +def test_from_pipeline_config_derives_has_sampling_extra_args_from_stage_defaults(): + stage = _from_pipeline_key("voxtral_tts").stage_by_id(0) assert (stage.model_config.default_sampling_params or {}).get("extra_args") assert stage.model_config.has_sampling_extra_args is True @@ -709,7 +804,7 @@ def test_diffusion_config_from_kwargs_reuses_legacy_normalization(monkeypatch): assert cfg.diffusers_call_kwargs == {} -def test_from_registry_normalizes_diffusion_config_aliases_from_engine_args(tmp_path): +def test_from_pipeline_config_normalizes_diffusion_config_aliases_from_engine_args(tmp_path): deploy_path = tmp_path / "dreamzero_diffusion_aliases.yaml" deploy_path.write_text( "\n".join( @@ -723,7 +818,7 @@ def test_from_registry_normalizes_diffusion_config_aliases_from_engine_args(tmp_ ) ) - stage = VllmOmniConfig.from_registry( + stage = _from_pipeline_key( "dreamzero", deploy_config_path=str(deploy_path), ).stage_by_id(0) diff --git a/tests/e2e/accuracy/test_hunyuan_image3.py b/tests/e2e/accuracy/test_hunyuan_image3.py index 00e77fc8f..88c7e255d 100644 --- a/tests/e2e/accuracy/test_hunyuan_image3.py +++ b/tests/e2e/accuracy/test_hunyuan_image3.py @@ -298,7 +298,7 @@ def _run_offline(deploy_config_path: str, output_path: Path) -> tuple[Image.Imag system_prompt_type = result.system_prompt_type ar_stop_token_ids = resolve_stop_token_ids(task="it2i", bot_task="think_recaption", tokenizer=tokenizer) - with OmniRunner(MODEL_PATH, deploy_config=deploy_config_path) as runner: + with OmniRunner(MODEL_PATH, deploy_config=deploy_config_path, trust_remote_code=True) as runner: params_list = list(runner.omni.default_sampling_params_list) for sp in params_list: if isinstance(sp, OmniDiffusionSamplingParams): @@ -491,7 +491,12 @@ def _run_dit_model( os.environ["VLLM_NVFP4_GEMM_BACKEND"] = nvfp4_backend try: - with OmniRunner(model, deploy_config=deploy_config_path, mode="text-to-image") as runner: + with OmniRunner( + model, + deploy_config=deploy_config_path, + mode="text-to-image", + trust_remote_code=True, + ) as runner: generator = torch.Generator(device=current_omni_platform.device_type or "cuda").manual_seed(SEED) params = OmniDiffusionSamplingParams( height=QUANT_HEIGHT, diff --git a/tests/e2e/offline_inference/test_covo_audio_expansion.py b/tests/e2e/offline_inference/test_covo_audio_expansion.py index ca05932ef..11672686e 100644 --- a/tests/e2e/offline_inference/test_covo_audio_expansion.py +++ b/tests/e2e/offline_inference/test_covo_audio_expansion.py @@ -14,7 +14,7 @@ from vllm_omni.model_executor.models.covo_audio.prompt_utils import ( models = ["tencent/Covo-Audio-Chat"] stage_config = get_deploy_config_path("covo_audio.yaml") -test_params = [(model, stage_config) for model in models] +test_params = [(model, stage_config, {"trust_remote_code": True}) for model in models] def get_question(): diff --git a/tests/e2e/offline_inference/test_dynin_omni_expansion.py b/tests/e2e/offline_inference/test_dynin_omni_expansion.py index 13b9e5cb3..19a847332 100644 --- a/tests/e2e/offline_inference/test_dynin_omni_expansion.py +++ b/tests/e2e/offline_inference/test_dynin_omni_expansion.py @@ -29,7 +29,7 @@ _DEFAULT_STAGE_CONFIG_PATH = Path(get_deploy_config_path("dynin_omni_ci.yaml")) models = ["snu-aidas/Dynin-Omni"] stage_configs = [str(_DEFAULT_STAGE_CONFIG_PATH)] -test_params = [(model, stage_config) for model in models for stage_config in stage_configs] +test_params = [(model, stage_config, {"trust_remote_code": True}) for model in models for stage_config in stage_configs] DYNIN_CONFIG_PATH = str(_DEFAULT_DYNIN_CONFIG_PATH) if _DEFAULT_DYNIN_CONFIG_PATH is not None else None diff --git a/tests/e2e/offline_inference/test_glm_tts_expansion.py b/tests/e2e/offline_inference/test_glm_tts_expansion.py index d6a8729f0..e54dd5480 100644 --- a/tests/e2e/offline_inference/test_glm_tts_expansion.py +++ b/tests/e2e/offline_inference/test_glm_tts_expansion.py @@ -98,6 +98,7 @@ def test_offline_voice_clone_zh(async_chunk: bool) -> None: MODEL, stage_configs_path=_get_deploy_config(async_chunk=async_chunk), stage_init_timeout=600, + trust_remote_code=True, ) as omni_runner: outputs = omni_runner.omni.generate( [ diff --git a/tests/e2e/offline_inference/test_hunyuanimage3.py b/tests/e2e/offline_inference/test_hunyuanimage3.py index ac1cb13cb..ebd338fe8 100644 --- a/tests/e2e/offline_inference/test_hunyuanimage3.py +++ b/tests/e2e/offline_inference/test_hunyuanimage3.py @@ -275,6 +275,7 @@ def omni() -> Generator[Omni, None, None]: MODEL_NAME, deploy_config=str(DEPLOY_CONFIG_PATH), mode="text-to-image", + trust_remote_code=True, ) as runner: yield runner.omni diff --git a/tests/e2e/offline_inference/test_hunyuanimage3_i2t_expansion.py b/tests/e2e/offline_inference/test_hunyuanimage3_i2t_expansion.py index 5178a40a6..008f0d2a9 100644 --- a/tests/e2e/offline_inference/test_hunyuanimage3_i2t_expansion.py +++ b/tests/e2e/offline_inference/test_hunyuanimage3_i2t_expansion.py @@ -53,6 +53,7 @@ def omni() -> Generator[Omni, None, None]: with OmniRunner( MODEL_NAME, stage_configs_path=str(STAGE_CONFIG_PATH), + trust_remote_code=True, ) as runner: yield runner.omni diff --git a/tests/e2e/offline_inference/test_moss_tts_expansion.py b/tests/e2e/offline_inference/test_moss_tts_expansion.py index 73e919d1c..81a64fcd9 100644 --- a/tests/e2e/offline_inference/test_moss_tts_expansion.py +++ b/tests/e2e/offline_inference/test_moss_tts_expansion.py @@ -76,7 +76,11 @@ pytestmark = [ pytest.mark.slow, pytest.mark.skip(reason="https://github.com/vllm-project/vllm-omni/issues/4700"), pytest.mark.tts, - pytest.mark.parametrize("omni_runner", [(MODEL, _get_test_config())], indirect=True), + pytest.mark.parametrize( + "omni_runner", + [(MODEL, _get_test_config(), {"trust_remote_code": True})], + indirect=True, + ), ] diff --git a/tests/e2e/offline_inference/test_moss_tts_nano_expansion.py b/tests/e2e/offline_inference/test_moss_tts_nano_expansion.py index 2d247f691..d3e7fcb0f 100644 --- a/tests/e2e/offline_inference/test_moss_tts_nano_expansion.py +++ b/tests/e2e/offline_inference/test_moss_tts_nano_expansion.py @@ -29,6 +29,7 @@ STAGE_CONFIG = get_deploy_config_path("moss_tts_nano.yaml") _OMNI_RUNNER_PARAM = ( MODEL_NAME, STAGE_CONFIG, + {"trust_remote_code": True}, ) pytestmark = [ diff --git a/tests/e2e/offline_inference/test_moss_tts_realtime.py b/tests/e2e/offline_inference/test_moss_tts_realtime.py index 5034f0f1d..892a395ba 100644 --- a/tests/e2e/offline_inference/test_moss_tts_realtime.py +++ b/tests/e2e/offline_inference/test_moss_tts_realtime.py @@ -98,7 +98,11 @@ pytestmark = [ pytest.mark.skip(reason="https://github.com/vllm-project/vllm-omni/issues/4700"), pytest.mark.full_model, pytest.mark.tts, - pytest.mark.parametrize("omni_runner", [(MODEL, _get_test_config())], indirect=True), + pytest.mark.parametrize( + "omni_runner", + [(MODEL, _get_test_config(), {"trust_remote_code": True})], + indirect=True, + ), ] # --------------------------------------------------------------------------- diff --git a/tests/e2e/offline_inference/test_moss_tts_realtime_expansion.py b/tests/e2e/offline_inference/test_moss_tts_realtime_expansion.py index 80b12e2e4..b84e5a0ff 100644 --- a/tests/e2e/offline_inference/test_moss_tts_realtime_expansion.py +++ b/tests/e2e/offline_inference/test_moss_tts_realtime_expansion.py @@ -24,7 +24,11 @@ from tests.helpers.stage_config import get_deploy_config_path MODEL = "OpenMOSS-Team/MOSS-TTS-Realtime" DEPLOY_CONFIG = get_deploy_config_path("moss_tts_realtime.yaml") -_OMNI_RUNNER_PARAM = (MODEL, DEPLOY_CONFIG, {"stage_init_timeout": 300}) +_OMNI_RUNNER_PARAM = ( + MODEL, + DEPLOY_CONFIG, + {"stage_init_timeout": 300, "trust_remote_code": True}, +) pytestmark = [ pytest.mark.slow, diff --git a/tests/e2e/offline_inference/test_moss_tts_v1_5_expansion.py b/tests/e2e/offline_inference/test_moss_tts_v1_5_expansion.py index 43bf80415..25c671e05 100644 --- a/tests/e2e/offline_inference/test_moss_tts_v1_5_expansion.py +++ b/tests/e2e/offline_inference/test_moss_tts_v1_5_expansion.py @@ -22,7 +22,11 @@ from tests.helpers.stage_config import get_deploy_config_path MODEL = "OpenMOSS-Team/MOSS-TTS-v1.5" DEPLOY_CONFIG = get_deploy_config_path("moss_tts.yaml") -_OMNI_RUNNER_PARAM = (MODEL, DEPLOY_CONFIG, {"stage_init_timeout": 600}) +_OMNI_RUNNER_PARAM = ( + MODEL, + DEPLOY_CONFIG, + {"stage_init_timeout": 600, "trust_remote_code": True}, +) pytestmark = [ pytest.mark.skip(reason="https://github.com/vllm-project/vllm-omni/issues/4643"), diff --git a/tests/e2e/offline_inference/test_voxcpm2_tts.py b/tests/e2e/offline_inference/test_voxcpm2_tts.py index 5c91db45a..c10694a40 100644 --- a/tests/e2e/offline_inference/test_voxcpm2_tts.py +++ b/tests/e2e/offline_inference/test_voxcpm2_tts.py @@ -14,8 +14,8 @@ VOXCPM2_MODEL = "openbmb/VoxCPM2" DEPLOY_CONFIG = get_deploy_config_path("voxcpm2.yaml") SAMPLE_RATE = 48000 -# (model, stage_config_path) — see ``omni_runner`` in tests.helpers.fixtures.runtime -_OMNI_RUNNER_PARAM = (VOXCPM2_MODEL, DEPLOY_CONFIG) +# VoxCPM2 ships a custom tokenizer, so remote code must be explicitly enabled. +_OMNI_RUNNER_PARAM = (VOXCPM2_MODEL, DEPLOY_CONFIG, {"trust_remote_code": True}) pytestmark = pytest.mark.parametrize("omni_runner", [_OMNI_RUNNER_PARAM], indirect=True) diff --git a/tests/engine/test_async_omni_engine_stage_init.py b/tests/engine/test_async_omni_engine_stage_init.py index d5a50cf29..e98c58d74 100644 --- a/tests/engine/test_async_omni_engine_stage_init.py +++ b/tests/engine/test_async_omni_engine_stage_init.py @@ -745,6 +745,7 @@ def test_resolve_stage_configs_injects_global_diffusion_attention_when_missing(m _config_path, stage_configs = engine._resolve_stage_configs( model="dummy-model", kwargs={"diffusion_attention_backend": "FLASH_ATTN"}, + trust_remote_code=False, ) diffusion_attention_config = stage_configs[0].engine_args.diffusion_attention_config @@ -778,6 +779,7 @@ def test_resolve_stage_configs_preserves_stage_diffusion_attention(monkeypatch): _config_path, stage_configs = engine._resolve_stage_configs( model="dummy-model", kwargs={"diffusion_attention_backend": "FLASH_ATTN"}, + trust_remote_code=False, ) assert stage_configs[0].engine_args.diffusion_attention_config is existing_attention @@ -804,6 +806,7 @@ def test_resolve_stage_configs_does_not_inject_diffusion_attention_into_llm_stag _config_path, stage_configs = engine._resolve_stage_configs( model="dummy-model", kwargs={"diffusion_attention_backend": "TORCH_SDPA"}, + trust_remote_code=False, ) assert stage_configs[0].engine_args.attention_config == {"backend": "FLASH_ATTN"} diff --git a/tests/entrypoints/test_async_omni_diffusion_config.py b/tests/entrypoints/test_async_omni_diffusion_config.py index ba132dbd2..d6b33dc89 100644 --- a/tests/entrypoints/test_async_omni_diffusion_config.py +++ b/tests/entrypoints/test_async_omni_diffusion_config.py @@ -355,6 +355,7 @@ def test_resolve_stage_configs_injects_additional_config_into_diffusion_stage(mo "stage_configs_path": "dummy.yaml", "additional_config": {"torchair_graph_config": {"enabled": True}}, }, + trust_remote_code=False, ) assert not hasattr(stage_configs[0].engine_args, "additional_config") @@ -392,6 +393,7 @@ def test_resolve_stage_configs_injects_quantization_config_into_diffusion_stage( "stage_configs_path": "dummy.yaml", "quantization_config": {"method": "bitsandbytes"}, }, + trust_remote_code=False, ) assert stage_configs[0].engine_args.quantization_config == {"method": "bitsandbytes"} diff --git a/tests/entrypoints/test_resolve_dreamzero_config.py b/tests/entrypoints/test_resolve_dreamzero_config.py index 4dcaddd63..cc459c269 100644 --- a/tests/entrypoints/test_resolve_dreamzero_config.py +++ b/tests/entrypoints/test_resolve_dreamzero_config.py @@ -28,11 +28,18 @@ def test_dreamzero_config_sets_model_class_and_policy_config(monkeypatch): classmethod(lambda _cls, model, trust_remote_code=True: "vla"), ) monkeypatch.setattr( - "vllm_omni.diffusion.utils.hf_utils._looks_like_dreamzero", + "vllm_omni.config.config_factory.StageConfigFactory.get_hf_config", + classmethod(lambda _cls, model, trust_remote_code=True: None), + ) + monkeypatch.setattr( + "vllm_omni.config.config_factory._looks_like_dreamzero", lambda _model: True, ) - stage_configs, _ = load_stage_configs_from_model("GEAR-Dreams/DreamZero-DROID") + stage_configs, _ = load_stage_configs_from_model( + "GEAR-Dreams/DreamZero-DROID", + trust_remote_code=False, + ) engine_args = stage_configs[0].engine_args assert engine_args.model_class_name == "DreamZeroPipeline" diff --git a/tests/entrypoints/test_utils.py b/tests/entrypoints/test_utils.py index b48ed2d91..4c8522ae0 100644 --- a/tests/entrypoints/test_utils.py +++ b/tests/entrypoints/test_utils.py @@ -315,13 +315,17 @@ class TestResolveModelConfigPath: class TestLoadAndResolveStageConfigs: - def test_load_and_resolve_with_kwargs(self): + def test_load_and_resolve_with_kwargs(self, mocker: MockerFixture): """Ensure that dtype survives default stage creation.""" kwargs = {"dtype": torch.float32} + mocker.patch("vllm_omni.entrypoints.utils.resolve_model_config_path", return_value=None) + mocker.patch("vllm_omni.entrypoints.utils.load_stage_configs_from_model", return_value=([], None)) + config_path, stage_configs, _ = load_and_resolve_stage_configs( model="black-forest-labs/FLUX.2-klein-4B", stage_configs_path=None, kwargs=kwargs, + trust_remote_code=False, default_stage_cfg_factory=lambda: AsyncOmniEngine._create_default_diffusion_stage_cfg(kwargs), ) assert config_path is None @@ -356,10 +360,12 @@ class TestLoadAndResolveStageConfigs: model="dummy-model", stage_configs_path=str(deploy_path), kwargs={}, + trust_remote_code=True, ) load_stage_configs.assert_called_once_with( "dummy-model", + trust_remote_code=True, base_engine_args={}, deploy_config_path=str(deploy_path), stage_overrides=None, diff --git a/tests/test_config_factory.py b/tests/test_config_factory.py index 54683e67c..e160a9b10 100644 --- a/tests/test_config_factory.py +++ b/tests/test_config_factory.py @@ -5,7 +5,6 @@ Unit tests for StageConfigFactory and related classes. """ import importlib -import inspect import warnings from dataclasses import dataclass, fields from pathlib import Path @@ -17,8 +16,10 @@ from transformers import PretrainedConfig, Qwen3OmniMoeConfig from tests.helpers.stage_config import get_deploy_config_path, get_deploy_config_stage from vllm_omni.config.config_factory import StageConfigFactory from vllm_omni.config.endpoint_policy import EndpointRestriction, OmniServingCapability -from vllm_omni.config.pipeline_registry import OMNI_PIPELINES, register_pipeline +from vllm_omni.config.omni_config import VllmOmniConfig +from vllm_omni.config.pipeline_registry import OMNI_PIPELINES, register_pipeline, resolve_pipeline_config from vllm_omni.config.stage_config import ( + _DEPLOY_DIR, DeployConfig, PipelineConfig, StageConfig, @@ -40,13 +41,21 @@ from vllm_omni.engine.arg_utils import SHARED_FIELDS, EngineArgs, internal_black pytestmark = [pytest.mark.core_model, pytest.mark.cpu] +@pytest.fixture(autouse=True) +def _stable_test_platform(monkeypatch): + from vllm_omni import platforms + + platform = platforms.current_omni_platform + monkeypatch.setattr(platform, "device_name", "cpu", raising=False) + monkeypatch.setattr(platform, "device_type", "cpu", raising=False) + + @pytest.fixture(autouse=True) def clear_config_factory_caches(): """Clear cached classmethods from the StageConfigFactory to prevent test pollution.""" yield StageConfigFactory.get_hf_config.cache_clear() StageConfigFactory.try_infer_model_type.cache_clear() - StageConfigFactory.get_pipeline_config.cache_clear() Q3_OMNI_ALL_STAGES_HF_CONFIG = Qwen3OmniMoeConfig(enable_audio_output=True) @@ -534,29 +543,41 @@ class TestPipelineDiscovery: def test_registry_resolver_qwen3_omni_all_stages(self): """Test that providing the HF config for qwen3 omni with audio enabled uses all stages.""" - pipeline = StageConfigFactory.resolve_pipeline_config( + pipeline = resolve_pipeline_config( "qwen3_omni_moe", Q3_OMNI_ALL_STAGES_HF_CONFIG, ) assert isinstance(pipeline, PipelineConfig) assert pipeline.model_type == "qwen3_omni_moe" + assert pipeline.default_deploy_config_name == "qwen3_omni_moe.yaml" assert len(pipeline.stages) == 3 # thinker + talker + code2wav def test_registry_resolver_qwen3_omni_thinker_only(self): """Test that providing the HF config for qwen3 omni without audio is thinker only.""" - pipeline = StageConfigFactory.resolve_pipeline_config( + pipeline = resolve_pipeline_config( "qwen3_omni_moe", Q3_OMNI_THINKER_HF_CONFIG, ) assert isinstance(pipeline, PipelineConfig) assert pipeline.model_type == "qwen3_omni_moe_thinker_only" + assert pipeline.default_deploy_config_name is None assert len(pipeline.stages) == 1 # thinker only + @pytest.mark.parametrize( + "pipeline", + [registered for registered in OMNI_PIPELINES.values() if isinstance(registered, PipelineConfig)], + ids=lambda pipeline: pipeline.model_type, + ) + def test_registered_default_deploy_config_exists(self, pipeline): + if pipeline.default_deploy_config_name is None: + return + assert Path(get_deploy_config_path(pipeline.default_deploy_config_name)).is_file() + def test_registry_returns_none_for_unknown(self): """Unknown model_types aren't found and resolve to `None`.""" assert "definitely_not_a_real_model" not in OMNI_PIPELINES assert OMNI_PIPELINES.get("definitely_not_a_real_model") is None - assert StageConfigFactory.resolve_pipeline_config("definitely_not_a_real_model") is None + assert resolve_pipeline_config("definitely_not_a_real_model") is None def test_pipeline_config_supports_hf_architectures(self): """PipelineConfig accepts hf_architectures for HF-arch fallback @@ -607,6 +628,236 @@ class TestPipelineConfigNew: class TestPipelineRegistration: + def test_resolve_pipeline_prefers_deploy_pipeline_key(self, clean_pipeline_registry, tmp_path): + deploy_key = "deploy_selected_pipeline" + model_type_key = "hf_model_type_pipeline" + deploy_pipe = PipelineConfig(model_type=deploy_key) + model_type_pipe = PipelineConfig(model_type=model_type_key) + register_pipeline(deploy_pipe) + register_pipeline(model_type_pipe) + + deploy_path = tmp_path / "deploy.yaml" + deploy_path.write_text(f"pipeline: {deploy_key}\n", encoding="utf-8") + + class FakeConfig(PretrainedConfig): + model_type = model_type_key + + fake_config = FakeConfig() + with patch("vllm_omni.config.config_factory.get_config", return_value=fake_config): + pipeline_cfg = StageConfigFactory.get_pipeline_config( + "fake/model", + trust_remote_code=True, + deploy_config_path=str(deploy_path), + ) + + assert pipeline_cfg is deploy_pipe + + def test_resolve_pipeline_rejects_invalid_deploy_pipeline_key( + self, + clean_pipeline_registry, + tmp_path, + ): + model_type_key = "registered_model_type_pipeline" + register_pipeline(PipelineConfig(model_type=model_type_key)) + + deploy_path = tmp_path / "deploy.yaml" + deploy_path.write_text("pipeline: missing_pipeline\n", encoding="utf-8") + + class FakeConfig(PretrainedConfig): + model_type = model_type_key + + with patch("vllm_omni.config.config_factory.get_config", return_value=FakeConfig()): + with pytest.raises(KeyError, match="missing_pipeline"): + StageConfigFactory.get_pipeline_config( + "fake/model", + trust_remote_code=True, + deploy_config_path=str(deploy_path), + ) + with pytest.raises(KeyError, match="missing_pipeline"): + StageConfigFactory.create_from_model( + "fake/model", + trust_remote_code=False, + cli_overrides={}, + deploy_config_path=str(deploy_path), + ) + with pytest.raises(KeyError, match="missing_pipeline"): + StageConfigFactory.create_legacy_stage_configs_from_model( + "fake/model", + trust_remote_code=False, + cli_overrides={}, + deploy_config_path=str(deploy_path), + ) + + def test_resolve_pipeline_matches_hf_architecture_fallback(self, clean_pipeline_registry): + pipeline_key = "architecture_fallback_pipeline" + pipe_cfg = PipelineConfig( + model_type=pipeline_key, + hf_architectures=("ArchitectureFallbackForTest",), + ) + register_pipeline(pipe_cfg) + + class FakeConfig(PretrainedConfig): + model_type = "unregistered_model_type" + + fake_config = FakeConfig() + fake_config.architectures = ["ArchitectureFallbackForTest"] + with patch("vllm_omni.config.config_factory.get_config", return_value=fake_config): + pipeline_cfg = StageConfigFactory.get_pipeline_config( + "fake/model", + trust_remote_code=True, + deploy_config_path=None, + ) + + assert pipeline_cfg is pipe_cfg + + def test_resolve_pipeline_architecture_fallback_respects_predicates(self, clean_pipeline_registry): + shared_arch = "SharedPredicateArchitectureForTest" + + def rejecting_predicate(_hf_config): + return False + + def raising_predicate(_hf_config): + raise ValueError("predicate failure") + + reject_cfg = PipelineConfig( + model_type="rejected_by_predicate", + hf_architectures=(shared_arch,), + hf_config_predicate=rejecting_predicate, + ) + raises_cfg = PipelineConfig( + model_type="raises_in_predicate", + hf_architectures=(shared_arch,), + hf_config_predicate=raising_predicate, + ) + accept_cfg = PipelineConfig( + model_type="accepted_by_predicate", + hf_architectures=(shared_arch,), + ) + register_pipeline(reject_cfg) + register_pipeline(raises_cfg) + register_pipeline(accept_cfg) + + class FakeConfig(PretrainedConfig): + model_type = "unregistered_model_type" + + fake_config = FakeConfig() + fake_config.architectures = [shared_arch] + with patch("vllm_omni.config.config_factory.get_config", return_value=fake_config): + pipeline_cfg = StageConfigFactory.get_pipeline_config( + "fake/model", + trust_remote_code=True, + deploy_config_path=None, + ) + + assert pipeline_cfg is accept_cfg + + def test_resolve_pipeline_architecture_fallback_rejects_non_matching_architecture( + self, + clean_pipeline_registry, + ): + predicate_calls = [] + + def predicate(_hf_config): + predicate_calls.append(_hf_config) + return True + + pipe_cfg = PipelineConfig( + model_type="predicate_without_arch_match", + hf_architectures=("DifferentArchitectureForTest",), + hf_config_predicate=predicate, + ) + register_pipeline(pipe_cfg) + + class FakeConfig(PretrainedConfig): + model_type = "unregistered_model_type" + + fake_config = FakeConfig() + fake_config.architectures = ["RequestedArchitectureForTest"] + with patch("vllm_omni.config.config_factory.get_config", return_value=fake_config): + pipeline_cfg = StageConfigFactory.get_pipeline_config( + "fake/model", + trust_remote_code=True, + deploy_config_path=None, + ) + + assert pipeline_cfg is None + assert predicate_calls == [fake_config] + + def test_resolve_pipeline_architecture_fallback_resolves_callable_pipeline(self, clean_pipeline_registry): + pipeline_key = "callable_architecture_fallback" + resolved_cfg = PipelineConfig( + model_type="callable_resolved_pipeline", + hf_architectures=("CallableArchitectureForTest",), + ) + seen_hf_configs = [] + + def resolver(hf_config): + seen_hf_configs.append(hf_config) + return resolved_cfg + + register_pipeline(resolver, model_type=pipeline_key) + + class FakeConfig(PretrainedConfig): + model_type = "unregistered_model_type" + + fake_config = FakeConfig() + fake_config.architectures = ["CallableArchitectureForTest"] + with patch("vllm_omni.config.config_factory.get_config", return_value=fake_config): + pipeline_cfg = StageConfigFactory.get_pipeline_config( + "fake/model", + trust_remote_code=True, + deploy_config_path=None, + ) + + assert pipeline_cfg is resolved_cfg + assert seen_hf_configs == [fake_config] + + def test_resolve_pipeline_returns_none_without_registry_or_architecture_match(self, clean_pipeline_registry): + class FakeConfig(PretrainedConfig): + model_type = "unregistered_model_type" + + fake_config = FakeConfig() + fake_config.architectures = ["UnmatchedArchitectureForTest"] + with patch("vllm_omni.config.config_factory.get_config", return_value=fake_config): + pipeline_cfg = StageConfigFactory.get_pipeline_config( + "fake/model", + trust_remote_code=True, + deploy_config_path=None, + ) + + assert pipeline_cfg is None + + def test_create_from_model_returns_structured_omni_config(self): + class FakeConfig(PretrainedConfig): + model_type = "qwen3_tts" + + with patch("vllm_omni.config.config_factory.get_config", return_value=FakeConfig()): + omni_config = StageConfigFactory.create_from_model( + "fake/model", + trust_remote_code=False, + cli_overrides={}, + deploy_config_path=None, + ) + + assert isinstance(omni_config, VllmOmniConfig) + assert omni_config.pipeline_config is OMNI_PIPELINES["qwen3_tts"] + assert len(omni_config.stage_configs) == 2 + + def test_create_from_model_preserves_model_on_structured_diffusion_stage(self): + class FakeConfig(PretrainedConfig): + model_type = "dreamzero" + + with patch("vllm_omni.config.config_factory.get_config", return_value=FakeConfig()): + omni_config = StageConfigFactory.create_from_model( + "fake/model", + trust_remote_code=False, + cli_overrides={}, + deploy_config_path=None, + ) + + assert isinstance(omni_config, VllmOmniConfig) + assert omni_config.stage_by_id(0).diffusion_config.model == "fake/model" + def test_pipeline_registration(self, clean_pipeline_registry): """Ensure that we can register and create a custom pipeline config.""" new_model_type = "new_model_type" @@ -622,15 +873,23 @@ class TestPipelineRegistration: model_type = new_model_type # Create the model + fake_config = FakeConfig() with ( - patch("vllm_omni.config.config_factory.get_config", return_value=FakeConfig()), - patch.object(StageConfigFactory, "_create_from_registry") as mock_create, + patch("vllm_omni.config.config_factory.get_config", return_value=fake_config), + patch.object(VllmOmniConfig, "from_pipeline_config") as mock_create, ): - StageConfigFactory.create_from_model("fake/model") + StageConfigFactory.create_from_model( + "fake/model", + trust_remote_code=False, + cli_overrides={}, + deploy_config_path=None, + ) mock_create.assert_called_once() - # Ensure that the passed pipeline_config is the right type - pipeline_cfg = mock_create.call_args[0][1] - assert isinstance(pipeline_cfg, PipelineConfig) + assert mock_create.call_args.args == (pipe_cfg,) + assert mock_create.call_args.kwargs["cli_overrides"] == { + "trust_remote_code": False, + "model": "fake/model", + } assert pipe_cfg.model_type == new_model_type def test_resolver_registration(self, clean_pipeline_registry): @@ -655,26 +914,125 @@ class TestPipelineRegistration: assert OMNI_PIPELINES[new_model_type] is custom_resolver # Create the model + fake_config = FakeConfig() with ( - patch("vllm_omni.config.config_factory.get_config", return_value=FakeConfig()), - patch.object(StageConfigFactory, "_create_from_registry") as mock_create, + patch("vllm_omni.config.config_factory.get_config", return_value=fake_config) as mock_get_config, + patch.object(VllmOmniConfig, "from_pipeline_config") as mock_create, ): - StageConfigFactory.create_from_model("fake/model") + StageConfigFactory.create_from_model( + "fake/model", + trust_remote_code=True, + cli_overrides={}, + deploy_config_path=None, + ) mock_create.assert_called_once() - # Ensure that the passed pipeline_config is the right type - pipeline_cfg = mock_create.call_args[0][1] - assert isinstance(pipeline_cfg, PipelineConfig) - assert pipeline_cfg.model_type == resolved_type + assert mock_create.call_args.args[0].model_type == resolved_type + assert mock_create.call_args.kwargs["cli_overrides"] == { + "trust_remote_code": True, + "model": "fake/model", + } + mock_get_config.assert_called_once_with("fake/model", trust_remote_code=True) + + with patch.object(StageConfigFactory, "_create_legacy_from_registry", return_value=([], None)) as mock_legacy: + StageConfigFactory.create_legacy_stage_configs_from_model( + "fake/model", + trust_remote_code=True, + cli_overrides={}, + deploy_config_path=None, + ) + + assert mock_legacy.call_args.args[0].model_type == resolved_type + assert mock_legacy.call_args.args[1]["trust_remote_code"] is True def test_resolve_when_autodetect_resolves_none(self): """Regression test for: https://github.com/vllm-project/vllm-omni/issues/4726""" deploy_path = get_deploy_config_path("ming_tts.yaml") resolved_config = StageConfigFactory.create_from_model( model="inclusionAI/Ming-omni-tts-0.5B", + trust_remote_code=False, + cli_overrides={}, deploy_config_path=deploy_path, ) assert resolved_config is not None - assert len(resolved_config) > 0 + assert len(resolved_config.stage_configs) > 0 + + def test_legacy_and_structured_paths_share_deploy_pipeline_override(self, clean_pipeline_registry, tmp_path): + pipeline_key = "deploy_only_pipeline" + pipe_cfg = PipelineConfig( + model_type=pipeline_key, + model_arch="DeployOnlyArch", + stages=(StagePipelineConfig(stage_id=0, model_stage="diffusion"),), + ) + register_pipeline(pipe_cfg) + + deploy_path = tmp_path / "deploy_only.yaml" + deploy_path.write_text(f"pipeline: {pipeline_key}\n", encoding="utf-8") + + class FakeConfig(PretrainedConfig): + model_type = "dense" + architectures = ("UnregisteredArch",) + + with patch("vllm_omni.config.config_factory.get_config", return_value=FakeConfig()): + omni_config = StageConfigFactory.create_from_model( + "fake/model", + trust_remote_code=False, + cli_overrides={}, + deploy_config_path=str(deploy_path), + ) + legacy_configs, _ = StageConfigFactory.create_legacy_stage_configs_from_model( + "fake/model", + trust_remote_code=False, + cli_overrides={}, + deploy_config_path=str(deploy_path), + ) + + assert omni_config is not None + assert omni_config.pipeline_config is pipe_cfg + assert legacy_configs is not None + assert len(legacy_configs) == 1 + assert legacy_configs[0].yaml_engine_args["model_arch"] == "DeployOnlyArch" + + def test_structured_path_loads_explicit_deploy_config_once(self, clean_pipeline_registry, tmp_path): + pipeline_key = "single_load_pipeline" + register_pipeline(PipelineConfig(model_type=pipeline_key)) + deploy_path = tmp_path / "single_load.yaml" + deploy_path.write_text(f"pipeline: {pipeline_key}\n", encoding="utf-8") + + class FakeConfig(PretrainedConfig): + model_type = "unregistered" + + with ( + patch("vllm_omni.config.config_factory.get_config", return_value=FakeConfig()), + patch( + "vllm_omni.config.config_factory.load_deploy_config", + wraps=load_deploy_config, + ) as mock_load, + ): + StageConfigFactory.create_from_model( + "fake/model", + trust_remote_code=False, + cli_overrides={}, + deploy_config_path=str(deploy_path), + ) + + mock_load.assert_called_once_with(deploy_path) + + def test_legacy_default_deploy_name_ignores_cwd(self, monkeypatch, tmp_path): + default_name = "pipeline_default.yaml" + (tmp_path / default_name).write_text("stages: []\n", encoding="utf-8") + monkeypatch.chdir(tmp_path) + pipeline = PipelineConfig( + model_type="pipeline_with_default", + default_deploy_config_name=default_name, + ) + + with patch( + "vllm_omni.config.config_factory.load_deploy_config", + return_value=DeployConfig(), + ) as mock_load: + StageConfigFactory._create_legacy_from_registry(pipeline, {}) + + mock_load.assert_called_once_with(_DEPLOY_DIR / default_name) def test_deploy_override_uses_correct_endpoint_restrictions(self, clean_pipeline_registry, tmp_path): """Ensure endpoint restrictions must come from the final pipeline @@ -756,7 +1114,7 @@ stages: ) deploy = load_deploy_config(deploy_path) - pipeline = StageConfigFactory.resolve_pipeline_config("qwen3_tts") + pipeline = resolve_pipeline_config("qwen3_tts") stages = merge_pipeline_deploy(pipeline, deploy) assert isinstance(pipeline, PipelineConfig) @@ -827,7 +1185,7 @@ stages: if not deploy_path.exists(): pytest.skip("Deploy config not found") - pipeline = StageConfigFactory.resolve_pipeline_config( + pipeline = resolve_pipeline_config( "qwen3_omni_moe", Q3_OMNI_ALL_STAGES_HF_CONFIG, ) @@ -844,7 +1202,7 @@ stages: assert s0.yaml_extras["default_sampling_params"]["detokenize"] is True def test_merge_pipeline_deploy_preserves_num_replicas(self, tmp_path): - pipeline = StageConfigFactory.resolve_pipeline_config( + pipeline = resolve_pipeline_config( "qwen3_omni_moe", Q3_OMNI_ALL_STAGES_HF_CONFIG, ) @@ -1131,7 +1489,7 @@ stages: class TestQwen3OmniPipeline: def test_registered(self): - p = StageConfigFactory.resolve_pipeline_config( + p = resolve_pipeline_config( "qwen3_omni_moe", Q3_OMNI_ALL_STAGES_HF_CONFIG, ) @@ -1141,7 +1499,7 @@ class TestQwen3OmniPipeline: assert p.validate() == [] def test_thinker(self): - p = StageConfigFactory.resolve_pipeline_config( + p = resolve_pipeline_config( "qwen3_omni_moe", Q3_OMNI_ALL_STAGES_HF_CONFIG, ) @@ -1156,7 +1514,7 @@ class TestQwen3OmniPipeline: assert s.sampling_constraints["detokenize"] is True def test_talker(self): - p = StageConfigFactory.resolve_pipeline_config( + p = resolve_pipeline_config( "qwen3_omni_moe", Q3_OMNI_ALL_STAGES_HF_CONFIG, ) @@ -1173,7 +1531,7 @@ class TestQwen3OmniPipeline: assert s.custom_process_next_stage_input_func is not None def test_code2wav(self): - p = StageConfigFactory.resolve_pipeline_config( + p = resolve_pipeline_config( "qwen3_omni_moe", Q3_OMNI_ALL_STAGES_HF_CONFIG, ) @@ -1188,14 +1546,14 @@ class TestQwen3OmniPipeline: class TestQwen2_5OmniPipeline: def test_registered(self): - p = StageConfigFactory.resolve_pipeline_config("qwen2_5_omni") + p = resolve_pipeline_config("qwen2_5_omni") assert isinstance(p, PipelineConfig) assert p.model_arch == "Qwen2_5OmniForConditionalGeneration" assert len(p.stages) == 3 assert p.validate() == [] def test_thinker(self): - p = StageConfigFactory.resolve_pipeline_config("qwen2_5_omni") + p = resolve_pipeline_config("qwen2_5_omni") assert isinstance(p, PipelineConfig) s = p.get_stage(0) @@ -1207,7 +1565,7 @@ class TestQwen2_5OmniPipeline: assert s.requires_multimodal_data is True def test_talker(self): - p = StageConfigFactory.resolve_pipeline_config("qwen2_5_omni") + p = resolve_pipeline_config("qwen2_5_omni") assert isinstance(p, PipelineConfig) s = p.get_stage(1) @@ -1221,7 +1579,7 @@ class TestQwen2_5OmniPipeline: assert s.sync_process_input_func is not None def test_code2wav(self): - p = StageConfigFactory.resolve_pipeline_config("qwen2_5_omni") + p = resolve_pipeline_config("qwen2_5_omni") assert isinstance(p, PipelineConfig) s = p.get_stage(2) @@ -1233,7 +1591,7 @@ class TestQwen2_5OmniPipeline: class TestQwen3TTSPipeline: def test_registered(self): - p = StageConfigFactory.resolve_pipeline_config("qwen3_tts") + p = resolve_pipeline_config("qwen3_tts") assert isinstance(p, PipelineConfig) assert p is not None assert p.model_arch == "Qwen3TTSTalkerForConditionalGeneration" @@ -1241,7 +1599,7 @@ class TestQwen3TTSPipeline: assert p.validate() == [] def test_talker_stage(self): - p = StageConfigFactory.resolve_pipeline_config("qwen3_tts") + p = resolve_pipeline_config("qwen3_tts") assert isinstance(p, PipelineConfig) s = p.get_stage(0) @@ -1255,7 +1613,7 @@ class TestQwen3TTSPipeline: assert s.model_arch is None def test_code2wav_stage_has_per_stage_model_arch(self): - p = StageConfigFactory.resolve_pipeline_config("qwen3_tts") + p = resolve_pipeline_config("qwen3_tts") assert isinstance(p, PipelineConfig) s = p.get_stage(1) @@ -1314,14 +1672,14 @@ class TestQwen3TTSPipeline: class TestMingFlashOmniPipeline: def test_registered(self): - p = StageConfigFactory.resolve_pipeline_config("ming_flash_omni") + p = resolve_pipeline_config("ming_flash_omni") assert isinstance(p, PipelineConfig) assert p.model_arch == "MingFlashOmniForConditionalGeneration" assert len(p.stages) == 2 assert p.validate() == [] def test_thinker_stage(self): - p = StageConfigFactory.resolve_pipeline_config("ming_flash_omni") + p = resolve_pipeline_config("ming_flash_omni") assert isinstance(p, PipelineConfig) s = p.get_stage(0) @@ -1335,7 +1693,7 @@ class TestMingFlashOmniPipeline: assert s.sampling_constraints["detokenize"] is True def test_talker_stage(self): - p = StageConfigFactory.resolve_pipeline_config("ming_flash_omni") + p = resolve_pipeline_config("ming_flash_omni") assert isinstance(p, PipelineConfig) s = p.get_stage(1) @@ -1361,7 +1719,7 @@ class TestMingFlashOmniPipeline: Lazy string references only fail at first inference otherwise — this catches typos in the pipeline declaration at import / registration time. """ - p = StageConfigFactory.resolve_pipeline_config("ming_flash_omni") + p = resolve_pipeline_config("ming_flash_omni") assert isinstance(p, PipelineConfig) s = p.get_stage(1) @@ -1371,14 +1729,14 @@ class TestMingFlashOmniPipeline: assert callable(getattr(module, attr)) def test_tts_pipeline_registered(self): - p = StageConfigFactory.resolve_pipeline_config("ming_flash_omni_tts") + p = resolve_pipeline_config("ming_flash_omni_tts") assert isinstance(p, PipelineConfig) assert p.model_arch == "MingFlashOmniTalkerForConditionalGeneration" assert len(p.stages) == 1 assert p.validate() == [] def test_tts_stage(self): - p = StageConfigFactory.resolve_pipeline_config("ming_flash_omni_tts") + p = resolve_pipeline_config("ming_flash_omni_tts") assert isinstance(p, PipelineConfig) s = p.get_stage(0) @@ -1405,7 +1763,7 @@ class TestMingFlashOmniPipeline: # We won't test stage 0/1 colocation contract here, # as there could exist more variant of custom device setup - pipeline = StageConfigFactory.resolve_pipeline_config("ming_flash_omni") + pipeline = resolve_pipeline_config("ming_flash_omni") assert isinstance(pipeline, PipelineConfig) stages = merge_pipeline_deploy(pipeline, deploy) assert len(stages) == 2 @@ -1428,14 +1786,14 @@ class TestMingFlashOmniPipeline: assert stages[0].yaml_engine_args["model_arch"] == "MingFlashOmniTalkerForConditionalGeneration" def test_thinker_only_pipeline_registered(self): - p = StageConfigFactory.resolve_pipeline_config("ming_flash_omni_thinker_only") + p = resolve_pipeline_config("ming_flash_omni_thinker_only") assert isinstance(p, PipelineConfig) assert p.model_arch == "MingFlashOmniForConditionalGeneration" assert len(p.stages) == 1 assert p.validate() == [] def test_thinker_only_stage(self): - p = StageConfigFactory.resolve_pipeline_config("ming_flash_omni_thinker_only") + p = resolve_pipeline_config("ming_flash_omni_thinker_only") assert isinstance(p, PipelineConfig) s = p.get_stage(0) @@ -1460,7 +1818,7 @@ class TestMingFlashOmniPipeline: assert len(deploy.stages) == 1 assert deploy.pipeline == "ming_flash_omni_thinker_only" - pipeline = StageConfigFactory.resolve_pipeline_config("ming_flash_omni_thinker_only") + pipeline = resolve_pipeline_config("ming_flash_omni_thinker_only") assert isinstance(pipeline, PipelineConfig) stages = merge_pipeline_deploy(pipeline, deploy) assert len(stages) == 1 @@ -1474,7 +1832,7 @@ class TestMingFlashOmniPipeline: assert p.validate() == [] def test_image_thinker_stage(self): - s = StageConfigFactory.resolve_pipeline_config("ming_flash_omni_image").get_stage(0) + s = resolve_pipeline_config("ming_flash_omni_image").get_stage(0) assert s.model_stage == "thinker" assert s.execution_type == StageExecutionType.LLM_AR assert s.input_sources == () @@ -1488,7 +1846,7 @@ class TestMingFlashOmniPipeline: assert s.prompt_expand_func is not None def test_image_dit_stage(self): - s = StageConfigFactory.resolve_pipeline_config("ming_flash_omni_image").get_stage(1) + s = resolve_pipeline_config("ming_flash_omni_image").get_stage(1) assert s.model_stage == "dit" assert s.execution_type == StageExecutionType.DIFFUSION assert s.input_sources == (0,) @@ -1500,7 +1858,7 @@ class TestMingFlashOmniPipeline: def test_image_processor_wiring_resolves(self): """The prompt_expand_func and custom_process_input_func strings must point to real callables.""" - pipeline = StageConfigFactory.resolve_pipeline_config("ming_flash_omni_image") + pipeline = resolve_pipeline_config("ming_flash_omni_image") assert isinstance(pipeline, PipelineConfig) thinker = pipeline.get_stage(0) @@ -1523,7 +1881,7 @@ class TestMingFlashOmniPipeline: assert deploy.connectors is not None assert "shared_memory_connector" in deploy.connectors - pipeline = StageConfigFactory.resolve_pipeline_config("ming_flash_omni_image") + pipeline = resolve_pipeline_config("ming_flash_omni_image") assert isinstance(pipeline, PipelineConfig) stages = merge_pipeline_deploy(pipeline, deploy) assert len(stages) == 2 @@ -1680,7 +2038,7 @@ class TestCLIOverrideFlow: pytest.skip("Deploy config not found") deploy = load_deploy_config(deploy_path) - pipeline = StageConfigFactory.resolve_pipeline_config( + pipeline = resolve_pipeline_config( "qwen3_omni_moe", Q3_OMNI_ALL_STAGES_HF_CONFIG, ) @@ -1698,7 +2056,7 @@ class TestCLIOverrideFlow: pytest.skip("Deploy config not found") deploy = load_deploy_config(deploy_path) - pipeline = StageConfigFactory.resolve_pipeline_config( + pipeline = resolve_pipeline_config( "qwen3_omni_moe", Q3_OMNI_ALL_STAGES_HF_CONFIG, ) @@ -1719,10 +2077,9 @@ class TestAuraOmniDeploy: assert deploy.pipeline == "aura_omni" def test_aura_omni_deploy_resolves_four_native_stages(self): - pipeline_cfg = StageConfigFactory.resolve_pipeline_config("aura_omni") + pipeline_cfg = resolve_pipeline_config("aura_omni") - stages, _ = StageConfigFactory._create_from_registry( - "qwen3_tts", + stages, _ = StageConfigFactory._create_legacy_from_registry( pipeline_cfg, cli_overrides={}, deploy_config_path=str(Path(__file__).parent.parent / "vllm_omni" / "deploy" / "aura_omni.yaml"), @@ -1939,12 +2296,11 @@ class TestSentinelDefaultPrecedence: def _stages(self, cli_overrides): model_type = "qwen3_omni_moe" - pipeline_cfg = StageConfigFactory.resolve_pipeline_config( + pipeline_cfg = resolve_pipeline_config( model_type, Q3_OMNI_ALL_STAGES_HF_CONFIG, ) - stages, _ = StageConfigFactory._create_from_registry( - "qwen3_omni_moe", + stages, _ = StageConfigFactory._create_legacy_from_registry( pipeline_cfg, cli_overrides=cli_overrides, ) @@ -2002,16 +2358,11 @@ class TestSentinelDefaultPrecedence: for stage in stages: assert stage.runtime_overrides == {} - def test_create_from_registry_no_cli_explicit_keys_param(self): - sig = inspect.signature(StageConfigFactory._create_from_registry) - named = [p for p in sig.parameters.values() if p.kind != p.VAR_KEYWORD] - assert "cli_explicit_keys" not in {p.name for p in named} - def test_async_chunk_dispatches_processors(self): """A single ``qwen3_tts`` pipeline picks per-chunk vs end-to-end processors based on ``deploy.async_chunk``, without needing a separate variant pipeline registration.""" - pipeline = StageConfigFactory.resolve_pipeline_config("qwen3_tts") + pipeline = resolve_pipeline_config("qwen3_tts") assert isinstance(pipeline, PipelineConfig) # async_chunk=True → stage 0's per-chunk processor wires up, stage 1 @@ -2082,7 +2433,7 @@ class TestSentinelDefaultPrecedence: called ``_build_talker_inputs`` identically. Merge under either async_chunk mode must not re-introduce a stage-0 full-payload hook.""" - pipeline = StageConfigFactory.resolve_pipeline_config("ming_flash_omni") + pipeline = resolve_pipeline_config("ming_flash_omni") assert isinstance(pipeline, PipelineConfig) stage0, stage1 = pipeline.stages @@ -2120,7 +2471,7 @@ class TestSamplingConstraintsPrecedence: pytest.skip("Deploy config not found") deploy = load_deploy_config(deploy_path) - pipeline = StageConfigFactory.resolve_pipeline_config( + pipeline = resolve_pipeline_config( "qwen3_omni_moe", Q3_OMNI_ALL_STAGES_HF_CONFIG, ) diff --git a/tests/utils/test_tracking_parser.py b/tests/utils/test_tracking_parser.py index 5bee3e3b0..00882744c 100644 --- a/tests/utils/test_tracking_parser.py +++ b/tests/utils/test_tracking_parser.py @@ -49,6 +49,11 @@ _TEST_DEPLOY = DeployConfig( @pytest.fixture() def mock_stages(monkeypatch): """Register a fake pipeline and mock deploy YAML loading.""" + from vllm_omni import platforms + + platform = platforms.current_omni_platform + monkeypatch.setattr(platform, "device_name", "cpu", raising=False) + monkeypatch.setattr(platform, "device_type", "cpu", raising=False) monkeypatch.setitem(OMNI_PIPELINES, _TEST_MODEL, _TEST_PIPELINE) monkeypatch.setattr( "vllm_omni.config.config_factory.load_deploy_config", @@ -584,6 +589,12 @@ def test_cli_overrides_config(tmp_path): ### Integration tests for arg resolution through StageConfigFactory +# +# These tests still target the transitional legacy StageConfig path because the +# current runtime reads explicit CLI values from ``stage.runtime_overrides``. +# After RFC #4021 migrates engine startup to consume ``VllmOmniConfig`` directly, +# these assertions should move to the structured config/runtime boundary and the +# ``_create_legacy_from_registry`` calls should be removed. def test_explicit_cli_arg_reaches_runtime_overrides(mock_stages): """Explicitly passed CLI values reach runtime_overrides on all stages.""" p = TrackingArgumentParser() @@ -591,8 +602,7 @@ def test_explicit_cli_arg_reaches_runtime_overrides(mock_stages): ns = p.parse_args(["--max-num-seqs", "999"]) explicit_kwargs = ns.get_explicit_kwargs_dict() - stages, _ = StageConfigFactory._create_from_registry( - _TEST_MODEL, + stages, _ = StageConfigFactory._create_legacy_from_registry( _TEST_PIPELINE, explicit_kwargs, deploy_config_path=mock_stages, @@ -608,8 +618,7 @@ def test_omitted_default_not_in_runtime_overrides(mock_stages): ns = p.parse_args([]) explicit_kwargs = ns.get_explicit_kwargs_dict() - stages, _ = StageConfigFactory._create_from_registry( - _TEST_MODEL, + stages, _ = StageConfigFactory._create_legacy_from_registry( _TEST_PIPELINE, explicit_kwargs, deploy_config_path=mock_stages, @@ -630,8 +639,7 @@ def test_config_file_args_reach_runtime_overrides(mock_stages): ns = p.parse_args(["--config", "fake.yaml"]) explicit_kwargs = ns.get_explicit_kwargs_dict() - stages, _ = StageConfigFactory._create_from_registry( - _TEST_MODEL, + stages, _ = StageConfigFactory._create_legacy_from_registry( _TEST_PIPELINE, explicit_kwargs, deploy_config_path=mock_stages, @@ -648,8 +656,7 @@ def test_per_stage_override_routes_correctly(mock_stages): ns = p.parse_args(["--stage-0-gpu-memory-utilization", "0.42"]) explicit_kwargs = ns.get_explicit_kwargs_dict() - stages, _ = StageConfigFactory._create_from_registry( - _TEST_MODEL, + stages, _ = StageConfigFactory._create_legacy_from_registry( _TEST_PIPELINE, explicit_kwargs, deploy_config_path=mock_stages, @@ -670,8 +677,7 @@ def test_explicit_args_omitted_from_yaml(mock_stages): ns = p.parse_args(["--enforce-eager"]) explicit_kwargs = ns.get_explicit_kwargs_dict() - stages, _ = StageConfigFactory._create_from_registry( - _TEST_MODEL, + stages, _ = StageConfigFactory._create_legacy_from_registry( _TEST_PIPELINE, explicit_kwargs, deploy_config_path=mock_stages, diff --git a/vllm_omni/config/config_factory.py b/vllm_omni/config/config_factory.py index 1b38a3d34..d43a7a48f 100644 --- a/vllm_omni/config/config_factory.py +++ b/vllm_omni/config/config_factory.py @@ -17,7 +17,8 @@ from vllm.transformers_utils.config import get_config from vllm.transformers_utils.repo_utils import get_hf_file_to_dict from vllm_omni.config.endpoint_policy import EndpointRestriction -from vllm_omni.config.pipeline_registry import OMNI_PIPELINES +from vllm_omni.config.omni_config import VllmOmniConfig +from vllm_omni.config.pipeline_registry import OMNI_PIPELINES, resolve_pipeline_config from vllm_omni.config.stage_config import ( _DEPLOY_DIR, DeployConfig, @@ -197,12 +198,12 @@ class StageConfigFactory: return None @classmethod - @functools.cache def get_pipeline_config( cls, model: str, trust_remote_code: bool, deploy_config_path: str | None = None, + user_deploy_config: DeployConfig | None = None, ) -> PipelineConfig | None: """Resolve the PipelineConfig for a model path/name.""" model_type = cls.try_infer_model_type(model=model, trust_remote_code=trust_remote_code) @@ -210,13 +211,15 @@ class StageConfigFactory: # Resolve the deploy config & check if the user set the pipeline; # If the pipeline is explicitly set, it takes highest priority - deploy_config_pipe = cls._get_deploy_override_pipe_config(hf_config, deploy_config_path) + if user_deploy_config is None: + user_deploy_config = cls._load_user_deploy_config(deploy_config_path) + deploy_config_pipe = cls._get_deploy_override_pipe_config(hf_config, user_deploy_config) if deploy_config_pipe is not None: return deploy_config_pipe # Pipeline isn't set in the yaml spec, so we need infer it ourselves. if model_type and model_type in OMNI_PIPELINES: - pipeline_cfg = cls.resolve_pipeline_config(model_type, hf_config) + pipeline_cfg = resolve_pipeline_config(model_type, hf_config) if pipeline_cfg is not None: return pipeline_cfg @@ -257,78 +260,114 @@ class StageConfigFactory: def _get_deploy_override_pipe_config( cls, hf_config: PretrainedConfig | None, - deploy_config_path: str | None, + deploy_config: DeployConfig | None, ) -> PipelineConfig | None: - """Load the deploy config and extract + resolve its pipeline field.""" - if deploy_config_path is None: + """Resolve an explicit pipeline override from a loaded deploy config.""" + if deploy_config is None or deploy_config.pipeline is None: return None + pipeline_cfg = resolve_pipeline_config(deploy_config.pipeline, hf_config) + if pipeline_cfg is None: + raise KeyError( + f"Pipeline {deploy_config.pipeline!r} from deploy config is not registered " + f"to OMNI_PIPELINES. Available: {sorted(OMNI_PIPELINES)}" + ) + return pipeline_cfg + + @staticmethod + def _load_user_deploy_config(deploy_config_path: str | None) -> DeployConfig | None: + """Load an explicit deploy YAML once for resolution and construction.""" + if deploy_config_path is None: + return None deploy_path = Path(deploy_config_path) - if deploy_path.exists(): - deploy_cfg = load_deploy_config(deploy_path) - if deploy_cfg.pipeline is not None: - return cls.resolve_pipeline_config(deploy_cfg.pipeline, hf_config) - return None + if not deploy_path.exists() and deploy_path.parent == Path("."): + candidate = _DEPLOY_DIR / deploy_path + if candidate.exists(): + deploy_path = candidate + if not deploy_path.exists(): + raise FileNotFoundError(f"Deploy config not found: {deploy_path}") + return load_deploy_config(deploy_path) @classmethod def create_from_model( cls, model: str, *, - trust_remote_code: bool = False, - cli_overrides: dict[str, Any] | None = None, - deploy_config_path: str | None = None, - strategy_specs: Mapping[Any, Any] | None = None, - **deprecated_kwargs: Any, - ) -> tuple[list[StageConfig] | None, str | None]: - """Load pipeline + deploy config, merge with CLI overrides. - - Checks OMNI_PIPELINES first, since supported models should be explicitly - registered. If a model is not registered in OMNI_PIPELINES, tries to fall - back to using the Transformers config & finding pipelines that have overlapping - supported architectures. - - When ``strategy_specs`` is provided (a mapping of role -> list of - ``StrategySpec``), the derived parallel sizing is overlaid onto the - merged stages (see ``vllm_omni.config.composable_parallel``). This is - opt-in: omitting it leaves the existing merge path untouched. - - Returns ``(stages, omni_lb_policy)``: the merged stages (``None`` when the - model is not in the pipeline registry and the caller should fall back to - the legacy YAML path) and the strategy-derived, pipeline-wide - ``omni_lb_policy`` (``None`` when no stage_replica axis set one). The - policy is returned rather than threaded through a mutable out-param so the - engine can apply it without every intermediate call carrying the dict. - """ - if cli_overrides is None: - cli_overrides = {} - + trust_remote_code: bool, + cli_overrides: dict[str, Any], + deploy_config_path: str | None, + ) -> VllmOmniConfig | None: + """Build the structured Omni config for a model/deploy pair.""" + user_deploy_config = cls._load_user_deploy_config(deploy_config_path) pipeline_cfg = cls.get_pipeline_config( model=model, trust_remote_code=trust_remote_code, deploy_config_path=deploy_config_path, + user_deploy_config=user_deploy_config, + ) + if pipeline_cfg is None: + return None + + registry_cli_overrides = { + **cli_overrides, + "trust_remote_code": trust_remote_code, + "model": model, + } + return VllmOmniConfig.from_pipeline_config( + pipeline_cfg, + user_deploy_config=user_deploy_config, + deploy_config_path=deploy_config_path, + cli_overrides=registry_cli_overrides, ) - if pipeline_cfg is not None: - return cls._create_from_registry( - pipeline_cfg.model_type, - pipeline_cfg, - cli_overrides, - deploy_config_path, - strategy_specs=strategy_specs, - ) - return None, None @classmethod - def _create_from_registry( + def create_legacy_stage_configs_from_model( + cls, + model: str, + *, + trust_remote_code: bool, + cli_overrides: dict[str, Any], + deploy_config_path: str | None, + strategy_specs: Mapping[Any, Any] | None = None, + ) -> tuple[list[StageConfig] | None, str | None]: + """Build current runtime stage configs from the shared resolution. + + The engine still consumes the legacy StageConfig/OmegaConf shape. + RFC #4021 will replace this transitional path as runtime consumers move + to VllmOmniConfig. + """ + user_deploy_config = cls._load_user_deploy_config(deploy_config_path) + pipeline_cfg = cls.get_pipeline_config( + model=model, + trust_remote_code=trust_remote_code, + deploy_config_path=deploy_config_path, + user_deploy_config=user_deploy_config, + ) + if pipeline_cfg is None: + return None, None + + legacy_cli_overrides = { + **cli_overrides, + "trust_remote_code": trust_remote_code, + } + return cls._create_legacy_from_registry( + pipeline_cfg, + legacy_cli_overrides, + deploy_config_path, + user_deploy_config=user_deploy_config, + strategy_specs=strategy_specs, + ) + + @classmethod + def _create_legacy_from_registry( cls, - model_type: str, pipeline_cfg: PipelineConfig, cli_overrides: dict[str, Any], deploy_config_path: str | None = None, + user_deploy_config: DeployConfig | None = None, strategy_specs: Mapping[Any, Any] | None = None, - **deprecated_kwargs: Any, ) -> tuple[list[StageConfig], str | None]: - """Create StageConfigs from pipeline registry + deploy YAML. + """Create current runtime StageConfigs from registry + deploy YAML. Precedence: caller-typed (non-None) value > deploy YAML > StageDeployConfig dataclass default. @@ -337,30 +376,15 @@ class StageConfigFactory: load-balancer policy (``None`` when no strategy set one) travels with the stages instead of through a mutable out-param. """ - # Resolve deploy config path - if deploy_config_path is None: - deploy_path = _DEPLOY_DIR / f"{model_type}.yaml" + if user_deploy_config is not None: + deploy_cfg = user_deploy_config + elif deploy_config_path is not None: + deploy_cfg = cls._load_user_deploy_config(deploy_config_path) + assert deploy_cfg is not None + elif pipeline_cfg.default_deploy_config_name is not None: + deploy_cfg = load_deploy_config(_DEPLOY_DIR / pipeline_cfg.default_deploy_config_name) else: - deploy_path = Path(deploy_config_path) - - if not deploy_path.exists(): - logger.warning( - "Deploy config not found: %s — using pipeline defaults only", - deploy_path, - ) deploy_cfg = DeployConfig() - else: - deploy_cfg = load_deploy_config(deploy_path) - # Fallback to using the deploy config pipeline class if it's a mismatch - if deploy_cfg.pipeline and deploy_cfg.pipeline != model_type: - resolved = cls.resolve_pipeline_config(deploy_cfg.pipeline) - if resolved is None: - raise KeyError( - f"Pipeline {deploy_cfg.pipeline!r} from {deploy_path.name!r} " - f"not found in OMNI_PIPELINES. Available: " - f"{sorted(OMNI_PIPELINES.keys())}" - ) - pipeline_cfg = resolved cli_async_chunk = cli_overrides.get("async_chunk") if cli_async_chunk is not None: @@ -565,13 +589,3 @@ class StageConfigFactory: ``filter_dataclass_kwargs(OmniEngineArgs, ...)``. """ return build_stage_runtime_overrides(stage.stage_id, cli_overrides) - - @staticmethod - def resolve_pipeline_config(model_type: str, hf_config: PretrainedConfig | None = None) -> PipelineConfig | None: - """Given a model type, resolve to the pipeline to be used. If the pipeline - maps to a callable we resolve based on the HF config.""" - if model_type not in OMNI_PIPELINES: - logger.warning("Model type %s is not registered to OMNI_PIPELINES", model_type) - return None - obj = OMNI_PIPELINES[model_type] - return obj(hf_config) if callable(obj) else obj diff --git a/vllm_omni/config/omni_config.py b/vllm_omni/config/omni_config.py index 256dcd5c7..4c2154401 100644 --- a/vllm_omni/config/omni_config.py +++ b/vllm_omni/config/omni_config.py @@ -2,9 +2,10 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """Structured vLLM-Omni configuration classes. -This module is additive for Phase 2 of RFC #4021. ``VllmOmniConfig.from_registry`` -builds the structured view directly from the pipeline registry and deploy config -so parity can be proven before later PRs cut consumers over to these classes. +This module is additive for Phase 2 of RFC #4021. +``VllmOmniConfig.from_pipeline_config`` builds the structured view from an +already-resolved pipeline and deploy config so parity can be proven before +later PRs cut consumers over to these classes. """ from __future__ import annotations @@ -215,10 +216,7 @@ def _stage_cli_overrides(stage_id: int, cli_overrides: Mapping[str, Any]) -> dic return result -def _resolve_deploy_path(model_type: str, deploy_config_path: str | None = None) -> Path: - if deploy_config_path is None: - return _DEPLOY_DIR / f"{model_type}.yaml" - +def _resolve_deploy_path(deploy_config_path: str) -> Path: deploy_path = Path(deploy_config_path) if not deploy_path.exists() and deploy_path.parent == Path("."): bare_name = deploy_path.name @@ -230,17 +228,27 @@ def _resolve_deploy_path(model_type: str, deploy_config_path: str | None = None) return deploy_path -def _registered_pipeline_keys() -> list[str]: - from vllm_omni.config.pipeline_registry import OMNI_PIPELINES +def _get_deploy_config( + pipeline_cfg: PipelineConfig, + user_deploy_config: DeployConfig | None, + deploy_config_path: str | None, +) -> tuple[DeployConfig, str | None]: + """Select user-provided, pipeline-default, or empty deploy settings.""" + if user_deploy_config is not None: + loaded_path = str(_resolve_deploy_path(deploy_config_path)) if deploy_config_path is not None else None + return copy.deepcopy(user_deploy_config), loaded_path - return sorted(OMNI_PIPELINES) + if deploy_config_path is not None: + resolved_path = _resolve_deploy_path(deploy_config_path) + if not resolved_path.exists(): + raise FileNotFoundError(f"Deploy config not found: {resolved_path}") + return load_deploy_config(resolved_path), str(resolved_path) + if pipeline_cfg.default_deploy_config_name is not None: + default_path = _DEPLOY_DIR / pipeline_cfg.default_deploy_config_name + return load_deploy_config(default_path), str(default_path) -def _resolve_registered_pipeline(model_type: str) -> PipelineConfig | None: - from vllm_omni.config.pipeline_registry import OMNI_PIPELINES - - registered = OMNI_PIPELINES[model_type] - return registered(None) if callable(registered) else registered + return DeployConfig(), None @config @@ -1225,52 +1233,41 @@ class VllmOmniConfig: raise KeyError(f"no stage {stage_id}") @classmethod - def from_registry( + def from_pipeline_config( cls, - model_type: str, + pipeline_cfg: PipelineConfig, + *, + user_deploy_config: DeployConfig | None = None, deploy_config_path: str | None = None, cli_overrides: dict[str, Any] | None = None, ) -> VllmOmniConfig: - """Create a structured config from a registered pipeline and deploy YAML.""" + """Create a structured config from a resolved pipeline and deploy YAML.""" if cli_overrides is None: cli_overrides = {} - deploy_path = _resolve_deploy_path(model_type, deploy_config_path) - loaded_deploy_config_path = str(deploy_path) if deploy_path.exists() else None - if loaded_deploy_config_path is not None: - deploy = load_deploy_config(deploy_path) - else: - deploy = DeployConfig() + deploy, loaded_deploy_config_path = _get_deploy_config( + pipeline_cfg, + user_deploy_config, + deploy_config_path, + ) - pipeline_key = deploy.pipeline or model_type - if pipeline_key not in _registered_pipeline_keys(): - raise KeyError( - f"Pipeline {pipeline_key!r} not in registry " - f"(resolved from {deploy_path.name!r}). Available: " - f"{_registered_pipeline_keys()}" - ) - pipeline = _resolve_registered_pipeline(pipeline_key) - if pipeline is None: - raise ValueError( - f"Pipeline {pipeline_key!r} did not resolve to a concrete PipelineConfig without an HF config" - ) - - deploy_for_registry = copy.deepcopy(deploy) if cli_overrides.get("async_chunk") is not None: - deploy_for_registry.async_chunk = bool(cli_overrides["async_chunk"]) + deploy.async_chunk = bool(cli_overrides["async_chunk"]) for name in _PIPELINE_DEPLOY_CLI_FIELDS: if cli_overrides.get(name) is not None: - setattr(deploy_for_registry, name, _copy_value(cli_overrides[name])) + setattr(deploy, name, _copy_value(cli_overrides[name])) - deploy_for_registry = _apply_platform_overrides(deploy_for_registry) - _validate_async_chunk_support(pipeline, deploy_for_registry) - deploy_by_id = {stage.stage_id: stage for stage in deploy_for_registry.stages} + deploy = _apply_platform_overrides(deploy) + if len(pipeline_cfg.stages) <= 1: + deploy.async_chunk = False + _validate_async_chunk_support(pipeline_cfg, deploy) + deploy_by_id = {stage.stage_id: stage for stage in deploy.stages} model = cli_overrides.get("model") stage_configs = tuple( _build_stage_config( - pipeline, - deploy_for_registry, + pipeline_cfg, + deploy, topology, deploy_by_id.get(topology.stage_id), _stage_engine_values( @@ -1279,7 +1276,7 @@ class VllmOmniConfig: ), model=model, ) - for topology in pipeline.stages + for topology in pipeline_cfg.stages ) orchestrator_config = VllmOmniOrchestratorConfig( @@ -1287,7 +1284,7 @@ class VllmOmniConfig: **_orchestrator_cli_overrides(cli_overrides), ) return cls( - pipeline_config=pipeline, + pipeline_config=pipeline_cfg, stage_configs=stage_configs, orchestrator_config=orchestrator_config, ) diff --git a/vllm_omni/config/pipeline_registry.py b/vllm_omni/config/pipeline_registry.py index 934d15d18..98468ff81 100644 --- a/vllm_omni/config/pipeline_registry.py +++ b/vllm_omni/config/pipeline_registry.py @@ -157,3 +157,15 @@ def register_pipeline(pipeline: PipelineConfig | PipelineResolverFunc, model_typ if errors: logger.warning("Registration for pipeline of type %s produced the following issues: %s", model_type, errors) OMNI_PIPELINES[model_type] = pipeline + + +def resolve_pipeline_config( + model_type: str, + hf_config: PretrainedConfig | None = None, +) -> PipelineConfig | None: + """Resolve a registry key to a concrete pipeline config.""" + if model_type not in OMNI_PIPELINES: + logger.warning("Model type %s is not registered to OMNI_PIPELINES", model_type) + return None + pipeline = OMNI_PIPELINES[model_type] + return pipeline(hf_config) if callable(pipeline) else pipeline diff --git a/vllm_omni/config/stage_config.py b/vllm_omni/config/stage_config.py index 6b8a09423..367e1b317 100644 --- a/vllm_omni/config/stage_config.py +++ b/vllm_omni/config/stage_config.py @@ -270,6 +270,9 @@ class PipelineConfig: # multi-component repos (e.g. GLM-Image). ``None`` = not a diffusers model. diffusers_class_name: str | None = None endpoint_restrictions: tuple[EndpointRestriction, ...] = () + # Bundled deploy defaults for this concrete pipeline topology. The file is + # loaded from vllm_omni/deploy; None uses DeployConfig defaults. + default_deploy_config_name: str | None = None def get_stage(self, stage_id: int) -> StagePipelineConfig | None: """Look up a stage by its ID.""" diff --git a/vllm_omni/engine/async_omni_engine.py b/vllm_omni/engine/async_omni_engine.py index ce9662884..fd3d63d7e 100644 --- a/vllm_omni/engine/async_omni_engine.py +++ b/vllm_omni/engine/async_omni_engine.py @@ -289,7 +289,11 @@ class AsyncOmniEngine: ) kwargs["trust_remote_code"] = trust_remote_code - self.config_path, self.stage_configs = self._resolve_stage_configs(model, kwargs) + self.config_path, self.stage_configs = self._resolve_stage_configs( + model, + kwargs, + trust_remote_code=trust_remote_code, + ) self.num_stages = len(self.stage_configs) stage0_args = getattr(self.stage_configs[0], "engine_args", None) if self.num_stages > 0 else None @@ -1156,7 +1160,13 @@ class AsyncOmniEngine: ) self._omni_lb_policy = str(derived) - def _resolve_stage_configs(self, model: str, kwargs: dict[str, Any]) -> tuple[str, list[Any]]: + def _resolve_stage_configs( + self, + model: str, + kwargs: dict[str, Any], + *, + trust_remote_code: bool, + ) -> tuple[str, list[Any]]: """Resolve stage configs and inject defaults shared by orchestrator/headless.""" stage_configs_path = kwargs.get("stage_configs_path", None) @@ -1182,6 +1192,7 @@ class AsyncOmniEngine: model, stage_configs_path, base_kwargs, + trust_remote_code=trust_remote_code, default_stage_cfg_factory=lambda: self._create_default_diffusion_stage_cfg(kwargs), deploy_config_path=deploy_config_path, stage_overrides=stage_overrides, diff --git a/vllm_omni/entrypoints/cli/serve.py b/vllm_omni/entrypoints/cli/serve.py index c1ca73e44..71483445a 100644 --- a/vllm_omni/entrypoints/cli/serve.py +++ b/vllm_omni/entrypoints/cli/serve.py @@ -824,6 +824,7 @@ def run_headless(args: TrackingNamespace) -> None: model, stage_configs_path, args_dict, + trust_remote_code=bool(getattr(args, "trust_remote_code", False)), deploy_config_path=args_dict.get("deploy_config"), stage_overrides=stage_overrides, strategy_config_path=args_dict.get("strategy_config"), diff --git a/vllm_omni/entrypoints/omni_base.py b/vllm_omni/entrypoints/omni_base.py index 304070dfa..d8b2c0291 100644 --- a/vllm_omni/entrypoints/omni_base.py +++ b/vllm_omni/entrypoints/omni_base.py @@ -144,7 +144,7 @@ class OmniBase(PDDisaggregationMixin): log_stats = kwargs.pop("log_stats", False) self._enable_ar_profiler = kwargs.pop("enable_ar_profiler", False) # NOTE: read-only lookup — must NOT pop. Popping here drops the key - # before it reaches ``StageConfigFactory._create_from_registry``, so + # before it reaches ``StageConfigFactory._create_legacy_from_registry``, so # ``--no-async-chunk`` (``async_chunk=False``) silently fails to # override the deploy YAML's ``async_chunk: true`` default. async_chunk = kwargs.get("async_chunk") diff --git a/vllm_omni/entrypoints/utils.py b/vllm_omni/entrypoints/utils.py index a2c598b31..6d32f4ab1 100644 --- a/vllm_omni/entrypoints/utils.py +++ b/vllm_omni/entrypoints/utils.py @@ -314,6 +314,8 @@ def resolve_model_config_path(model: str) -> str: def load_stage_configs_from_model( model: str, + *, + trust_remote_code: bool, base_engine_args: dict | None = None, deploy_config_path: str | None = None, stage_overrides: dict[str, dict[str, Any]] | None = None, @@ -322,13 +324,14 @@ def load_stage_configs_from_model( """Load stage configurations from model's default config file. For models registered in the pipeline registry (new path), uses - ``StageConfigFactory.create_from_model()`` which merges + ``StageConfigFactory.create_legacy_stage_configs_from_model()`` which merges PipelineConfig + DeployConfig + CLI overrides. For other models (legacy path), loads stage configs from YAML. Args: model: Model name or path (used to determine model_type) + trust_remote_code: Whether to trust remote code while resolving the model config. base_engine_args: Base engine args to merge as CLI overrides. deploy_config_path: Optional explicit deploy config path. stage_overrides: Per-stage overrides from --stage-overrides. @@ -346,20 +349,25 @@ def load_stage_configs_from_model( base_engine_args = {} cli_overrides = _convert_dataclasses_to_dict(dict(base_engine_args)) + cli_overrides["trust_remote_code"] = trust_remote_code if stage_overrides: for stage_id_str, overrides in stage_overrides.items(): for key, val in overrides.items(): cli_overrides[f"stage_{stage_id_str}_{key}"] = val + # Current runtime initialization still consumes legacy OmegaConf stage + # configs. ``StageConfigFactory.create_from_model`` now produces the + # structured ``VllmOmniConfig`` object; future RFC #4021 changes will + # migrate the engine/runtime consumers and replace this legacy resolver. strategy_specs = None if strategy_config_path is not None: from vllm_omni.config.composable_parallel.strategy_loader import load_strategy_specs strategy_specs = load_strategy_specs(strategy_config_path) - stages, omni_lb_policy = StageConfigFactory.create_from_model( + stages, omni_lb_policy = StageConfigFactory.create_legacy_stage_configs_from_model( model, - trust_remote_code=cli_overrides.get("trust_remote_code", False), + trust_remote_code=trust_remote_code, cli_overrides=cli_overrides, deploy_config_path=deploy_config_path, strategy_specs=strategy_specs, @@ -550,6 +558,8 @@ def load_and_resolve_stage_configs( model: str, stage_configs_path: str | None, kwargs: dict | None, + *, + trust_remote_code: bool, default_stage_cfg_factory: Any = None, deploy_config_path: str | None = None, stage_overrides: dict[str, dict[str, Any]] | None = None, @@ -561,6 +571,7 @@ def load_and_resolve_stage_configs( model: Model name or path stage_configs_path: Optional path to legacy YAML (stage_args format) kwargs: Engine arguments to merge with stage configs + trust_remote_code: Whether to trust remote code while resolving the model config. default_stage_cfg_factory: Optional callable that takes no args and returns default stage config list when no configs are found deploy_config_path: Optional path to deploy YAML (new format). @@ -606,6 +617,7 @@ def load_and_resolve_stage_configs( config_path = deploy_config_path stage_configs, omni_lb_policy = load_stage_configs_from_model( model, + trust_remote_code=trust_remote_code, base_engine_args=kwargs, deploy_config_path=deploy_config_path, stage_overrides=stage_overrides, @@ -621,6 +633,7 @@ def load_and_resolve_stage_configs( config_path = resolve_model_config_path(model) stage_configs, omni_lb_policy = load_stage_configs_from_model( model, + trust_remote_code=trust_remote_code, base_engine_args=kwargs, stage_overrides=stage_overrides, strategy_config_path=strategy_config_path, diff --git a/vllm_omni/model_executor/models/aura_omni/pipeline.py b/vllm_omni/model_executor/models/aura_omni/pipeline.py index 273589e1b..14aa5ff30 100644 --- a/vllm_omni/model_executor/models/aura_omni/pipeline.py +++ b/vllm_omni/model_executor/models/aura_omni/pipeline.py @@ -24,6 +24,7 @@ _QWEN3_TTS_PROC = "vllm_omni.model_executor.stage_input_processors.qwen3_tts" AURA_OMNI_PIPELINE = PipelineConfig( model_type="aura_omni", + default_deploy_config_name="aura_omni.yaml", model_arch="Qwen3ASRForConditionalGeneration", stages=( StagePipelineConfig( diff --git a/vllm_omni/model_executor/models/bagel/pipeline.py b/vllm_omni/model_executor/models/bagel/pipeline.py index c68a531c2..b0544222f 100644 --- a/vllm_omni/model_executor/models/bagel/pipeline.py +++ b/vllm_omni/model_executor/models/bagel/pipeline.py @@ -27,6 +27,7 @@ _PROC = "vllm_omni.model_executor.stage_input_processors.bagel" BAGEL_PIPELINE = PipelineConfig( model_type="bagel", + default_deploy_config_name="bagel.yaml", model_arch="OmniBagelForConditionalGeneration", hf_architectures=("BagelForConditionalGeneration",), stages=( @@ -63,6 +64,7 @@ BAGEL_PIPELINE = PipelineConfig( BAGEL_THINK_PIPELINE = PipelineConfig( model_type="bagel_think", + default_deploy_config_name="bagel_think.yaml", model_arch="OmniBagelForConditionalGeneration", hf_architectures=(), stages=( @@ -96,6 +98,7 @@ BAGEL_THINK_PIPELINE = PipelineConfig( BAGEL_SINGLE_STAGE_PIPELINE = PipelineConfig( model_type="bagel_single_stage", + default_deploy_config_name="bagel_single_stage.yaml", model_arch="BagelForConditionalGeneration", hf_architectures=(), stages=( diff --git a/vllm_omni/model_executor/models/cosyvoice3/pipeline.py b/vllm_omni/model_executor/models/cosyvoice3/pipeline.py index a8d51fdfe..413dc1dc3 100644 --- a/vllm_omni/model_executor/models/cosyvoice3/pipeline.py +++ b/vllm_omni/model_executor/models/cosyvoice3/pipeline.py @@ -22,6 +22,7 @@ _PROC = "vllm_omni.model_executor.stage_input_processors.cosyvoice3" COSYVOICE3_PIPELINE = PipelineConfig( model_type="cosyvoice3", + default_deploy_config_name="cosyvoice3.yaml", model_arch="CosyVoice3Model", stages=( StagePipelineConfig( diff --git a/vllm_omni/model_executor/models/covo_audio/pipeline.py b/vllm_omni/model_executor/models/covo_audio/pipeline.py index f604bc57d..c2e1911ef 100644 --- a/vllm_omni/model_executor/models/covo_audio/pipeline.py +++ b/vllm_omni/model_executor/models/covo_audio/pipeline.py @@ -17,6 +17,7 @@ _PROC = "vllm_omni.model_executor.stage_input_processors.covo_audio" COVO_AUDIO_PIPELINE = PipelineConfig( model_type="covo_audio", + default_deploy_config_name="covo_audio.yaml", model_arch="CovoAudioForConditionalGeneration", stages=( StagePipelineConfig( diff --git a/vllm_omni/model_executor/models/dreamzero/pipeline.py b/vllm_omni/model_executor/models/dreamzero/pipeline.py index bc815dedc..58ba5b24d 100644 --- a/vllm_omni/model_executor/models/dreamzero/pipeline.py +++ b/vllm_omni/model_executor/models/dreamzero/pipeline.py @@ -10,6 +10,7 @@ from vllm_omni.config.stage_config import ( DREAMZERO_PIPELINE = PipelineConfig( model_type="dreamzero", + default_deploy_config_name="dreamzero.yaml", model_arch="DreamZeroPipeline", stages=( StagePipelineConfig( diff --git a/vllm_omni/model_executor/models/dynin_omni/pipeline.py b/vllm_omni/model_executor/models/dynin_omni/pipeline.py index db0c51e35..584b6ab5c 100644 --- a/vllm_omni/model_executor/models/dynin_omni/pipeline.py +++ b/vllm_omni/model_executor/models/dynin_omni/pipeline.py @@ -23,6 +23,7 @@ _PROC = "vllm_omni.model_executor.stage_input_processors.dynin_omni" DYNIN_OMNI_PIPELINE = PipelineConfig( model_type="dynin_omni", + default_deploy_config_name="dynin_omni.yaml", model_arch="DyninOmniForConditionalGeneration", # Arch-fallback safety net: route by hf_config.architectures when the # auto-detected model_type does not match the registry key exactly. diff --git a/vllm_omni/model_executor/models/fish_speech/pipeline.py b/vllm_omni/model_executor/models/fish_speech/pipeline.py index baee1b5e7..1cb113094 100644 --- a/vllm_omni/model_executor/models/fish_speech/pipeline.py +++ b/vllm_omni/model_executor/models/fish_speech/pipeline.py @@ -20,6 +20,7 @@ _PROC = "vllm_omni.model_executor.stage_input_processors.fish_speech" FISH_SPEECH_PIPELINE = PipelineConfig( model_type="fish_qwen3_omni", + default_deploy_config_name="fish_qwen3_omni.yaml", model_arch="FishSpeechSlowARForConditionalGeneration", stages=( StagePipelineConfig( diff --git a/vllm_omni/model_executor/models/glm_image/pipeline.py b/vllm_omni/model_executor/models/glm_image/pipeline.py index bd041292c..fd74bfad8 100644 --- a/vllm_omni/model_executor/models/glm_image/pipeline.py +++ b/vllm_omni/model_executor/models/glm_image/pipeline.py @@ -14,6 +14,7 @@ from vllm_omni.config.stage_config import ( GLM_IMAGE_PIPELINE = PipelineConfig( model_type="glm_image", + default_deploy_config_name="glm_image.yaml", model_arch="GlmImageForConditionalGeneration", hf_architectures=("GlmImageForConditionalGeneration",), diffusers_class_name="GlmImagePipeline", diff --git a/vllm_omni/model_executor/models/glm_tts/pipeline.py b/vllm_omni/model_executor/models/glm_tts/pipeline.py index f016a993e..2a1fedd3f 100644 --- a/vllm_omni/model_executor/models/glm_tts/pipeline.py +++ b/vllm_omni/model_executor/models/glm_tts/pipeline.py @@ -12,6 +12,7 @@ _PROC = "vllm_omni.model_executor.stage_input_processors.glm_tts" GLM_TTS_PIPELINE = PipelineConfig( model_type="glm_tts", + default_deploy_config_name="glm_tts.yaml", model_arch="GLMTTSForConditionalGeneration", stages=( StagePipelineConfig( diff --git a/vllm_omni/model_executor/models/gr00t/pipeline.py b/vllm_omni/model_executor/models/gr00t/pipeline.py index 38a76f297..a2e4c8133 100644 --- a/vllm_omni/model_executor/models/gr00t/pipeline.py +++ b/vllm_omni/model_executor/models/gr00t/pipeline.py @@ -10,6 +10,7 @@ from vllm_omni.config.stage_config import ( GR00T_N1D7_PIPELINE = PipelineConfig( model_type="Gr00tN1d7", + default_deploy_config_name="Gr00tN1d7.yaml", model_arch="Gr00tN1d7Pipeline", hf_architectures=("Gr00tN1d7",), stages=( diff --git a/vllm_omni/model_executor/models/higgs_audio_v2/pipeline.py b/vllm_omni/model_executor/models/higgs_audio_v2/pipeline.py index fb6832569..1d3db52d6 100644 --- a/vllm_omni/model_executor/models/higgs_audio_v2/pipeline.py +++ b/vllm_omni/model_executor/models/higgs_audio_v2/pipeline.py @@ -12,6 +12,7 @@ _PROC = "vllm_omni.model_executor.stage_input_processors.higgs_audio_v2" HIGGS_AUDIO_V2_PIPELINE = PipelineConfig( model_type="higgs_audio_v2", + default_deploy_config_name="higgs_audio_v2.yaml", model_arch="HiggsAudioV2ForConditionalGeneration", hf_architectures=("HiggsAudioV2ForConditionalGeneration",), stages=( diff --git a/vllm_omni/model_executor/models/higgs_audio_v3/pipeline.py b/vllm_omni/model_executor/models/higgs_audio_v3/pipeline.py index 2f35c497e..9bfc411df 100644 --- a/vllm_omni/model_executor/models/higgs_audio_v3/pipeline.py +++ b/vllm_omni/model_executor/models/higgs_audio_v3/pipeline.py @@ -24,6 +24,7 @@ _PROC = "vllm_omni.model_executor.stage_input_processors.higgs_audio_v3" HIGGS_AUDIO_V3_PIPELINE = PipelineConfig( model_type="higgs_multimodal_qwen3", + default_deploy_config_name="higgs_multimodal_qwen3.yaml", model_arch="HiggsAudioV3TalkerForConditionalGeneration", hf_architectures=("HiggsMultimodalQwen3ForConditionalGeneration",), stages=( diff --git a/vllm_omni/model_executor/models/hunyuan_image3/pipeline.py b/vllm_omni/model_executor/models/hunyuan_image3/pipeline.py index 06ebff9c7..fd53043bd 100644 --- a/vllm_omni/model_executor/models/hunyuan_image3/pipeline.py +++ b/vllm_omni/model_executor/models/hunyuan_image3/pipeline.py @@ -18,6 +18,7 @@ _HUNYUAN_IMAGE3_INPUT_PROCESSOR = "vllm_omni.model_executor.stage_input_processo HUNYUAN_IMAGE3_PIPELINE = PipelineConfig( model_type="hunyuan_image_3_moe", + default_deploy_config_name="hunyuan_image_3_moe.yaml", model_arch=_HUNYUAN_IMAGE3_MODEL_ARCH, hf_architectures=_HUNYUAN_IMAGE3_HF_ARCHS, stages=( @@ -50,6 +51,7 @@ HUNYUAN_IMAGE3_PIPELINE = PipelineConfig( HUNYUAN_IMAGE3_AR_PIPELINE = PipelineConfig( model_type="hunyuan_image3_ar", + default_deploy_config_name="hunyuan_image3_ar.yaml", model_arch=_HUNYUAN_IMAGE3_MODEL_ARCH, hf_architectures=(), stages=( @@ -71,6 +73,7 @@ HUNYUAN_IMAGE3_AR_PIPELINE = PipelineConfig( HUNYUAN_IMAGE3_DIT_PIPELINE = PipelineConfig( model_type="hunyuan_image3_dit", + default_deploy_config_name="hunyuan_image3_dit.yaml", model_arch=_HUNYUAN_IMAGE3_MODEL_ARCH, hf_architectures=(), stages=( diff --git a/vllm_omni/model_executor/models/indextts2/pipeline.py b/vllm_omni/model_executor/models/indextts2/pipeline.py index 2ce02cee8..50f866ffd 100644 --- a/vllm_omni/model_executor/models/indextts2/pipeline.py +++ b/vllm_omni/model_executor/models/indextts2/pipeline.py @@ -16,6 +16,7 @@ _PROC = "vllm_omni.model_executor.stage_input_processors.indextts2" INDEXTTS2_PIPELINE = PipelineConfig( model_type="indextts2", + default_deploy_config_name="indextts2.yaml", model_arch="IndexTTS2TalkerForConditionalGeneration", stages=( StagePipelineConfig( diff --git a/vllm_omni/model_executor/models/lance/pipeline.py b/vllm_omni/model_executor/models/lance/pipeline.py index 777f8942d..126a23acb 100644 --- a/vllm_omni/model_executor/models/lance/pipeline.py +++ b/vllm_omni/model_executor/models/lance/pipeline.py @@ -15,6 +15,7 @@ from vllm_omni.config.stage_config import ( LANCE_PIPELINE = PipelineConfig( model_type="lance", + default_deploy_config_name="lance.yaml", model_arch="LancePipeline", hf_architectures=(), stages=( diff --git a/vllm_omni/model_executor/models/mammoth_moda2/pipeline.py b/vllm_omni/model_executor/models/mammoth_moda2/pipeline.py index be35eb562..830c63aff 100644 --- a/vllm_omni/model_executor/models/mammoth_moda2/pipeline.py +++ b/vllm_omni/model_executor/models/mammoth_moda2/pipeline.py @@ -18,6 +18,7 @@ _PROC = "vllm_omni.model_executor.stage_input_processors.mammoth_moda2" MAMMOTH_MODA2_PIPELINE = PipelineConfig( model_type="mammoth_moda2", + default_deploy_config_name="mammoth_moda2.yaml", model_arch="MammothModa2ForConditionalGeneration", hf_architectures=("Mammothmoda2Model", "MammothModa2ForConditionalGeneration"), stages=( @@ -49,6 +50,7 @@ MAMMOTH_MODA2_PIPELINE = PipelineConfig( # Single-stage AR variant for understanding / summarization tasks MAMMOTH_MODA2_AR_PIPELINE = PipelineConfig( model_type="mammoth_moda2_ar", + default_deploy_config_name="mammoth_moda2_ar.yaml", model_arch="MammothModa2ForConditionalGeneration", stages=( StagePipelineConfig( diff --git a/vllm_omni/model_executor/models/mimo_audio/pipeline.py b/vllm_omni/model_executor/models/mimo_audio/pipeline.py index 9de6575d9..9e54979de 100644 --- a/vllm_omni/model_executor/models/mimo_audio/pipeline.py +++ b/vllm_omni/model_executor/models/mimo_audio/pipeline.py @@ -23,6 +23,7 @@ _IM_END_TOKEN_ID = 151645 MIMO_AUDIO_PIPELINE = PipelineConfig( model_type="mimo_audio", + default_deploy_config_name="mimo_audio.yaml", # HF ``architectures: ["MiMoAudioModel"]`` is also the registry key in # ``model_executor/models/registry.py``; it resolves to the internal # class ``MiMoAudioForConditionalGeneration`` in ``mimo_audio.py``. diff --git a/vllm_omni/model_executor/models/ming_flash_omni/pipeline.py b/vllm_omni/model_executor/models/ming_flash_omni/pipeline.py index e416a117e..da8c24843 100644 --- a/vllm_omni/model_executor/models/ming_flash_omni/pipeline.py +++ b/vllm_omni/model_executor/models/ming_flash_omni/pipeline.py @@ -21,6 +21,7 @@ _PROC = "vllm_omni.model_executor.stage_input_processors.ming_flash_omni" MING_FLASH_OMNI_PIPELINE = PipelineConfig( model_type="ming_flash_omni", + default_deploy_config_name="ming_flash_omni.yaml", model_arch="MingFlashOmniForConditionalGeneration", # Upstream HF config applies model_type="bailingmm_moe_v2_lite" # (the thinker sub-config name) rather than "ming_flash_omni". @@ -66,6 +67,7 @@ MING_FLASH_OMNI_PIPELINE = PipelineConfig( # Standalone TTS variant: talker only. MING_FLASH_OMNI_TTS_PIPELINE = PipelineConfig( model_type="ming_flash_omni_tts", + default_deploy_config_name="ming_flash_omni_tts.yaml", model_arch="MingFlashOmniTalkerForConditionalGeneration", stages=( StagePipelineConfig( @@ -87,6 +89,7 @@ MING_FLASH_OMNI_TTS_PIPELINE = PipelineConfig( # Thinker-only variant: multimodal understanding with text output MING_FLASH_OMNI_THINKER_ONLY_PIPELINE = PipelineConfig( model_type="ming_flash_omni_thinker_only", + default_deploy_config_name="ming_flash_omni_thinker_only.yaml", model_arch="MingFlashOmniForConditionalGeneration", stages=( StagePipelineConfig( @@ -109,6 +112,7 @@ MING_FLASH_OMNI_THINKER_ONLY_PIPELINE = PipelineConfig( # Thinker + image-generation (diffusion) variant: text-to-image / img2img. MING_FLASH_OMNI_IMAGE_PIPELINE = PipelineConfig( model_type="ming_flash_omni_image", + default_deploy_config_name="ming_flash_omni_image.yaml", model_arch="MingFlashOmniForConditionalGeneration", stages=( StagePipelineConfig( diff --git a/vllm_omni/model_executor/models/ming_tts/pipeline.py b/vllm_omni/model_executor/models/ming_tts/pipeline.py index 762dcf6fd..86eecd41e 100644 --- a/vllm_omni/model_executor/models/ming_tts/pipeline.py +++ b/vllm_omni/model_executor/models/ming_tts/pipeline.py @@ -12,6 +12,7 @@ _PROC = "vllm_omni.model_executor.stage_input_processors.ming_tts" MING_TTS_PIPELINE = PipelineConfig( model_type="ming_tts", + default_deploy_config_name="ming_tts.yaml", model_arch="MingTTSForConditionalGeneration", hf_architectures=("MingTTSForConditionalGeneration", "BailingMMNativeForConditionalGeneration"), # The dense (0.5B) and MoE (16.8B) share architectures=["BailingMMNativeForConditionalGeneration"] @@ -63,6 +64,7 @@ MING_TTS_PIPELINE = PipelineConfig( # Keep this for auto-detection of deploy config yaml. MING_TTS_MOE_PIPELINE = PipelineConfig( model_type="ming_tts_moe", + default_deploy_config_name="ming_tts_moe.yaml", model_arch="MingTTSForConditionalGeneration", hf_architectures=("MingTTSForConditionalGeneration", "BailingMMNativeForConditionalGeneration"), # Disambiguate from the dense-0.5B ming_tts pipeline diff --git a/vllm_omni/model_executor/models/minicpmo_4_5/pipeline.py b/vllm_omni/model_executor/models/minicpmo_4_5/pipeline.py index 701e014e6..f966555d0 100644 --- a/vllm_omni/model_executor/models/minicpmo_4_5/pipeline.py +++ b/vllm_omni/model_executor/models/minicpmo_4_5/pipeline.py @@ -22,6 +22,7 @@ _PROC = "vllm_omni.model_executor.stage_input_processors.minicpmo_4_5_omni" MINICPMO_4_5_PIPELINE = PipelineConfig( model_type="minicpmo_4_5", + default_deploy_config_name="minicpmo_4_5.yaml", model_arch="MiniCPMO45OmniForConditionalGeneration", # MiniCPM-o 4.5's HF config.json reports `model_type="minicpmo"` and # `architectures=["MiniCPMO"]` — both shared verbatim with older MiniCPM-o diff --git a/vllm_omni/model_executor/models/moss_tts/pipeline.py b/vllm_omni/model_executor/models/moss_tts/pipeline.py index c6cf5c86c..254bc18fe 100644 --- a/vllm_omni/model_executor/models/moss_tts/pipeline.py +++ b/vllm_omni/model_executor/models/moss_tts/pipeline.py @@ -22,6 +22,7 @@ _PROC = "vllm_omni.model_executor.stage_input_processors.moss_tts" MOSS_TTS_PIPELINE = PipelineConfig( model_type="moss_tts", + default_deploy_config_name="moss_tts.yaml", model_arch="MossTTSDelayModel", # HF architectures string stages=( StagePipelineConfig( @@ -53,6 +54,7 @@ MOSS_TTS_PIPELINE = PipelineConfig( MOSS_TTS_REALTIME_PIPELINE = PipelineConfig( model_type="moss_tts_realtime", + default_deploy_config_name="moss_tts_realtime.yaml", model_arch="MossTTSRealtime", # different talker class from the delay variant stages=( StagePipelineConfig( @@ -82,6 +84,7 @@ MOSS_TTS_REALTIME_PIPELINE = PipelineConfig( MOSS_TTS_LOCAL_PIPELINE = PipelineConfig( model_type="moss_tts_local", + default_deploy_config_name="moss_tts_local.yaml", model_arch="MossTTSLocalModel", # different talker class: GPT2-style local depth transformer stages=( StagePipelineConfig( diff --git a/vllm_omni/model_executor/models/moss_tts_nano/pipeline.py b/vllm_omni/model_executor/models/moss_tts_nano/pipeline.py index 749c4f980..c089ea90e 100644 --- a/vllm_omni/model_executor/models/moss_tts_nano/pipeline.py +++ b/vllm_omni/model_executor/models/moss_tts_nano/pipeline.py @@ -18,6 +18,7 @@ from vllm_omni.config.stage_config import ( MOSS_TTS_NANO_PIPELINE = PipelineConfig( model_type="moss_tts_nano", + default_deploy_config_name="moss_tts_nano.yaml", model_arch="MossTTSNanoForCausalLM", stages=( StagePipelineConfig( diff --git a/vllm_omni/model_executor/models/omnivoice/pipeline.py b/vllm_omni/model_executor/models/omnivoice/pipeline.py index 88d479459..760d5314f 100644 --- a/vllm_omni/model_executor/models/omnivoice/pipeline.py +++ b/vllm_omni/model_executor/models/omnivoice/pipeline.py @@ -14,6 +14,7 @@ from vllm_omni.config.stage_config import ( OMNIVOICE_PIPELINE = PipelineConfig( model_type="omnivoice", + default_deploy_config_name="omnivoice.yaml", model_arch="OmniVoicePipeline", stages=( StagePipelineConfig( diff --git a/vllm_omni/model_executor/models/qwen2_5_omni/pipeline.py b/vllm_omni/model_executor/models/qwen2_5_omni/pipeline.py index b4ed8efee..4ce5ef207 100644 --- a/vllm_omni/model_executor/models/qwen2_5_omni/pipeline.py +++ b/vllm_omni/model_executor/models/qwen2_5_omni/pipeline.py @@ -17,6 +17,7 @@ _PROC = "vllm_omni.model_executor.stage_input_processors.qwen2_5_omni" QWEN2_5_OMNI_PIPELINE = PipelineConfig( model_type="qwen2_5_omni", + default_deploy_config_name="qwen2_5_omni.yaml", model_arch="Qwen2_5OmniForConditionalGeneration", stages=( StagePipelineConfig( diff --git a/vllm_omni/model_executor/models/qwen3_omni/pipeline.py b/vllm_omni/model_executor/models/qwen3_omni/pipeline.py index ca3a09f89..9baecdd3e 100644 --- a/vllm_omni/model_executor/models/qwen3_omni/pipeline.py +++ b/vllm_omni/model_executor/models/qwen3_omni/pipeline.py @@ -21,6 +21,7 @@ _PROC = "vllm_omni.model_executor.stage_input_processors.qwen3_omni" QWEN3_OMNI_PIPELINE = PipelineConfig( model_type="qwen3_omni_moe", + default_deploy_config_name="qwen3_omni_moe.yaml", model_arch="Qwen3OmniMoeForConditionalGeneration", endpoint_restrictions=( EndpointRestriction( diff --git a/vllm_omni/model_executor/models/qwen3_tts/pipeline.py b/vllm_omni/model_executor/models/qwen3_tts/pipeline.py index 7f50931dd..8a68e63cd 100644 --- a/vllm_omni/model_executor/models/qwen3_tts/pipeline.py +++ b/vllm_omni/model_executor/models/qwen3_tts/pipeline.py @@ -15,6 +15,7 @@ _PROC = "vllm_omni.model_executor.stage_input_processors.qwen3_tts" QWEN3_TTS_PIPELINE = PipelineConfig( model_type="qwen3_tts", + default_deploy_config_name="qwen3_tts.yaml", # Pipeline-level default; the code2wav stage overrides per-stage below. model_arch="Qwen3TTSTalkerForConditionalGeneration", stages=( diff --git a/vllm_omni/model_executor/models/voxcpm2/pipeline.py b/vllm_omni/model_executor/models/voxcpm2/pipeline.py index 0579ebc8c..c92cf61ba 100644 --- a/vllm_omni/model_executor/models/voxcpm2/pipeline.py +++ b/vllm_omni/model_executor/models/voxcpm2/pipeline.py @@ -15,6 +15,7 @@ from vllm_omni.config.stage_config import ( VOXCPM2_PIPELINE = PipelineConfig( model_type="voxcpm2", + default_deploy_config_name="voxcpm2.yaml", model_arch="VoxCPM2TalkerForConditionalGeneration", stages=( StagePipelineConfig( diff --git a/vllm_omni/model_executor/models/voxtral_tts/pipeline.py b/vllm_omni/model_executor/models/voxtral_tts/pipeline.py index 3bc2743c6..e6bc712fe 100644 --- a/vllm_omni/model_executor/models/voxtral_tts/pipeline.py +++ b/vllm_omni/model_executor/models/voxtral_tts/pipeline.py @@ -16,6 +16,7 @@ _PROC = "vllm_omni.model_executor.stage_input_processors.voxtral_tts" VOXTRAL_TTS_PIPELINE = PipelineConfig( model_type="voxtral_tts", + default_deploy_config_name="voxtral_tts.yaml", model_arch="VoxtralTTSForConditionalGeneration", stages=( StagePipelineConfig( From 4fbbdfd250e472ae14c304d1f45408b76c9488e7 Mon Sep 17 00:00:00 2001 From: Yukim1 <121286183+zwhzzz0821@users.noreply.github.com> Date: Sat, 18 Jul 2026 17:34:33 +0800 Subject: [PATCH 015/240] [Refactor] Migrate Legacy Stage Configs to the Pipeline Registry (#5031) Signed-off-by: zwhzzz0821 <2831474076@qq.com> Co-authored-by: zwhzzz0821 Co-authored-by: SYLAR <125541396+lishunyang12@users.noreply.github.com> Co-authored-by: Hongsheng Liu --- docs/cli/serve.md | 3 +- docs/configuration/stage_configs.md | 2 +- docs/contributing/model/adding_omni_model.md | 9 +- docs/contributing/model/adding_tts_model.md | 17 ++- .../image_to_video/image_to_video.py | 7 + .../offline_inference/mimo_audio/end2end.py | 20 +-- .../offline_inference/step_audio2/README.md | 73 +++++------ .../offline_inference/step_audio2/end2end.py | 58 ++++++--- .../text_to_video/text_to_video.py | 7 + examples/online_serving/step_audio2/README.md | 12 +- .../openai_chat_completion_client.py | 4 +- .../step_audio2/openai_speech_client.py | 2 +- .../online_serving/step_audio2/run_curl.sh | 2 +- pyproject.toml | 1 - .../test_step_audio2_async_chunk.py | 30 ++--- tests/test_config_factory.py | 81 +++++++++++- vllm_omni/config/pipeline_registry.py | 10 ++ vllm_omni/deploy/hunyuan_video_15.yaml | 18 +++ vllm_omni/deploy/step_audio_2.yaml | 39 ++++++ vllm_omni/deploy/step_audio_2_asr.yaml | 22 ++++ .../deploy/step_audio_2_async_chunk.yaml | 37 ++++++ vllm_omni/deploy/wan2_2_ti2v.yaml | 17 +++ .../models/hunyuan_video/__init__.py | 2 + .../models/hunyuan_video/pipeline.py | 26 ++++ .../models/step_audio2/__init__.py | 3 - .../models/step_audio2/pipeline.py | 71 ++++++++++ .../model_executor/models/wan2_2/__init__.py | 2 + .../model_executor/models/wan2_2/pipeline.py | 26 ++++ .../model_executor/stage_configs/__init__.py | 0 .../hunyuan_video_15_dit_fp8.yaml | 38 ------ .../stage_configs/mimo_audio.yaml | 61 --------- .../stage_configs/step_audio_2.yaml | 122 ------------------ .../stage_configs/step_audio_2_asr.yaml | 35 ----- .../step_audio_2_async_chunk.yaml | 78 ----------- .../stage_configs/wan2_2_ti2v_dit_fp8.yaml | 41 ------ .../stage_input_processors/mimo_audio.py | 4 +- .../stage_input_processors/step_audio2.py | 48 ++++--- vllm_omni/platforms/cuda/platform.py | 2 +- vllm_omni/platforms/musa/platform.py | 2 +- vllm_omni/platforms/npu/platform.py | 2 +- vllm_omni/platforms/rocm/platform.py | 2 +- 41 files changed, 520 insertions(+), 516 deletions(-) create mode 100644 vllm_omni/deploy/hunyuan_video_15.yaml create mode 100644 vllm_omni/deploy/step_audio_2.yaml create mode 100644 vllm_omni/deploy/step_audio_2_asr.yaml create mode 100644 vllm_omni/deploy/step_audio_2_async_chunk.yaml create mode 100644 vllm_omni/deploy/wan2_2_ti2v.yaml create mode 100644 vllm_omni/model_executor/models/hunyuan_video/__init__.py create mode 100644 vllm_omni/model_executor/models/hunyuan_video/pipeline.py create mode 100644 vllm_omni/model_executor/models/step_audio2/pipeline.py create mode 100644 vllm_omni/model_executor/models/wan2_2/__init__.py create mode 100644 vllm_omni/model_executor/models/wan2_2/pipeline.py delete mode 100644 vllm_omni/model_executor/stage_configs/__init__.py delete mode 100644 vllm_omni/model_executor/stage_configs/hunyuan_video_15_dit_fp8.yaml delete mode 100644 vllm_omni/model_executor/stage_configs/mimo_audio.yaml delete mode 100644 vllm_omni/model_executor/stage_configs/step_audio_2.yaml delete mode 100644 vllm_omni/model_executor/stage_configs/step_audio_2_asr.yaml delete mode 100644 vllm_omni/model_executor/stage_configs/step_audio_2_async_chunk.yaml delete mode 100644 vllm_omni/model_executor/stage_configs/wan2_2_ti2v_dit_fp8.yaml diff --git a/docs/cli/serve.md b/docs/cli/serve.md index 9d1747e0b..ff9e561e0 100644 --- a/docs/cli/serve.md +++ b/docs/cli/serve.md @@ -8,8 +8,7 @@ The stage-based CLI is designed for deployments that require launching each pipe - For **migrated models** that utilize the bundled deployment YAML configurations located in `vllm_omni/deploy/`, the `--deploy-config` flag is only required to override the default configuration. By default, executing `vllm serve MODEL --omni ...` automatically loads the bundled deployment configuration. -- For **legacy models** utilizing configuration files located in - `vllm_omni/model_executor/stage_configs/`, the `--stage-configs-path` parameter remains mandatory. +- For custom legacy `stage_args` YAMLs, pass the file with `--stage-configs-path`. Example: Initializing Stage 0 (Orchestrator and API Server): The commands below show a common device mapping where Stage 0 uses GPU 0 and diff --git a/docs/configuration/stage_configs.md b/docs/configuration/stage_configs.md index 1f14d21fd..1aa4ca1b3 100644 --- a/docs/configuration/stage_configs.md +++ b/docs/configuration/stage_configs.md @@ -3,7 +3,7 @@ In vLLM-Omni, a model's `PipelineConfig` defines its fixed stage topology, while a deploy configuration controls how those stages run. Models that have not migrated to this split still use legacy stage configurations with a `stage_args` schema. !!! note - Default deploy config YAMLs (for example, `vllm_omni/deploy/qwen2_5_omni.yaml`, `vllm_omni/deploy/qwen3_omni_moe.yaml`, and `vllm_omni/deploy/qwen3_tts.yaml`) are bundled and loaded automatically when neither `--stage-configs-path` nor `--deploy-config` is provided — the model registry resolves the right pipeline + deploy YAML by `model_type`. The bundled defaults have been verified on 1xH100 for Qwen2.5-Omni and 2xH100 for Qwen3-Omni. Models that have not yet migrated to the new schema continue to use the legacy `vllm_omni/model_executor/stage_configs/.yaml` files via `--stage-configs-path`. + Default deploy config YAMLs (for example, `vllm_omni/deploy/qwen2_5_omni.yaml`, `vllm_omni/deploy/qwen3_omni_moe.yaml`, and `vllm_omni/deploy/qwen3_tts.yaml`) are bundled and loaded automatically when neither `--stage-configs-path` nor `--deploy-config` is provided — the model registry resolves the right pipeline + deploy YAML by `model_type`. Custom legacy `stage_args` YAMLs are still accepted via `--stage-configs-path`. ## Deploy configuration schema diff --git a/docs/contributing/model/adding_omni_model.md b/docs/contributing/model/adding_omni_model.md index c20e94eda..5032c9805 100644 --- a/docs/contributing/model/adding_omni_model.md +++ b/docs/contributing/model/adding_omni_model.md @@ -34,6 +34,7 @@ vllm_omni/model_executor/models/ └── your_model_name/ # Model directory (e.g., qwen3_omni) ├── __init__.py # Exports main model class ├── your_model.py # Main unified model class + ├── pipeline.py # Pipeline topology registry entry ├── your_model_stage1_implementation.py # Stage 1 implementation (e.g., thinker) ├── your_model_stage2_implementation.py # Stage 2 implementation (e.g., talker) └── your_model_stage3_implementation.py # Stage 3 implementation (e.g., code2wav) @@ -42,10 +43,14 @@ vllm_omni/model_executor/models/ vllm_omni/model_executor/stage_input_processors/ └── your_model_name.py # Stage transition processors -vllm_omni/model_executor/stage_configs/ -└── your_model_name.yaml # Stage configuration file +vllm_omni/deploy/ +└── your_model_name.yaml # Deployment configuration file ``` +New in-tree models should register their `PipelineConfig` in +`vllm_omni/config/pipeline_registry.py`; use `--stage-configs-path` only for +custom legacy `stage_args` YAMLs. + ## Step-by-Step Implementation ### Step 1: Create the Model Directory diff --git a/docs/contributing/model/adding_tts_model.md b/docs/contributing/model/adding_tts_model.md index 3e5ae30df..b9bb9484f 100644 --- a/docs/contributing/model/adding_tts_model.md +++ b/docs/contributing/model/adding_tts_model.md @@ -175,15 +175,16 @@ vllm_omni/model_executor/models/ your_model_name/ __init__.py your_model.py # Unified class (stage dispatch) + pipeline.py # Pipeline topology registry entry your_model_ar_stage.py # Stage 0: AR stage your_model_decoder.py # Stage 1: audio decoder vllm_omni/model_executor/stage_input_processors/ your_model_name.py # Stage 0 -> Stage 1 transition -vllm_omni/model_executor/stage_configs/ - your_model_name.yaml # Batch mode config - your_model_name_async_chunk.yaml # Streaming mode config +vllm_omni/deploy/ + your_model_name.yaml # Batch/deploy config + your_model_name_async_chunk.yaml # Streaming/deploy config ``` ### Example placement @@ -445,6 +446,13 @@ Key configuration fields: | `final_output` | Whether this stage produces the final user-facing output | | `final_output_type` | Type of final output (`audio`, `text`, etc.) | +!!! note + New in-tree models should define frozen topology in `models//pipeline.py`, + register it in `vllm_omni/config/pipeline_registry.py`, and put deployment + knobs in `vllm_omni/deploy/.yaml`. Legacy `stage_args` YAMLs are still + accepted for custom configs via `--stage-configs-path`, but should not be used + for new bundled defaults. + ### Batch mode ```yaml @@ -771,8 +779,9 @@ directly from the model's own generator. vllm_omni/model_executor/models/your_model_name/ __init__.py modeling_your_model_name.py # unified class: load_weights + forward + streaming + pipeline.py # single-stage pipeline topology -vllm_omni/model_executor/stage_configs/your_model_name.yaml +vllm_omni/deploy/your_model_name.yaml ``` No stage input processor is needed. diff --git a/examples/offline_inference/image_to_video/image_to_video.py b/examples/offline_inference/image_to_video/image_to_video.py index 2d1e70697..d64367bc2 100644 --- a/examples/offline_inference/image_to_video/image_to_video.py +++ b/examples/offline_inference/image_to_video/image_to_video.py @@ -102,6 +102,11 @@ def parse_args() -> argparse.Namespace: default=None, help="Override model class name (e.g., LTX2ImageToVideoPipeline or LTX23ImageToVideoPipeline).", ) + parser.add_argument( + "--deploy-config", + default=None, + help="Optional deploy config YAML to use for pipeline-backed runs.", + ) parser.add_argument("--image", help="Path to the first-frame or source image.") parser.add_argument("--last-image", help="Path to a last-frame condition (used by models such as VACE).") parser.add_argument("--mask-image", help="Path to an inpainting mask (used by models such as VACE).") @@ -453,6 +458,8 @@ def main(): enable_diffusion_pipeline_profiler=args.enable_diffusion_pipeline_profiler, profiler_config=args.profiler_config, ) + if args.deploy_config: + omni_kwargs["deploy_config"] = args.deploy_config if args.quantization is not None: omni_kwargs["quantization"] = args.quantization # Cosmos3 loads its (gated) guardrail models at build time, so the guardrails diff --git a/examples/offline_inference/mimo_audio/end2end.py b/examples/offline_inference/mimo_audio/end2end.py index 2fe46c972..139351aab 100644 --- a/examples/offline_inference/mimo_audio/end2end.py +++ b/examples/offline_inference/mimo_audio/end2end.py @@ -219,31 +219,31 @@ def main(args): # Build query result based on query type # Notice: The audio files used in this example are available at: https://github.com/XiaomiMiMo/MiMo-Audio/tree/main/examples if args.query_type == "tts_sft": - # python3 -u end2end.py --stage-configs-path ${config_file} --model ${MODEL_PATH} --query-type tts_sft + # python3 -u end2end.py --deploy-config ${config_file} --model-name ${MODEL_PATH} --query-type tts_sft query_result = query_func(text=text, read_text_only=True) elif args.query_type == "tts_sft_with_instruct": - # python3 -u end2end.py --stage-configs-path ${config_file} --model ${MODEL_PATH} --query-type tts_sft_with_instruct --instruct "Speak happily in a child's voice" + # python3 -u end2end.py --deploy-config ${config_file} --model-name ${MODEL_PATH} --query-type tts_sft_with_instruct --instruct "Speak happily in a child's voice" query_result = query_func(text=text, instruct=instruct, read_text_only=True) elif args.query_type == "tts_sft_with_audio": - # python3 -u end2end.py --stage-configs-path ${config_file} --model ${MODEL_PATH} --query-type tts_sft_with_audio --audio_path "./spoken_dialogue_assistant_turn_1.wav" + # python3 -u end2end.py --deploy-config ${config_file} --model-name ${MODEL_PATH} --query-type tts_sft_with_audio --audio-path "./spoken_dialogue_assistant_turn_1.wav" audio_list = [get_audio_data(audio_path)] query_result = query_func(text=text, read_text_only=True, prompt_speech=audio_path, audio_list=audio_list) elif args.query_type == "tts_sft_with_natural_instruction": - # python3 -u end2end.py --stage-configs-path ${config_file} --model ${MODEL_PATH} --query-type tts_sft_with_natural_instruction --text "In a panting young male voice, he said: I can't run anymore, wait for me!" + # python3 -u end2end.py --deploy-config ${config_file} --model-name ${MODEL_PATH} --query-type tts_sft_with_natural_instruction --text "In a panting young male voice, he said: I can't run anymore, wait for me!" query_result = query_func(text=text, read_text_only=False) elif args.query_type == "audio_trancribing_sft": - # python3 -u end2end.py --stage-configs-path ${config_file} --model ${MODEL_PATH} --query-type audio_trancribing_sft --audio_path "./spoken_dialogue_assistant_turn_1.wav" + # python3 -u end2end.py --deploy-config ${config_file} --model-name ${MODEL_PATH} --query-type audio_trancribing_sft --audio-path "./spoken_dialogue_assistant_turn_1.wav" audio_path = "spoken_dialogue_assistant_turn_1.wav" text = "Please transcribe this audio and repeat it once." query_result = query_func(text=text, audio_path=audio_path, use_sostm=True) elif args.query_type == "audio_understanding_sft": - # python3 -u end2end.py --stage-configs-path ${config_file} --model ${MODEL_PATH} --query-type audio_understanding_sft --text "Summarize the audio." --audio_path "./spoken_dialogue_assistant_turn_1.wav" + # python3 -u end2end.py --deploy-config ${config_file} --model-name ${MODEL_PATH} --query-type audio_understanding_sft --text "Summarize the audio." --audio-path "./spoken_dialogue_assistant_turn_1.wav" query_result = query_func(text=text, audio_path=audio_path) elif args.query_type == "audio_understanding_sft_with_thinking": - # python3 -u end2end.py --stage-configs-path ${config_file} --model ${MODEL_PATH} --query-type audio_understanding_sft_with_thinking --text "Summarize the audio." --audio_path "./spoken_dialogue_assistant_turn_1.wav" + # python3 -u end2end.py --deploy-config ${config_file} --model-name ${MODEL_PATH} --query-type audio_understanding_sft_with_thinking --text "Summarize the audio." --audio-path "./spoken_dialogue_assistant_turn_1.wav" query_result = query_func(text=text, audio_path=audio_path, thinking=True) elif args.query_type == "spoken_dialogue_sft_multiturn": - # python3 -u end2end.py --stage-configs-path ${config_file} --model ${MODEL_PATH} --query-type spoken_dialogue_sft_multiturn --audio_path "./prompt_speech_zh_m.wav" + # python3 -u end2end.py --deploy-config ${config_file} --model-name ${MODEL_PATH} --query-type spoken_dialogue_sft_multiturn --audio-path "./prompt_speech_zh_m.wav" first_turn_text_response = "我没办法获取实时的天气信息。不过呢,你可以试试几个方法来查看今天的天气。首先,你可以用手机自带的天气功能,比如苹果手机的天气应用,或者直接在系统设置里查看。其次,你也可以用一些专业的天气服务,像是国外的AccuWeather、Weather.com,或者国内的中国天气网、墨迹天气等等。再有就是,你还可以在谷歌或者百度里直接搜索你所在的城市加上天气这两个字。如果你能告诉我你所在的城市,我也可以帮你分析一下历史天气趋势,不过最新的数据还是需要你通过官方渠道去获取哦。" audio_list = [] s1_audio_path = "weather_of_today.mp3" @@ -261,7 +261,7 @@ def main(args): ] query_result = query_func(message_list, system_prompt=None, ref_audio_path=audio_path, audio_list=audio_list) elif args.query_type == "speech2text_dialogue_sft_multiturn": - # python3 -u end2end.py --stage-configs-path ${config_file} --model ${MODEL_PATH} --query-type speech2text_dialogue_sft_multiturn + # python3 -u end2end.py --deploy-config ${config_file} --model-name ${MODEL_PATH} --query-type speech2text_dialogue_sft_multiturn s1_audio_path = "weather_of_today.mp3" s2_audio_path = "beijing.mp3" audio_list = [] @@ -277,7 +277,7 @@ def main(args): ] query_result = query_func(message_list, thinking=True, audio_list=audio_list) elif args.query_type == "text_dialogue_sft_multiturn": - # python3 -u end2end.py --stage-configs-path ${config_file} --model ${MODEL_PATH} --query-type text_dialogue_sft_multiturn + # python3 -u end2end.py --deploy-config ${config_file} --model-name ${MODEL_PATH} --query-type text_dialogue_sft_multiturn message_list = [ {"role": "user", "content": "Could you recommend some tourist attractions in China?"}, {"role": "assistant", "content": "Hello, which city would you like to travel to?"}, diff --git a/examples/offline_inference/step_audio2/README.md b/examples/offline_inference/step_audio2/README.md index 84d0ecdde..abd21f371 100644 --- a/examples/offline_inference/step_audio2/README.md +++ b/examples/offline_inference/step_audio2/README.md @@ -51,7 +51,7 @@ Single request latency comparison between vLLM-Omni and official Step-Audio2 imp **Benchmark environment:** - GPU: NVIDIA H100 80GB (single card) -- Model: Step-Audio2-mini +- Model: Step-Audio-2-mini - Warmup: 1 run, Measured: 3 runs (averaged) ### Async Chunk Streaming Performance @@ -73,7 +73,7 @@ Comparison between sequential (non-async) and async chunk modes via `/v1/audio/s **Benchmark environment:** - GPU: 4x NVIDIA RTX 3090 24GB (TP=2 for Thinker, 1 GPU for Token2Wav) -- Model: Step-Audio2-mini +- Model: Step-Audio-2-mini - Endpoint: `/v1/audio/speech` (10 prompts, concurrency=1) - Measured via `bench_tts_serve.py` @@ -91,23 +91,22 @@ pip install step-audio2 ## Model Setup +Step-Audio2 ships a custom Transformers configuration, so this model-specific +example enables remote code loading when it initializes the model. + ### Option 1: Auto-download from HuggingFace (Recommended) -The script will **automatically download** the model on first run: +The script will **automatically download** the model on first run when a +HuggingFace model id is passed: ```bash -# Just run without specifying --model, it will auto-download stepfun-ai/Step-Audio2-mini -python end2end.py --query-type audio_to_text - -# Or explicitly specify the HuggingFace model -python end2end.py --query-type audio_to_text --model stepfun-ai/Step-Audio2-mini +python end2end.py --query-type audio_to_text --model stepfun-ai/Step-Audio-2-mini ``` Models will be cached in `~/.cache/huggingface/hub/` for future use. -**Available models**: -- `stepfun-ai/Step-Audio2-mini` (smaller, faster) -- `stepfun-ai/Step-Audio2-7B` (larger, better quality) +**Supported model**: +- `stepfun-ai/Step-Audio-2-mini` ### Option 2: Manual Download (for offline use) @@ -115,15 +114,15 @@ Download and use locally: ```bash # Download from HuggingFace -huggingface-cli download stepfun-ai/Step-Audio2-mini --local-dir ./models/Step-Audio2-mini +hf download stepfun-ai/Step-Audio-2-mini --local-dir ./models/Step-Audio-2-mini # Then use the local path -python end2end.py --query-type audio_to_text --model ./models/Step-Audio2-mini +python end2end.py --query-type audio_to_text --model ./models/Step-Audio-2-mini ``` Ensure the model directory contains: ``` -Step-Audio2-mini/ +Step-Audio-2-mini/ ├── config.json ├── model.safetensors (or pytorch_model.bin) ├── tokenizer.json @@ -144,20 +143,18 @@ Transcribe audio to text: ```bash # Quick start - Using default model and test audio -python end2end.py --query-type audio_to_text +python end2end.py --query-type audio_to_text \ + --model stepfun-ai/Step-Audio-2-mini # Using your own audio file (model will auto-download) -python end2end.py --query-type audio_to_text \ - --audio-path /path/to/input.wav - -# With specific model python end2end.py --query-type audio_to_text \ --audio-path /path/to/input.wav \ - --model stepfun-ai/Step-Audio2-7B + --model stepfun-ai/Step-Audio-2-mini # With custom question python end2end.py --query-type audio_to_text \ --audio-path input.wav \ + --model stepfun-ai/Step-Audio-2-mini \ --question "What is the speaker saying?" ``` @@ -170,12 +167,8 @@ Convert text to speech: ```bash # Basic TTS (model auto-downloads) python end2end.py --query-type text_to_audio \ - --text "Hello, this is a test of Step Audio 2 synthesis." - -# With specific model -python end2end.py --query-type text_to_audio \ - --text "Hello, this is a test." \ - --model stepfun-ai/Step-Audio2-7B + --text "Hello, this is a test of Step Audio 2 synthesis." \ + --model stepfun-ai/Step-Audio-2-mini ``` **Note**: Speaker voice is controlled by the `STEP_AUDIO2_DEFAULT_PROMPT_WAV` environment variable or the default prompt wav bundled with the model. @@ -191,12 +184,8 @@ Process input audio and generate output audio: ```bash # Basic voice conversion (model auto-downloads) python end2end.py --query-type audio_to_audio \ - --audio-path /path/to/source_audio.wav - -# With specific model -python end2end.py --query-type audio_to_audio \ - --audio-path source.wav \ - --model stepfun-ai/Step-Audio2-7B + --audio-path /path/to/source_audio.wav \ + --model stepfun-ai/Step-Audio-2-mini ``` This mode: @@ -210,8 +199,14 @@ This mode: ```bash # Use custom stage configuration python end2end.py --query-type audio_to_text \ + --model stepfun-ai/Step-Audio-2-mini \ --stage-configs-path /path/to/custom_config.yaml +# Use custom deploy configuration +python end2end.py --query-type audio_to_text \ + --model stepfun-ai/Step-Audio-2-mini \ + --deploy-config /path/to/step_audio_2_asr.yaml + # Multiple prompts (for batch testing) python end2end.py --query-type audio_to_text \ --audio-path input.wav \ @@ -246,14 +241,14 @@ python end2end.py --query-type text_to_audio \ ## Configuration -### Stage Configuration +### Deploy Configuration -The default configuration (`step_audio_2.yaml`) uses: +The default configuration (`vllm_omni/deploy/step_audio_2.yaml`) uses: -- **Stage 0 (Thinker)**: GPU 0, 80% memory -- **Stage 1 (Token2Wav)**: GPU 1, 30% memory +- **Stage 0 (Thinker)**: GPUs 0-1 with tensor parallel size 2, 70% memory +- **Stage 1 (Token2Wav)**: GPU 1, 20% memory -For **single GPU** setup, edit the config to use `devices: "0"` for both stages. +For **single GPU** setup, edit a deploy config copy to use `devices: "0"` for both stages. ### Sampling Parameters @@ -351,7 +346,7 @@ Complete example from audio to final output: # 1. ASR: Transcribe audio python end2end.py --query-type audio_to_text \ --audio-path interview.wav \ - --model ./models/Step-Audio2-7B \ + --model ./models/Step-Audio-2-mini \ --output-dir ./outputs # 2. Check the transcription @@ -361,7 +356,7 @@ cat ./outputs/00000_text.txt STEP_AUDIO2_DEFAULT_PROMPT_WAV=./speaker_samples/female_voice.wav \ python end2end.py --query-type text_to_audio \ --text "The quick brown fox jumps over the lazy dog" \ - --model ./models/Step-Audio2-7B \ + --model ./models/Step-Audio-2-mini \ --output-dir ./outputs # 4. Listen to the result diff --git a/examples/offline_inference/step_audio2/end2end.py b/examples/offline_inference/step_audio2/end2end.py index c67f44926..8cc7e2a26 100644 --- a/examples/offline_inference/step_audio2/end2end.py +++ b/examples/offline_inference/step_audio2/end2end.py @@ -232,32 +232,43 @@ def main(args): raise ValueError(f"Unknown query type: {args.query_type}") # Initialize vLLM-Omni with Step-Audio2 - # Resolve stage config path + # Resolve deploy config path. Keep explicit stage_configs_path for legacy + # custom configs, but use bundled deploy configs by default. + deploy_config_path = None + stage_config_path = None + if args.stage_configs_path and args.deploy_config: + raise ValueError("--stage-configs-path and --deploy-config are mutually exclusive") if args.stage_configs_path: stage_config_path = args.stage_configs_path + elif args.deploy_config: + deploy_config_path = args.deploy_config else: - configs_dir = Path(__file__).parent.parent.parent.parent / "vllm_omni/model_executor/stage_configs" + configs_dir = Path(__file__).parent.parent.parent.parent / "vllm_omni/deploy" if args.query_type == "audio_to_text": # ASR only needs Thinker (Stage 0), no Token2Wav - stage_config_path = str(configs_dir / "step_audio_2_asr.yaml") - if not os.path.exists(stage_config_path): - stage_config_path = str(configs_dir / "step_audio_2.yaml") + deploy_config_path = str(configs_dir / "step_audio_2_asr.yaml") else: # TTS/S2ST need both stages - stage_config_path = str(configs_dir / "step_audio_2.yaml") + deploy_config_path = str(configs_dir / "step_audio_2.yaml") - omni_llm = Omni( - model=model_name, - log_stats=args.enable_stats, - log_file=("step_audio2_pipeline.log" if args.enable_stats else None), - init_sleep_seconds=args.init_sleep_seconds, - batch_timeout=args.batch_timeout, - init_timeout=args.init_timeout, - shm_threshold_bytes=args.shm_threshold_bytes, - stage_configs_path=stage_config_path, - worker_backend=args.worker_backend, - ray_address=args.ray_address, - ) + omni_kwargs = { + "model": model_name, + "log_stats": args.enable_stats, + "log_file": ("step_audio2_pipeline.log" if args.enable_stats else None), + "init_sleep_seconds": args.init_sleep_seconds, + "batch_timeout": args.batch_timeout, + "init_timeout": args.init_timeout, + "shm_threshold_bytes": args.shm_threshold_bytes, + "worker_backend": args.worker_backend, + "ray_address": args.ray_address, + "trust_remote_code": True, + } + if stage_config_path is not None: + omni_kwargs["stage_configs_path"] = stage_config_path + else: + omni_kwargs["deploy_config"] = deploy_config_path + + omni_llm = Omni(**omni_kwargs) # Configure sampling parameters for each stage # Stage 0 (Thinker): Generate text and audio tokens @@ -405,16 +416,21 @@ def parse_args(): help=( "Model path (required). Can be:\n" " - Local path: /path/to/step-audio-2\n" - " - HuggingFace ID: stepfun-ai/Step-Audio2-mini" + " - HuggingFace ID: stepfun-ai/Step-Audio-2-mini" ), ) parser.add_argument( "--stage-configs-path", type=str, default=None, - help="Path to stage config YAML file (default: use built-in step_audio_2.yaml)", + help="Path to a legacy stage config YAML file (default: use bundled deploy config)", + ) + parser.add_argument( + "--deploy-config", + type=str, + default=None, + help="Path to a deploy config YAML file (default: use bundled Step-Audio2 deploy config)", ) - # Query configuration parser.add_argument( "--query-type", diff --git a/examples/offline_inference/text_to_video/text_to_video.py b/examples/offline_inference/text_to_video/text_to_video.py index 0f9bb8c8e..addf7cb7a 100644 --- a/examples/offline_inference/text_to_video/text_to_video.py +++ b/examples/offline_inference/text_to_video/text_to_video.py @@ -124,6 +124,11 @@ def parse_args() -> argparse.Namespace: default=None, help="Override model class name (e.g., LTX2TwoStagesVideoPipeline).", ) + parser.add_argument( + "--deploy-config", + default=None, + help="Optional deploy config YAML to use for pipeline-backed runs.", + ) parser.add_argument("--prompt", default="A serene lakeside sunrise with mist over the water.", help="Text prompt.") parser.add_argument("--negative-prompt", default="", help="Negative prompt.") parser.add_argument( @@ -375,6 +380,8 @@ def main(): enable_diffusion_pipeline_profiler=args.enable_diffusion_pipeline_profiler, profiler_config=args.profiler_config, ) + if args.deploy_config: + omni_kwargs["deploy_config"] = args.deploy_config if args.boundary_ratio is not None: omni_kwargs["boundary_ratio"] = args.boundary_ratio if args.flow_shift is not None: diff --git a/examples/online_serving/step_audio2/README.md b/examples/online_serving/step_audio2/README.md index e93f58ccc..25e2a6785 100644 --- a/examples/online_serving/step_audio2/README.md +++ b/examples/online_serving/step_audio2/README.md @@ -10,15 +10,15 @@ Please refer to [README.md](../../../README.md) ```bash # Async chunk mode (recommended — lower first-packet latency for TTS) -vllm serve stepfun-ai/Step-Audio2-mini --omni --port 8092 \ - --stage-configs-path vllm_omni/model_executor/stage_configs/step_audio_2_async_chunk.yaml \ +vllm serve stepfun-ai/Step-Audio-2-mini --omni --port 8092 \ + --deploy-config vllm_omni/deploy/step_audio_2_async_chunk.yaml \ --trust-remote-code --enforce-eager ``` Sequential mode: ```bash -vllm serve stepfun-ai/Step-Audio2-mini --omni --port 8092 \ - --stage-configs-path vllm_omni/model_executor/stage_configs/step_audio_2.yaml \ +vllm serve stepfun-ai/Step-Audio-2-mini --omni --port 8092 \ + --deploy-config vllm_omni/deploy/step_audio_2.yaml \ --trust-remote-code --enforce-eager ``` @@ -51,7 +51,7 @@ Or via curl: ```bash curl -X POST http://localhost:8092/v1/audio/speech \ -H "Content-Type: application/json" \ - -d '{"model":"stepfun-ai/Step-Audio2-mini","input":"你好世界","voice":"default"}' \ + -d '{"model":"stepfun-ai/Step-Audio-2-mini","input":"你好世界","voice":"default"}' \ --output output.wav ``` @@ -139,7 +139,7 @@ Step-Audio2 uses the OpenAI-compatible chat completions API: ```json { - "model": "stepfun-ai/Step-Audio2-mini", + "model": "stepfun-ai/Step-Audio-2-mini", "messages": [ { "role": "system", diff --git a/examples/online_serving/step_audio2/openai_chat_completion_client.py b/examples/online_serving/step_audio2/openai_chat_completion_client.py index 2b91dacb2..94acb5533 100644 --- a/examples/online_serving/step_audio2/openai_chat_completion_client.py +++ b/examples/online_serving/step_audio2/openai_chat_completion_client.py @@ -9,7 +9,7 @@ Step-Audio2 is a two-stage model: Usage examples: # Start server first: - vllm serve stepfun-ai/Step-Audio2-mini --omni --port 8092 + vllm serve stepfun-ai/Step-Audio-2-mini --omni --port 8092 # Audio to text (ASR) python openai_chat_completion_client.py --query-type audio_to_text @@ -311,7 +311,7 @@ def parse_args(): "--model", "-m", type=str, - default="stepfun-ai/Step-Audio2-mini", + default="stepfun-ai/Step-Audio-2-mini", help="Model name", ) parser.add_argument( diff --git a/examples/online_serving/step_audio2/openai_speech_client.py b/examples/online_serving/step_audio2/openai_speech_client.py index 740b36805..762ca72a6 100644 --- a/examples/online_serving/step_audio2/openai_speech_client.py +++ b/examples/online_serving/step_audio2/openai_speech_client.py @@ -87,7 +87,7 @@ def parse_args(): "--model", "-m", type=str, - default="stepfun-ai/Step-Audio2-mini", + default="stepfun-ai/Step-Audio-2-mini", help="Model name/path", ) parser.add_argument( diff --git a/examples/online_serving/step_audio2/run_curl.sh b/examples/online_serving/step_audio2/run_curl.sh index 3811308e7..3a3b1c9f4 100755 --- a/examples/online_serving/step_audio2/run_curl.sh +++ b/examples/online_serving/step_audio2/run_curl.sh @@ -110,7 +110,7 @@ output=$(curl -sS -X POST "${API_BASE}/v1/chat/completions" \ -H "Content-Type: application/json" \ -d @- < token2wav. +# +# Sequential mode waits for the full Thinker output before running Token2Wav. +async_chunk: false +trust_remote_code: true +enable_prefix_caching: false + +stages: + - stage_id: 0 + devices: "0,1" + tensor_parallel_size: 2 + max_num_seqs: 1 + gpu_memory_utilization: 0.7 + enforce_eager: true + async_scheduling: false + max_num_batched_tokens: 8192 + default_sampling_params: + temperature: 0.7 + top_p: 0.9 + top_k: -1 + max_tokens: 1024 + seed: 42 + repetition_penalty: 1.05 + + - stage_id: 1 + devices: "1" + max_num_seqs: 1 + gpu_memory_utilization: 0.2 + enforce_eager: true + async_scheduling: false + max_num_batched_tokens: 4096 + hf_overrides: + architectures: ["StepAudio2Token2WavForConditionalGeneration"] + default_sampling_params: + temperature: 0.0 + top_p: 1.0 + top_k: -1 + max_tokens: 1 + seed: 42 diff --git a/vllm_omni/deploy/step_audio_2_asr.yaml b/vllm_omni/deploy/step_audio_2_asr.yaml new file mode 100644 index 000000000..0eb0a8523 --- /dev/null +++ b/vllm_omni/deploy/step_audio_2_asr.yaml @@ -0,0 +1,22 @@ +# Step-Audio2 ASR deploy: thinker only. +pipeline: step_audio_2_asr +async_chunk: false +trust_remote_code: true +enable_prefix_caching: false + +stages: + - stage_id: 0 + devices: "0,1" + tensor_parallel_size: 2 + max_num_seqs: 1 + gpu_memory_utilization: 0.8 + enforce_eager: true + async_scheduling: false + max_num_batched_tokens: 2048 + default_sampling_params: + temperature: 0.7 + top_p: 0.9 + top_k: -1 + max_tokens: 1024 + seed: 42 + repetition_penalty: 1.05 diff --git a/vllm_omni/deploy/step_audio_2_async_chunk.yaml b/vllm_omni/deploy/step_audio_2_async_chunk.yaml new file mode 100644 index 000000000..75b5cf901 --- /dev/null +++ b/vllm_omni/deploy/step_audio_2_async_chunk.yaml @@ -0,0 +1,37 @@ +# Step-Audio2 deploy: thinker -> token2wav with async chunk streaming. +async_chunk: true +trust_remote_code: true +enable_prefix_caching: false + +stages: + - stage_id: 0 + devices: "0,1" + tensor_parallel_size: 2 + max_num_seqs: 1 + gpu_memory_utilization: 0.7 + enforce_eager: true + async_scheduling: false + max_num_batched_tokens: 1024 + default_sampling_params: + temperature: 0.7 + top_p: 0.9 + top_k: -1 + max_tokens: 1024 + seed: 42 + repetition_penalty: 1.05 + + - stage_id: 1 + devices: "1" + max_num_seqs: 1 + gpu_memory_utilization: 0.2 + enforce_eager: true + async_scheduling: false + max_num_batched_tokens: 2048 + hf_overrides: + architectures: ["StepAudio2Token2WavForConditionalGeneration"] + default_sampling_params: + temperature: 0.0 + top_p: 1.0 + top_k: -1 + max_tokens: 1 + seed: 42 diff --git a/vllm_omni/deploy/wan2_2_ti2v.yaml b/vllm_omni/deploy/wan2_2_ti2v.yaml new file mode 100644 index 000000000..bdb5cfecb --- /dev/null +++ b/vllm_omni/deploy/wan2_2_ti2v.yaml @@ -0,0 +1,17 @@ +# Wan2.2 TI2V-5B deploy. +# +# Single GPU by default. Increase tensor_parallel_size and devices for A14B MoE variants. +async_chunk: false +trust_remote_code: true +distributed_executor_backend: mp + +stages: + - stage_id: 0 + devices: "0" + max_num_seqs: 1 + tensor_parallel_size: 1 + enforce_eager: true + # Keep VAE tiling on by default for higher resolutions / longer clips. + vae_use_tiling: true + default_sampling_params: + seed: 42 diff --git a/vllm_omni/model_executor/models/hunyuan_video/__init__.py b/vllm_omni/model_executor/models/hunyuan_video/__init__.py new file mode 100644 index 000000000..208f01a7c --- /dev/null +++ b/vllm_omni/model_executor/models/hunyuan_video/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project diff --git a/vllm_omni/model_executor/models/hunyuan_video/pipeline.py b/vllm_omni/model_executor/models/hunyuan_video/pipeline.py new file mode 100644 index 000000000..341d14e5d --- /dev/null +++ b/vllm_omni/model_executor/models/hunyuan_video/pipeline.py @@ -0,0 +1,26 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""HunyuanVideo-1.5 pipeline topology.""" + +from vllm_omni.config.stage_config import ( + PipelineConfig, + StageExecutionType, + StagePipelineConfig, +) + +HUNYUAN_VIDEO_15_PIPELINE = PipelineConfig( + model_type="hunyuan_video_15", + model_arch="HunyuanVideo15Pipeline", + diffusers_class_name="HunyuanVideo15Pipeline", + stages=( + StagePipelineConfig( + stage_id=0, + model_stage="dit", + execution_type=StageExecutionType.DIFFUSION, + input_sources=(), + final_output=True, + final_output_type="video", + model_arch="HunyuanVideo15Pipeline", + ), + ), +) diff --git a/vllm_omni/model_executor/models/step_audio2/__init__.py b/vllm_omni/model_executor/models/step_audio2/__init__.py index 0b6f07e28..e69de29bb 100644 --- a/vllm_omni/model_executor/models/step_audio2/__init__.py +++ b/vllm_omni/model_executor/models/step_audio2/__init__.py @@ -1,3 +0,0 @@ -from .step_audio2 import StepAudio2ForConditionalGeneration - -__all__ = ["StepAudio2ForConditionalGeneration"] diff --git a/vllm_omni/model_executor/models/step_audio2/pipeline.py b/vllm_omni/model_executor/models/step_audio2/pipeline.py new file mode 100644 index 000000000..7ba6d9ee1 --- /dev/null +++ b/vllm_omni/model_executor/models/step_audio2/pipeline.py @@ -0,0 +1,71 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Step-Audio2 pipeline topologies (frozen).""" + +from vllm_omni.config.stage_config import ( + PipelineConfig, + StageExecutionType, + StagePipelineConfig, +) + +_PROC = "vllm_omni.model_executor.stage_input_processors.step_audio2" +_HF_ARCHITECTURES = ( + "StepAudio2ForCausalLM", + "StepAudio2ForConditionalGeneration", +) + + +STEP_AUDIO2_PIPELINE = PipelineConfig( + model_type="step_audio_2", + model_arch="StepAudio2ForConditionalGeneration", + hf_architectures=_HF_ARCHITECTURES, + stages=( + StagePipelineConfig( + stage_id=0, + model_stage="step_audio2_thinker", + execution_type=StageExecutionType.LLM_AR, + input_sources=(), + final_output=True, + final_output_type="text", + owns_tokenizer=True, + requires_multimodal_data=True, + engine_output_type="latent", + model_arch="StepAudio2ThinkerForConditionalGeneration", + async_chunk_process_next_stage_input_func=f"{_PROC}.thinker2token2wav_async_chunk", + sampling_constraints={"detokenize": True}, + ), + StagePipelineConfig( + stage_id=1, + model_stage="token2wav", + execution_type=StageExecutionType.LLM_GENERATION, + input_sources=(0,), + final_output=True, + final_output_type="audio", + engine_output_type="audio", + model_arch="StepAudio2Token2WavModel", + sync_process_input_func=f"{_PROC}.thinker2token2wav", + sampling_constraints={"detokenize": False}, + ), + ), +) + + +STEP_AUDIO2_ASR_PIPELINE = PipelineConfig( + model_type="step_audio_2_asr", + model_arch="StepAudio2ForConditionalGeneration", + stages=( + StagePipelineConfig( + stage_id=0, + model_stage="step_audio2_thinker", + execution_type=StageExecutionType.LLM_AR, + input_sources=(), + final_output=True, + final_output_type="text", + owns_tokenizer=True, + requires_multimodal_data=True, + engine_output_type="text", + model_arch="StepAudio2ThinkerForConditionalGeneration", + sampling_constraints={"detokenize": True}, + ), + ), +) diff --git a/vllm_omni/model_executor/models/wan2_2/__init__.py b/vllm_omni/model_executor/models/wan2_2/__init__.py new file mode 100644 index 000000000..208f01a7c --- /dev/null +++ b/vllm_omni/model_executor/models/wan2_2/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project diff --git a/vllm_omni/model_executor/models/wan2_2/pipeline.py b/vllm_omni/model_executor/models/wan2_2/pipeline.py new file mode 100644 index 000000000..7059845fa --- /dev/null +++ b/vllm_omni/model_executor/models/wan2_2/pipeline.py @@ -0,0 +1,26 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Wan2.2 TI2V pipeline topology.""" + +from vllm_omni.config.stage_config import ( + PipelineConfig, + StageExecutionType, + StagePipelineConfig, +) + +WAN2_2_TI2V_PIPELINE = PipelineConfig( + model_type="wan2_2_ti2v", + model_arch="WanPipeline", + diffusers_class_name="WanPipeline", + stages=( + StagePipelineConfig( + stage_id=0, + model_stage="dit", + execution_type=StageExecutionType.DIFFUSION, + input_sources=(), + final_output=True, + final_output_type="video", + model_arch="WanPipeline", + ), + ), +) diff --git a/vllm_omni/model_executor/stage_configs/__init__.py b/vllm_omni/model_executor/stage_configs/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/vllm_omni/model_executor/stage_configs/hunyuan_video_15_dit_fp8.yaml b/vllm_omni/model_executor/stage_configs/hunyuan_video_15_dit_fp8.yaml deleted file mode 100644 index 097afc020..000000000 --- a/vllm_omni/model_executor/stage_configs/hunyuan_video_15_dit_fp8.yaml +++ /dev/null @@ -1,38 +0,0 @@ -# Stage config for running HunyuanVideo-1.5 DiT with ModelOpt FP8 auto-detect. -# Single GPU. Bump `tensor_parallel_size` and `devices` for multi-GPU TP. -# -# Use with a ModelOpt FP8 checkpoint (e.g. produced by -# examples/quantization/quantize_hunyuanvideo_15_modelopt_fp8.py). - -stage_args: - - stage_id: 0 - stage_type: diffusion - runtime: - devices: "0" - max_batch_size: 1 - engine_args: - model_stage: dit - model_class_name: HunyuanVideo15Pipeline - max_num_seqs: 1 - enforce_eager: true - trust_remote_code: true - distributed_executor_backend: "mp" - # HunyuanVideo-1.5 peak VRAM is dominated by the single-shot VAE decode of - # the full video (the DiT contributes little), so tiling the decode is the - # main memory lever — far more than FP8 (e.g. 480x832x33: ~117 -> ~47 GiB). - # Default it on here; override with --vae-use-tiling/false if you have headroom. - vae_use_tiling: true - parallel_config: - tensor_parallel_size: 1 - - final_output: true - final_output_type: video - is_comprehension: false - default_sampling_params: - seed: 42 - -runtime: - enabled: true - defaults: - window_size: -1 - max_inflight: 1 diff --git a/vllm_omni/model_executor/stage_configs/mimo_audio.yaml b/vllm_omni/model_executor/stage_configs/mimo_audio.yaml deleted file mode 100644 index b81f27fe4..000000000 --- a/vllm_omni/model_executor/stage_configs/mimo_audio.yaml +++ /dev/null @@ -1,61 +0,0 @@ - -# MiMo Audio deploy: 2-stage thinker+talker → code2wav. -# -# Default mode is async-chunk streaming on a single GPU (both stages on -# device 0) through SharedMemoryConnector. For the legacy 2-GPU sync -# pipeline (stage 1 on a second card), pass ``--no-async-chunk -# --stage-overrides '{"1": {"devices": "1", "max_model_len": 18192, "max_num_batched_tokens": 18192}}'`` -# to the serve CLI. -async_chunk: true -dtype: bfloat16 - -connectors: - connector_of_shared_memory: - name: SharedMemoryConnector - extra: - shm_threshold_bytes: 365536 - codec_streaming: true - connector_get_sleep_s: 0.001 - connector_get_max_wait_first_chunk: 3000 - connector_get_max_wait: 300 - codec_chunk_frames: 30 - codec_left_context_frames: 40 - -stages: - - stage_id: 0 - max_num_seqs: 1 - gpu_memory_utilization: 0.5 - enforce_eager: true - trust_remote_code: true - enable_prefix_caching: false - max_num_batched_tokens: 8192 - max_model_len: 8192 - devices: "0" - output_connectors: - to_stage_1: connector_of_shared_memory - default_sampling_params: - temperature: 0.6 - top_p: 0.95 - top_k: 50 - max_tokens: 18192 - seed: 42 - repetition_penalty: 1.1 - - - stage_id: 1 - max_num_seqs: 1 - gpu_memory_utilization: 0.4 - enforce_eager: true - trust_remote_code: true - enable_prefix_caching: false - async_scheduling: false - max_num_batched_tokens: 8192 - max_model_len: 8192 - devices: "0" - input_connectors: - from_stage_0: connector_of_shared_memory - default_sampling_params: - temperature: 0.0 - top_p: 1.0 - top_k: -1 - max_tokens: 18192 - seed: 42 diff --git a/vllm_omni/model_executor/stage_configs/step_audio_2.yaml b/vllm_omni/model_executor/stage_configs/step_audio_2.yaml deleted file mode 100644 index b560fea9b..000000000 --- a/vllm_omni/model_executor/stage_configs/step_audio_2.yaml +++ /dev/null @@ -1,122 +0,0 @@ -# Stage config for Step-Audio2 with vLLM-Omni -# -# Two-stage pipeline: -# Stage 0 (Thinker): Audio understanding → Text + Audio tokens -# Stage 1 (Token2Wav): Audio tokens → Waveform (24kHz) -# -# Hardware requirement: 1-2 GPUs (can run on single GPU with devices: "0" for both) - -# Model configuration reference (for documentation) -# Actual values are set in hf_overrides below -# -# Token configuration: -# text_max: 151688 # Maximum text token ID (inclusive) -# audio_start: 151696 # First audio token ID in absolute vocabulary -# audio_vocab_size: 6562 # Total number of audio tokens -# audio_eos: 6561 # Audio EOS token ID (relative to audio_start) -# audio_patch_token_id: 151690 # placeholder token ID -# -# Encoder configuration: -# n_mels: 128 # Number of mel frequency bins -# n_audio_ctx: 1500 # Audio context length -# n_audio_state: 512 # Audio encoder hidden state dimension -# n_audio_head: 8 # Number of attention heads -# n_audio_layer: 6 # Number of encoder layers -# kernel_size: 3 # Convolution kernel size for adapter -# adapter_stride: 2 # Stride for adapter convolution - -stage_args: - # Stage 0: Thinker (Audio Understanding + Token Generation) - - stage_id: 0 - runtime: - process: true # Run in separate process - devices: "0,1,2,3" # Use 3 GPUs for tensor parallelism - max_batch_size: 1 # Single sample at a time - engine_args: - model_stage: step_audio2_thinker - model_arch: StepAudio2ThinkerForConditionalGeneration - tensor_parallel_size: 4 # Split model across 3 GPUs - # NOTE: All default values are defined in step_audio2_constants.py (DEFAULT_MODEL_CONFIG) - # Only override here if your model differs from the defaults - # - # Default values (for reference, from constants.py): - # hidden_size: 4096 # Qwen2-7B (change to 1536 for 1.5B, 896 for 0.5B) - # text_max: 151688 - # audio_start: 151696 - # audio_vocab_size: 6562 - # audio_eos: 6561 - # audio_patch_token_id: 151690 - # n_mels: 128 - # n_audio_ctx: 1500 - # n_audio_state: 512 - # n_audio_head: 8 - # n_audio_layer: 6 - # kernel_size: 3 - # adapter_stride: 2 - # - # Uncomment and modify only if needed: - # hidden_size: 4096 - worker_cls: vllm_omni.worker.gpu_ar_worker.GPUARWorker - scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler - gpu_memory_utilization: 0.8 - enforce_eager: true # Required for now - trust_remote_code: true - enable_prefix_caching: false - max_num_batched_tokens: 8192 - engine_output_type: text # Outputs text + audio tokens - is_comprehension: true # This stage does comprehension - final_output: true # Can be final output (for ASR mode) - final_output_type: text - default_sampling_params: - temperature: 0.7 - top_p: 0.9 - top_k: -1 - max_tokens: 1024 - seed: 42 - detokenize: true - repetition_penalty: 1.05 - - # Stage 1: Token2Wav (Audio Synthesis) - - stage_id: 1 - runtime: - process: true - devices: "3" # Use GPU 3 (separate from Stage 0's 0-2) - max_batch_size: 1 - engine_args: - model_stage: token2wav - model_arch: StepAudio2Token2WavModel - hf_overrides: - architectures: ["StepAudio2Token2WavForConditionalGeneration"] - worker_cls: vllm_omni.worker.gpu_generation_worker.GPUGenerationWorker - scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler - - gpu_memory_utilization: 0.3 # Token2Wav needs less memory - enforce_eager: true - trust_remote_code: true - async_scheduling: false - enable_prefix_caching: false - max_num_batched_tokens: 4096 - engine_output_type: audio # Outputs waveform - engine_input_source: [0] # Takes input from Stage 0 - custom_process_input_func: vllm_omni.model_executor.stage_input_processors.step_audio2.thinker2token2wav - final_output: true # Final output stage - final_output_type: audio - default_sampling_params: - temperature: 0.0 - top_p: 1.0 - top_k: -1 - max_tokens: 1 - seed: 42 - detokenize: false - -# Runtime configuration -runtime: - enabled: true - defaults: - window_size: -1 # Wait for full input before triggering - max_inflight: 1 # Process one at a time - - edges: - - from: 0 # Thinker → Token2Wav - to: 1 - window_size: -1 # Wait for complete output from Thinker diff --git a/vllm_omni/model_executor/stage_configs/step_audio_2_asr.yaml b/vllm_omni/model_executor/stage_configs/step_audio_2_asr.yaml deleted file mode 100644 index 1cecfddc5..000000000 --- a/vllm_omni/model_executor/stage_configs/step_audio_2_asr.yaml +++ /dev/null @@ -1,35 +0,0 @@ -# Stage config for Step-Audio2 ASR (speech-to-text only) -# -# Single-stage pipeline: only Thinker, no Token2Wav. -# Use this for audio_to_text tasks where no audio output is needed. - -stage_args: - - stage_id: 0 - stage_type: llm - runtime: - process: true - devices: "0,1" - max_batch_size: 1 - engine_args: - model_stage: step_audio2_thinker - model_arch: StepAudio2ThinkerForConditionalGeneration - worker_type: ar - scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler - gpu_memory_utilization: 0.8 - enforce_eager: true - trust_remote_code: true - enable_prefix_caching: false - max_num_batched_tokens: 2048 - tensor_parallel_size: 2 - engine_output_type: text - is_comprehension: true - final_output: true - final_output_type: text - default_sampling_params: - temperature: 0.7 - top_p: 0.9 - top_k: -1 - max_tokens: 1024 - seed: 42 - detokenize: true - repetition_penalty: 1.05 diff --git a/vllm_omni/model_executor/stage_configs/step_audio_2_async_chunk.yaml b/vllm_omni/model_executor/stage_configs/step_audio_2_async_chunk.yaml deleted file mode 100644 index 98c484072..000000000 --- a/vllm_omni/model_executor/stage_configs/step_audio_2_async_chunk.yaml +++ /dev/null @@ -1,78 +0,0 @@ -# Stage config for Step-Audio2 with async_chunk support -# -# Two-stage pipeline with streaming between stages: -# Stage 0 (Thinker): Audio understanding → Text + Audio tokens (streamed in chunks) -# Stage 1 (Token2Wav): Audio tokens → Waveform (24kHz) -# -# Key difference from step_audio_2.yaml: -# - async_chunk: true enables streaming from Thinker to Token2Wav -# - Thinker uses custom_process_next_stage_input_func to send audio token chunks -# - Token2Wav receives chunks incrementally instead of waiting for full output -# -# Hardware requirement: 1-2 GPUs (can run on single GPU with devices: "0" for both) - -async_chunk: true -stage_args: - # Stage 0: Thinker (Audio Understanding + Token Generation) - - stage_id: 0 - stage_type: llm - runtime: - process: true - devices: "0,1" - max_batch_size: 1 - engine_args: - model_stage: step_audio2_thinker - model_arch: StepAudio2ThinkerForConditionalGeneration - worker_type: ar - scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler - gpu_memory_utilization: 0.8 - enforce_eager: true - trust_remote_code: true - enable_prefix_caching: false - max_num_batched_tokens: 1024 - tensor_parallel_size: 2 - engine_output_type: latent - custom_process_next_stage_input_func: vllm_omni.model_executor.stage_input_processors.step_audio2.thinker2token2wav_async_chunk - is_comprehension: true - final_output: true - final_output_type: text - default_sampling_params: - temperature: 0.7 - top_p: 0.9 - top_k: -1 - max_tokens: 1024 - seed: 42 - detokenize: true - repetition_penalty: 1.05 - - # Stage 1: Token2Wav (Audio Synthesis) - - stage_id: 1 - stage_type: llm - runtime: - process: true - devices: "3" - max_batch_size: 1 - engine_args: - model_stage: token2wav - model_arch: StepAudio2Token2WavModel - worker_type: generation - hf_overrides: - architectures: ["StepAudio2Token2WavForConditionalGeneration"] - scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler - gpu_memory_utilization: 0.3 - enforce_eager: true - trust_remote_code: true - async_scheduling: false - enable_prefix_caching: false - max_num_batched_tokens: 2048 - engine_output_type: audio - engine_input_source: [0] - final_output: true - final_output_type: audio - default_sampling_params: - temperature: 0.0 - top_p: 1.0 - top_k: -1 - max_tokens: 1 - seed: 42 - detokenize: false diff --git a/vllm_omni/model_executor/stage_configs/wan2_2_ti2v_dit_fp8.yaml b/vllm_omni/model_executor/stage_configs/wan2_2_ti2v_dit_fp8.yaml deleted file mode 100644 index 1b1c40537..000000000 --- a/vllm_omni/model_executor/stage_configs/wan2_2_ti2v_dit_fp8.yaml +++ /dev/null @@ -1,41 +0,0 @@ -# Stage config for running Wan2.2 TI2V-5B DiT with ModelOpt FP8 auto-detect. -# Single GPU (TI2V-5B fits 80GB BF16; FP8 drops by ~half). -# For the A14B MoE variants, bump `tensor_parallel_size` and `devices`. -# -# Use with a ModelOpt FP8 checkpoint (e.g. produced by -# examples/quantization/quantize_wan2_2_modelopt_fp8.py). - -stage_args: - - stage_id: 0 - stage_type: diffusion - runtime: - devices: "0" - max_batch_size: 1 - engine_args: - model_stage: dit - # Wan2.2 TI2V-5B is served by the base Wan22Pipeline, which auto-detects - # expand_timesteps (TI2V-5B mode) from model_index.json. (The standalone - # Wan22TI2VPipeline was consolidated into Wan22Pipeline upstream.) - model_class_name: Wan22Pipeline - max_num_seqs: 1 - enforce_eager: true - trust_remote_code: true - distributed_executor_backend: "mp" - # Tile the VAE decode by default. Wan2.2's 480p peak is DiT-dominated (so - # FP8 is the main lever here), but tiling adds headroom for higher - # resolutions / longer clips at negligible cost. Override with --vae-use-tiling/false. - vae_use_tiling: true - parallel_config: - tensor_parallel_size: 1 - - final_output: true - final_output_type: video - is_comprehension: false - default_sampling_params: - seed: 42 - -runtime: - enabled: true - defaults: - window_size: -1 - max_inflight: 1 diff --git a/vllm_omni/model_executor/stage_input_processors/mimo_audio.py b/vllm_omni/model_executor/stage_input_processors/mimo_audio.py index 3a1a4441f..cd9ca7bf2 100644 --- a/vllm_omni/model_executor/stage_input_processors/mimo_audio.py +++ b/vllm_omni/model_executor/stage_input_processors/mimo_audio.py @@ -17,8 +17,8 @@ logger = init_logger(__name__) # sequence fed to stage-1 must not exceed this, otherwise gpu_input_batch # add_request will fail with a broadcast error when copying prompt_token_ids # into token_ids_cpu. Keep in sync with the stage-1 ``max_model_len`` in -# ``vllm_omni/model_executor/stage_configs/mimo_audio.yaml`` and the offline -# example ``examples/offline_inference/mimo_audio/end2end.py``. +# ``vllm_omni/deploy/mimo_audio.yaml`` and the offline example +# ``examples/offline_inference/mimo_audio/end2end.py``. MAX_CODE2WAV_TOKENS = 18192 # Minimum safe values for codec streaming parameters. diff --git a/vllm_omni/model_executor/stage_input_processors/step_audio2.py b/vllm_omni/model_executor/stage_input_processors/step_audio2.py index 5e015aa7c..c0c0932b2 100644 --- a/vllm_omni/model_executor/stage_input_processors/step_audio2.py +++ b/vllm_omni/model_executor/stage_input_processors/step_audio2.py @@ -7,6 +7,7 @@ import torch from vllm.inputs import TextPrompt from vllm.logger import init_logger +from vllm_omni.data_entry_keys import CodesStruct, MetaStruct, OmniPayloadStruct from vllm_omni.engine import OmniEngineCoreRequest from vllm_omni.inputs.data import OmniTokensPrompt from vllm_omni.model_executor.models.step_audio2.step_audio2_constants import ( @@ -35,14 +36,14 @@ def _ensure_list(x): def thinker2token2wav_async_chunk( transfer_manager: Any, - pooling_output: dict[str, Any], + multimodal_output: dict[str, Any] | None, request: OmniEngineCoreRequest, is_finished: bool = False, -) -> dict[str, Any] | None: +) -> OmniPayloadStruct | None: """ Async chunk processor: stream audio tokens from Thinker to Token2Wav. - Unlike Qwen3-Omni which passes hidden states via pooling_output, + Unlike Qwen3-Omni which passes hidden states via multimodal_output, Step Audio2 generates audio tokens as part of the LLM autoregressive output (token IDs >= audio_start). This processor extracts those audio tokens from the running token stream and sends them in chunks. @@ -62,14 +63,13 @@ def thinker2token2wav_async_chunk( Args: transfer_manager: OmniChunkTransferAdapter instance (has ``code_prompt_token_ids`` defaultdict for state tracking). - pooling_output: Pooler output dict (unused — audio tokens are in + multimodal_output: Multimodal output dict (unused — audio tokens are in the token ID stream, not in hidden states). request: Current engine request with access to all_token_ids. is_finished: Whether the upstream request has finished generating. Returns: - dict with framework-compatible fields, or None if the chunk is - not yet ready. + Structured connector payload, or None if the chunk is not yet ready. """ audio_start = DEFAULT_TOKEN_CONFIG.audio_start audio_eos = DEFAULT_TOKEN_CONFIG.audio_eos @@ -100,18 +100,22 @@ def thinker2token2wav_async_chunk( # Last chunk: send all remaining tokens if available <= 0: # No audio tokens at all (text-only response) — send EOF marker - return { - "code_predictor_codes": [], - "left_context_size": 1, # 1 = last chunk - "finished": torch.tensor(True, dtype=torch.bool), - } + return OmniPayloadStruct( + codes=CodesStruct(audio=torch.empty(0, dtype=torch.long)), + meta=MetaStruct( + left_context_size=1, # 1 = last chunk + finished=torch.tensor(True, dtype=torch.bool), + ), + ) remaining_tokens = audio_tokens[consumed:] transfer_manager.code_prompt_token_ids[request_id].extend(remaining_tokens) - return { - "code_predictor_codes": remaining_tokens, - "left_context_size": 1, # 1 = last chunk - "finished": torch.tensor(True, dtype=torch.bool), - } + return OmniPayloadStruct( + codes=CodesStruct(audio=torch.tensor(remaining_tokens, dtype=torch.long)), + meta=MetaStruct( + left_context_size=1, # 1 = last chunk + finished=torch.tensor(True, dtype=torch.bool), + ), + ) else: # Non-last chunk: need chunk_size + pre_lookahead_len tokens required = chunk_size + pre_lookahead_len @@ -120,11 +124,13 @@ def thinker2token2wav_async_chunk( chunk_tokens = audio_tokens[consumed : consumed + required] # Only consume chunk_size tokens; the lookahead portion is re-used transfer_manager.code_prompt_token_ids[request_id].extend(audio_tokens[consumed : consumed + chunk_size]) - return { - "code_predictor_codes": chunk_tokens, - "left_context_size": 0, # 0 = not last chunk - "finished": torch.tensor(False, dtype=torch.bool), - } + return OmniPayloadStruct( + codes=CodesStruct(audio=torch.tensor(chunk_tokens, dtype=torch.long)), + meta=MetaStruct( + left_context_size=0, # 0 = not last chunk + finished=torch.tensor(False, dtype=torch.bool), + ), + ) def thinker2token2wav( diff --git a/vllm_omni/platforms/cuda/platform.py b/vllm_omni/platforms/cuda/platform.py index a05bb289a..7be135352 100644 --- a/vllm_omni/platforms/cuda/platform.py +++ b/vllm_omni/platforms/cuda/platform.py @@ -36,7 +36,7 @@ class CudaOmniPlatform(OmniPlatform, CudaPlatformBase): @classmethod def get_default_stage_config_path(cls) -> str: - return "vllm_omni/model_executor/stage_configs" + return "vllm_omni/deploy" @classmethod def has_flash_attn_package(cls) -> bool: diff --git a/vllm_omni/platforms/musa/platform.py b/vllm_omni/platforms/musa/platform.py index 92a194d59..818338da4 100644 --- a/vllm_omni/platforms/musa/platform.py +++ b/vllm_omni/platforms/musa/platform.py @@ -32,7 +32,7 @@ class MUSAOmniPlatform(OmniPlatform, MUSAPlatformBase): @classmethod def get_default_stage_config_path(cls) -> str: - return "vllm_omni/model_executor/stage_configs" + return "vllm_omni/deploy" @classmethod def has_flash_attn_package(cls) -> bool: diff --git a/vllm_omni/platforms/npu/platform.py b/vllm_omni/platforms/npu/platform.py index fe49455d5..5133fa98e 100644 --- a/vllm_omni/platforms/npu/platform.py +++ b/vllm_omni/platforms/npu/platform.py @@ -75,7 +75,7 @@ class NPUOmniPlatform(OmniPlatform, NPUPlatform): @classmethod def get_default_stage_config_path(cls) -> str: - return "vllm_omni/platforms/npu/stage_configs" + return "vllm_omni/deploy" @classmethod def get_diffusion_model_impl_qualname(cls, op_name: str) -> str: diff --git a/vllm_omni/platforms/rocm/platform.py b/vllm_omni/platforms/rocm/platform.py index f905fd73c..38a14c325 100644 --- a/vllm_omni/platforms/rocm/platform.py +++ b/vllm_omni/platforms/rocm/platform.py @@ -122,7 +122,7 @@ class RocmOmniPlatform(OmniPlatform, RocmPlatform): @classmethod def get_default_stage_config_path(cls) -> str: - return "vllm_omni/platforms/rocm/stage_configs" + return "vllm_omni/deploy" @classmethod def get_torch_device(cls, local_rank: int | None = None) -> torch.device: From e9991a59077dc5e32795fc4cbf2f097d0d0fe5a5 Mon Sep 17 00:00:00 2001 From: psv666 <150513104+psv666@users.noreply.github.com> Date: Sat, 18 Jul 2026 19:51:38 +0800 Subject: [PATCH 016/240] [Doc] Fix Qwen3-Omni modality documentation examples (#5173) Signed-off-by: psv666 <2693925048@qq.com> Co-authored-by: amy-why-3459 --- .../examples/online_serving/qwen3_omni.md | 13 ++++++++----- examples/online_serving/qwen3_omni/README.md | 13 ++++++++----- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/docs/user_guide/examples/online_serving/qwen3_omni.md b/docs/user_guide/examples/online_serving/qwen3_omni.md index ec509d597..e372692a8 100644 --- a/docs/user_guide/examples/online_serving/qwen3_omni.md +++ b/docs/user_guide/examples/online_serving/qwen3_omni.md @@ -140,7 +140,7 @@ You can control output modalities to specify which types of output the model sho | Modalities | Output | |------------|--------| | `["text"]` | Text only | -| `["audio"]` | Text + Audio | +| `["audio"]` | Audio only | | `["text", "audio"]` | Text + Audio | | Not specified | Text + Audio (default) | @@ -161,13 +161,16 @@ curl http://localhost:8091/v1/chat/completions \ #### Text + Audio ```bash -curl http://localhost:8091/v1/chat/completions \ +response=$(curl -s http://localhost:8091/v1/chat/completions \ -H "Content-Type: application/json" \ -d '{ "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct", "messages": [{"role": "user", "content": "Describe vLLM in brief."}], - "modalities": ["audio"] - }' + "modalities": ["text", "audio"] + }') + +echo "$response" | jq -r '.choices[0].message.content' +echo "$response" | jq -r '.choices[1].message.audio.data' | base64 -d > output.wav ``` ### Using Python client @@ -205,7 +208,7 @@ client = OpenAI(base_url="http://localhost:8091/v1", api_key="EMPTY") response = client.chat.completions.create( model="Qwen/Qwen3-Omni-30B-A3B-Instruct", messages=[{"role": "user", "content": "Describe vLLM in brief."}], - modalities=["audio"] + modalities=["text", "audio"] ) # Response contains two choices: one with text, one with audio print(response.choices[0].message.content) # Text response diff --git a/examples/online_serving/qwen3_omni/README.md b/examples/online_serving/qwen3_omni/README.md index 3c3ad2b16..74a18af32 100644 --- a/examples/online_serving/qwen3_omni/README.md +++ b/examples/online_serving/qwen3_omni/README.md @@ -353,13 +353,16 @@ curl http://localhost:8091/v1/chat/completions \ #### Text + Audio ```bash -curl -s http://localhost:8091/v1/chat/completions \ +response=$(curl -s http://localhost:8091/v1/chat/completions \ -H "Content-Type: application/json" \ -d '{ "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct", "messages": [{"role": "user", "content": "Describe vLLM in brief."}], - "modalities": ["audio"] - }' | jq -r '.choices[0].message.audio.data' | base64 -d > output.wav + "modalities": ["text", "audio"] + }') + +echo "$response" | jq -r '.choices[0].message.content' +echo "$response" | jq -r '.choices[1].message.audio.data' | base64 -d > output.wav ``` ### Using Python client @@ -456,8 +459,8 @@ response = client.chat.completions.create( modalities=["audio"], extra_body={"speaker": "chelsie"} ) -# Audio uses the specified speaker -print(response.choices[1].message.audio) +# Audio-only responses contain a single audio choice +print(response.choices[0].message.audio) ``` Supported speaker names depend on the model (e.g. `Ethan`, `Chelsie`, `Aiden`). Omit `speaker` to use the default. From 109942465fa4d8e140f9ccac3430bfd90b7cf6de Mon Sep 17 00:00:00 2001 From: Yuxiang Liang Date: Sat, 18 Jul 2026 19:52:44 +0800 Subject: [PATCH 017/240] Support Fish Speech TTS inference on XPU platform (#4856) Signed-off-by: Liangyx2 Signed-off-by: Yuxiang Liang Signed-off-by: Yuxiang Co-authored-by: Chendi.Xue --- docs/models/supported_models.md | 2 +- .../text_to_speech/fish_speech/end2end.py | 78 ++++++++++++++----- vllm_omni/attention/fish_kvcache_attn.py | 2 + vllm_omni/deploy/fish_qwen3_omni.yaml | 12 +++ 4 files changed, 75 insertions(+), 19 deletions(-) diff --git a/docs/models/supported_models.md b/docs/models/supported_models.md index 99b17d529..1ab76588f 100644 --- a/docs/models/supported_models.md +++ b/docs/models/supported_models.md @@ -83,7 +83,7 @@ th { | `MiMoAudioModel` | MiMo-Audio-7B-Instruct | `XiaomiMiMo/MiMo-Audio-7B-Instruct` | ✅︎ | ✅︎ | | | | `MiMoV2ASRForCausalLM` | MiMo-V2.5-ASR | `XiaomiMiMo/MiMo-V2.5-ASR` | ✅︎ | ✅︎ | | | | `Flux2Pipeline` | FLUX.2-dev | `black-forest-labs/FLUX.2-dev` | ✅︎ | ✅︎ | | | -| `FishSpeechSlowARForConditionalGeneration` | Fish Speech S2 Pro | `fishaudio/s2-pro` | ✅︎ | ✅︎ | | | +| `FishSpeechSlowARForConditionalGeneration` | Fish Speech S2 Pro | `fishaudio/s2-pro` | ✅︎ | ✅︎ | | ✅︎ | | `DreamIDOmniPipeline` | DreamID-Omni | `XuGuo699/DreamID-Omni` | ✅︎ | ✅︎ | | | | `SenseNovaU1Pipeline` | SenseNova-U1 (DiT-only) | `SenseNova/SenseNova-U1-8B-MoT` | ✅︎ | | | | | `LancePipeline` | Lance | `bytedance-research/Lance` | ✅︎ | | | | diff --git a/examples/offline_inference/text_to_speech/fish_speech/end2end.py b/examples/offline_inference/text_to_speech/fish_speech/end2end.py index 4d6855246..5661cccd3 100644 --- a/examples/offline_inference/text_to_speech/fish_speech/end2end.py +++ b/examples/offline_inference/text_to_speech/fish_speech/end2end.py @@ -122,12 +122,48 @@ def _save_wav(output_dir: str, request_id: str, mm: dict) -> None: logger.info("Request %s: saved audio to %s (sr=%d)", request_id, out_wav, sr) +def _collect_audio_chunk( + mm: dict, + all_audio_chunks: list[torch.Tensor], + sample_rate: int | None, +) -> tuple[int | None, int]: + """Extract audio and sample rate from a multimodal output chunk. + + Returns updated sample_rate and the number of audio segments in this chunk. + """ + audio = mm.get("audio") if mm else None + if audio is not None: + if isinstance(audio, list): + all_audio_chunks.extend(audio) + else: + all_audio_chunks.append(audio) + sr_raw = mm.get("sr") if mm else None + if sr_raw is not None and not (isinstance(sr_raw, (list, tuple)) and len(sr_raw) == 0): + sr_val = sr_raw[-1] if isinstance(sr_raw, list) and sr_raw else sr_raw + sample_rate = sr_val.item() if hasattr(sr_val, "item") else int(sr_val) + n = len(audio) if isinstance(audio, list) else (0 if audio is None else 1) + return sample_rate, n + + +def _build_omni_kwargs(args, model_name: str) -> dict: + """Build keyword arguments for Omni/AsyncOmni initialization.""" + omni_kwargs = dict( + model=model_name, + log_stats=args.log_stats, + stage_init_timeout=args.stage_init_timeout, + ) + if args.deploy_config: + omni_kwargs["deploy_config"] = args.deploy_config + else: + omni_kwargs["stage_configs_path"] = args.stage_configs_path or DEFAULT_STAGE_CONFIG + return omni_kwargs + + def main(args): """Run offline inference with Omni.""" output_dir = args.output_dir os.makedirs(output_dir, exist_ok=True) - stage_configs_path = args.stage_configs_path or DEFAULT_STAGE_CONFIG model_name = args.model or DEFAULT_MODEL inputs = [ @@ -139,12 +175,7 @@ def main(args): ) ] - omni = Omni( - model=model_name, - stage_configs_path=stage_configs_path, - log_stats=args.log_stats, - stage_init_timeout=args.stage_init_timeout, - ) + omni = Omni(**_build_omni_kwargs(args, model_name)) t_start = time.perf_counter() for stage_outputs in omni.generate(inputs): @@ -165,7 +196,6 @@ async def main_streaming(args): output_dir = args.output_dir os.makedirs(output_dir, exist_ok=True) - stage_configs_path = args.stage_configs_path or DEFAULT_STAGE_CONFIG model_name = args.model or DEFAULT_MODEL prompt = build_prompt( @@ -175,24 +205,20 @@ async def main_streaming(args): model_name=model_name, ) - omni = AsyncOmni( - model=model_name, - stage_configs_path=stage_configs_path, - log_stats=args.log_stats, - stage_init_timeout=args.stage_init_timeout, - ) + omni = AsyncOmni(**_build_omni_kwargs(args, model_name)) request_id = "0" t_start = time.perf_counter() t_prev = t_start chunk_idx = 0 + all_audio_chunks: list[torch.Tensor] = [] + sample_rate = None async for stage_output in omni.generate(prompt, request_id=request_id): mm = stage_output.request_output.outputs[0].multimodal_output if not stage_output.finished: t_now = time.perf_counter() - audio = mm.get("audio") - n = len(audio) if isinstance(audio, list) else (0 if audio is None else 1) + sample_rate, n = _collect_audio_chunk(mm, all_audio_chunks, sample_rate) dt_ms = (t_now - t_prev) * 1000 ttfa_ms = (t_now - t_start) * 1000 if chunk_idx == 0: @@ -202,10 +228,20 @@ async def main_streaming(args): t_prev = t_now chunk_idx += 1 else: + # Collect any remaining audio in the final message + sample_rate, _ = _collect_audio_chunk(mm, all_audio_chunks, sample_rate) + t_end = time.perf_counter() total_ms = (t_end - t_start) * 1000 logger.info("Request %s: done total=%.1fms chunks=%d", request_id, total_ms, chunk_idx) - _save_wav(output_dir, request_id, mm) + + if all_audio_chunks and sample_rate: + audio_tensor = torch.cat(all_audio_chunks, dim=-1) + out_wav = os.path.join(output_dir, f"output_{request_id}.wav") + sf.write(out_wav, audio_tensor.float().cpu().numpy().flatten(), samplerate=sample_rate, format="WAV") + logger.info("Request %s: saved audio to %s (sr=%d)", request_id, out_wav, sample_rate) + else: + logger.warning("Request %s: no audio chunks received", request_id) def parse_args(): @@ -236,11 +272,17 @@ def parse_args(): default=None, help=f"HuggingFace model path (default: {DEFAULT_MODEL}).", ) + parser.add_argument( + "--deploy-config", + type=str, + default=None, + help="Deploy config YAML path (under vllm_omni/deploy/).", + ) parser.add_argument( "--stage-configs-path", type=str, default=None, - help="Path to stage configs YAML.", + help="Path to stage configs YAML (legacy, prefer --deploy-config).", ) parser.add_argument( "--output-dir", diff --git a/vllm_omni/attention/fish_kvcache_attn.py b/vllm_omni/attention/fish_kvcache_attn.py index 9e552e762..e7235168c 100644 --- a/vllm_omni/attention/fish_kvcache_attn.py +++ b/vllm_omni/attention/fish_kvcache_attn.py @@ -111,6 +111,8 @@ def can_use_fish_kvcache_attn( def _is_cuda_graph_capturing() -> bool: + if not torch.cuda.is_available(): + return False return bool(torch.cuda.is_current_stream_capturing()) diff --git a/vllm_omni/deploy/fish_qwen3_omni.yaml b/vllm_omni/deploy/fish_qwen3_omni.yaml index 5b0c44988..539b37775 100644 --- a/vllm_omni/deploy/fish_qwen3_omni.yaml +++ b/vllm_omni/deploy/fish_qwen3_omni.yaml @@ -63,3 +63,15 @@ stages: max_tokens: 65536 seed: 42 repetition_penalty: 1.0 + +platforms: + xpu: + stages: + # Verified on Intel Arc B60. + - stage_id: 0 + gpu_memory_utilization: 0.8 + max_model_len: 2048 + max_num_batched_tokens: 2048 + - stage_id: 1 + max_model_len: 2048 + max_num_batched_tokens: 2048 From ffd4e1f97e10c63245ec12c6659816aa7ff53061 Mon Sep 17 00:00:00 2001 From: Wang fuyin Date: Sun, 19 Jul 2026 13:38:35 +0800 Subject: [PATCH 018/240] [Bugfix] Fix stale pipeline resolver test call (#5193) Signed-off-by: Acerak01-fy --- tests/test_config_factory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_config_factory.py b/tests/test_config_factory.py index 0c8dec69c..b8777dcce 100644 --- a/tests/test_config_factory.py +++ b/tests/test_config_factory.py @@ -1232,7 +1232,7 @@ stages: assert deploy.stages[1].devices == "1" def test_step_audio2_dispatches_sync_and_async_chunk_processors(self): - pipeline = StageConfigFactory.resolve_pipeline_config("step_audio_2") + pipeline = resolve_pipeline_config("step_audio_2") assert isinstance(pipeline, PipelineConfig) sync_stages = merge_pipeline_deploy(pipeline, DeployConfig(async_chunk=False)) From c23dbfe111b22e768e1482db0fb53cb68f63718a Mon Sep 17 00:00:00 2001 From: Alex Brooks Date: Sun, 19 Jul 2026 00:03:07 -0600 Subject: [PATCH 019/240] [CI/Build] Add Precommit Hooks for Catching Missing PytestMarks (#4953) Signed-off-by: Alex Brooks --- .github/workflows/pre-commit.yml | 6 ++ .pre-commit-config.yaml | 7 ++ tools/pre_commit/check_test_marks.py | 111 +++++++++++++++++++++++++++ 3 files changed, 124 insertions(+) create mode 100644 tools/pre_commit/check_test_marks.py diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index 964fb5abc..2bd0cc133 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -24,3 +24,9 @@ jobs: python-version: "3.12" cache: 'pip' - uses: pre-commit/action@v3.0.1 + # Don't run the pytest mark check against everything in the CI + # for now since a lot of files are still missing them. This check + # is used to help contributors remember to add the marks when they + # commit. + env: + SKIP: check-test-ci-coverage diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6db093e0e..be649785d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -54,6 +54,13 @@ repos: verbose: true stages: [commit-msg] + - id: check-test-ci-coverage + name: Ensure test files have pytestmarks + entry: python tools/pre_commit/check_test_marks.py + language: python + types: [python] + files: ^tests/ + # Keep `suggestion` last - id: suggestion name: Suggestion diff --git a/tools/pre_commit/check_test_marks.py b/tools/pre_commit/check_test_marks.py new file mode 100644 index 000000000..9a727ab72 --- /dev/null +++ b/tools/pre_commit/check_test_marks.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Run a pre-commit hook that fails if test files are modified or added +that (probably) never run in the CI. For now, this means that every tests file +needs to have a CI level marker (e.g., core_model, advanced_model, full_model, +etc) and hardware mark / helper so that we ensure mutated tests will actually +be selected as long as there are pytest commands pointing at the right paths. +""" + +import os +import re +import sys + +# CI level markers +LEVEL_MARKERS = ("core_model", "advanced_model", "full_model", "slow") + +# Hardware markers. These are platforms and resources that CI filters on. +# NOTE: If new hardware marks are added to pyproject.toml etc, we need +# to add them here as well. +HARDWARE_MARKERS = ( + "cpu", + "gpu", + "cuda", + "rocm", + "xpu", + "npu", + "musa", + "H100", + "L4", + "MI325", + "B60", + "S5000", + "A2", + "A3", +) + +# Helpers from tests/helpers/mark.py that auto-apply hardware marks. +HARDWARE_HELPERS = ("hardware_test", "hardware_marks") + +# Match mark.X since we could also do `from pytest import mark`. +# \b prevents matching prefixes (e.g., mark.slow vs mark.slow_test). +LEVEL_RE = re.compile(r"mark\.(?:" + "|".join(LEVEL_MARKERS) + r")\b") +HARDWARE_RE = re.compile(r"mark\.(?:" + "|".join(HARDWARE_MARKERS) + r")\b") +HELPER_RE = re.compile(r"(?:" + "|".join(HARDWARE_HELPERS) + r")\s*\(") + +MISSING_LEVEL_MARKER = "Level" +MISSING_HARDWARE_MARKER = "Hardware" + +# Check if a file is located under tests/ and matches test_.py +# or _test.py, since pytest technically collects on both. +# Note that we use the former everywhere in this repo by convention. +TEST_FILE_RE = re.compile(r"^tests/(?:.*/)?(?:test_[^/]*\.py$|[^/]*_test\.py$)") + + +def is_test_file(path: str) -> bool: + """Determine whether or not a path is pointing at a test file or not.""" + return bool(TEST_FILE_RE.match(path)) + + +def read_test_file(path: str) -> str | None: + """Read a test file's contents, or return None if it doesn't exist.""" + if not os.path.isfile(path): + return None + with open(path, encoding="utf-8") as f: + return f.read() + + +def has_level_marker(contents: str) -> bool: + """Check if file contents contain at least one CI level marker.""" + return bool(LEVEL_RE.search(contents)) + + +def has_hardware_marker(contents: str) -> bool: + """Check if file contents contain any hardware marker or helpers.""" + return bool(HARDWARE_RE.search(contents) or HELPER_RE.search(contents)) + + +def get_files_missing_markers( + staged_files: list[str], +) -> dict[str, list[str]]: + """Return a dict mapping file path to list of missing marker types.""" + results: dict[str, list[str]] = {} + for path in staged_files: + if is_test_file(path) and (contents := read_test_file(path)) is not None: + missing = [] + if not has_level_marker(contents): + missing.append(MISSING_LEVEL_MARKER) + if not has_hardware_marker(contents): + missing.append(MISSING_HARDWARE_MARKER) + if missing: + results[path] = missing + return results + + +if __name__ == "__main__": + missing = get_files_missing_markers(sys.argv[1:]) + + if missing: + file_lines = "\n".join(f" - {path} [{' and '.join(problems)}]" for path, problems in missing.items()) + print( + "\033[91merror:\033[0m test files are missing pytest marks " + "required for Buildkite CI collection.\n\n" + f"Level marks, e.g.: {', '.join(LEVEL_MARKERS[:4])}\n" + f"Hardware marks, e.g.: {', '.join(HARDWARE_MARKERS[:4])}, ...\n" + f" or helpers: {', '.join(HARDWARE_HELPERS)}\n\n" + "The following files are missing marks:\n" + f"{file_lines}\n\n" + "To skip: SKIP=check-test-ci-coverage git commit ..." + ) + sys.exit(1) From 0d8c63268896bebeb473e396ebbc882f0994a041 Mon Sep 17 00:00:00 2001 From: wangyu <53896905+yenuo26@users.noreply.github.com> Date: Sun, 19 Jul 2026 15:06:54 +0800 Subject: [PATCH 020/240] [Refactor][CI] Reorganize .buildkite directory by platform (#5119) --- .../{ => amd/scripts}/bootstrap-amd-omni.sh | 14 +- .../scripts}/run-amd-test.sh | 2 +- .buildkite/{ => amd}/test-amd-merge.yml | 0 .../test-amd-ready.yml} | 0 .../{ => amd}/test-template-amd-omni.j2 | 4 +- .buildkite/{ => cuda}/pipeline.yml | 10 +- .../rebase-pipeline.yml} | 6 +- .../{ => cuda}/scripts/upload_pipeline.py | 16 +- .buildkite/{ => cuda}/test-merge.yml | 0 .buildkite/{ => cuda}/test-nightly.yml | 0 .buildkite/{ => cuda}/test-ready.yml | 0 .buildkite/{ => cuda}/test-weekly.yml | 0 .../pipeline-intel.yml} | 4 +- .../scripts}/bootstrap-intel-omni.sh | 2 +- .../scripts}/run-xpu-test.sh | 0 .../pipeline-npu-a3.yml} | 4 +- .../pipeline-npu.yml} | 4 +- .../test-npu-nightly.yml} | 0 .../test-npu-ready.yml} | 0 .../release-pipeline.yml} | 8 +- .../generate-and-upload-nightly-index.sh | 6 +- .../scripts/generate-nightly-index.py | 0 .../scripts/publish-release-images.sh | 0 .../scripts/publish-release-wheels-pypi.sh | 0 .../scripts/upload-nightly-wheels.sh | 0 .buildkite/scripts/docker_login_ecr_public.sh | 73 --------- .../scripts/hardware_ci/run_npu_test.sh | 145 ------------------ .buildkite/test-amd.yaml | 118 -------------- .claude/skills/vllm-omni-test/SKILL.md | 28 ++-- .../vllm-omni-test/references/test-routing.md | 14 +- docs/contributing/README.md | 2 +- docs/contributing/ci/CI_5levels.md | 20 +-- docs/contributing/ci/test_guide.md | 10 +- .../fault_injection_reliability_matrix.md | 2 +- tools/nightly/run_nightly_jobs.sh | 24 +-- tools/run_jobs_common.sh | 12 +- tools/run_merge_jobs.sh | 12 +- tools/run_ready_jobs.sh | 12 +- 38 files changed, 114 insertions(+), 438 deletions(-) rename .buildkite/{ => amd/scripts}/bootstrap-amd-omni.sh (94%) rename .buildkite/{scripts/hardware_ci => amd/scripts}/run-amd-test.sh (98%) rename .buildkite/{ => amd}/test-amd-merge.yml (100%) rename .buildkite/{test-amd-ready.yaml => amd/test-amd-ready.yml} (100%) rename .buildkite/{ => amd}/test-template-amd-omni.j2 (96%) rename .buildkite/{ => cuda}/pipeline.yml (87%) rename .buildkite/{rebase-pipeline.yaml => cuda/rebase-pipeline.yml} (76%) rename .buildkite/{ => cuda}/scripts/upload_pipeline.py (97%) rename .buildkite/{ => cuda}/test-merge.yml (100%) rename .buildkite/{ => cuda}/test-nightly.yml (100%) rename .buildkite/{ => cuda}/test-ready.yml (100%) rename .buildkite/{ => cuda}/test-weekly.yml (100%) rename .buildkite/{pipeline-intel.yaml => intel/pipeline-intel.yml} (79%) rename .buildkite/{ => intel/scripts}/bootstrap-intel-omni.sh (94%) rename .buildkite/{scripts/hardware_ci => intel/scripts}/run-xpu-test.sh (100%) rename .buildkite/{pipeline-npu-a3.yaml => npu/pipeline-npu-a3.yml} (97%) rename .buildkite/{pipeline-npu.yaml => npu/pipeline-npu.yml} (98%) rename .buildkite/{test-npu-nightly.yaml => npu/test-npu-nightly.yml} (100%) rename .buildkite/{test-npu-ready.yaml => npu/test-npu-ready.yml} (100%) rename .buildkite/{release-pipeline.yaml => release/release-pipeline.yml} (92%) rename .buildkite/{ => release}/scripts/generate-and-upload-nightly-index.sh (92%) rename .buildkite/{ => release}/scripts/generate-nightly-index.py (100%) rename .buildkite/{ => release}/scripts/publish-release-images.sh (100%) rename .buildkite/{ => release}/scripts/publish-release-wheels-pypi.sh (100%) rename .buildkite/{ => release}/scripts/upload-nightly-wheels.sh (100%) delete mode 100755 .buildkite/scripts/docker_login_ecr_public.sh delete mode 100644 .buildkite/scripts/hardware_ci/run_npu_test.sh delete mode 100644 .buildkite/test-amd.yaml diff --git a/.buildkite/bootstrap-amd-omni.sh b/.buildkite/amd/scripts/bootstrap-amd-omni.sh similarity index 94% rename from .buildkite/bootstrap-amd-omni.sh rename to .buildkite/amd/scripts/bootstrap-amd-omni.sh index 16ed02b26..ee858068a 100755 --- a/.buildkite/bootstrap-amd-omni.sh +++ b/.buildkite/amd/scripts/bootstrap-amd-omni.sh @@ -154,14 +154,14 @@ upload_pipeline() { fi # Use local template file for vllm-omni - cp .buildkite/test-template-amd-omni.j2 .buildkite/test-template.j2 + cp .buildkite/amd/test-template-amd-omni.j2 .buildkite/amd/test-template.j2 # (WIP) Use pipeline generator instead of jinja template - if [ -e ".buildkite/pipeline_generator/pipeline_generator.py" ]; then + if [ -e ".buildkite/amd/pipeline_generator/pipeline_generator.py" ]; then python -m pip install click pydantic - python .buildkite/pipeline_generator/pipeline_generator.py --run_all=$RUN_ALL --list_file_diff="$LIST_FILE_DIFF" --nightly="$NIGHTLY" --mirror_hw="$AMD_MIRROR_HW" - buildkite-agent pipeline upload .buildkite/pipeline.yaml + python .buildkite/amd/pipeline_generator/pipeline_generator.py --run_all=$RUN_ALL --list_file_diff="$LIST_FILE_DIFF" --nightly="$NIGHTLY" --mirror_hw="$AMD_MIRROR_HW" + buildkite-agent pipeline upload .buildkite/amd/pipeline.yaml exit 0 fi echo "List file diff: $LIST_FILE_DIFF" @@ -171,7 +171,7 @@ upload_pipeline() { FAIL_FAST=$(fail_fast) - cd .buildkite + cd .buildkite/amd # Select test definition file: merge suite for main, ready suite for PRs. # For debugging, DEBUG_TEST_YAML can override the selection — accepts @@ -182,7 +182,7 @@ upload_pipeline() { TEST_YAML="test-amd-merge.yml" ;; ready) - TEST_YAML="test-amd-ready.yaml" + TEST_YAML="test-amd-ready.yml" ;; *) echo "ERROR: DEBUG_TEST_YAML must be 'merge' or 'ready', got '$DEBUG_TEST_YAML'" >&2 @@ -193,7 +193,7 @@ upload_pipeline() { elif [[ $BUILDKITE_BRANCH == "main" ]]; then TEST_YAML="test-amd-merge.yml" else - TEST_YAML="test-amd-ready.yaml" + TEST_YAML="test-amd-ready.yml" fi ( diff --git a/.buildkite/scripts/hardware_ci/run-amd-test.sh b/.buildkite/amd/scripts/run-amd-test.sh similarity index 98% rename from .buildkite/scripts/hardware_ci/run-amd-test.sh rename to .buildkite/amd/scripts/run-amd-test.sh index f880455d4..c49614e70 100755 --- a/.buildkite/scripts/hardware_ci/run-amd-test.sh +++ b/.buildkite/amd/scripts/run-amd-test.sh @@ -56,7 +56,7 @@ container_name="rocm_${BUILDKITE_COMMIT}_$(tr -dc A-Za-z0-9 < /dev/urandom | hea # # Install AWS CLI to authenticate to ECR Public Gallery to get higher rate limit for pulling images # sudo apt-get update && sudo apt-get install -y awscli ## Use safe docker login helper to prevent race conditions -# source "$(dirname "${BASH_SOURCE[0]}")/../docker_login_ecr_public.sh" +# source "$(dirname "${BASH_SOURCE[0]}")/../../common/scripts/docker_login_ecr_public.sh" # safe_docker_login_ecr_public ## Pull the container from AMD Docker Hub diff --git a/.buildkite/test-amd-merge.yml b/.buildkite/amd/test-amd-merge.yml similarity index 100% rename from .buildkite/test-amd-merge.yml rename to .buildkite/amd/test-amd-merge.yml diff --git a/.buildkite/test-amd-ready.yaml b/.buildkite/amd/test-amd-ready.yml similarity index 100% rename from .buildkite/test-amd-ready.yaml rename to .buildkite/amd/test-amd-ready.yml diff --git a/.buildkite/test-template-amd-omni.j2 b/.buildkite/amd/test-template-amd-omni.j2 similarity index 96% rename from .buildkite/test-template-amd-omni.j2 rename to .buildkite/amd/test-template-amd-omni.j2 index aa5ca8bcd..2f1a9aed3 100644 --- a/.buildkite/test-template-amd-omni.j2 +++ b/.buildkite/amd/test-template-amd-omni.j2 @@ -57,7 +57,7 @@ steps: {% endif %} {% set cmd_body = (step.command or (step.commands | join("\n"))) | trim %} {% set indented_cmd = cmd_body | replace("\n", "\n ") %} - command: bash .buildkite/scripts/hardware_ci/run-amd-test.sh + command: bash .buildkite/amd/scripts/run-amd-test.sh env: DOCKER_BUILDKIT: "1" {% if step.env %} @@ -104,7 +104,7 @@ steps: {% endif %} {% set cmd_body = (step.command or (step.commands | join("\n"))) | trim %} {% set indented_cmd = cmd_body | replace("\n", "\n ") %} - command: bash .buildkite/scripts/hardware_ci/run-amd-test.sh + command: bash .buildkite/amd/scripts/run-amd-test.sh env: DOCKER_BUILDKIT: "1" {% if step.env %} diff --git a/.buildkite/pipeline.yml b/.buildkite/cuda/pipeline.yml similarity index 87% rename from .buildkite/pipeline.yml rename to .buildkite/cuda/pipeline.yml index 040393f6c..3b460c1c6 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/cuda/pipeline.yml @@ -9,7 +9,7 @@ steps: - label: ":github: Resolve skip-ci (docs / skip marks) & upload pipeline" key: upload-ci-pipeline commands: - - "python3 .buildkite/scripts/upload_pipeline.py --upload .buildkite/pipeline.yml" + - "python3 .buildkite/cuda/scripts/upload_pipeline.py --upload .buildkite/cuda/pipeline.yml" agents: queue: "cpu_queue_premerge" @@ -37,7 +37,7 @@ steps: if [ "${NIGHTLY:-}" = "1" ] && [ "${BUILDKITE_BRANCH}" = "main" ]; then EXTRA="--e2e" fi - python3 .buildkite/scripts/upload_pipeline.py --upload $$EXTRA .buildkite/test-ready.yml + python3 .buildkite/cuda/scripts/upload_pipeline.py --upload $$EXTRA .buildkite/cuda/test-ready.yml agents: queue: "cpu_queue_premerge" @@ -52,7 +52,7 @@ steps: if [ "${NIGHTLY:-}" = "1" ] && [ "${BUILDKITE_BRANCH}" = "main" ]; then EXTRA="--e2e" fi - python3 .buildkite/scripts/upload_pipeline.py --upload $$EXTRA .buildkite/test-merge.yml + python3 .buildkite/cuda/scripts/upload_pipeline.py --upload $$EXTRA .buildkite/cuda/test-merge.yml agents: queue: "cpu_queue_premerge" @@ -70,7 +70,7 @@ steps: build.pull_request.labels includes "diffusion-x2v-test" )) commands: - - buildkite-agent pipeline upload .buildkite/test-nightly.yml + - buildkite-agent pipeline upload .buildkite/cuda/test-nightly.yml agents: queue: "cpu_queue_premerge" @@ -82,6 +82,6 @@ steps: (build.branch == "main" && build.env("WEEKLY") == "1") || (build.branch != "main" && build.pull_request.labels includes "weekly-test") commands: - - buildkite-agent pipeline upload .buildkite/test-weekly.yml + - buildkite-agent pipeline upload .buildkite/cuda/test-weekly.yml agents: queue: "cpu_queue_premerge" diff --git a/.buildkite/rebase-pipeline.yaml b/.buildkite/cuda/rebase-pipeline.yml similarity index 76% rename from .buildkite/rebase-pipeline.yaml rename to .buildkite/cuda/rebase-pipeline.yml index 796ad19a7..514c9cc1e 100644 --- a/.buildkite/rebase-pipeline.yaml +++ b/.buildkite/cuda/rebase-pipeline.yml @@ -14,7 +14,7 @@ steps: depends_on: image-build key: upload-ready-pipeline commands: - - python3 .buildkite/scripts/upload_pipeline.py --upload --all .buildkite/test-ready.yml + - python3 .buildkite/cuda/scripts/upload_pipeline.py --upload --all .buildkite/cuda/test-ready.yml agents: queue: "cpu_queue_premerge" @@ -23,7 +23,7 @@ steps: depends_on: image-build key: upload-merge-pipeline commands: - - python3 .buildkite/scripts/upload_pipeline.py --upload --all .buildkite/test-merge.yml + - python3 .buildkite/cuda/scripts/upload_pipeline.py --upload --all .buildkite/cuda/test-merge.yml agents: queue: "cpu_queue_premerge" @@ -33,6 +33,6 @@ steps: key: upload-nightly-pipeline if: build.env("NIGHTLY") == "1" commands: - - python3 .buildkite/scripts/upload_pipeline.py --upload --all .buildkite/test-nightly.yml + - python3 .buildkite/cuda/scripts/upload_pipeline.py --upload --all .buildkite/cuda/test-nightly.yml agents: queue: "cpu_queue_premerge" diff --git a/.buildkite/scripts/upload_pipeline.py b/.buildkite/cuda/scripts/upload_pipeline.py similarity index 97% rename from .buildkite/scripts/upload_pipeline.py rename to .buildkite/cuda/scripts/upload_pipeline.py index d9220ed0f..dc233af7a 100644 --- a/.buildkite/scripts/upload_pipeline.py +++ b/.buildkite/cuda/scripts/upload_pipeline.py @@ -16,16 +16,16 @@ Usage: python3 upload_pipeline.py [--upload] [--all | --e2e] # Bootstrap (replaces upload_pipeline_with_skip_ci.sh): - python3 upload_pipeline.py --upload .buildkite/pipeline.yml + python3 upload_pipeline.py --upload .buildkite/cuda/pipeline.yml # Test pipeline (replaces upload_test_pipeline_with_diff_skip.py): - python3 upload_pipeline.py --upload .buildkite/test-merge.yml + python3 upload_pipeline.py --upload .buildkite/cuda/test-merge.yml # Full suite (rebase pipeline): keep all steps, still strip the uploader-only key: - python3 upload_pipeline.py --upload --all .buildkite/test-merge.yml + python3 upload_pipeline.py --upload --all .buildkite/cuda/test-merge.yml # Nightly L2 E2E only: keep the E2E Test group, ignore source_file_dependencies: - python3 upload_pipeline.py --upload --e2e .buildkite/test-ready.yml + python3 upload_pipeline.py --upload --e2e .buildkite/cuda/test-merge.yml """ from __future__ import annotations @@ -47,7 +47,7 @@ except ImportError: import yaml LOG = "upload_pipeline" -ROOT = Path(__file__).resolve().parent.parent.parent +ROOT = Path(__file__).resolve().parent.parent.parent.parent DOC_SEP = "\n---\n" BOOTSTRAP_MARKER = "__IMAGE_BUILD_IF__" E2E_GROUP_MARKER = "E2E Test" @@ -475,7 +475,7 @@ def render_pipeline( # ``--all`` forces the keep-all-steps path (no diff-aware skipping) while still # stripping ``source_file_dependencies``. Used by the rebase pipeline so main builds - # run the full e2e suite (see .buildkite/rebase-pipeline.yaml). + # run the full e2e suite (see .buildkite/cuda/rebase-pipeline.yml). if force_all or e2e_only: changed_files = None @@ -509,8 +509,8 @@ def main() -> int: parser.add_argument( "pipeline", nargs="?", - default=".buildkite/pipeline.yml", - help="Pipeline YAML path (default: .buildkite/pipeline.yml)", + default=".buildkite/cuda/pipeline.yml", + help="Pipeline YAML path (default: .buildkite/cuda/pipeline.yml)", ) parser.add_argument( "--upload", diff --git a/.buildkite/test-merge.yml b/.buildkite/cuda/test-merge.yml similarity index 100% rename from .buildkite/test-merge.yml rename to .buildkite/cuda/test-merge.yml diff --git a/.buildkite/test-nightly.yml b/.buildkite/cuda/test-nightly.yml similarity index 100% rename from .buildkite/test-nightly.yml rename to .buildkite/cuda/test-nightly.yml diff --git a/.buildkite/test-ready.yml b/.buildkite/cuda/test-ready.yml similarity index 100% rename from .buildkite/test-ready.yml rename to .buildkite/cuda/test-ready.yml diff --git a/.buildkite/test-weekly.yml b/.buildkite/cuda/test-weekly.yml similarity index 100% rename from .buildkite/test-weekly.yml rename to .buildkite/cuda/test-weekly.yml diff --git a/.buildkite/pipeline-intel.yaml b/.buildkite/intel/pipeline-intel.yml similarity index 79% rename from .buildkite/pipeline-intel.yaml rename to .buildkite/intel/pipeline-intel.yml index d3a2655bf..e68f59e53 100644 --- a/.buildkite/pipeline-intel.yaml +++ b/.buildkite/intel/pipeline-intel.yml @@ -1,9 +1,9 @@ -# .buildkite/pipeline-intel.yaml +# .buildkite/intel/pipeline-intel.yml steps: - group: "INTEL Tests" steps: - label: "Intel: XPU Test" - command: "bash .buildkite/scripts/hardware_ci/run-xpu-test.sh" + command: "bash .buildkite/intel/scripts/run-xpu-test.sh" agents: queue: "intel-gpu-omni" env: diff --git a/.buildkite/bootstrap-intel-omni.sh b/.buildkite/intel/scripts/bootstrap-intel-omni.sh similarity index 94% rename from .buildkite/bootstrap-intel-omni.sh rename to .buildkite/intel/scripts/bootstrap-intel-omni.sh index b99568141..e51252bd9 100755 --- a/.buildkite/bootstrap-intel-omni.sh +++ b/.buildkite/intel/scripts/bootstrap-intel-omni.sh @@ -11,7 +11,7 @@ fi upload_pipeline() { echo "--- 🛠 Preparing Intel pipeline" - buildkite-agent pipeline upload .buildkite/pipeline-intel.yaml + buildkite-agent pipeline upload .buildkite/intel/pipeline-intel.yml } get_diff() { diff --git a/.buildkite/scripts/hardware_ci/run-xpu-test.sh b/.buildkite/intel/scripts/run-xpu-test.sh similarity index 100% rename from .buildkite/scripts/hardware_ci/run-xpu-test.sh rename to .buildkite/intel/scripts/run-xpu-test.sh diff --git a/.buildkite/pipeline-npu-a3.yaml b/.buildkite/npu/pipeline-npu-a3.yml similarity index 97% rename from .buildkite/pipeline-npu-a3.yaml rename to .buildkite/npu/pipeline-npu-a3.yml index 8776219e1..a57f5f8e5 100644 --- a/.buildkite/pipeline-npu-a3.yaml +++ b/.buildkite/npu/pipeline-npu-a3.yml @@ -88,7 +88,7 @@ # # queue: "ascend-a3" # # resource_class: "npu-2" # # command: | -# # buildkite-agent pipeline upload .buildkite/test-npu-ready.yaml +# # buildkite-agent pipeline upload .buildkite/npu/test-npu-ready.yml # # L4 Test — main+NIGHTLY=1 (scheduled), or PR with specific label (e.g. add label then Rebuild) # - label: "Upload Nightly Pipeline" @@ -104,7 +104,7 @@ # build.pull_request.labels includes "diffusion-x2v-test" # )) # commands: - # - buildkite-agent pipeline upload .buildkite/test-npu-nightly.yaml + # - buildkite-agent pipeline upload .buildkite/npu/test-npu-nightly.yml # agents: # queue: "ascend-a3" # resource_class: "npu-2" diff --git a/.buildkite/pipeline-npu.yaml b/.buildkite/npu/pipeline-npu.yml similarity index 98% rename from .buildkite/pipeline-npu.yaml rename to .buildkite/npu/pipeline-npu.yml index d035ffc94..fd67711f0 100644 --- a/.buildkite/pipeline-npu.yaml +++ b/.buildkite/npu/pipeline-npu.yml @@ -171,7 +171,7 @@ steps: queue: "ascend-a2b3" resource_class: "npu-1" command: | - buildkite-agent pipeline upload .buildkite/test-npu-ready.yaml + buildkite-agent pipeline upload .buildkite/npu/test-npu-ready.yml # L4 Test — main+NIGHTLY=1 (scheduled), or PR with specific label (e.g. add label then Rebuild) - label: "Upload Nightly Pipeline" @@ -189,7 +189,7 @@ steps: build.pull_request.labels includes "diffusion-x2v-test" )) commands: - - buildkite-agent pipeline upload .buildkite/test-npu-nightly.yaml + - buildkite-agent pipeline upload .buildkite/npu/test-npu-nightly.yml agents: queue: "ascend-a2b3" resource_class: "npu-1" diff --git a/.buildkite/test-npu-nightly.yaml b/.buildkite/npu/test-npu-nightly.yml similarity index 100% rename from .buildkite/test-npu-nightly.yaml rename to .buildkite/npu/test-npu-nightly.yml diff --git a/.buildkite/test-npu-ready.yaml b/.buildkite/npu/test-npu-ready.yml similarity index 100% rename from .buildkite/test-npu-ready.yaml rename to .buildkite/npu/test-npu-ready.yml diff --git a/.buildkite/release-pipeline.yaml b/.buildkite/release/release-pipeline.yml similarity index 92% rename from .buildkite/release-pipeline.yaml rename to .buildkite/release/release-pipeline.yml index c560fefe1..351e105cc 100644 --- a/.buildkite/release-pipeline.yaml +++ b/.buildkite/release/release-pipeline.yml @@ -16,7 +16,7 @@ steps: - "uv venv --python=3.12 && source .venv/bin/activate" - "uv pip install --upgrade build" - "python3 -m build" - - "bash .buildkite/scripts/upload-nightly-wheels.sh" + - "bash .buildkite/release/scripts/upload-nightly-wheels.sh" - label: "Generate and upload wheel indices" depends_on: "build-wheel" @@ -24,7 +24,7 @@ steps: agents: queue: small_cpu_queue_release commands: - - "bash .buildkite/scripts/generate-and-upload-nightly-index.sh" + - "bash .buildkite/release/scripts/generate-and-upload-nightly-index.sh" - block: "Upload release wheels to PyPI" depends_on: "build-wheel" @@ -38,7 +38,7 @@ steps: agents: queue: small_cpu_queue_release commands: - - "bash .buildkite/scripts/publish-release-wheels-pypi.sh" + - "bash .buildkite/release/scripts/publish-release-wheels-pypi.sh" - block: "Unblock to build release Docker images" depends_on: ~ @@ -100,7 +100,7 @@ steps: agents: queue: small_cpu_queue_release commands: - - "bash .buildkite/scripts/publish-release-images.sh" + - "bash .buildkite/release/scripts/publish-release-images.sh" plugins: - docker-login#v3.0.0: username: vllmbot diff --git a/.buildkite/scripts/generate-and-upload-nightly-index.sh b/.buildkite/release/scripts/generate-and-upload-nightly-index.sh similarity index 92% rename from .buildkite/scripts/generate-and-upload-nightly-index.sh rename to .buildkite/release/scripts/generate-and-upload-nightly-index.sh index b09c13f5c..63f5da596 100755 --- a/.buildkite/scripts/generate-and-upload-nightly-index.sh +++ b/.buildkite/release/scripts/generate-and-upload-nightly-index.sh @@ -36,12 +36,12 @@ mkdir -p "$INDICES_OUTPUT_DIR" # HACK: we do not need regex module here, but it is required by pre-commit hook # To avoid any external dependency, we simply replace it back to the stdlib re module -sed -i.bak 's/import regex as re/import re/g' .buildkite/scripts/generate-nightly-index.py && rm -f .buildkite/scripts/generate-nightly-index.py.bak +sed -i.bak 's/import regex as re/import re/g' .buildkite/release/scripts/generate-nightly-index.py && rm -f .buildkite/release/scripts/generate-nightly-index.py.bak # Generate indices -- the version is just the commit hash (not omni/{commit}) # because relative paths are computed between the index and wheel directories, # both of which live under the omni/ prefix in S3. -$PYTHON .buildkite/scripts/generate-nightly-index.py \ +$PYTHON .buildkite/release/scripts/generate-nightly-index.py \ --version "$BUILDKITE_COMMIT" \ --current-objects "$obj_json" \ --output-dir "$INDICES_OUTPUT_DIR" \ @@ -78,7 +78,7 @@ if [[ "$version" != *"dev"* ]]; then rm -rf "${INDICES_OUTPUT_DIR:?}" mkdir -p "$INDICES_OUTPUT_DIR" # wheel-dir is overridden to be the commit directory, so that the indices point to the correct wheel path - $PYTHON .buildkite/scripts/generate-nightly-index.py \ + $PYTHON .buildkite/release/scripts/generate-nightly-index.py \ --version "$s3_version" \ --wheel-dir "$BUILDKITE_COMMIT" \ --current-objects "$obj_json" \ diff --git a/.buildkite/scripts/generate-nightly-index.py b/.buildkite/release/scripts/generate-nightly-index.py similarity index 100% rename from .buildkite/scripts/generate-nightly-index.py rename to .buildkite/release/scripts/generate-nightly-index.py diff --git a/.buildkite/scripts/publish-release-images.sh b/.buildkite/release/scripts/publish-release-images.sh similarity index 100% rename from .buildkite/scripts/publish-release-images.sh rename to .buildkite/release/scripts/publish-release-images.sh diff --git a/.buildkite/scripts/publish-release-wheels-pypi.sh b/.buildkite/release/scripts/publish-release-wheels-pypi.sh similarity index 100% rename from .buildkite/scripts/publish-release-wheels-pypi.sh rename to .buildkite/release/scripts/publish-release-wheels-pypi.sh diff --git a/.buildkite/scripts/upload-nightly-wheels.sh b/.buildkite/release/scripts/upload-nightly-wheels.sh similarity index 100% rename from .buildkite/scripts/upload-nightly-wheels.sh rename to .buildkite/release/scripts/upload-nightly-wheels.sh diff --git a/.buildkite/scripts/docker_login_ecr_public.sh b/.buildkite/scripts/docker_login_ecr_public.sh deleted file mode 100755 index 51c5e1a5d..000000000 --- a/.buildkite/scripts/docker_login_ecr_public.sh +++ /dev/null @@ -1,73 +0,0 @@ -#!/bin/bash -# Helper function to safely login to ECR Public with per-job config isolation -# Uses DOCKER_CONFIG environment variable to prevent race conditions -# -# This script prevents the "device or resource busy" error by giving each -# Buildkite job its own isolated Docker config directory. -# -# Usage: -# source docker_login_ecr_public.sh && safe_docker_login_ecr_public - -set -euo pipefail - -# Configuration -ECR_REGISTRY="public.ecr.aws" - -setup_isolated_docker_config() { - # Use BUILDKITE_JOB_ID for job-specific isolation - # Fallback to PID if running outside Buildkite - local job_id="${BUILDKITE_JOB_ID:-$$}" - - # Set Docker config to job-specific directory - export DOCKER_CONFIG="/tmp/docker-config-${job_id}" - - # Create directory if it doesn't exist - mkdir -p "$DOCKER_CONFIG" - - echo "[docker-config] Using isolated Docker config: $DOCKER_CONFIG" -} - -check_docker_auth() { - # Check if already authenticated to the given registry - # Returns 0 if authenticated, 1 if not - local registry="$1" - - # Check if credentials exist in the isolated config - if [[ -f "$DOCKER_CONFIG/config.json" ]]; then - # Check if registry is present in config - if grep -q "$registry" "$DOCKER_CONFIG/config.json" 2>/dev/null; then - return 0 - fi - fi - - return 1 -} - -safe_docker_login_ecr_public() { - # Setup isolated config first - setup_isolated_docker_config - - local registry="$ECR_REGISTRY" - - # Check if already authenticated (within this job) - if check_docker_auth "$registry"; then - echo "[docker-login] Already authenticated to $registry in this job" - return 0 - fi - - # Perform login to isolated config directory - echo "[docker-login] Logging in to $ECR_REGISTRY (isolated config)..." - if aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin "$ECR_REGISTRY"; then - echo "[docker-login] Login successful (config: $DOCKER_CONFIG)" - return 0 - else - local exit_code=$? - echo "[docker-login] ERROR: Login failed with exit code $exit_code" >&2 - return $exit_code - fi -} - -# Execute if run as script (not sourced) -if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then - safe_docker_login_ecr_public -fi diff --git a/.buildkite/scripts/hardware_ci/run_npu_test.sh b/.buildkite/scripts/hardware_ci/run_npu_test.sh deleted file mode 100644 index 492699c5f..000000000 --- a/.buildkite/scripts/hardware_ci/run_npu_test.sh +++ /dev/null @@ -1,145 +0,0 @@ -#!/bin/bash - -# This script build the Ascend NPU docker image and run the offline inference inside the container. -# It serves a sanity check for compilation and basic model usage. -set -ex - -# Base ubuntu image with basic ascend development libraries and python installed -VLLM_OMNI_REPO="https://github.com/vllm-project/vllm-omni.git" -BASE_IMAGE_NAME="quay.nju.edu.cn/ascend/vllm-ascend:v0.11.0rc2" -image_name="npu/vllm-omni-ci:${BUILDKITE_COMMIT}_${EPOCHSECONDS}" -# image_name="npu/vllm-ci:${BUILDKITE_COMMIT}_${EPOCHSECONDS}" -container_name="npu_${BUILDKITE_COMMIT}_$(tr -dc A-Za-z0-9 < /dev/urandom | head -c 10; echo)" - -# BUILDKITE_AGENT_NAME format is {hostname}-{agent_idx}-{npu_card_num}cards -agent_idx=$(echo "${BUILDKITE_AGENT_NAME}" | awk -F'-' '{print $(NF-1)}') -echo "agent_idx: ${agent_idx}" -builder_name="cachebuilder${agent_idx}" -builder_cache_dir="/mnt/docker-cache${agent_idx}" -mkdir -p ${builder_cache_dir} - -# Try building the docker image -cat <=6.0 pytest-cov modelscope - -COPY . . - -# Install vllm-omni -WORKDIR /workspace -ARG VLLM_OMNI_REPO=https://github.com/vllm-project/vllm-omni.git -ARG VLLM_OMNI_TAG=main -ARG BUILDKITE_PULL_REQUEST -ARG BUILDKITE_PULL_REQUEST_REPO -RUN git config --global url."https://gh-proxy.test.osinfra.cn/https://github.com/".insteadOf "https://github.com/" && \ - if [ "\$BUILDKITE_PULL_REQUEST" != "false" ] && [ -n "\$BUILDKITE_PULL_REQUEST" ]; then \ - echo "Cloning and checking out PR #\$BUILDKITE_PULL_REQUEST..." && \ - git clone \$VLLM_OMNI_REPO /workspace/vllm-omni && \ - cd /workspace/vllm-omni && \ - git fetch origin pull/\$BUILDKITE_PULL_REQUEST/head:pr-\$BUILDKITE_PULL_REQUEST && \ - git checkout pr-\$BUILDKITE_PULL_REQUEST; \ - else \ - echo "Not a PR build, using main branch" && \ - git clone --depth 1 \$VLLM_OMNI_REPO /workspace/vllm-omni; \ - fi - -RUN --mount=type=cache,target=/root/.cache/pip \ - export PIP_EXTRA_INDEX_URL=https://mirrors.huaweicloud.com/ascend/repos/pypi && \ - source /usr/local/Ascend/ascend-toolkit/set_env.sh && \ - source /usr/local/Ascend/nnal/atb/set_env.sh && \ - export LD_LIBRARY_PATH=\$LD_LIBRARY_PATH:/usr/local/Ascend/ascend-toolkit/latest/`uname -i`-linux/devlib && \ - python3 -m pip install -v -e /workspace/vllm-omni/ - -ENV VLLM_WORKER_MULTIPROC_METHOD=spawn - -WORKDIR /workspace/vllm-omni -CMD ["/bin/bash"] - -EOF - -# Setup cleanup -remove_docker_container() { - docker rm -f "${container_name}" || true; - docker image rm -f "${image_name}" || true; - docker system prune -f || true; -} -trap remove_docker_container EXIT - -# Generate corresponding --device args based on BUILDKITE_AGENT_NAME -# Ascend NPU BUILDKITE_AGENT_NAME format is {hostname}-{agent_idx}-{npu_card_num}cards, and agent_idx starts from 1. -# e.g. atlas-a2-001-1-2cards means this is the 1-th agent on atlas-a2-001 host, and it has 2 NPU cards. -# returns --device /dev/davinci0 --device /dev/davinci1 -parse_and_gen_devices() { - local input="$1" - local index cards_num - if [[ "$input" =~ ([0-9]+)-([0-9]+)cards$ ]]; then - index="${BASH_REMATCH[1]}" - cards_num="${BASH_REMATCH[2]}" - else - echo "parse error" >&2 - return 1 - fi - - local devices="" - local i=0 - while (( i < cards_num )); do - local dev_idx=$(((index - 1)*cards_num + i )) - devices="$devices --device /dev/davinci${dev_idx}" - ((i++)) - done - - # trim leading space - devices="${devices#"${devices%%[![:space:]]*}"}" - # Output devices: assigned to the caller variable - printf '%s' "$devices" -} - -devices=$(parse_and_gen_devices "${BUILDKITE_AGENT_NAME}") || exit 1 - -# Run the image and execute the Out-Of-Tree (OOT) platform interface test case on Ascend NPU hardware. -# This test checks whether the OOT platform interface is functioning properly in conjunction with -# the hardware plugin vllm-ascend. -hf_model_cache_dir=/mnt/hf_cache${agent_idx} -ms_model_cache_dir=/mnt/modelscope${agent_idx} -mkdir -p ${hf_model_cache_dir} -mkdir -p ${ms_model_cache_dir} -docker run \ - --init \ - ${devices} \ - --device /dev/davinci_manager \ - --device /dev/devmm_svm \ - --device /dev/hisi_hdc \ - -v /usr/local/dcmi:/usr/local/dcmi \ - -v /usr/local/bin/npu-smi:/usr/local/bin/npu-smi \ - -v /usr/local/Ascend/driver/lib64/:/usr/local/Ascend/driver/lib64/ \ - -v /usr/local/Ascend/driver/version.info:/usr/local/Ascend/driver/version.info \ - -v /etc/ascend_install.info:/etc/ascend_install.info \ - -v ${hf_model_cache_dir}:/root/.cache/huggingface \ - -v ${ms_model_cache_dir}:/root/.cache/modelscope \ - --network host \ - --entrypoint="" \ - --name "${container_name}" \ - "${image_name}" \ - bash -c ' - set -e - VLLM_USE_MODELSCOPE=True pytest -s -v tests/e2e/offline_inference/test_qwen2_5_omni_expansion.py -' diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml deleted file mode 100644 index 520c1bbae..000000000 --- a/.buildkite/test-amd.yaml +++ /dev/null @@ -1,118 +0,0 @@ -steps: - -- label: "Diffusion Model Test" - timeout_in_minutes: 30 - agent_pool: mi325_2 - depends_on: amd-build - mirror_hardwares: [amdproduction] - grade: Blocking - commands: - - export GPU_ARCHS=gfx942 - - pytest -s -v tests/e2e/offline_inference/test_z_image.py - -- label: "Diffusion Images API LoRA E2E" - timeout_in_minutes: 30 - agent_pool: mi325_1 - depends_on: amd-build - mirror_hardwares: [amdproduction] - grade: Blocking - commands: - - export GPU_ARCHS=gfx942 - - export VLLM_LOGGING_LEVEL=DEBUG - - export VLLM_WORKER_MULTIPROC_METHOD=spawn - - pytest -s -v tests/diffusion/lora/test_images_generations_lora.py - -- label: "Diffusion Model CPU offloading Test" - timeout_in_minutes: 20 - agent_pool: mi325_1 - depends_on: amd-build - mirror_hardwares: [amdproduction] - grade: Blocking - commands: - - export GPU_ARCHS=gfx942 - - export VLLM_LOGGING_LEVEL=DEBUG - - export VLLM_WORKER_MULTIPROC_METHOD=spawn - - pytest -s -v tests/diffusion/offloader/test_diffusion_cpu_offload.py - - pytest -s -v tests/diffusion/offloader/test_diffusion_layerwise_offload.py - -- label: "Diffusion Cache Backend Test" - timeout_in_minutes: 15 - agent_pool: mi325_1 - depends_on: amd-build - mirror_hardwares: [amdproduction] - grade: Blocking - commands: - - export GPU_ARCHS=gfx942 - - export VLLM_LOGGING_LEVEL=DEBUG - - export VLLM_WORKER_MULTIPROC_METHOD=spawn - - pytest -s -v -m 'core_model and cache and diffusion and not distributed_rocm and MI325' - -- label: "Diffusion Sequence Parallelism Test" - timeout_in_minutes: 20 - agent_pool: mi325_2 - depends_on: amd-build - mirror_hardwares: [amdproduction] - grade: Blocking - commands: - - export GPU_ARCHS=gfx942 - - export VLLM_LOGGING_LEVEL=DEBUG - - export VLLM_WORKER_MULTIPROC_METHOD=spawn - - pytest -s -v tests/diffusion/distributed/test_sequence_parallel.py -m core_model - -- label: "Diffusion Tensor Parallelism Test" - timeout_in_minutes: 20 - agent_pool: mi325_2 - depends_on: amd-build - commands: - - export GPU_ARCHS=gfx942 - - export VLLM_LOGGING_LEVEL=DEBUG - - export VLLM_WORKER_MULTIPROC_METHOD=spawn - - pytest -s -v tests/diffusion/distributed/test_zimage_parallelism.py - -- label: "Diffusion GPU Worker Test" - timeout_in_minutes: 20 - agent_pool: mi325_2 - depends_on: amd-build - mirror_hardwares: [amdproduction] - grade: Blocking - commands: - - pytest -s -v tests/diffusion/test_diffusion_worker.py - -- label: "Omni Model Test Qwen2-5-Omni" - timeout_in_minutes: 15 - agent_pool: mi325_2 - depends_on: amd-build - mirror_hardwares: [amdproduction] - grade: Blocking - commands: - - export GPU_ARCHS=gfx942 - - export VLLM_LOGGING_LEVEL=DEBUG - - export VLLM_WORKER_MULTIPROC_METHOD=spawn - - pytest -s -v tests/e2e/offline_inference/test_qwen2_5_omni_expansion.py - - pytest -s -v tests/engine/test_async_omni_engine_abort.py - -- label: "Omni Model Test Qwen3-Omni" - timeout_in_minutes: 15 - agent_pool: mi325_2 - depends_on: amd-build - mirror_hardwares: [amdproduction] - grade: Blocking - commands: - - export VLLM_LOGGING_LEVEL=DEBUG - - export VLLM_WORKER_MULTIPROC_METHOD=spawn - - export VLLM_ROCM_USE_AITER=0 - - pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py - - pytest -s -v tests/e2e/online_serving/test_qwen3_omni.py - - -- label: "Omni Sleep Mode Test" - timeout_in_minutes: 40 - agent_pool: mi325_2 - depends_on: amd-build - mirror_hardwares: [amdproduction] - grade: Blocking - commands: - - export GPU_ARCHS=gfx942 - - export VLLM_LOGGING_LEVEL=DEBUG - - export VLLM_WORKER_MULTIPROC_METHOD=spawn - - pytest -s -v tests/entrypoints/test_omni_sleep_mode.py -m "advanced_model and omni and MI325" --run-level "advanced_model" diff --git a/.claude/skills/vllm-omni-test/SKILL.md b/.claude/skills/vllm-omni-test/SKILL.md index dcc2506da..db5068a50 100644 --- a/.claude/skills/vllm-omni-test/SKILL.md +++ b/.claude/skills/vllm-omni-test/SKILL.md @@ -9,7 +9,7 @@ description: Generate and run tests for vllm-project/vllm-omni with CI-aligned l Use this skill to generate minimal, stable test cases and run them with the correct marker/level strategy for [vllm-project/vllm-omni](https://github.com/vllm-project/vllm-omni). -**Link convention:** Paths such as `.buildkite/` and `docs/contributing/` live at the **vllm-omni repo root**. Markdown links use repo-relative paths from this skill file (e.g. `../../../.buildkite/test-ready.yml`, `../../../docs/contributing/ci/CI_5levels.md`). +**Link convention:** Paths such as `.buildkite/` and `docs/contributing/` live at the **vllm-omni repo root**. Markdown links use repo-relative paths from this skill file (e.g. `../../../.buildkite/cuda/test-ready.yml`, `../../../docs/contributing/ci/CI_5levels.md`). Default priorities: @@ -96,7 +96,7 @@ Existing references: `tests/e2e/offline_inference/test_qwen2_5_omni.py` (L2-styl |----------|------------------|---------------------------|---------------------------| | **Offline inference e2e** | `tests/e2e/offline_inference/` | **Module (default):** `omni_runner` + `omni_runner_handler`. **Function (isolation only):** `omni_runner_function` + `omni_runner_handler_function`. Diffusion/TTS may use `Omni(...).generate` directly | L2: `core_model` + **one of** `omni` / `tts` / `diffusion`; `@hardware_test(...)` when GPU/NPU is required | | **Online serving e2e** | `tests/e2e/online_serving/` | **Module (default):** `omni_server` + `openai_client`. **Function (isolation only):** `omni_server_function` + `openai_client_function`. Clients: `send_omni_request` (omni), `send_audio_speech_request` (tts), `send_diffusion_request` / `send_video_diffusion_request` / `send_images_generations_request` (diffusion) | Baseline smoke: **`core_model` + `advanced_model`**; heavier paths: `advanced_model` only; L4 expansion: `full_model` | -| **Documentation / runnable examples** | `tests/examples/offline_inference/`, `tests/examples/online_serving/` | **Offline docs (preferred):** extract Python/Bash blocks from the doc README (e.g. `ReadmeSnippet.extract_readme_snippets`), `pytest.mark.parametrize` each snippet, run via `example_runner.run` with a stable `output_subfolder`. **Online docs:** copy client/request scripts into dedicated tests and keep them in sync with the doc page. | Usually **L4**: `advanced_model`, often `example` plus hardware marks matching the nightly docs-example job (see `.buildkite/test-nightly.yml`). Full conventions: [docs/contributing/ci/test_examples/l4_doc_example_tests.inc.md](../../../docs/contributing/ci/test_examples/l4_doc_example_tests.inc.md) (introduced in [PR #1910](https://github.com/vllm-project/vllm-omni/pull/1910): naming, output directory layout, skip rules, avoid trimming `num_inference_steps` without a strong CI reason). | +| **Documentation / runnable examples** | `tests/examples/offline_inference/`, `tests/examples/online_serving/` | **Offline docs (preferred):** extract Python/Bash blocks from the doc README (e.g. `ReadmeSnippet.extract_readme_snippets`), `pytest.mark.parametrize` each snippet, run via `example_runner.run` with a stable `output_subfolder`. **Online docs:** copy client/request scripts into dedicated tests and keep them in sync with the doc page. | Usually **L4**: `advanced_model`, often `example` plus hardware marks matching the nightly docs-example job (see `.buildkite/cuda/test-nightly.yml`). Full conventions: [docs/contributing/ci/test_examples/l4_doc_example_tests.inc.md](../../../docs/contributing/ci/test_examples/l4_doc_example_tests.inc.md) (introduced in [PR #1910](https://github.com/vllm-project/vllm-omni/pull/1910): naming, output directory layout, skip rules, avoid trimming `num_inference_steps` without a strong CI reason). | | **Performance / benchmark** | `tests/dfx/perf/tests/*.json` + `run_*_benchmark.py` | JSON or script-driven server + load config; assert explicit metrics / baselines | L4 Perf: JSON `mark` with `full_model` + `omni`/`tts`/`diffusion`; wire `test-nightly.yml` Perf steps | | **Invalid parameter / negative HTTP validation** | `tests/dfx/reliability/invalid_param_test/` | Live `omni_server` + low-level `send_*_http_request` with `err_code` / `err_message` | `pytest.mark.slow` + `omni` / `tts` / `diffusion` + `@hardware_marks` (`H100` or `L4`); CI in **`test-weekly.yml`** (not ready/merge/nightly) | @@ -460,7 +460,7 @@ When the user asks for **invalid parameter validation**, **invalid request bodie 4. **Adding a new model** to invalid-param coverage: append a `pytest.param(OmniServerParams(model="...", stage_config_path=..., server_args=...), ...)` entry to the script’s `_PARAMS` list — do **not** create `test_invalid_.py` unless the route is new. -5. **CI — `.buildkite/test-weekly.yml` only** (not `test-ready.yml` / `test-merge.yml` / `test-nightly.yml`): +5. **CI — `.buildkite/cuda/test-weekly.yml` only** (not `test-ready.yml` / `test-merge.yml` / `test-nightly.yml`): | Weekly step | Command | When your cases run | |-------------|---------|---------------------| @@ -629,10 +629,10 @@ If the test is not already collected by an existing pipeline command (for exampl | Test level | Edit this file | Typical trigger / intent | |------------|----------------|---------------------------| -| **L1** and **L2** | [`.buildkite/test-ready.yml`](../../../.buildkite/test-ready.yml) | PR **ready** label; L1 CPU + L2 GPU/basic e2e (steps labeled **Omni ·**, **TTS ·**, **Diffusion ·**) | -| **L3** | [`.buildkite/test-merge.yml`](../../../.buildkite/test-merge.yml) | Post-merge; `advanced_model` integration per model type | -| **L4** | [`.buildkite/test-nightly.yml`](../../../.buildkite/test-nightly.yml) | Nightly; grouped by model type (see below) | -| **Invalid param / reliability (weekly)** | [`.buildkite/test-weekly.yml`](../../../.buildkite/test-weekly.yml) | Weekly; `tests/dfx/reliability/invalid_param_test/` — **not** L1–L4 e2e pipelines | +| **L1** and **L2** | [`.buildkite/cuda/test-ready.yml`](../../../.buildkite/cuda/test-ready.yml) | PR **ready** label; L1 CPU + L2 GPU/basic e2e (steps labeled **Omni ·**, **TTS ·**, **Diffusion ·**) | +| **L3** | [`.buildkite/cuda/test-merge.yml`](../../../.buildkite/cuda/test-merge.yml) | Post-merge; `advanced_model` integration per model type | +| **L4** | [`.buildkite/cuda/test-nightly.yml`](../../../.buildkite/cuda/test-nightly.yml) | Nightly; grouped by model type (see below) | +| **Invalid param / reliability (weekly)** | [`.buildkite/cuda/test-weekly.yml`](../../../.buildkite/cuda/test-weekly.yml) | Weekly; `tests/dfx/reliability/invalid_param_test/` — **not** L1–L4 e2e pipelines | **Level-specific Buildkite delivery (follow the user’s requested level):** @@ -809,21 +809,21 @@ Use **repo-relative paths** (no leading `./`). Prefer **directory** entries (`.. When extending an **existing** E2E step to run an additional test file, **append** that file (and any new model/deploy paths) to `source_file_dependencies` on the same step. When authoring a PR, mention the dependency list in the **Buildkite change** section of your test plan. -#### L3 / L4: validating Buildkite from a feature branch (root `pipeline.yml`) +#### L3 / L4: validating Buildkite from a feature branch (`cuda/pipeline.yml`) -Production behavior is driven by [`.buildkite/pipeline.yml`](../../../.buildkite/pipeline.yml): it builds the CI image, then uploads **L2** (`test-ready.yml` on non-`main`), **L3** (`test-merge.yml` when `build.branch == "main"`), and **L4** (`test-nightly.yml` when `build.env("NIGHTLY") == "1"`). +Production behavior is driven by [`.buildkite/cuda/pipeline.yml`](../../../.buildkite/cuda/pipeline.yml): it builds the CI image, then uploads **L2** (`test-ready.yml` on non-`main`), **L3** (`test-merge.yml` when `build.branch == "main"`), and **L4** (`test-nightly.yml` when `build.env("NIGHTLY") == "1"`). When you add or debug **L3** or **L4** tests and need those child pipelines to run **off `main`** or **without** `NIGHTLY=1`, apply **temporary** edits (revert before merge): 1. **Point the upload at the right file (optional shortcut)** - In the **Upload Ready Pipeline** step, the command is `buildkite-agent pipeline upload .buildkite/test-ready.yml`. For a one-off run you may change that path to `.buildkite/test-merge.yml` (L3) or `.buildkite/test-nightly.yml` (L4), and adjust the step’s `if:` so it does not conflict with the other upload steps (avoid double-uploading unless intentional). + In the **Upload Ready Pipeline** step, the command is `buildkite-agent pipeline upload .buildkite/cuda/test-ready.yml`. For a one-off run you may change that path to `.buildkite/cuda/test-merge.yml` (L3) or `.buildkite/cuda/test-nightly.yml` (L4), and adjust the step’s `if:` so it does not conflict with the other upload steps (avoid double-uploading unless intentional). 2. **L3 — merge pipeline** - The gate `if: build.branch == "main"` lives on the **Upload Merge Pipeline** step in `pipeline.yml` (not inside `test-merge.yml`). **Comment out that `if` line** so `test-merge.yml` is uploaded on your feature branch. Alternatively rely on step (1) instead of the dedicated merge upload step. + The gate `if: build.branch == "main"` lives on the **Upload Merge Pipeline** step in `cuda/pipeline.yml` (not inside `test-merge.yml`). **Comment out that `if` line** so `test-merge.yml` is uploaded on your feature branch. Alternatively rely on step (1) instead of the dedicated merge upload step. 3. **L4 — nightly pipeline** - **Comment out** `if: build.env("NIGHTLY") == "1"` on the **Upload Nightly Pipeline** step in `pipeline.yml` so the nightly definition is uploaded without setting the env var. - Child steps in [`test-nightly.yml`](../../../.buildkite/test-nightly.yml) often repeat `if: build.env("NIGHTLY") == "1"`; **comment out those lines on the steps you need to run**, otherwise they will still be skipped after upload. + **Comment out** `if: build.env("NIGHTLY") == "1"` on the **Upload Nightly Pipeline** step in `cuda/pipeline.yml` so the nightly definition is uploaded without setting the env var. + Child steps in [`test-nightly.yml`](../../../.buildkite/cuda/test-nightly.yml) often repeat `if: build.env("NIGHTLY") == "1"`; **comment out those lines on the steps you need to run**, otherwise they will still be skipped after upload. ### Step 6: Run Tests @@ -890,5 +890,5 @@ When completing a request, return: ## Additional Resources - Marker and command routing: [references/test-routing.md](references/test-routing.md) -- CI pipelines (vllm-omni): [test-ready.yml](../../../.buildkite/test-ready.yml) (L1/L2), [test-merge.yml](../../../.buildkite/test-merge.yml) (L3), [test-nightly.yml](../../../.buildkite/test-nightly.yml) (L4), [test-weekly.yml](../../../.buildkite/test-weekly.yml) (invalid param / reliability) +- CI pipelines (vllm-omni): [test-ready.yml](../../../.buildkite/cuda/test-ready.yml) (L1/L2), [test-merge.yml](../../../.buildkite/cuda/test-merge.yml) (L3), [test-nightly.yml](../../../.buildkite/cuda/test-nightly.yml) (L4), [test-weekly.yml](../../../.buildkite/cuda/test-weekly.yml) (invalid param / reliability) - L4 documentation example tests (naming, extraction vs copied scripts, output dirs, skips): [docs/contributing/ci/test_examples/l4_doc_example_tests.inc.md](../../../docs/contributing/ci/test_examples/l4_doc_example_tests.inc.md) — see also [PR #1910](https://github.com/vllm-project/vllm-omni/pull/1910) diff --git a/.claude/skills/vllm-omni-test/references/test-routing.md b/.claude/skills/vllm-omni-test/references/test-routing.md index f78e1c77d..83b9b2007 100644 --- a/.claude/skills/vllm-omni-test/references/test-routing.md +++ b/.claude/skills/vllm-omni-test/references/test-routing.md @@ -2,7 +2,7 @@ Use this reference to map testing goals to levels, markers, and runnable commands. -**Repo paths** (`.buildkite/`, `docs/contributing/…`): link with repo-relative paths from this file (e.g. `../../../../.buildkite/test-ready.yml`, `../../../../docs/contributing/ci/CI_5levels.md`). +**Repo paths** (`.buildkite/`, `docs/contributing/…`): link with repo-relative paths from this file (e.g. `../../../../.buildkite/cuda/test-ready.yml`, `../../../../docs/contributing/ci/CI_5levels.md`). ## Model-centric e2e filename convention (L2–L4) @@ -224,7 +224,7 @@ When the user’s test plan includes **invalid parameter validation**, **invalid | Sleep / wakeup / server control | `test_invalid_server_control.py` | 3. **Style:** `pytestmark = [pytest.mark.slow, pytest.mark.]`; `_PARAMS` + `hardware_marks`; `send_*_http_request` with `err_code` + `err_message`; parametrized `body_spec` rows with `id=`; `_minimal_*_json()` helpers; `_SKIP_ISSUE_3649` when tracked in [#3649](https://github.com/vllm-project/vllm-omni/issues/3649). -4. **CI:** [`.buildkite/test-weekly.yml`](../../../../.buildkite/test-weekly.yml) group **Reliability Test - Invalid parameters Test** — **not** ready/merge/nightly. +4. **CI:** [`.buildkite/cuda/test-weekly.yml`](../../../../.buildkite/cuda/test-weekly.yml) group **Reliability Test - Invalid parameters Test** — **not** ready/merge/nightly. ```bash # Weekly H100 (diffusion / omni / video invalid-param) @@ -291,10 +291,10 @@ When adding or modifying tests, do not stop at “where the file lives” — al | Level | Repo file | Model-type grouping | |-------|-----------|---------------------| -| L1, L2 | [`.buildkite/test-ready.yml`](../../../../.buildkite/test-ready.yml) | Steps prefixed **Omni ·**, **TTS ·**, **Diffusion ·** under **E2E Test** — **`source_file_dependencies` required** | -| L3 | [`.buildkite/test-merge.yml`](../../../../.buildkite/test-merge.yml) | Per-model E2E steps; `-m "advanced_model and …"` — **`source_file_dependencies` required** | -| L4 | [`.buildkite/test-nightly.yml`](../../../../.buildkite/test-nightly.yml) | **Omni / TTS / Diffusion X2I(&A&T) / X2V** — each group may have **Function**, **Accuracy**, **Perf**, **Doc** steps | -| Invalid param (weekly) | [`.buildkite/test-weekly.yml`](../../../../.buildkite/test-weekly.yml) | **Invalid parameters Test · H100** / **· L4** — sweeps `tests/dfx/reliability/invalid_param_test/` | +| L1, L2 | [`.buildkite/cuda/test-ready.yml`](../../../../.buildkite/cuda/test-ready.yml) | Steps prefixed **Omni ·**, **TTS ·**, **Diffusion ·** under **E2E Test** — **`source_file_dependencies` required** | +| L3 | [`.buildkite/cuda/test-merge.yml`](../../../../.buildkite/cuda/test-merge.yml) | Per-model E2E steps; `-m "advanced_model and …"` — **`source_file_dependencies` required** | +| L4 | [`.buildkite/cuda/test-nightly.yml`](../../../../.buildkite/cuda/test-nightly.yml) | **Omni / TTS / Diffusion X2I(&A&T) / X2V** — each group may have **Function**, **Accuracy**, **Perf**, **Doc** steps | +| Invalid param (weekly) | [`.buildkite/cuda/test-weekly.yml`](../../../../.buildkite/cuda/test-weekly.yml) | **Invalid parameters Test · H100** / **· L4** — sweeps `tests/dfx/reliability/invalid_param_test/` | ### `source_file_dependencies` (E2E Test only — ready & merge) @@ -380,7 +380,7 @@ Common patterns in `test-ready.yml` / `test-merge.yml`: When extending an existing step (e.g. add `tests/e2e/offline_inference/test_qwen_image.py` to **Diffusion · Qwen Image Test**), update `source_file_dependencies` and `commands` only; **keep the existing `agents` + `plugins` block unchanged** unless the hardware tier changes. -The root [`.buildkite/pipeline.yml`](../../../../.buildkite/pipeline.yml) decides **which** child file is uploaded. To run **L3** on a feature branch, comment out `if: build.branch == "main"` on the merge upload step. To run **L4** without `NIGHTLY=1`, comment out the nightly upload step’s `if: build.env("NIGHTLY") == "1"` and the same `if` on relevant steps inside `test-nightly.yml`. Revert such edits before merging. +[`.buildkite/cuda/pipeline.yml`](../../../../.buildkite/cuda/pipeline.yml) decides **which** child file is uploaded. To run **L3** on a feature branch, comment out `if: build.branch == "main"` on the merge upload step. To run **L4** without `NIGHTLY=1`, comment out the nightly upload step’s `if: build.env("NIGHTLY") == "1"` and the same `if` on relevant steps inside `test-nightly.yml`. Revert such edits before merging. Platform-specific pipelines (e.g. AMD) follow the same level → file pairing under `.buildkite/`. diff --git a/docs/contributing/README.md b/docs/contributing/README.md index e10d8d3b3..63a62eabf 100644 --- a/docs/contributing/README.md +++ b/docs/contributing/README.md @@ -131,7 +131,7 @@ Please run the L1 and L2 test cases locally first and attach the results before ### Automatic skip-ci (docs and pytest skip marks) -On pull requests and `main` pushes, the bootstrap step in [`.buildkite/pipeline.yml`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/pipeline.yml) runs [`.buildkite/scripts/upload_pipeline.py`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/scripts/upload_pipeline.py) against the git diff. When every changed file qualifies, **L2 (`ready`) and L3 (`merge-test`) pipelines are not uploaded**, so the default GPU CI jobs are skipped. +On pull requests and `main` pushes, the bootstrap step in [`.buildkite/cuda/pipeline.yml`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/cuda/pipeline.yml) runs [`.buildkite/cuda/scripts/upload_pipeline.py`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/cuda/scripts/upload_pipeline.py) against the git diff. When every changed file qualifies, **L2 (`ready`) and L3 (`merge-test`) pipelines are not uploaded**, so the default GPU CI jobs are skipped. | Change per file | Examples | | --- | --- | diff --git a/docs/contributing/ci/CI_5levels.md b/docs/contributing/ci/CI_5levels.md index c9d86e1c4..d3c164d9b 100644 --- a/docs/contributing/ci/CI_5levels.md +++ b/docs/contributing/ci/CI_5levels.md @@ -319,7 +319,7 @@ Before entering specific testing levels, the project establishes two common spec ### Diff-aware Buildkite uploads (`source_file_dependencies`) -L2 (`.buildkite/test-ready.yml`) and L3 (`.buildkite/test-merge.yml`) pipelines can **skip unrelated GPU jobs at upload time** based on the PR diff. This is implemented by `.buildkite/scripts/upload_pipeline.py`, which filters steps before calling `buildkite-agent pipeline upload`. +L2 (`.buildkite/cuda/test-ready.yml`) and L3 (`.buildkite/cuda/test-merge.yml`) pipelines can **skip unrelated GPU jobs at upload time** based on the PR diff. This is implemented by `.buildkite/cuda/scripts/upload_pipeline.py`, which filters steps before calling `buildkite-agent pipeline upload`. #### What `source_file_dependencies` is @@ -334,14 +334,14 @@ L2 (`.buildkite/test-ready.yml`) and L3 (`.buildkite/test-merge.yml`) pipelines | `main` branch push | `git diff --name-only ^..` | | Other (e.g. local dry-run, non-PR branch) | Diff cannot be resolved → **no filtering**; all steps are uploaded and `source_file_dependencies` is still stripped | -Docs-only PRs are handled earlier in bootstrap (`pipeline.yml`) via skip-ci logic; `source_file_dependencies` applies only to the **uploaded** L2/L3 test pipelines. +Docs-only PRs are handled earlier in bootstrap (`.buildkite/cuda/pipeline.yml`) via skip-ci logic; `source_file_dependencies` applies only to the **uploaded** L2/L3 test pipelines. #### Where it is configured | Level | Pipeline file | Upload entry | | --- | --- | --- | -| L2 | `.buildkite/test-ready.yml` | `upload_pipeline.py --upload .buildkite/test-ready.yml` (from `pipeline.yml`) | -| L3 | `.buildkite/test-merge.yml` | `upload_pipeline.py --upload .buildkite/test-merge.yml` (from `pipeline.yml`) | +| L2 | `.buildkite/cuda/test-ready.yml` | `upload_pipeline.py --upload .buildkite/cuda/test-ready.yml` (from `cuda/pipeline.yml`) | +| L3 | `.buildkite/cuda/test-merge.yml` | `upload_pipeline.py --upload .buildkite/cuda/test-merge.yml` (from `cuda/pipeline.yml`) | Steps **without** `source_file_dependencies` are always uploaded (subject to the usual label conditions: `ready` for L2, `merge-test` for L3). @@ -393,10 +393,10 @@ A **group** may also define `source_file_dependencies`; nested steps inherit fil ```bash # Render filtered YAML to stdout (no upload) -python3 .buildkite/scripts/upload_pipeline.py .buildkite/test-ready.yml +python3 .buildkite/cuda/scripts/upload_pipeline.py .buildkite/cuda/test-ready.yml # Confirm uploader-only keys are stripped -python3 .buildkite/scripts/upload_pipeline.py .buildkite/test-merge.yml | grep source_file_dependencies +python3 .buildkite/cuda/scripts/upload_pipeline.py .buildkite/cuda/test-merge.yml | grep source_file_dependencies # (no output expected) ``` @@ -404,7 +404,7 @@ On a PR build, Buildkite logs from `upload_pipeline.py` include lines such as `s #### Related -- Implementation: `.buildkite/scripts/upload_pipeline.py` +- Implementation: `.buildkite/cuda/scripts/upload_pipeline.py` - L2/L3 diff skip does **not** replace label-based triggers (`ready`, `merge-test`); it only reduces which steps appear **after** the pipeline is already scheduled. ### Test helper environment variables @@ -871,7 +871,7 @@ Reliability tests are **short fault-injection** integration runs (L5 **(b)** in #### CI trigger -Weekly Buildkite (`.buildkite/test-weekly.yml`) runs one step per model suite (trigger: `WEEKLY=1` or PR label `weekly-test`), for example: +Weekly Buildkite (`.buildkite/cuda/test-weekly.yml`) runs one step per model suite (trigger: `WEEKLY=1` or PR label `weekly-test`), for example: | Buildkite step | Test file | CI hardware | | -------------- | --------- | ----------- | @@ -902,12 +902,12 @@ pytest -s -v tests/dfx/reliability/test_reliability_voxcpm2.py -m slow 1. Add `test_reliability_.py` under `tests/dfx/reliability/`. 2. Define **`RELIABILITY_SCENARIOS`** and pass them through **`create_reliability_omni_server_params()`** with the correct deploy or e2e stage-config directory (same pattern as existing files). 3. Reuse **`helpers`** for OOM / kill / raw HTTP; prefer **`assert_fault_exception()`** and **`resolve_oom_device_spec()`** from `tests/dfx/conftest.py` for consistent device selection vs stage YAML. -4. Register **`slow`** (and **`hardware_test`** if needed); extend **`.buildkite/test-weekly.yml`** when the suite should run in weekly L5. +4. Register **`slow`** (and **`hardware_test`** if needed); extend **`.buildkite/cuda/test-weekly.yml`** when the suite should run in weekly L5. - - ***Stability***: `pytest -s -v tests/dfx/stability/scripts/test_stability_qwen3_omni.py` or `pytest -s -v tests/dfx/stability/scripts/test_stability_wan22.py` (or add `test_stability_.py` alongside a matching JSON config) - - ***Reliability***: `pytest -s -v tests/dfx/reliability/test_reliability_.py -m slow` (current suites: `qwen3_omni`, `wan22`, `hunyuan_image`, `voxcpm2`; add `test_reliability_.py` and a matching step in `.buildkite/test-weekly.yml` for new models) + - ***Reliability***: `pytest -s -v tests/dfx/reliability/test_reliability_.py -m slow` (current suites: `qwen3_omni`, `wan22`, `hunyuan_image`, `voxcpm2`; add `test_reliability_.py` and a matching step in `.buildkite/cuda/test-weekly.yml` for new models) ## Summary diff --git a/docs/contributing/ci/test_guide.md b/docs/contributing/ci/test_guide.md index ad6a63963..d1179e832 100644 --- a/docs/contributing/ci/test_guide.md +++ b/docs/contributing/ci/test_guide.md @@ -49,11 +49,11 @@ Failed jobs: 1/2 cd tests pytest -s -v -m "core_model and cpu" ``` - The latest test command is available in the "Simple Unit Test" step of this [pipeline](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/test-ready.yml). + The latest test command is available in the "Simple Unit Test" step of this [pipeline](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/cuda/test-ready.yml). === "L2 level" - **Recommended:** run CI-aligned jobs from the repo root via [`tools/run_ready_jobs.sh`](https://github.com/vllm-project/vllm-omni/blob/main/tools/run_ready_jobs.sh). The script reads [`.buildkite/test-ready.yml`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/test-ready.yml), generates per-step bash wrappers, runs pytest, and tees logs under `logs/ready_jobs/` (see [CI job runners](#ci-job-runners-l2l4-logs-timing-and-timeouts) for timing and timeout behavior). Requires `bash`, `python3`, and PyYAML (`pip install pyyaml`). + **Recommended:** run CI-aligned jobs from the repo root via [`tools/run_ready_jobs.sh`](https://github.com/vllm-project/vllm-omni/blob/main/tools/run_ready_jobs.sh). The script reads [`.buildkite/cuda/test-ready.yml`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/cuda/test-ready.yml), generates per-step bash wrappers, runs pytest, and tees logs under `logs/ready_jobs/` (see [CI job runners](#ci-job-runners-l2l4-logs-timing-and-timeouts) for timing and timeout behavior). Requires `bash`, `python3`, and PyYAML (`pip install pyyaml`). ```bash # All L2 and L1 jobs (default: every pytest step in test-ready.yml) @@ -85,7 +85,7 @@ Failed jobs: 1/2 === "L3 level" - **Recommended:** run CI-aligned jobs from the repo root via [`tools/run_merge_jobs.sh`](https://github.com/vllm-project/vllm-omni/blob/main/tools/run_merge_jobs.sh). The script reads [`.buildkite/test-merge.yml`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/test-merge.yml), generates per-step bash wrappers, runs pytest, and tees logs under `logs/merge_jobs/` (see [CI job runners](#ci-job-runners-l2l4-logs-timing-and-timeouts)). Requires `bash`, `python3`, and PyYAML (`pip install pyyaml`). + **Recommended:** run CI-aligned jobs from the repo root via [`tools/run_merge_jobs.sh`](https://github.com/vllm-project/vllm-omni/blob/main/tools/run_merge_jobs.sh). The script reads [`.buildkite/cuda/test-merge.yml`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/cuda/test-merge.yml), generates per-step bash wrappers, runs pytest, and tees logs under `logs/merge_jobs/` (see [CI job runners](#ci-job-runners-l2l4-logs-timing-and-timeouts)). Requires `bash`, `python3`, and PyYAML (`pip install pyyaml`). ```bash # All L3 and L1 jobs (default: every pytest step in test-merge.yml) @@ -117,7 +117,7 @@ Failed jobs: 1/2 === "L4 level" - **Recommended:** run CI-aligned nightly jobs from the repo root via [`tools/nightly/run_nightly_jobs.sh`](https://github.com/vllm-project/vllm-omni/blob/main/tools/nightly/run_nightly_jobs.sh). The script reads [`.buildkite/test-nightly.yml`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/test-nightly.yml), generates per-step bash wrappers, runs pytest (perf jobs first, then aggregates perf JSON into Excel), and tees logs under `logs/nightly_jobs/` (see [CI job runners](#ci-job-runners-l2l4-logs-timing-and-timeouts); `generate_nightly_perf_excel` is included in the timing summary). Requires `bash`, `python3`, and PyYAML (`pip install pyyaml`). + **Recommended:** run CI-aligned nightly jobs from the repo root via [`tools/nightly/run_nightly_jobs.sh`](https://github.com/vllm-project/vllm-omni/blob/main/tools/nightly/run_nightly_jobs.sh). The script reads [`.buildkite/cuda/test-nightly.yml`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/cuda/test-nightly.yml), generates per-step bash wrappers, runs pytest (perf jobs first, then aggregates perf JSON into Excel), and tees logs under `logs/nightly_jobs/` (see [CI job runners](#ci-job-runners-l2l4-logs-timing-and-timeouts); `generate_nightly_perf_excel` is included in the timing summary). Requires `bash`, `python3`, and PyYAML (`pip install pyyaml`). ```bash # All nightly jobs (default: test-type all, model-type all) @@ -189,7 +189,7 @@ Failed jobs: 1/2 ``` - The latest L5 CI jobs (reliability + invalid-parameter weekly steps) are in [test-weekly.yml](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/test-weekly.yml). + The latest L5 CI jobs (reliability + invalid-parameter weekly steps) are in [test-weekly.yml](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/cuda/test-weekly.yml). You can find more information about markers in the documentation: [marker doc](./tests_markers.md) diff --git a/docs/user_guide/fault_injection_reliability_matrix.md b/docs/user_guide/fault_injection_reliability_matrix.md index 974982b3b..5e5122bfc 100644 --- a/docs/user_guide/fault_injection_reliability_matrix.md +++ b/docs/user_guide/fault_injection_reliability_matrix.md @@ -65,7 +65,7 @@ More failure modes (for example, network jitter and network interruption) will b ## Source of Conclusions -The behaviors and conclusions above are summarized from current fault injection validation results on **`Qwen3-Omni`**, **`Wan2.2`**, **`HunyuanImage-3.0-Instruct`** (DiT-only, `/v1/images/generations`), and **`VoxCPM2`** (`/v1/audio/speech`). Automated coverage lives under `tests/dfx/reliability/` (`test_reliability_qwen3_omni.py`, `test_reliability_wan22.py`, `test_reliability_hunyuan_image.py`, `test_reliability_voxcpm2.py`) and runs weekly via `.buildkite/test-weekly.yml`. +The behaviors and conclusions above are summarized from current fault injection validation results on **`Qwen3-Omni`**, **`Wan2.2`**, **`HunyuanImage-3.0-Instruct`** (DiT-only, `/v1/images/generations`), and **`VoxCPM2`** (`/v1/audio/speech`). Automated coverage lives under `tests/dfx/reliability/` (`test_reliability_qwen3_omni.py`, `test_reliability_wan22.py`, `test_reliability_hunyuan_image.py`, `test_reliability_voxcpm2.py`) and runs weekly via `.buildkite/cuda/test-weekly.yml`. ### Example: `SIGTERM` fault injection log (Qwen3-Omni) diff --git a/tools/nightly/run_nightly_jobs.sh b/tools/nightly/run_nightly_jobs.sh index 7134a36a1..a95eb8876 100644 --- a/tools/nightly/run_nightly_jobs.sh +++ b/tools/nightly/run_nightly_jobs.sh @@ -2,7 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project # -# Extract steps from .buildkite/test-nightly.yml that contain pytest, synthesize +# Extract steps from .buildkite/cuda/test-nightly.yml that contain pytest, synthesize # small bash wrappers (exports + pytest), run them, and tee output to logs named # after each step's Buildkite "key" when present (otherwise a slug of the label). # YAML steps whose label contains "Perf Test" run first, then @@ -51,17 +51,17 @@ # Usage: # bash path/to/run_nightly_jobs.sh # REPO_ROOT=/path/to/vllm-omni bash path/to/run_nightly_jobs.sh --test-type perf,acc --dry-run -# YML=/path/to/vllm-omni/.buildkite/test-nightly.yml bash path/to/run_nightly_jobs.sh +# YML=/path/to/vllm-omni/.buildkite/cuda/test-nightly.yml bash path/to/run_nightly_jobs.sh # # Repository / YAML (no dependency on where this script lives): -# • Set REPO_ROOT (or pass --repo-root) — default YAML is $REPO_ROOT/.buildkite/test-nightly.yml +# • Set REPO_ROOT (or pass --repo-root) — default YAML is $REPO_ROOT/.buildkite/cuda/test-nightly.yml # • Or set YML (or --yaml) — repo root is inferred as parent of the .buildkite directory # • Or run from inside the clone: git rev-parse --show-toplevel, else walk up from $PWD, -# then from the script's directory, until .buildkite/test-nightly.yml exists +# then from the script's directory, until .buildkite/cuda/test-nightly.yml exists # # Optional environment: # REPO_ROOT - vllm-omni root (working directory for pytest); see above -# YML - path to test-nightly.yml (default: $REPO_ROOT/.buildkite/test-nightly.yml) +# YML - path to test-nightly.yml (default: $REPO_ROOT/.buildkite/cuda/test-nightly.yml) # LOG_DIR - logs + generated job scripts; when unset, a timestamped directory under # $REPO_ROOT/logs/ is created: # nightly_jobs_YYYYMMDD-HHMMSS (default / YAML nightly steps) @@ -247,7 +247,7 @@ _nightly_log_dir_basename() { fi } -BUILDKITE_REL=".buildkite/test-nightly.yml" +BUILDKITE_REL=".buildkite/cuda/test-nightly.yml" _find_repo_containing_nightly() { local dir="${1:-}" @@ -267,8 +267,14 @@ _derive_repo_root_from_yml() { local yml="$1" local d d="$(cd "$(dirname "${yml}")" && pwd)" - [[ "$(basename "${d}")" == ".buildkite" ]] || return 1 - printf '%s\n' "$(dirname "${d}")" + while [[ "${d}" != "/" ]]; do + if [[ "$(basename "${d}")" == ".buildkite" ]]; then + printf '%s\n' "$(dirname "${d}")" + return 0 + fi + d="$(dirname "${d}")" + done + return 1 } # Resolve REPO_ROOT, YML (no relative path between script and repo) @@ -278,7 +284,7 @@ if [[ -n "${YML:-}" && -n "${REPO_ROOT:-}" ]]; then elif [[ -n "${YML:-}" ]]; then YML="$(cd "$(dirname "${YML}")" && pwd)/$(basename "${YML}")" if ! REPO_ROOT="$(_derive_repo_root_from_yml "${YML}")"; then - echo "Could not derive REPO_ROOT from YML=${YML} (expected file at /.buildkite/test-nightly.yml)." >&2 + echo "Could not derive REPO_ROOT from YML=${YML} (expected file at /.buildkite/cuda/test-nightly.yml)." >&2 echo "Set REPO_ROOT explicitly (or pass --repo-root) for pytest working directory." >&2 exit 2 fi diff --git a/tools/run_jobs_common.sh b/tools/run_jobs_common.sh index ddc1794dc..7aa0861e4 100644 --- a/tools/run_jobs_common.sh +++ b/tools/run_jobs_common.sh @@ -181,7 +181,7 @@ _run_jobs_print_timing_summary() { # tools/run_merge_jobs.sh). Nightly does not use this entrypoint. # # Entry scripts must set before calling run_yaml_ci_jobs_main: -# BUILDKITE_REL - e.g. .buildkite/test-ready.yml +# BUILDKITE_REL - e.g. .buildkite/cuda/test-ready.yml # DEFAULT_LOG_SUBDIR - e.g. ready_jobs # --------------------------------------------------------------------------- @@ -246,8 +246,14 @@ _run_jobs_derive_repo_root_from_yml() { local yml="$1" local d d="$(cd "$(dirname "${yml}")" && pwd)" - [[ "$(basename "${d}")" == ".buildkite" ]] || return 1 - printf '%s\n' "$(dirname "${d}")" + while [[ "${d}" != "/" ]]; do + if [[ "$(basename "${d}")" == ".buildkite" ]]; then + printf '%s\n' "$(dirname "${d}")" + return 0 + fi + d="$(dirname "${d}")" + done + return 1 } run_yaml_ci_jobs_main() { diff --git a/tools/run_merge_jobs.sh b/tools/run_merge_jobs.sh index 35b38a8e0..9f9ab13b0 100644 --- a/tools/run_merge_jobs.sh +++ b/tools/run_merge_jobs.sh @@ -2,7 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project # -# Extract steps from .buildkite/test-merge.yml that contain pytest, synthesize +# Extract steps from .buildkite/cuda/test-merge.yml that contain pytest, synthesize # small bash wrappers (exports + pytest), run them, and tee output to logs named # after each step's Buildkite "key" when present (otherwise a slug of the label). # @@ -21,17 +21,17 @@ # Usage: # bash tools/run_merge_jobs.sh # REPO_ROOT=/path/to/vllm-omni bash tools/run_merge_jobs.sh --model-type diffusion --dry-run -# YML=/path/to/vllm-omni/.buildkite/test-merge.yml bash tools/run_merge_jobs.sh +# YML=/path/to/vllm-omni/.buildkite/cuda/test-merge.yml bash tools/run_merge_jobs.sh # # Repository / YAML (no dependency on where this script lives): -# • Set REPO_ROOT (or pass --repo-root) — default YAML is $REPO_ROOT/.buildkite/test-merge.yml +# • Set REPO_ROOT (or pass --repo-root) — default YAML is $REPO_ROOT/.buildkite/cuda/test-merge.yml # • Or set YML (or --yaml) — repo root is inferred as parent of the .buildkite directory # • Or run from inside the clone: git rev-parse --show-toplevel, else walk up from $PWD, -# then from the script's directory, until .buildkite/test-merge.yml exists +# then from the script's directory, until .buildkite/cuda/test-merge.yml exists # # Optional environment: # REPO_ROOT - vllm-omni root (working directory for pytest); see above -# YML - path to test-merge.yml (default: $REPO_ROOT/.buildkite/test-merge.yml) +# YML - path to test-merge.yml (default: $REPO_ROOT/.buildkite/cuda/test-merge.yml) # LOG_DIR - logs + generated job scripts (default: $REPO_ROOT/logs/merge_jobs); # per-job *.log plus timing_summary.log after the run # MODEL_TYPE - comma-separated and/or repeated flags (default: all); see above @@ -41,7 +41,7 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -BUILDKITE_REL=".buildkite/test-merge.yml" +BUILDKITE_REL=".buildkite/cuda/test-merge.yml" DEFAULT_LOG_SUBDIR="merge_jobs" # shellcheck source=tools/run_jobs_common.sh diff --git a/tools/run_ready_jobs.sh b/tools/run_ready_jobs.sh index 89f082436..b854faa07 100644 --- a/tools/run_ready_jobs.sh +++ b/tools/run_ready_jobs.sh @@ -2,7 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project # -# Extract steps from .buildkite/test-ready.yml that contain pytest, synthesize +# Extract steps from .buildkite/cuda/test-ready.yml that contain pytest, synthesize # small bash wrappers (exports + pytest), run them, and tee output to logs named # after each step's Buildkite "key" when present (otherwise a slug of the label). # @@ -21,17 +21,17 @@ # Usage: # bash tools/run_ready_jobs.sh # REPO_ROOT=/path/to/vllm-omni bash tools/run_ready_jobs.sh --model-type omni --dry-run -# YML=/path/to/vllm-omni/.buildkite/test-ready.yml bash tools/run_ready_jobs.sh +# YML=/path/to/vllm-omni/.buildkite/cuda/test-ready.yml bash tools/run_ready_jobs.sh # # Repository / YAML (no dependency on where this script lives): -# • Set REPO_ROOT (or pass --repo-root) — default YAML is $REPO_ROOT/.buildkite/test-ready.yml +# • Set REPO_ROOT (or pass --repo-root) — default YAML is $REPO_ROOT/.buildkite/cuda/test-ready.yml # • Or set YML (or --yaml) — repo root is inferred as parent of the .buildkite directory # • Or run from inside the clone: git rev-parse --show-toplevel, else walk up from $PWD, -# then from the script's directory, until .buildkite/test-ready.yml exists +# then from the script's directory, until .buildkite/cuda/test-ready.yml exists # # Optional environment: # REPO_ROOT - vllm-omni root (working directory for pytest); see above -# YML - path to test-ready.yml (default: $REPO_ROOT/.buildkite/test-ready.yml) +# YML - path to test-ready.yml (default: $REPO_ROOT/.buildkite/cuda/test-ready.yml) # LOG_DIR - logs + generated job scripts (default: $REPO_ROOT/logs/ready_jobs); # per-job *.log plus timing_summary.log after the run # MODEL_TYPE - comma-separated and/or repeated flags (default: all); see above @@ -41,7 +41,7 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -BUILDKITE_REL=".buildkite/test-ready.yml" +BUILDKITE_REL=".buildkite/cuda/test-ready.yml" DEFAULT_LOG_SUBDIR="ready_jobs" # shellcheck source=tools/run_jobs_common.sh From eef3bd5c4a35fced3e73d0b70464d6f48ec5ed3e Mon Sep 17 00:00:00 2001 From: Lancer Date: Mon, 20 Jul 2026 10:05:05 +0800 Subject: [PATCH 021/240] [Feat] support CFG Parallel for FLUX kontext (#2281) Signed-off-by: Lancer --- .../test_flux_kontext_expansion.py | 19 +- .../diffusion/distributed/cfg_parallel.py | 25 +++ .../models/flux/pipeline_flux_kontext.py | 194 +++++++++++++----- 3 files changed, 186 insertions(+), 52 deletions(-) diff --git a/tests/e2e/online_serving/test_flux_kontext_expansion.py b/tests/e2e/online_serving/test_flux_kontext_expansion.py index 5fa4eb2a1..17d4b0263 100644 --- a/tests/e2e/online_serving/test_flux_kontext_expansion.py +++ b/tests/e2e/online_serving/test_flux_kontext_expansion.py @@ -5,6 +5,8 @@ and are supported by the FluxKontext model. import pytest +from tests.helpers import skip_if_gated_repo_inaccessible +from tests.helpers.mark import hardware_marks from tests.helpers.media import generate_synthetic_image from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler, dummy_messages_from_mix_data @@ -13,6 +15,8 @@ pytestmark = [pytest.mark.diffusion, pytest.mark.slow] EDIT_PROMPT = "Transform this modern, geometrist image into a Vincent van Gogh style impressionist painting." NEGATIVE_PROMPT = "blurry, low quality, modern, geometrist" MODEL = "black-forest-labs/FLUX.1-Kontext-dev" +skip_if_gated_repo_inaccessible(MODEL) +PARALLEL_FEATURE_MARKS = hardware_marks(res={"cuda": "L4"}, num_cards=2) def _get_diffusion_feature_cases(model: str): @@ -26,7 +30,20 @@ def _get_diffusion_feature_cases(model: str): "--enable-cpu-offload", ], ), - id="parallel_001", + id="parallel_tp_2", + marks=PARALLEL_FEATURE_MARKS, + ), + pytest.param( + OmniServerParams( + model=model, + server_args=[ + "--enable-cpu-offload", + "--cfg-parallel-size", + "2", + ], + ), + id="parallel_cfg_2", + marks=PARALLEL_FEATURE_MARKS, ), ] diff --git a/vllm_omni/diffusion/distributed/cfg_parallel.py b/vllm_omni/diffusion/distributed/cfg_parallel.py index 83ce73b16..55130e0cf 100644 --- a/vllm_omni/diffusion/distributed/cfg_parallel.py +++ b/vllm_omni/diffusion/distributed/cfg_parallel.py @@ -463,6 +463,31 @@ class CFGParallelMixin(metaclass=ABCMeta): """ raise NotImplementedError("Subclasses must implement diffuse") + def check_cfg_parallel_validity(self, true_cfg_scale: float, has_neg_prompt: bool) -> bool: + """ + Check if CFG parallel configuration is valid. + + Args: + true_cfg_scale: The classifier-free guidance scale value + has_neg_prompt: Whether a negative prompt is provided + + Returns: + True if CFG parallel configuration is valid, False otherwise + """ + if get_classifier_free_guidance_world_size() == 1: + return True + + if true_cfg_scale <= 1: + logger.warning("CFG parallel is NOT working correctly when true_cfg_scale <= 1.") + return False + + if not has_neg_prompt: + logger.warning( + "CFG parallel is NOT working correctly when there is no negative prompt or negative prompt embeddings." + ) + return False + return True + def scheduler_step( self, noise_pred: torch.Tensor | tuple[torch.Tensor, ...], diff --git a/vllm_omni/diffusion/models/flux/pipeline_flux_kontext.py b/vllm_omni/diffusion/models/flux/pipeline_flux_kontext.py index 0b3b80cbd..15f0e4e0c 100644 --- a/vllm_omni/diffusion/models/flux/pipeline_flux_kontext.py +++ b/vllm_omni/diffusion/models/flux/pipeline_flux_kontext.py @@ -24,6 +24,7 @@ from transformers import CLIPTextModel, CLIPTokenizer, T5EncoderModel, T5Tokeniz from vllm.model_executor.models.utils import AutoWeightsLoader from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig +from vllm_omni.diffusion.distributed.cfg_parallel import CFGParallelMixin from vllm_omni.diffusion.distributed.utils import get_local_device from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader from vllm_omni.diffusion.models.flux import ( @@ -72,6 +73,7 @@ def get_flux_kontext_post_process_func(od_config: OmniDiffusionConfig) -> Callab class FluxKontextPipeline( nn.Module, FluxPipelineMixin, + CFGParallelMixin, SupportImageInput, ProgressBarMixin, DiffusionPipelineProfilerMixin, @@ -157,6 +159,7 @@ class FluxKontextPipeline( self._attention_kwargs = None self._joint_attention_kwargs = None self._num_timesteps = None + self._current_timestep = None self._interrupt = False self._callback_tensor_inputs = ["latents", "prompt_embeds"] self.latent_channels = self.vae.config.latent_channels if hasattr(self.vae, "config") else 16 @@ -465,6 +468,113 @@ class FluxKontextPipeline( def num_timesteps(self): return self._num_timesteps + @property + def current_timestep(self) -> torch.Tensor | None: + return self._current_timestep + + def predict_noise( + self, + hidden_states: torch.Tensor, + timestep: torch.Tensor, + guidance: torch.Tensor | None, + pooled_projections: torch.Tensor, + encoder_hidden_states: torch.Tensor, + txt_ids: torch.Tensor, + img_ids: torch.Tensor, + joint_attention_kwargs: dict[str, Any] | None, + return_dict: bool = False, + ) -> torch.Tensor: + return self.transformer( + hidden_states=hidden_states, + timestep=timestep, + guidance=guidance, + pooled_projections=pooled_projections, + encoder_hidden_states=encoder_hidden_states, + txt_ids=txt_ids, + img_ids=img_ids, + joint_attention_kwargs=joint_attention_kwargs, + return_dict=return_dict, + )[0] + + def diffuse( + self, + prompt_embeds: torch.Tensor, + pooled_prompt_embeds: torch.Tensor, + negative_prompt_embeds: torch.Tensor | None, + negative_pooled_prompt_embeds: torch.Tensor | None, + latents: torch.Tensor, + image_latents: torch.Tensor | None, + latent_ids: torch.Tensor, + text_ids: torch.Tensor, + negative_text_ids: torch.Tensor | None, + timesteps: torch.Tensor, + do_true_cfg: bool, + guidance: torch.Tensor | None, + true_cfg_scale: float, + callback_on_step_end: Callable[[int, int, dict], None] | None = None, + callback_on_step_end_tensor_inputs: list[str] = ["latents"], + ) -> torch.Tensor: + self.scheduler.set_begin_index(0) + self.transformer.do_true_cfg = do_true_cfg + with self.progress_bar(total=len(timesteps)) as pbar: + for i, t in enumerate(timesteps): + if self.interrupt: + continue + + self._current_timestep = t + latent_model_input = latents + if image_latents is not None: + latent_model_input = torch.cat([latents, image_latents], dim=1) + timestep = t.expand(latents.shape[0]).to(device=latents.device, dtype=latents.dtype) + + positive_kwargs = { + "hidden_states": latent_model_input, + "timestep": timestep / 1000, + "guidance": guidance, + "pooled_projections": pooled_prompt_embeds, + "encoder_hidden_states": prompt_embeds, + "txt_ids": text_ids, + "img_ids": latent_ids, + "joint_attention_kwargs": self.joint_attention_kwargs, + "return_dict": False, + } + negative_kwargs = None + if do_true_cfg: + negative_kwargs = { + "hidden_states": latent_model_input, + "timestep": timestep / 1000, + "guidance": guidance, + "pooled_projections": negative_pooled_prompt_embeds, + "encoder_hidden_states": negative_prompt_embeds, + "txt_ids": negative_text_ids, + "img_ids": latent_ids, + "joint_attention_kwargs": self.joint_attention_kwargs, + "return_dict": False, + } + + noise_pred = self.predict_noise_maybe_with_cfg( + do_true_cfg, + true_cfg_scale, + positive_kwargs, + negative_kwargs, + cfg_normalize=False, + output_slice=latents.size(1), + ) + + latents = self.scheduler_step_maybe_with_cfg(noise_pred, t, latents, do_true_cfg) + + if callback_on_step_end is not None: + callback_kwargs = {} + for k in callback_on_step_end_tensor_inputs: + callback_kwargs[k] = locals()[k] + callback_outputs = callback_on_step_end(self, i, t, callback_kwargs) + + latents = callback_outputs.pop("latents", latents) + prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds) + + pbar.update() + return latents + @torch.no_grad() def forward( self, @@ -503,8 +613,13 @@ class FluxKontextPipeline( if isinstance(first_prompt, str): prompt = first_prompt + # Keep negative_prompt from function parameter (passed via API) elif first_prompt: prompt = first_prompt.get("prompt") or "" + # Prefer negative_prompt from prompt dict, fallback to function parameter + np_from_dict = first_prompt.get("negative_prompt") + if np_from_dict is not None: + negative_prompt = np_from_dict # else: keep original prompt unchanged # Handle image from prompt data @@ -524,8 +639,17 @@ class FluxKontextPipeline( height = req.sampling_params.height or height width = req.sampling_params.width or width num_inference_steps = req.sampling_params.num_inference_steps or num_inference_steps + sigmas = req.sampling_params.sigmas or sigmas guidance_scale = req.sampling_params.guidance_scale or guidance_scale + true_cfg_scale = ( + req.sampling_params.true_cfg_scale if req.sampling_params.true_cfg_scale is not None else true_cfg_scale + ) generator = req.sampling_params.generator or generator + num_images_per_prompt = ( + req.sampling_params.num_outputs_per_prompt + if req.sampling_params.num_outputs_per_prompt > 0 + else num_images_per_prompt + ) latents = ( req.sampling_params.extra_args.get("latents") if req.sampling_params.extra_args.get("latents") is not None @@ -568,6 +692,8 @@ class FluxKontextPipeline( ) do_true_cfg = true_cfg_scale > 1.0 and has_neg_prompt + do_true_cfg = do_true_cfg and self.check_cfg_parallel_validity(true_cfg_scale, has_neg_prompt) + # 1. Prepare text embeddings prompt_embeds, pooled_prompt_embeds, text_ids = self.encode_prompt( prompt=prompt, @@ -651,57 +777,23 @@ class FluxKontextPipeline( self._joint_attention_kwargs = {} # 5. Denoising loop - self.scheduler.set_begin_index(0) - with self.progress_bar(total=len(timesteps)) as pbar: - for i, t in enumerate(timesteps): - if self.interrupt: - continue - - latent_model_input = latents - if image_latents is not None: - latent_model_input = torch.cat([latents, image_latents], dim=1) - timestep = t.expand(latents.shape[0]).to(latents.dtype) - - noise_pred = self.transformer( - hidden_states=latent_model_input, - timestep=timestep / 1000, - guidance=guidance, - pooled_projections=pooled_prompt_embeds, - encoder_hidden_states=prompt_embeds, - txt_ids=text_ids, - img_ids=latent_ids, - joint_attention_kwargs=self.joint_attention_kwargs, - return_dict=False, - )[0] - noise_pred = noise_pred[:, : latents.size(1)] - - if do_true_cfg: - neg_noise_pred = self.transformer( - hidden_states=latent_model_input, - timestep=timestep / 1000, - guidance=guidance, - pooled_projections=negative_pooled_prompt_embeds, - encoder_hidden_states=negative_prompt_embeds, - txt_ids=negative_text_ids, - img_ids=latent_ids, - joint_attention_kwargs=self.joint_attention_kwargs, - return_dict=False, - )[0] - neg_noise_pred = neg_noise_pred[:, : latents.size(1)] - noise_pred = neg_noise_pred + true_cfg_scale * (noise_pred - neg_noise_pred) - - latents = self.scheduler.step(noise_pred, t, latents, return_dict=False)[0] - - if callback_on_step_end is not None: - callback_kwargs = {} - for k in callback_on_step_end_tensor_inputs: - callback_kwargs[k] = locals()[k] - callback_outputs = callback_on_step_end(self, i, t, callback_kwargs) - - latents = callback_outputs.pop("latents", latents) - prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds) - - pbar.update() + latents = self.diffuse( + prompt_embeds=prompt_embeds, + pooled_prompt_embeds=pooled_prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + negative_pooled_prompt_embeds=negative_pooled_prompt_embeds, + latents=latents, + image_latents=image_latents, + latent_ids=latent_ids, + text_ids=text_ids, + negative_text_ids=negative_text_ids if do_true_cfg else None, + timesteps=timesteps, + do_true_cfg=do_true_cfg, + guidance=guidance, + true_cfg_scale=true_cfg_scale, + callback_on_step_end=callback_on_step_end, + callback_on_step_end_tensor_inputs=callback_on_step_end_tensor_inputs, + ) if output_type == "latent": image = latents else: From 98e0a49eb09dc3cf2f9f36ea48795cb951f5c1ce Mon Sep 17 00:00:00 2001 From: WeiQing Chen <40507679+david6666666@users.noreply.github.com> Date: Mon, 20 Jul 2026 11:00:58 +0800 Subject: [PATCH 022/240] docs: update WeChat QR code (#5204) Signed-off-by: David Chen <530634352@qq.com> --- docs/assets/WeChat.jpg | Bin 183022 -> 292129 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/docs/assets/WeChat.jpg b/docs/assets/WeChat.jpg index cb7300de29fdfe8466ee1d689129218533e9bd15..fc5894e7a6f24b74c910238cbd98ca4c06fef0bc 100644 GIT binary patch literal 292129 zcmeFZS5%W-*ESji4ISwnR6vA)h=_C|B3(d5Kx(9kfP(ZEh*G6SKtO2`X(B?TcS5fs zy|)yKR1<1|5dJ*xx5wTm`(W??;6Hhvo3Yk_!43DC$y{sB>$>L5`Sdv&a9LklPa8l* z1prV{z5wTQ08PLJs(-G3<_rH^G}Qmh7inmyX)ay7bm>0>NJo1KNC&)hiI#zuj{cvE z@*N`sJ>x$={PR!#=T9$Oq^7<|54;5Y&n^GIrt?++EA1r|O(iweHNXW{Dr#1$^G<*; zWt|uQvpmXT|L3B*KuvS;5|FYC2FeZ9mnqAqrlu_VB4x#tyF)0y2hgxyWVaf$u0 z9q^hb$L)7XxwInp%iB1gjNnD(?7hP1=()Igc=^Pxi{Frtyd!^CK~YIr^MRJOj;@}* z$x~A^^Jf;84vtPQoL{=Qdi(hL`3D3Bg-1k2MZb@UO-}iknwI|QbH>-a{DQ)wZ^hp$ zDyyn%YU}D7+B-VCx_f&2`bWpcCnl$+XJ#>2-15rm+Mo3e!tUPw!Qm0{_~akEr~uUe zV-{upf3=I1V%LR#-UHx2c2QmMrwnRVnu}LuFR^Jn2HJVDU%UN|mg9a>Zh0G>h};uA zr@hw*J(uVmj2Ph`)Ba=G|2M*;piQC+VWLN`zyiK&{JIYee!GN;N`62mgkI}yPY|ClXc!wK3=~rKD5TJ z@MtxwCg7`D%3~viSN)WkOuBNeOjOK9m57G>@#^+I3gpaL$v)~wit~sWQ5{hu1>M%9 z4Y06SlgG`&O?uxJZYF(`b2*Y`ER>M$aEge#_Mw&3lE;gTI0w9HHt#wI(8?Q%8{(b2 zW%xTiUULP`Lbz;`bzfZBlNlKMf}R?5BDqEVdh@vRev08)CL(vmmd}1q%K8m?L1!@U zVCiD*#rzkD{o86F0y}zW`yIg2uc7C`Y z)UPCB?k821C;^e0wlf3+GChWnXE`1ZW~#zCyIz;3N#{KtOAK8vM(`QkUU}w!l(VDK zW{m?TJ{5T%7iWtoKFP{-v`I7w(`8MGpb2>iyipoouIMlv@5f1OCcw$`+^6~^2Yho) zv_jm@{+Kg}T|LPMe!SwN+mO`ld*3 zPF;A{-7izWHadvBpY$WXW6P7mtZWpH<&ILk9fGqihj{k1z_Jy!MLuZMQawXo0xIsf ztsscv@OE$viiva;7()gTan=*Q38C`R9wi$o)?!UG?hF;$SJ)$!sMXxIAW?TG{c|K^ z89qJHB^9aIP;LLUU3{E1_Pe;@gW1ZM_bzl(EX^b~^@i!1!S|7n9<>jidyMQKWCtZu zsEd(aaUCbDSJQQ>+T$Iz`p6sxUjsry598dX$RSfC(8OWvp{jxtL5N3yYvZNe{EOyh zA7hrC-qF{A+lRjA0Q%G_L0X-K=#)jI(0A!5E#k7num82MR8qdXwPVCjl_J-!_I|ab^ zB}N%LFJ&c{?&^#`xnv(|ZUVojWr;5=arTN4Ef>jR7S7oA=^pK6(<{7(dly-2GTCC| z8O2wCEml(5@S1O zIw|nMr;#=(TL&e=`75;d9Duii%!nVB5GHCn7#MW-Uz;8HTXlrcIEara$!~P1o&9oC zehvAx_ubAqqlni%gmsTmDkP#xiTm5;Mg$tFsNn>AwE-{l}Yt@7q9{6FqAS z7`sWgZD#mV{g(YMFgjjbVo!J+9N8k}X8G`Y!;fu$Dis_3?7n^EqmhGPX>Virlb`86 zUX1X^TOmv^IWXRUWmX(_4v0GkcnC+e@Dn_`;aslDA*(g(zD^-ngJSzst)?Qz_+^DF zLRv6Q%3fF1@pLK8=Cja?LbFk+@t*^b5Iwq6E$^NZgXu=0JS`4W-isWM@BKFpf%I;d z3}N~Q9EhBQr#KYFO)wC-&@o}&CTTnaMr^1P&aWx5K6&Ifx%=s3nu$~i&nrL>*ANDx z5A{&Q7ql1w4`ZFlXMGTQSeuoN`JvcUfrw;7VRp}1v)G##9}=v~nw-9`I9GQBK#-o7 zxS5fGgECDkP-7g7iu6#yvi6<4e8!U5D3n+j@R?IT)ugX=0kLUq-02b4a_KBjm5t0m zcnWixd_IPI&HY4{?OHvf?jvH4=naxv7nvIxJR%Q{coSfvGd68&xqWwcuI6r1ha;n- z4{gr_UGis?9E*4_G8hAROpi4 z9N`#bS zVJ^Xzx?%oiX=r$ABp?vc1K%)t%K0%H4u1dBEqZ+@JGhT$NdSe;4W-3jKLfSJofQW? z{XWS0>t#qU?x4)%5JW5{sNjma-Tv|tGtE6+oL}`zI}D{UToO7Uc*}7Qc5w75(v7rK zpHQ6xXlC&@hSfR1F+0+J1K;L_JSX1Bk~U9dVu%hPOsLRn(0F$0;@XM@mnZzUw+`>X zf|bw#t_e=F!Mk=Qd=h(%!r7Ld#h}em(&qq~56mrP7eoFzPcz$UTLED2KJj_;e&K4< z)(WDQYV0ux=BjxcM?J`k4mQUqWfh+TUYMme2$!rRJ?b_=X}dzz304muIeut$;9UQ@ zd-`-#?C0p;oW<2(7l(wgzaY+5_4%n*VPP0LSDp6kqdI2?ZeUK&_erh&Y(aq1DF+s*M8JT-we0gJ&co zKHK$O#P;ba4SBr5xQDVoT{9+*$89$srRh4~4&RlXr|R2`J)Y_iuB_b#ehH{G9($En zSB|op?#dqtE4=c|F?^{yG!^Q7R!XKLPpV&UB*x%TT~m68QN~p+yYpGybN7BL@a3Ty zjHA>g2sb}}2`K5TNrSuGu3jvbZWDUysv(Uigh?I=DSz+y?wjjw`o(UyNp$>0&S|>3 z+yXJBjZE`%xJmO{bJt4RXSxim-tA|fE_o9(NR`ASU%{cyx%l2^B0mA%y~t7H**GlX zEeT?WHODK&z4m=CCDl}&Gi$Tt^F5w?0VQ*gpwB<_ULoV-!)HQ|L#A^;oxST%eG8O8 zDw%6ux%8fgHY_DzbW(CUCH_Z2B`=~@pL+-3O5Jx5q7YsECf-c zEB^#e;xNYKX=hov$;hF7&H*~9uu_<|ElWGG(hy>JpYp@dR7H)!cvumj(Bg zz$Yb$6^I$hON7kI!-31z0D^b)Y?ZJR4quf^Zqd9gsS+YhrRPj_+U-7kuwHVC)Ew_E zm%nAPC{!(B>8ih2v?3l$iACv%Dnxjh`Vz**a`MbRMOM0MfY)A`k*n++&}Qk;xxjEA zMhAU^=N!!1?OL|^-D@jBg}bJ8?9aG{EC^{US#YtWxqj5CywDs0aCdPpqKgm0Hq~O` z+mBC0tX|dziO07{xEz)1@h%C*x43ur&`>87sda_2zx}oM%HV99;pW=DJ=S`T-SmDz zVaDW$`&utqil~I!_J3AAasXqqS#+6|$J+i0ig?J_Df!{*{3Z2A)5|cXpuf?Su(EJ( zZf>2dhM!3MIec>*@iZoR;kYX%ofiEpJPTSq-vE1Up&iYRT`^5T}LLdPL|B#<+@8QfOu zUz_t#`yB8%*)+kTtxv??+W)XD#hlpTjYbLl5#oZ-yIiiRmep!8aCEU3QB+0t%g*y1 zl$sn)#~lWYNJ}?4;IX?lH$#xGOQy*L2HCa&0oJ+1gMn<*uKRF#+M6D7zdMbh7=-yg zzrMb6vJv=q%baptuMlVOr-xy9L>Gc*!0BRQ(zmm(Rc)R9OUZ-gGi9h%zL0s^&gSW1#*W|2=o5;lVXP+u=pJBow;-S;Rj8hs@ zbKQQoZq=e3**JxL!yQ9ua@;mWxYoVYK#9b9v=vw|K>z4o&b;g{Pb=}yv?4f~bEs!t ziCnSFDY09mi1axY68OK=U`$#gtPEM>}szvq{=6fbN z2gEYTk8aOhO~?-W@oMJugiz--|Ls%)HzkYfSqkSXq9k!}zOF34a=o=@HhR@gu}(A| z4}-Y`SuBbEn4vZgqPTnWod)qvLw@tLr`~a;SLq|;eLtk|J0zYdB1c%}WDdau?yb=e zt}Ry#89L1xB5|cz$_ZapBcD~irhl~8gLEZ|93B#rFiT+w#xf5d)TQ-VKV<1_bC&F! zm8gmcDC&x2;vhsgBIjn5WWH#%pYDn3{aiEn9pi-1YovR41Z9D7mI+f^OJWB(an3%3 znZmp`48N(`WZu>b1~X6eL3Pf)wt$HJA;TNDpJ_@xw~q4{d84MWR^afb1vIu_qLMbQ zs$9GKt1Y|8dpCYfq*SMLbMX?YGV{@h&t_b3dg|M_zyQKrF*)_%XRDo^miG54?SRoP zY_4H9oATcX5|dUw@k<((2xJg3PF>StNKgY+!5>^iL%dtuG278zK% zVBP0;_t?&}KK6#d)c8^btV0DA2V>chzac%4X_1I8$pNL@%8SgEf09w5U1zPOKal3W zb%Y>Z?rSti}1>9u;#*zud#>l8J?5z!ZaE!-SUSbL!TR8W0~cmJ(E7V?k3*1o0So{>IOV3u6mx zyTTAnBK;w9Cj9UsvHbMjD}s6>@5miSYWeMbr-#M!+CMZz1ZjI$TRxtpqR^j1sfjP7 zRhE8UtEs&{B}*wPS~8WHAIH5ZOh=ianTOmz?%P?PlZaC3|eQ`Mf9 z%mYwuuS25j_joaq-?abslCuUC3sYTw+%(r`O`JcmK_JY@Dnu)M9QG-Mx`jUwVPA`q z-LOpjI)Cet=C@ZjcvZ=qo1u&ZV5241BVcmDjEptXN|MUCTzdFgBOC*`v%9c9(Q4})Y?wfV#*|Kge^J9!IwM5Uu6=OBG zom~Of5-q*5T69zf;?Iu>0pu3yA&WWi8!mwxBmM!?QK-T;oG~3MKocu5O$Jw)KmwXrawl| z=ozWc;z;MspNIL$t*rffSxJybp>3!damoS1u(Ymzfv{inYy6t54$Q~@hdIDk1=RY=-il zx=VFEV0|`YIIoO;pQj5f5N5n-)EU~W6#^G^0$!>GgN@ToR?Y#lCheXi3BoSaqO$tJ z%ondV;6kQ5b*Ke0T;AjAbMu3sL~m;-qc=sVm^Fa2I?n}G@pEY`r|7+ z2ZjCwV|jAab%KaruI-;|7m0~urkf2w)~AW}@^TJ9<%k++}?t04p%gKNm>E@#MuGNen)3* z+Mr~o-({5@ocz`jYYZg{%RQQ%J{y8`ns-9*kN?hE_8jV;rTfanpX9usa&alPv^q;N z1~-4a0@lcJTO%+(&fO7e5&Dg4GC%uCmI#o;xmUT<^uJP-DXN-^-K>86o+&G*XtfKb zCX9c=5*sQ@>>z}Bu&rC#epQw6@OnPH^Pl^ zxr;8;h!(?BZH_iI-Gs;+VkO;!O-mjvANmY+8N`j;WK_7F%gdOOQ>0eMY3*u%`vEIl zPLLMcvPHpQ`6Q{cY_bp$*fB}IG+Kq)c1HB)>=`PuJ)mYydKccl2VtqnEc%@(O?2O4 z3VM?TGMV0W9rJP3f_Te3Y`N`Ps1jcLX4!m`g>+d5lNe6P5|DBG+1jFkuZ!Z#FvGHT zXQ17tKWVxHcXa#vI`pzuQMM@bB7%$d3oP@Ow@cWjlQ?Qp5s%x)o+Og$1`Wvs16>fdDr33HHjxCk(c0yIzEgUR@QleIZ zgmx>HZkZ zz~97A&VEY3IYGC{7DWd#s|vIRjI~Fc+|3msiee6gUrv@LiayP(2vbN&*Xfk~x zfx1ITA+r_5*(Bi^Js~T28Qf62Fx--FQ~h#Ix8JmoE+_Y! zA!v~KXLuhbqS$qOCs^^O^e{T3NLUr?~ zsdO;0Q)9j=XOb%(KAVNHnvJX4Xir->_Eb%&Nj${Yqp-cKeC6SrZ#x%5y(qb40Lcai zr-rpD%b&q0Xdp)35M*Kl(|@+J-&h;wWa8D>>Uia?j;-6!)1P&GPf``<4F0%rnhVB0 z`dIt&G!hz!yRQIi*YMI%Y|nA&>B*6yDJF0ZgSYi!$tG2>*}21HB?~e*V99fz;Dk~2 zL>y#GPE2i!oj!C;KdfMxN4Wu zwhc#O#YH{pH>p8L>ns>sGwdeQsR?<+*h9LK6hdeb)McUY$y!T1=YX12q14kSkye(DFkJqooTWOfBbM)th- zo1a-CCT{2!K(1>>nR)T}pkvAJ$-HfXQ*pT1;DF`$_T?WX-EV^Ew+G5cy2d;{qJLFv z)bW+CCXXLmf{CIi68j6@i=*UjGK1a|3WqZG8*A**Y_&$F>pNxs62pJ!gs%_2ubO(d zh_1Fp!F+Owjm#F{fQ5%uvs`p#kp5^H5o(=o8T_d*ZX|U)OxaJK=t0@V3izW=kC~#w z!UjB;LchE`2XI1M!c=%15_|{VI|?~gNMIm9cPeq!)kfqXzg*hKr{5Rk_W8So@ee(w zxv@YBYe6tI zFc7rI@+-di_Rwt@%@_ib7c{_J!I-KR6m+Yf{9Hf$`AQ1_P~09xQT)o4g(&%Vn}zVVo3FM59x>Zo z^A3|I8?|W?HhCwdpW=7!+;3jtKInwH3QsTLP_gP416meO9}(Xn10u~H8MtQrngU&O zE|E%XHItXT?>@+{!*U~dz?kswF>IOQRvJo+4o2 zAEV~9)LH-hgx z>F^i5;9dPD_;hlu3&}Y{@+U5p1|Td^4u>kAx#pu!-}8q7(V%_`wKLaadxgN!!Z6~S z=fS~jUwaiGClu+Wv=0B>G9{DCn0~2b0xo|DR{16CAsqYhZw#B1JpkJ!&_>22B zmBUTuSf$9Wd8WGs{Q7P75g+{Ts`Zn7CyDUG;3^oSW{KN5fVI>eS5y9Z`+0fJymd0-qLdy-Rzs^th4VkL3{M@!RCs?)ChSG#TAaF+> z%#eG&-H-Dcyz0Py@isn`;SWy5&8$IoWcJl1cB}PoYFc zRZY74U~EW%Gx%V)x|G7EqKM6WmK3|`e*7hmxOiNab?2G&x&=hLsgN0RzQfQtd;}@I z*Z%q>@@Vz(p|>7-eIjhJXM+SzAdeu8N=9sCOz@lSi}b@g(n70Nrt>UMs(&sDjOKkL zV{g;g+&dY;(BYDQyS;34mZ^QdnNvMxMYQ`mHo5xplL}#P|3VADbo*4-R6PIqnFxPu zLDbioF4qRGOs1OEvCC2Xsjb}r^x9&j>e*Fm-a8BLb$ai=2`y}C#B(Klfj%J`sS9r^ z8)fGGInn3O*6d`nEXtNxNQmupJlk3J`7soW_*oPsrK$#uxx*5}WzJ{6UH`IzChS7! zi(?%gKu+}`gG{^Kl$f5S&d?+>zEl5p5r5uF$~OsOJauecY(sUI9D@%w3x^8$|gsznhL}o$B!{>VOM}Y zLl9bAu;Hu?yqF>>Wk83?3lPS_X|O+5b#3hEw)wsdzVs=vTa2mx0o&yu$5bWcng^Lq zCMcP_0AtF1L??7rz-O&5nG(NW56w|qokhc(n@tWQhE)|as()wudw6hWzN!Cw=y=LF zmD{m6`qZ{4q4RdplPZtlQ{CIo^@mc0{V8lIMQ;akSwWQ%xW%|HL>0qi58lO&28~z# z?tRD4&@j5bOZBm#9?q4M(8REgK}d_<=PKVO3^W|rG8xdxP2uH`0|MQosA7V zQl=@8z)6;XKA7B*U+|U63j@uyKZ_o47f}OVYqmTHysP?)Q(iLNN(geQNFH6fTy6D~ zs763_`4<sd?1TKqj_hP)!f-ZvpxvqHYGK-5-*BXnzE@9UYV+ z;7gS{QDl(IVDYrVbd#1HZ;DalyM9i)Da8%tsf(vl0SFvd0TwiLfw-r~HtQl$W260e z5A`UpN3mzqx`g%mw!=gOhopiuT;!}8DuO)+00KmeoAE>MU&MA>VK~ytTWF#k*psHJ zgNsOPo9(mFU9Z`CH0o-SBES6RVWSf{2b9{fo<4^@tRa>YZceodaV9za9J|?$Oz%y$ z-UeGc!rM{CFmw{CqNWH>9^0O+Nwd>%;4+Xg>u@KhRyp&k$nh8&Ro3+U0k;x&?hMTk zNOHa)maS8*hr#YZ;Zkt>6`8m@tvKS78-=e5T}CnxOTSyMLzt>d z8gE&g11fGhT!qz*+!Eit-aLkj>52GqG*01~?h!2@r5jKr|3jLEJzCMUYos-_srv&@{l^t-ffw{KNJxWA509&`r6wKbt` z#JyJa(xW-!i5g)2`_6XTCC?BeQbxSHF`s{O6DeBZjvkWV2TL9_&WnXD+gcG z=hp(?>}H$B3Vr77S+TUeLafGuqz&2-uSTUYHFY_PWq-HxhGx0!MLe@UTpH_DOK>Fd z!5>skIpcd0zfwvnDz1Xu`{`>;X=!qZ@=H+tOw8K5rYq||7Up`K-1c1bZikIDzA?sh zL9e^LL&kNFmKa%z7au}JOfH7^7GpAwQv6jl57U#SsrASAJs9U2EwT+W0q*q7JD=7k z>(2o*yF*r)n7#JE)47Ab=h<+9mA}_1?(=rJ zk}lS!Jis$WuI5{9?Q*59-blGF4&bE8d1K6Z82A9ADgQCQY6YyG&D+CdIyRP(cM5++ ziCQ6D!4xif_PW}I`N&i2uLh~WnnmC)lA;5Kp})d9S`J!))V8CC&ZrapNf{EmaT^$) zP0}R*yXZ>a*Vbx?$|ZF#&t?LPVExq;U|*kZKSZWm!P3o9yVn;2iiqBCn#}t=)vu+x zJYBY7nXJ=clH^#IP%q9Sm|z4yDp)QMmluDS-bv6`u61Y;V3A*A4obOn93-E!^8kLjuK@&{C!6@nn(T|m{ex2@OEr1l`(EW2 zS@j@_`nEylHZc_?m-!*Q^k_EdXV8HHnW6LwgebmP&fB1?y2QH9+B6#f!}Um?r*_zU zg@U~viyISfcyuT_YYHS05pK)=yJBsml7mb#Loj}`8^HlaBZS28LvNcDJijk>m8?%0 zU#!^i7EqvCs5zqlk_`HDoA%pqrgJ1>W~xHn`|KSIO-by4V{Ma=YL7&+nguZoWZ+v@ zT|Vp~lEPL?U%@F;%iGNiqHKM9v zlVqy#M-iobb595|8I?95=sFKWN)J>PSpjun(8R1YPH^4GxKeS)X)jXoV13-Y1BHK@ zdBek_N!t6x57oc2cfYN-KWEV#d~oIC$XHDA&Yh=}4X984eRvM2f^X}%IbiuARkQ0g ztK?08=q-WMm3PR+p)gDLl&@;PvmIWphI!M4O$$nln2d7!*sUD^{v zK3`)ce}z%v=Y{{m!K{FLN8RqX3_pHfM2<_YLq@%?AeM>QeHhWFJ-A3)Hib(BW7unl z&S)s{QmfAz(oR;jpat(yY7+L@%`g2p81ILE*WV&`km+SlZ&3gP3;CKJp`-Os-KjRY zo4hE7^$}$YAm7pk><@A7`k^q=Ky2ar7|&N_{X!cwacR{lyvOBZ7d3h=pP(8kOG zn;l2$0tC(of3Bk&PfHSwf|CB|REGwKjw0|6&jGQ*%p?a-0t4D~I9P)0R>D@V(`wJi@txuohvTq4?@`H3%A&MwqAbdlxUG8UjKz z604C=!v>0!;&f+F|Ip=XK&Sr?U9C0&yzBuolwceIHO&pFp#T%BZyG*4X~~n)gvy*E zMjgRmbC3)Eq^DgW&l(TwP;)(84Fr$+Q+;5uhLZftCL-G>ink?9E4I!7nr&Z79w9&l zf&S*@tt2p!%F|!!t}?FThjS*)N9B5P#VatDZ%DIdyk9dmxK5oPDmQ+Ouln@UUx%1? z&%W^VCo=;t)+FZd5BjvQ5@k-cNzS!gU`Dbq6Ptp4rL8gV?RSkp(Qx=TC1fMa4oT@0 zhzPxquP#Z5t0^ke=hF)v<57hAsvrMRrkDL3#Z9dzDP3>bqs=1GiQQB%1u29uryS~@ zWo?lGp+myY?}qUY#5pFv9vH}x+ZWw~P|#spdrg*hUNHMh|Or0AL?&NSK{oVefu zSfOH}f4<+hvb)Su--{ghOH};5Xf;F>TZ_RM_NZSQrJGwilW@by`kKD%f8vMI+?QgK zaZr8}OY{R@By$H8hJYc-evw5kkY!r6kdjT~8Jq=vs>4l5U4&S`7MvfE`7E^m_r|m$ zIvCq@4mealOANghkfIrfy@-ss-Vpl?Hj?SSJC6rQjZWu+9LEST+@IH{l_H z_I~sz6x;b)jCjA+(DSd>BYkbP!QNwK0F2WUpj& z_vOdx?(!Pl+a3x!HaNU;mfxgIsM1HeHl2Ds?K;fVVCz|aY^k0?}KSS;y=o}DBLp8}V@*A4T$X}ks*gS4;CVe3{!Z#;9{ByamU z#f;>SS!Eq}&J!f|!5k=b<5DGjs{{?6W6}RfbSmihVi=&h#4Vi#t1~3N#@su8w(OlU zK7n9@g76-1)!8VmDZJ~;vw~*7J=I3H(;oe9oL**rL-iYEeE55~iT%`s;=B(@rn$tp zCrb`?d}kH2*H>RU?6z62g*VBlPrNw7iFYWGf?@bM8$x7c6_o>@@7K45N~W1Fc?7%@ zK{ffog2Nr7MH-6D1_*%`I&0#Qk^1#YVmR#UAjiG=fs3uXJD0tv%as18B!;zk5zFy% zxSsBzttFoI4CzUvQ$dpIYv)(ns&#EL?~Dev@|PI*gWR+p68JBO63i~&4cu|SO-Gx%|?;s@4`*8qs{4E z%0uCl!$rDoqRA+&!Bvvh;QRYuiMo#zcY>zkt;G+8kW?18|6nB!vl`HQS%*J7T@B~Q zfOk?e)nEo03iz5I{w#}D)bNMOu|4>hp*WvWAgMF=ANXlF>Pu_&pdpr5Z^@l%?d*%o z;$@nE5(7%K_zya@!U$rD*KnfQ26_L~=FG3r-}w6Wzi}uoC0;?$S)sEid$o{)P_37d zw=2iptNL0N9K4tNOWzV|SX6zHhD0=wbj^=)-d%p^eGYiIjLcD2pNCjNUaMa03cwKlSUMZxDXA zxhOoM`iP&*CVhQzt5&o(FQcuB$d6C#_w;weAGTcwUVC5^)dH!XYI^yD56rP4A_yCE zJO}XQ`C>izwHXv*4~&V-xs*CCPE&HpVDRF2YW8q!Q?UvOUhC7?n80?iX0-5DcR1BG zn=hs-1bD38Mfv*Zi4wE{oIL^OO zscHVkvl!=nmy+*6h+G9B34$Q$(&(DiXDmX}Vd-t)J_!^>9Kfq!6t*v-UC2C8cl;#; z)|h3%SRiGcFZOS&b3!eMHRt;qx0R*0mgj(YWT+g`Z|D(0J5Roe@{Fcfw{>3K>G|gw zr~d74y6*m@`S#=PU24WdB)8lEIoCY-$*S5bE_s~DAkS{gS}IZa@X1-Wy5P&C01rx$ z4DB7m+wJ4_3@UCRal$|QN^|j680RxuW!Ty2Ibd_Dl7ZnG@I*lMCU7TGW?Bj^Z1)~E zCks)Ns+1|d>0+W*hb!L@Xc_H!zXSyk<%{nRZwNEr>0|8{(0v<4lCus9r97fMUtJX5 z{fPWqol802@Y7#&r4mI}?yc zQ`<>ZTe9+T*1ozg!GFu`REoGU<1@qqWgb4bA)*w^A9w8$`xL6a>@uwFIvAa~>QM#W z@cJoX= zL5Xe8s4>9FQ~|(RPpc?WgsgP z3P}FLPCWixY)70U?A)h@x)40oF0-&MzzF>XhGKyu zHE*Sa9tK0ds4!;i+)1!99X1&lxyd&Jbi9T_=!VKdgJ1q!;#IXtRtbpR6of4mixMXY z9Y)m(3J^Y3>)4`XvKS>j#eJ-o=(6F!De0Jno&=l&v`0fFUCJpUBVH2Ail z{O|h8ibovb9Ve0tYQ1B)7oTTlVucr-i6}mDsOVW4`Rcl!ls74u0IGY^&+@*^7&9;h zC+SmS^*g9bMdRu91Ol?Fm37ada|%-QSMghx$oRM7Hrus*ocq@arY6~s(rwmGEI;>4 z!L%1tO$*AlBFC=))@nahlNKqSf^w6YHNZkh90M4cXp$ZQ)>V6=H5DUjFsHI~*}9LW z@6e*xJRz%u@!hx|a-F4JL`Z6kgS9)4Ruwj;Va{U`Hr$=(<@`?r1oeMoboj?+#i2xT zA@RP-0TxIe@v$p&J&=EsU3iXplU4@gJ$lU+K;PUzd?UVl8(j`ZMJ? zFQgQM<SU(%s9PDvI@F02Gi!AmfZ=x&5bdj2EYQ+Io-W-VE3S_Le~Mi$?n#Rg|7l`udz} z>p-V9DTRwD+l7)bN@GIDLyAHpHTC3McGR_}yedP+9r6)--%R4B93Wd6sggxv^%N3i zHU!?i#57YCJKB;5Ppu3uuS3?5_>+Fz0MC1@Kr&Dji;bmj;HslYcMtJ?y%H`-@O;pj z6dtZT`qTPCyv_%ui^LJ{so3cjIJ z`$PUoMA7B0DfnUeXj5hzxb~w0S(O3@L0w#QG;jABu4_pK@IS+QhL6_E%!RidodY=B zKhRi?W(1<1mZab$uxEdn@#zOvEaj zN&Rk0Yw=BA;R^xUVws)(-y@T9?r5gj=TZ8rizqc3uECpY61ndZh&-+*1Bb$AF+JO? zcn>m{bC=a&tiiO6&nmUb?YxxZ*p(g!5u=>w>eT1zi0N@8e!GVn{_d2b-+4D=Flky zD5;+;4{YClvMB55KMh}>HMBS$QT-PVATOa4N8z1vz*Q#TqaA(x-Z*Fa(LTQ)2i%T-}+j^vlgFRJ3)|?t7bv8hJ zd!2kYi>Viv7P{E3h&Hc~;;sJBU8C>~2Z{LVgFEEz@Ms0i1yva=oH?M!$i~>P@lpJ( zCz`$T4rcI&wWF)`&5Z`ZS5WKzYy_?$Gl$l{lPxAAl+S2*G1k?!Kg3p@l{T|Ws=cmo z4tSdb)<5!K+EiyFqE1DKB}QW;OOLhjykyYuvgikV%A{CX9zj~qb(FZ+I)U|?OTT@+MYhh(EFHHaRCp(CMOqj&TQCT{-z z#$nr=dmkYU=aD}V$I#Kcxm7v7pYva_SV{T#t?`qC)P~4UX?R1k><2@J-Jg-&`~uXA zU&m#(eKwsNntXL9ytaA(=5bK%v+rhfFUFuu&WBO=AMZGxehVoXRk}o;Krl;J^Q3fn z@|eyY$%*FC5RZb@GxWO)PaC%^mm9qJ`gC_s90=oxGqbl)>|BahG(+|vP%C36wsJj!al zR^>u#wbFZ3iy0c@^91UWKmOt#R|A0?puHtZ6Fx@Ts(T4<#Bw~+>5I=WNAZ2LMlw# zZ(F>HtXyuVo>&XE!W3}lmTTMTE zu1Y$g7j6SLAxb21VxtpQsmvkeVW4WQ5av^!BRribY1MszbWw7#N+yRU*7yjX#c2;%mf_ zNd}M_TUJUV5|TnU|BM$tk#!@$C=OIu5jsN#e2!))sdrdEjd44KwFA|q#(;%+|ELLn zUSfj(LvnO}`EOO>;l#;!GRIpaeyJFI)G;-HsQw#0Mo9x{WcD2-w1Z0=8rLN=PUf)c zA5Ur{#bd>!2xP)jW<3CW6Cz6D44iR{lDaE3`&}d#jvZ6FadhqtB%x~7R=6_as5aF=94q+fq^kD27 zswY)-zRc@|j#kXiTl5-#YH}3#c_6=U5$q!wT&|OCOAq=iw`0r{(1VWdDrI?0`Irtm zjD=_hgPBDHCC5MtZ~<5NV?dL|lrqpl(nZ2ho2;8UM|p}-@}$2(foGLOYr6yWIz*Zz zLrE4tLvIr`@ZX^Fcu4!{m!#P{Zim@*QVY*WJ!)>1XNmhn_W#A+dxbR>MeCwLKtM|9 zNRg<3popk61tcg^6cHO;hzdv(A}SIkCKM^si-Lj@6af`5N|zQw6%ddvH3_JQQWDUx zAj$GhPxE01J9dN`F>^B z-zYwv5&tD#PVJoQ(!Csxe1PE^v4VDA&H0O7fQ33~*`?^|aff77U5lRNUDA%SfM3tjsulqXG<0`BI8W$y|GLww8zi-q=;n9~O0Msw2C?&8z|O+j|#@hJb+<&K7XyW=8y``6Dj8>&9dc(m1Y&^7iP>45}K z_1E5xYnN1hFJe$c>iJ}}=aeVvx7<5`e$F&DvE-F4YY?PA__T?%fB%23`7fen`%mhG z5a(QLnUUw5#GcptoL*&h+*}!I{?zqp^UQ(O-vv+v$9fE~Cp|cU(z^|ux8nj|%75$f z^4y+M_vdx~h3m4pCMkCV7n7pb+uBLk-70D7dvILK8?*1uw3XW1@9wCVb6oyFm!fhk z37?gos_qyG+!wfTmH7LPhQI@)b|M_|Cu^aNqg&KM5jUgEma4)_hjy+mO^^4Jx|%vW z5%HzWl-7;&xcF{HgiYf3qo4b-gv|wRsOGOs1}q>aQrfDBj|SWS(5LW&0cG|c1D6# z<~RKJ6hD=~4u1**`=EB;YhPBkS0bA(A7c?C+EL_DSvS-FvgBQW>b6IwIgcORztfi) z7QcZz6K#A_Yh7_>k5g9tZS=d-iB+V^j>uhGaKkxdUT`|)5?;O$n{aDE*A_}aEyr_H zc`^HzzVk{=rBvH?R65cVJpqMo0om-X^9LzsX|y-euSK#*nmwGJCxyB{|5Gf8GcSTuO`@QnK5vz26Mo z$OFsgGco_vA1$o!bBK!cHRkPkN-EP_=-cxsgHDQ#>#(?@{i~QJ3%RYXGby11LIwgbVRX3L;1iYrKbvHvWk>;vfiZ(_;NuEsZj~?0RiaZ^d<; zz~95pe-V2)GO(OeIOPZbUqr^azX;F*>Hqn%Hz#GFG|9QBTAwb}t-R#F2rv&p=uLNf zqa38bnAqlaI;W?6=+gU5HZ%cm`ydgydG=^l=FPqAQHf-|^wM}X@;`pG{XTW3WuZD4S))l zv*zw0^ASG_{YC65__v@_f=|AJip>A{^B?f!@L$9a4K#1f1DJ>Z@TUJbkmYBWTzlS! z;=hQ`CyAV;l*Jwl90}58w2)sY=9_v!K#N0hvjGeqpZ}kh#~r*f+z z{m#>KorakAMhnjhpiElZh7U-;GaiR1yYP>D?8`O%a#0K`PCAv+UW){HS@|ecH?gW} z1T711oj3k%7<)^~-?7;|F*yB*@s_bv%*%iL zrsUg2Ub{Qojs4QicBtHGL*Ro#wx&Gp@DjWSBXdls_kZUEl5-Ip_} zmxIkw!v>Bcm8HrpcP)Dat)jrl{16~_ z3$nr3zL3mOD1igq0_xF5jlyR$N8Eo;9r}UZ_NTadbwq9VWTXG0`-rkh_m-ZZ|wi z{)mALiEYG6>jh5^0T1RG8O*hXZj#-hE270@$Yyz=qSNag6E43hoV?;vwHPm2BXgx( z>bul6Yej<@jLLE)20C;e79A66p>6ipo8;>CW38TGiPzR`OmzxBx-PL4-BKz~tvn{Z zGD_WK=8Aw*xq35L3EW^yrs{qsp~2_Pym9GMr@L2+iY}Ib-jpj^xNq^tHQW_#E;Xed z6>#`jnm@1P6#F~Do&VV2^NOJFTXGrDn2>S3z+kQ;TLzQpZYi;X1 z^u|MgH5X~7g+rf)qM5X|@gPdxPj>gsf&k+qKQE3?AEVv2DM)-2awJY&3C7kR;Oqq= z8cLzpI@V%ZN#mk3xzAs4_YJ>woau07RlUeI3*V_*bHcC4{7q2Y8T9&lrOfh1URoTBS5sHe~{i`xd&3X zYUl~$x0S$Q62WlyVyf2Bm&)pEX>9DmkLo)ludx*Uz*W7>iAcVU2mj5xky{Vth!Clr zMCLe+d8R>zEe-}e0CIRf93O{~5(+~bIF1Qyn&yJMQ2MZdQsK>s@*LI6U*yBp_m2^_POx%&E!OX8y~* zlVV~A5IocW8&X`?gtr(0S$NHhBhsG?6h)iq;i}eLS*1Os@oQXBMnFgnWuUWRBS*}P1@(b!j)M=Mw&!NWc-FsI$(o&x6O*@-% z51e89r@!vRnbx&T;K}*@Hv8$T9 zoDLy1<4=v90JRKcNLE_6Xg%mZiIj4A_xZQRO%f(igU%;56!f*)9V<-A2AauOJC#~FhoImWKl2p z)~?Mb7QbGCoul?hAUagCxiIfJh5i!7ys5B|!*D|@!vjRA#_f={%}@xRr{}8xSYY5% zd{}zQ_6H}*zg1Z+rdaCp@$~yKJy_^#35XQyM_{5v)mfWF@WiO@C z#do=dk@omR+9mE?pV)h2$J#M%5DrGKg4%-H#Lq}zL15)1FzwVI{2SUYC|SBOD5f(K zI%GR*-!4M&0Y62dxSnKaWZLc39J4ov-m}xTA}nwkr?q$W;rK+u{(zq;UjV7ZiMwsv z8+e2wdW$kHY6TTGZu$Tvys(_TJ!HyP*b<~4L%Hvv=w<8RJNWTnkoWe8bYV4pK2G-; zkDqsgo-(ap`AeFQo{tN3^m8~hkm|hiE4L@wGon{EclZba}=?45zeU1OlzR+YqnKZu$rY# zxy)C0;ji81n&!KlZ9kq>KlR}-|1`MSpb-$sBJ;qyALAYdh<+#UG0=vErstWxYi*fP zt&Z8Yey)TUnPoxGyms0xCGIzEe7rmw&;9Wi5!clyOzcmFfFmd>iiHoDqY5yE-`a*?3-0#vxf4afMUNBs_T-(VbRU{cTq6(mL5aomSl-rU zu!YQbRY2xt`8XDq1k-=CuSxVK(-WKncxxz%L5v8^Rb_n#Jz?>JjM4y?38OUbF82$v5jo9R^7W@5y@w@9mpG-yf}w?#2<8+YAfc zt(KkdbWbWPJ|t)PDDEp&bWFNfa=qlTI)eYtoA^0`F8*i~L$?_#i+89URAuXmLATc& z)4L8IGiaV@THV?rc{@`ikv@AwdxalX+XI24WMwi#;qKL;Ky3)Z=g~f3btzmqUA7P} zrt)#UHs=&!z*RNsZEu<>VO#8{`k_rW*VB(Q)Yi2retn?6<*~qAJa_uP?gAzk#dJeM zdq_CcIT7d1}n-K@KkBiqnofonIlTe?eNa(flI?d-17qO$16%x4y0Wc4F4_QGeS9U>fkqH0z~eo zqbhQwv|`F5OeBBck8-!#P4UvsV>=%)Bw`$#_O$W8r3+7f_=xx*-m0*r2WxC2*LA>3 zNK8Wb&=PWM(derCEHMeibV-4X>ey|=^&k`2T5YD!>`Ufa8|Ig|`zuIKLd5oSCJLDs zuJL<^Rdb%Z8@|%sdy=1!3H8AezPvP3BaQ-O&+h)*D23~DE*s@K=*IjZO0>vczxZS0 z>FpQaX8pV;09IS6h>-s>|8gl_X0k(`|Kh6gXJPix9Yrns%BUjim%)gyA00 z2h4#7PR-?I0e@8`=F+3tT_dI@jBOsi{m;uD6JO3Je_lGvzc-_)rc>%52{0HqVlODF z#F+z>F*P_zZN>-|+lz+2-Ni2Wf$cLR^w6qe7tw%@6H29&`y~BNvTt{NURE*m$7I4V za_T;I*PQUTxMr!QpOy&1(|$`?#M%G7fB4q_`BlJxMn(huaD(DQoUnF7PQrE; zv86ju%|6F|qyl(G*_q|C8J$6;#-ha`S`(P}tT%n*}nEeI!Yo9#1UC@10O)f@pTBCWlc0^NGsss(*(#Ht|?{ug5 z+ebIbGyAx*K5TH!29m+k!=!@}tXfj@FP>l!R!l5Osr$yJuH)Xx)ILRWk(mIYnaBia zIkXSOl>tUzt(lZ?V4fsifr`35f14uqM^_%wJ2N;tTCrRI)$>T4`pU=p&4+e)9>m`_ zPFpLULqgg#u7Xc1is^`sR+WL=O9(xV1R;A-T#TKAZeaa|jff9A{1)FH1^8C+7Unby z4`52s^1xucR-$mF6JX)j94&5OKP={ko8nq9SE(Dyext^}oQ3&u!TU@<{}QW9Su$K~ z;*9le-Hu8nGB1FxqY52pNF$W`jDOa?4V6TYVHsq_nVzoIXvmEa?6Q?D#hSvNifLAp z<}JxLnj75Mr@$UxMElXkXW7@7m_=O%dfdM0u4B%u`<89zx=#-UVo$MIoln&sCceI$ z+#%OWq`DL74bwZpM!w~WGUuo|o@~!D=5c~5>q^s0HXcf3}1?>JKDepk`$OPb01 zn<=Ig`YLu^mEyC^5r8s)MVTMo070Zc4X-8y329Ll+!&R;usyCw-YyBzK`8(+Sv`JjQ{kX6hgZS{s7EF9Z3z52n7#H3fe^ zC3NIU^B&`Ob4-4D$ks7wi3ifsR3c(du4$><1%d0~0o zA(k%0i~J`}h3k7#xGE*+%OHm#FaQCsLu}kkme>p&bx{)=7#y+tD30DD%^PkXHto|} zpwzE$`?PN_XWjm^8V}|qc=|8mkuw27{Gv-mZ(!5I52EQWXe)2fO~j3P28pK-f>9$< z9sVM|NfMd%WR4sjoPvT{?|V1Fvp~uoorS@18np+3$d9@=>&oyJ;J0W+fqC5{ekq>C z@^l8Z_ZZYT6e%hEu9y(II9swjthS=2%ivSR( zxShD+J^X9K4H98usOa)6s;&rE01EbG2Z~asL2p?7n!f4%ROxob6vx{WA0DT>JWfBM zKDv4WNYE}ItI_{XjHeS2rr+JgpN0NN{ewg>-4w254(zDFwC5#x=>0_u?jvJ{EQ$Mo z6MPK2?hIbUHoyUUbE>uhK|K4fXQUiAnn?AkShN7=BC-r0RyMG(>%(KN_0VGrtkms2 zFUwUVuFFc_LughKJu11qAT;wQGDA$qg9?1)+8R0*mHIEb^=-ZNNa^CivfQnbHrts; z^N^o`i7?>6TjJ6Fn6Wm>67CAoOs~4ch}Z zF>fAmKZ5PGBKn{bNKEWs1XnK9oc9ua^_L%md9~!b zQ>-~#1%b`v?}O|uwnIFMnMmV{^an4Y4~!?ypCsmxuP63<=ANco@K36&k<0bPjb@g) zD!^qGOeZ-GIh`FOD9@LbWIR{#Rh6lHD0pA0dFo=h>Wwei-@bV7Z0hEHFjZz?Vlm7c zZ?ot~eqY0JkAxR^Po{WsM&VdTnYNp6UjM+Mw6HyI!p$D?@zDRnJK|fJFYTwXvXH-vxoN>5|c?|s5lhUf(EHy z1*B`iNT`3NZ=AyJUGMC`m4O(dB_V;0GP*DUIalBX9wDI zEwl9?pQ8D0;pF3HV&&Ox;RmAh%i0G{Pa?Cet{zfhVMK{k44OVc8OvcuLKdZXcU}xE zJ8y=7p5(hUOuda@)SV3|z(uQBd0y4=Y;_#0w)vWLL7};3M|IK9Bw0QPASs*wS=H0{ z5C%EE8X(1(WTl30?K!VILPD8ef1XF@yqunBUzzgDJKWXEdYS%4>{|NnL%TfCxIahi z?fP%6R|a~Xi5wUh*VAYl{#2|Iyj^kcr&d8kr~&r_VQk!;yZ}y>kg9)nO87p_fxLd$ zj;zO<^8g^(HDI9+BIz(OihT5Ol)thXoI zi;Y3c_E~3~G|bNjl`HaTeD=gTV$Nc^f#-&4p;VI>qXLV;+;xE#p~(V*gD*+)kH84` zj?uWsw3KJS)(5#OUfuW7@WY!m&UvTJo>TtDKTn)kg~hJ}5FTWN#)vMwLc9#e+?OWF zU=}$%<6`Sq+XT$@ih6D)&12ScAUwYdSe7~eq-4E2OoaEyR zNBIz`y{MI#OFc4ejdCo*RQyo}MaFL3AwKskJ1mg$1(=Q3%S54?gkUnW*dy+nM9MN6}<}xx2 zrwwoUm;BO)&H@C6s)P3KlNT3d^hOXvXOPMsJw%<#2b>G>&&gjC<+BgG6ApB+@iY1$ z6a+P<@m^8r{p9iU^y(%aW`;y1k2_gHu_eCQx3a1|WH4g3g?+cuB6~9jlpkce7>d2w zpAx)&wD%w%PpVk^)vZfM6$Ml)TL^;~K?0J{PekCInAWqrr{^J~*1M*ewZMg4m^p;Q z>f5G$D?3kau5SIPem54`Sj$|Mf840TDzPEMmzKCQ=WFpyUH+WgV@*{vwodQLX#i zHkZB)Fxy_^ryhIWQF5;9G z1@{U;?nFYSCh#deMm6uz7vk?xq6c|)88X&|ML>OdhwEqyMY@wE>2JKb^k$L&CZvd3 zUapAiZo_KQs(9D+uz2nY@W-NvbP!s;Cvs)U?2F9MX5Fzed--h`3l{fo8*}uxTxG&S zbNf%3%KPL@+QH!DCqP*a0uyW40iP*lV&Ly+p3R?=oNS6Xg`P}gUI)nNGFuhwcre}= z0E|Z6vzKR5ME@cJKt+$ZT8nKk7S1T~nCd9AIJm?cV$DTFrA-+8rg1Ha;DEjSi^v7Z z0%o#3Y?FuVB||%6xh0-67>)#P78u?Sb}qBR&v1Wr7%F~cBAL2TS;6su$bw>(30y{{$7XT~W7Z#P@2(BdKAUo;ewux>z;h$|^6o$=F1ePeTaC`qI_64xYD@GnXb z`1Rj^UJe#=Ru)f<#ayzBJ{pwTj1i1SLDQ%@$MQm+v|*KEpvgksUqq9K+Kj7HCrGH7V)XjMoWD$f5kt-K?G9w}BIzkr_6!>SMm z(E23q;e&q>D}pGNVs$cl9+hnmt2`mDZwf#3!xRX+P=sq!W>q=fJYMUwdT<9B7w8}> zhCpP63;DzMi#yPC5~UKL(bXIk9FqNd$W%cg_9`nk9rWg~zy~=FUR!CaIC&QAesS;Z zY$9N4u09xM$$kUZ|@YbFaSq7^}?$KG!W@3LQ*Bv+`J;0Eq7Iycaw+J|T3d*iAwxRJ zv9sv0wN_QWI_OF^Ia=xGT65tbXRB2VvQmDRd2bLa{k}Vn9!a%uH+p1By_|m zzEKL$thHG;+t%K&z%@rqkLZGn?cNB&%F>M1lfdVSioNc&J5>%~sF-n~CRABp3gM75 z9ys!m*Klq#v;>`{B|@jCQDWvmpoL3~dbmzWNnnfBV@2g>&mz{-4+@h9z+JG_TqOsV z_Ds?Ji5VnU7CB@1oGx2mEna{#X}&Sh^XB+z3C@R?Cw9c9tb0XT&3*UU{u%*wvyCBp zaP!-iV9wQ_pw#6rA_(+O3u6>DqXf6v>EcgF4e!<03r2Cv2X7T0=~6R~m+Y>%f3aLY z-Kd8ri`M~Pmo-5*OqX!!}kO#N*+d4 z?7Q}-PaMjz+H>+yI_=E+4+-y84w_?#?_H5^zD3fgO`!{FE*9G_uW*zGm^y!YgSB*OX4rrXIy5#tm|3jHmSc{3&6YzK`7 zcY)hYmltiZl~aJ->W`9%OsGFE^ne9a=k3vuZ^P zQlXj)42}j&1so2%1M7zND{w31_eSPA$&UHSytv~0{gBnSUYLi+OHAL{9u@#)spxk` zT$wCbq>Xct@DUc_J%_c3RLy!RVBlR?X31HhW~!8dd_4b}PlA}Ug;r}^nP956c*}G2 zBVY(<3ke+HlTmfy?Ko>TNMS^d{z|7ooYqjs?AT~C5w|g-@1)=DYME>B#OQ@*c{`+S zbQ1L7h4iu$kMrU5TkdYwqrU^}LR-d@8oral0#w*Wx6_N;O!qxc>(7dDJSVIcDGV|S z+lW*=UW%hnr0SbmFc*nF6i8jOrJEw|ff4PKfoK?JYRv#Pd^*b7I^J$%b^oUVn&-CH z-Qdyh133oyoCT`{3|i(j(yTTKF}o4^b8WyuJ698=>S0szn_<6D*habJ=S2`gK`@D zou_wJb(E^rp4tH{Cb;h7DL#H6 z`hlu}%T251&R#wan~e65H{4jDGt?+b_)HYPMUjJi92301XT70*I0`GO zU5<@+FfEC5_)IU}q-TyYA@riEH2+DLXaQ%x|1@9x;3?L&Bwi#r(M%clLfy{ZCY3jK z>QcI4^#S+h*3F3G;@1vK`oPFy7DO*|3?MYyb`*V;Cxl^IM~r>@e5+5=jrX#2D58F% z+B0eOqf7VUpU!l?@PD0oUZ4PQFkuagoM+{Buqj}Y{y0k8-L>e2KRDXS51&QN8W^wL zn@fzO+7I0EZ@qs|a8Fm;u0-7)%`cz?*?xQu7DaPR2(2g~Q)T)x((fE@bjCjB?Qg&^ z&L*|=iA_!VyPkge{@9F(VX~8RXq)<*l^4?L(c`-kWfA-{7)W2p4Krff&Gs?h#+Og&d}qG*9r19v>X#UOyv=HH;vFn$XN(&Bb*(2HYr!8onhe@SZe-ZmrWMAtA$E&OL2U_vHlr;j58% zh&jCfC9&J@#hMzc1s0A52mcBW1RV4%(1LC{pOwK5VQD7e9cGm$Y#0rL*`FrH=Gi~m zJkOeJQorM!`hj0Soe)E@q|iZXoOz5muE5>^V)F?i&aAZ)<3g4CnsTPLXg@o4_E1cq z>iEaJz|9(`=iw&P!8_@lkW@>L{}n<)lp`cK*y zpoC8Ho&jb8ITqf-BLO_1FV;hFSeKyH=6B~c&V*{HqUD{dQGXBz>jgofqnSxYoER=zAaeAx7EfW2>nTQ zxQVB$O+wZ{5FK#UY%7+~f)f)SW#d^!EeVTf&9qwA(jY^2vs-jE+|wXXQ>76Zw7YXw zJ+$qfAJDUjz)$1*7rp8=ZgB=(U5ltc48GJ0Uac)Lo-QcR8Mlk7IjIrI;Dz4AB%_#4 z_R#i}IIyP=F`KxDS=9itSg6OM;_%MdO7kPrvSIvu0WM3X1B*BJF*r$e*Ui{B+mY|# zb`sOSF98Hh%VVBRl!|Xzi^YXM_CvIm&;IX!3_#MhF%{DMoN}#tbEv~Tf8HELK?|Gh zf_rmysBkE_k>pwGo}?fgQSgjLLEuH8-YJaeKg7&j{av6<%8yYPx# zw%g}|`Ya;ZcT+W29tjH{V1l$o3P%-Z%g$vMElQO@Wy`I(dhhx*213NQX#3J_lg!u) zUE9>JoXj~>)3Kv-5n~4y-o-SGJ|zl^CxMb0p<-O}$hH9Y0sQt2tH%RZ!(K+6O8Xvi zw6|3-^l)?J553GY1TAnXfE!K?2K}#ef%#ux<#EMXnhU_IU*3X4HG@XSLoXi3;v6mw+45(;Dv^x0O77(~ciob<1K8GQQk!BZDB$|N;+Z{aTOxVq1LPb0|E3g(oB zAt4Qbo^^@nh-oyc;Ns1bR3~HIrCL0Bc@yQd!^o-p~4mhdQZ(+ROX7|7U(-byUbTA+Z!q&`;f zE|Ba}YZ{-vzZysV@bI`2Si}^(Q{j(nyQ28|%j|~pPL0Xq)7v(zN)j4R2s?C_z3Tk> zZX3Xde)`9+5}|9PF(Eja%&3MHjEUfCpL|$ZlMVcW{9*_M3DLd4o^-t8AJAS81#kH; zDX=i++yj8iZj%^og{4LpD%}sVcj#p17fz`&FYfr!xl{V>5$aArc7%2oCD!j?a$wP4 z43q-2)^<1O!OL-6aXTtt*dXl3Vkv|6(*I-K*VO2Xb$b$XZTW`c5pdxvvJs#KcY-|o zkDstIk#VyKRQvrzGs=iws10LE3=O(C50XuH!;kEwWTV!{2jL}Kp!uDC=QH}Gp_}c- z+sF^NYUDBhTJMEpZaUU>wI}E`7X9hRH7?JKv=z8A;G!$$ zxxO!>t!RY7S!}?JmIBAVsdn2zOi26AQ#Vb0`WpT zQaxB6fzs^NbA$RvupMVfGPo0YY2@carB7RQ62p}5XI{A9xz%x+-5e^ z143qn{^o1Xc}oxi>HoWSM7t1f7x6#+BZ6qBu*rfyu90O?E!3#Jz7AF&sQ#!{;*ZO$ zWq6yt7;zQy&H@{xC0%aNP_1Pijh^3NkEBTzP16%4HwoNZsg<9+1&F0!GWkn+!c?QJN?T z`+qE5Tha$~st*+BnBnxpfmh>qUGBBXbUcGwcSE&d3P{Xjr1%Y?b|j?UN8J``UhiV{ zi2kEOyLHQjw&0?=vD$Y>kV_UmHGtrD7MOry@FjPEa0qJfqC;p=W?LmS6K~nR9Oa7Y zf8A4+FK|Pv`YJ4!y;~zzGup=yeS@M%q=(a(E{DK9kpTYuv;1Y1-4bKeY=-zQnEz_X=N=#BEoCd|*)!f(i){KGEKjo9~4(d=~mx+9z3kZHuSd z-Hz--OH6AFXT^Hmnqb#``22XSqo&Nf;Mubahb%Q-@vorSi$ja(Ygp(I&2<1TD7#VN zO6hQyX-W9H3=Z(+^uE*2BKXZ*b6g}?EBD=-7L|)P#K2;Jt?2>)k1vX&?z#r2*SpQAet)m=G%#dyGy}R^qrLj_h8?~)k1ghzm`1zwn>>LX zN86VTQ+!fX=UpF_*$3XFH=NUs^HOw?D%B8sc~{bg+=hhoGy#8QTnO-2l>hNpk_h8m z0>GiAX0aUE5M7=|dN?}Q9@5vf8u){Tc2nY)LqxZJ%Q_8dzq#oAadwyD>DMn?-R~i@ z_9pzki5ajY9zrpFmbovhJ5X#312oU{M7gbfy|+}zs+t$yxEuFE+)#E1Yqnd|t+>s8+E&asu((a}$gu03xNV6^zc6Hn z^&KFF!-3M{82kwUyVM!M*$di#8pF{*DxsZ86&^C+e5O`l#h_~qxg=B)H^fV4`=dx@ z-0z-?FHh;MY8`lvn0t7Bh#ylA_QMIiZb|!>2JuSp_=_0W9RNCl{fjHLb?4FSv-QAM z7B=`(?w|qawvs>rtKBKz0Im)`_gUzh(f=}uQp^m6#X3RLm0#=#;FkUK&3*^vW9l#P zF{_h6es>vC_i06K{W*Pv<+QN+X3{_Vdy(1m*B5n@E4`#A?bJ@F3A&{gXI{CE;5XgE z1!g)5Bqef)?}WHAInAgo6__zw#-ov_8GD7Fu1_~|CZWs?s4}VKx|VW~M#!no>-)_G z7Pem(Ct9lNu&~iwLB>6c_Gv?wcy31iyx|S^pV@`^H{MbG1pHmTrF{vHe^~FdKHnrSsl+FCED}g)a$4IlxeSlTZaQ7dyvR>-moP9%Byuw1U zTMLv*fM`u%1bijX_VgzhAc>cV_eqMd30`#A04S&4mzxXV6>v^N`;Fh=$>a0@`)OGFKks#!Cfr*g1QSDVO_t=5gTf$01=k zp%=>Zcu(*2+mt=4Qz*uNnKR%3sgdjea;QxCF%g30>i$WAPmsGo!17;GL*a)>QE1_M zA*K++DAr7vyKf-3qa|0Szw~#hM3-CElf25v11i2=saZGV%qLqjCPz=`_q(-@DoDSv zig%fs;4}Z4Zh!PNxdRIw2A44$$j7Tf#5ds0Cg`gag)CNeTwp6&>d!qh`*ptgvhALi z-Wp%LsD_{ui=*Sv`fo1qGq8{rt(^wiao~vbvTWUjHKwKq$LGF|0hNK-H03s9i>rX1 zHa;)G7fF!6^_VS{ik%^!*-a`>Ni=l=r(_kp%i#|#f>sGgpSm)5li{znc0q1a(k2=A zUev$67E-tWC-DxduhyWKIIshK$Pv>IO7%=&5rF%Y^-rNy!LHRUABeN>oZ-9m=Odg{wPPHjcG~oGwg3vMj_rthLq2I8(7G0 z;h?)$%?30?DhhjVm_5Uh=ZLcgv%5ELNrBEZIZbFjSRQOwNAuJs9o?2jg@NsZw*rLkT41FG z&e%I1Qrvsz$_dl#PyY&oz|E*|AFKy0D@25#WQfh%7;c<~~1?UqHSE6t}#!}!?c<0vbQf@TX78+N!4o=s}Bhp7*{&maE;vPzY zB@P3X9ZTYxH3DMmtSqqw_-OQ5(uU0)kdFB^qx-LxLxV^KM~40kRTavW#=w#&$dy*P zFs%hbSAd#2Eufew0e9o?U6$?5zNMcZKTMV}OR2pf|g$;BUnvFC)^o?Fu#5Oz%<=jQDlmlp5Q|1;14;QY|00Y zkMGy;#B2jn_JP8@h!(>iVLrq#Pxnp4wxvQh(4ym$NIqOktCPp^Gj1&(z77P9pEJ#j z6WU)cStr?i`(Yw_9=+HPoX%8UIFU}uc340w5saLFs%ljwJ~kC%_yJ$RIro(GtA4e> zs$L0I#tVCPc{uSYX58%z)CQg-^BPRuSf1?tyttrix*>6GJ!`S`TB`KahzLrja#BNCOlF{Di81#G8LO%mF7R8Wz?XKLMeQrYZwWmWiOiy2( z>aI>P^mF-o?Yb!a5<+M-2Z+H*u(Pjl{IKvGE4Gb>r1$}5(H_Vhif1dMrEt%5URJZ@ zBV@g<_Adq*#)KO!D;@Y|<0hDxqh<-m5WP{5mJ>icgMQ4vA$&qtIK^gLxOrxZ)>DJ@x+wbV4-gO$ym4|XzqBCn;nH6LrTE=vTP)z-a z8B8>?_-FREN8L8Hx7L*Ixg1#HEV1_ zSqt^fsT#OvfZJVqLF{_)_My1NLCx3XNeZ;PdQpgh83Tss4+}H9t05HYpY0SuF1;1E zEpqSLnq+0=(=Gc6+?CcE1YJ5f!W{L8!t{wv!QY(4sNlow6>)=(!_ifb7=u9uN37gZ zv~SfWFV=lo?A;tZ%0KX7d5kzqY=33}OMD!HjV`#aa!Yn^}RT0hR-Kb+GmhVW*LXFTI6_kBMP(|gCF}~f=ib>=Y%>MJ|^{X-Ak&c&MjA*tsYp43( zc`E2AY?tw-_Hw#$yzT}JykiQ$f}_v^s3intnQQGm@fpv!g6GJ{u|Qy)5D*hWRUxnq zp*s6o07iKpvh8`$=7$XBuhCJgVUjX=ujoCMNTjYl=0n8Ic*MwnE`nP4rV6T}Hl?(= z5ISnmi=N7lKP@>b9L#qVwv4aTIC#Mhq1prpb?N`!KZI{7@U~?nM@|>sR!qZ5f~vbe z_4LmAK)la4N*_by&vr36{hcx$REU=8Uy*cG8@PF{m#F5aGgi7@`0ey zLkt3AwfcLCMobRGMrKGnPTS3`+|#WMm{qiG^=~b^h+&3QMdXNWc+I<{|Ib#ygfDPR?E6FBMe>1fJ{ z(Q9;H7rf)ArCluz&1q%Ljm@c6SIb3>YG3WTxkEh0gckyN6F)E5DPznfD2v$+C_ayY z=NGN{0pUVH8tXU<+W{~+{Lo9_VxGCy@S03_F5AZ|J-N7e(V zoNBM6fE$m_(+<9mAk71%c86IP&(1sy3O%{_g=+M? zNz?-yRl41QT#%^5daIL4d}6QeWAOo6^EEI zR&J>7zK>sx_Znps@8Jx>=)4J~c%_(BdfyRj^KT?Pw0{=l!^W$#_$;gFYN!u6?bH>u zP!+sxmcbX}zL0t0%$ctIxy;!pV>-I++HoO@8ONrYNrrQxyHJoLm<6ziqKSoPgX}e~ z9Rw_0$A0|9ljE@r>QZ0HukcIXs<3bIHS66&Q$^lP)5f}33SPAH@#o;iT$2;tCb1iz zK;pqd%q!kN{VF-#j=flv}-nwhg(Te z@0zq`xsM2@>psarS(HnC;dh{tTmH8;gDP<{f?GbY zY8I;!#ci!2hw1*Z_{ZYwu2iu~ey6rz9Bdre%o<#G-x`^8>fmTS0diSGNZ`5*t&=Q2 z$6miam7FptDexJ%d~@@T84DWGE%1L0Vf!6*CV&s>6d?M7PC;0WR?m=R^1_>vaLo~0-E*aXZ`23!otA8u!H%Ji_!>M%5y{Y{Hl^!Xg|jbWuD@Y!)U=*? z#HdW!gQ}}v=%Oq&7fj6+Z~NemQfC!G#X;XzxpU9(M&%SiL^@SO7yVf$l*#)!eFDKi>sX|_BjSxj@e-&%Cc#OcPd zIoUi*kLH^+$=cll356ec6?(6GOY?4k`Y68@SO{I{xBiJ94UxJzl!dhj1o{X&rWP(w zKPjqk3J!HH|6-^&XBqZ|bu-gf$))-Rm5JP_Qj zsuzaTwmkGSnpJq8NFus`Mht2%>Kp>viR>Xyj>5u0&)}Kg=T_$7kCL!QIKpSd_Nnjr zN34908x^R8>_1iarvfEF84UY^bcjnwXdrMaTC%6*Org4FtZEsbmQxK(7Rtv>Ma)XI zk2EZqsY@T)_E}YDKjr?!FgR*`M8**$q_LcdqTKnd;FfMJ95Y1gyGLW^h{p{6dQDgx zv~coEQXH8WnA2tYP>!fS*0gF`sZISJ}u>ktz`rZK2_Gw}t4ODC|4yA=l z1ZPpJyoF=tb)R{n52k7a34aqKYvo$gH-22zU#XBp$K=Yj? z$!X!wz8o5{4_@DEb;$k5&Q~X=I@HVwe}FYPK%}JLKrc5H$#4u8$Jmc!LYYJM%yejO z$SL`YNwuoiso+qRg|u+P>8Ry;QD1q|jgU1>@R*ElczY<+1ibc9sE9e=je9i|kCg~+ zLqAW~owsKk*_9ziU!yKqmgN0Ct_gK-=wH-%z!T>~bZSmoyD4NqBW%UUGncGonPHU= zy}e?>d!qN@0-g8$qFqdn%2B)Qz7tmH>_brJIg&!K*fg|CLz3)!QT3GoF5f9;Wi7^3 zo*c%Q(kz?q#=Q5;w)4eZF?{dnJ4fxVZ6Ybu+PGcJUvCG!oU7IVCF2eHQ%J^fgUlIC zF0ciEa&k+5Vwj2dg&&6}FlLN%eWo7rGNepPJLFr*oU~9uj^GNysNG_25`HO<|g)i23xsl=^3?&aj+*@ zHmUvdgjW>z)l9M+GN@lAytox<@MR-p1<#4E&-?#j8TxS=EShV_cgt6~%3T=kf zP>~yRdvib;gzF~Dx0#wO$L-16WBYEbZ0bOV<%H43sCM5^?4`eW>}8Q*Y*7@c)P;m8 z07A5@-!|?lHO@b1{$TuJcT&kpXi_=~c^5 zr?DA*6BQDkhG%%;RUkE%Eb~BL7W3pfZ8-}^l^v^&^sC8J&I>+ymKMG1)V^nz+kS~P zy}^0P*pa(ng}O0RL$#Hqugo|YEz}>Fq3N~XGXBG{V>hk_@-cE?c_(-~D2tXqm;fpJ z5(x(+IlFUs zfGQARM`v94WIgnan?Y~I)MBbc{{+3j%ym~CFG!aQNF}f zloHAxCjf{SO#Cg5rFGTg?={FqvAqby*Q^vpkpDZ|8>XDiFDRkD<6#^$hf6B5H0;t{MPF>a zcs|$u(UWN}uQOiu0RdsT-NO9o+POMqH{%~}=ivjQRaBtw0iuVb5K_{&%4~p5BCF&{ z8>hXrlA|by*8XXvbZPU}+bwxl6)t3DBi?IU0SElM!Oty4`QnpUL?M(vKFI=oi-QW} zWotf=FgoiXXf-u=b>!^PgL94wcQ+> zo>^}%Tk&)ik}+Dy^s z-ixyjq`Vp~?$5gLz_F>IPiR?WdNn~h3Qi1sh?rxuLbyO}=Nn;4i*<|PvGk61Z#nIF zWm=n4P}!70F9NZz^|3$g@;#WWebqNp*7w`lr>?edWw`wCun&sa)P?0+ec} zUaa~@jBZ?PS)k&9#It7?i-tS;6S@%c8n|yh9A$iYN{TjzXZvE#aS5ov zp%fL(nsJU4b*|1!InKV2d>rpORuNm7YpYXLeAs=(;1aiL$9&EnZZk0$H{A+v-@$#o z2Xz5|6De#8f~6RxZVRSRx98yf+y^_$@A`lG)sBi#)wOtG|1)eQ*D4!CybcX%Ac#oD zZ|7*y-bD2K$T9S9SB79U^AxO(R81}n_eMNxGu*gh+LQk1i8fxRP%DsfT&NV=g{~oX zyG$yvPBZXJt&CT(<37qYz3@{;>bFk@6_!5@)Oqfk{ zMk?fp{Xndr2gzVKF$LL_(~U@K$!?`q>vpyH?S4MBaPo`SWN-<#JmRWqi$sK?(R&eH z!{cC{+VQKmCjLY$uK7Fq29Sv!AiPG#Ku0wf{!+lgZC9E-q~)1H**U$vS!7Uv5IL+o6QZLgq3T{>`cM z|DDwR8SQr}+?W_g3yu2mwq~?iy`>HBYgQy3>^&Wva7n{|FDsqA z2Fk7**=^vS830L!`ZA>2#8tt;z66kGthB^i&y}@@b8`UB9m#M(Pm^ih>z$yEX7;aSs}YQ9p`PEesoaCS15ap zBI&ZyplG%U_Cx6&?vu92XKGM%7gh;>`gPvSMO1K>n!%1MW_Gb&7qr$(jwt`2=0H1f z99aVvNL3S_o{gDsfTkEC-Pe!dDOIOca0*b=Pn!Ho{V;W$>d;c%HZxSgj+ZNYI`n(F zldFO}D2M;#qHCA61Gk>=5ZCDlBR~+em*`(Ll#Q04IiEVy8db*#zvbH|K2%K(9V%1` zk*C?LI_i&aeNCxTeI+zJ=JAfma3Dh3Lj9nIX4VlhM6#!aFLhsJOm~}k%uRF&SUGBN zHb?r74qcV0rb%?Yv}(9i6LrR%*CzbM)BmXCKu)j-h`R&=xDP>idL<^He-pKI^E^l< zd7z}M`>?u-eqNum*$IX7p(Yo`kMrM77=G6}w_7!LUqS~mYS_n%tpHCDg>Yf`snKq7 zJ)wghLif~v6IvWvaejVb!l`FG|0%n+Z{wteM6!bW$s0%iTaMBEr{Qq@P$J^iPY$9g zXvX5x&f4K+la0_w-Ku+!Z%Qh-_D$VR70I0x8cUxf?Bc$`T|`5n%Tq%A=vq8S3(v^J zbp<`vC}OR%(9_ud#+Yy9udy|;_(f*ckBR+2DhdJ~S)c+~)e*p4Ci)IdiuB->P#3G- zX))itBs}q4*k!#-`Jm4GXvIH0rFMO`BJ2$i;!gzbt`lJ9bBT45IEf3xd;8TT;{VD$0=rHENEK>VNc1Q=PBE3X0!ovXRga0|VXbBfM2&k?qCihN) z94yiIA+D>{BK)5E$Fs-KALQ2MNrfLSALkprY%T;MqTjTp&X?Em^eMJdj2f%<6DZjj zK!?6DA3Xjv5EBl|3@id+M$31UtoDyB_iZ?DX^vOEJwqD@y*ue3SjkZ`CyTZNVz$4v z3FJ@i8CMLu`2^zYx$cv4<>_MJGxZZthLgi_hb8_@-#@Y)FG-!oGcKEPWI!1Uq@@B{ zjIb;!G6gAy)%=c^vfcre(ncn{Cah4E8U=%ihS@*Alqw%`wB}rSuVDDg4qf!;1uG=H z%@KS}ki{ntgQvd}Q^#@#$@4OZdL}qJkBQ^NCL&di9l)w&`PT&meBD3>XS`jt;<-9@>4{T@hI-wHWAHtOP@I#jndjW2E6T4U zv{-JLF8`c*T|l@GC`-ti=%rTXG%QlXv}c`%ni$c}6IKFc!C1qbql`~u zbzAicM^6}fC23RP!y9&|nG>e$a0(Gp1?AI9Dmn8SK2=4Sn`vcQ`NKlCQt(j)xjQnp zowTSR>+tyO2RpYuA4IzTr(RO#K)UxNkD#O0W1KOFgyph1Zs6J$1}kyJgm4SJ z-Sl1~6mk9;=#zc-xLh+8M4i?QGHBRo;qnlXz}*F@F&H` z=|v;gs*s|Aqeha^XWveEoIb$2WoGQk2lWHIU-{N;!O@DxC@}~fV34s(3U){R@R~B8 zbBfFGT9y3a%OtPEH`TJFFUzfv zNWq5Yy4IFFO9%C2ziWZtB#+%wIKB6huNDti5B&hixJK?#!U^I!)vWBP?+EUiOc0+Q z&{PlFQe3Q${qjnz`$}*AKAte?`&XI|qITeXLx5;5$P5LjUqX6`wPth!=5%jMbPC}@ z{kVQk|3dw*gjn&GhwQM&+al`ymP7^ZuH43h`glsr-&qQZ69G{94;w&F;nNKw2*UB- zVX5DUH2x10um-f+v+1APz89*(={@-HI9j>ooxq#N4_clE?fa>E_&&oyVF_qp_#m06 zWX_p$rl6CN+<&S;Ys_N}K^ic|%S-ne`JvU&E*VD@%L-_mB($Dxj#^cMU6@SCt0s9&v3 zpPw}l4kCTa+q8`iRqjHt=WGrtdrP1A7aF4f&0f1o-hnZukrg>I%#=h7-}4(O-k&&n zPtOE45rYD;L(zKmX`-bUE|!y)DzB~>-hTV?NgJ8Z65ETe8%hIhoUE3QbssUt zmGvujneCBF^WT!k1upOn?nyc!xAS7H15Yk5rva!X;R4W*w*As=rbstXZlgD|-M?^- ze2UbS9$%wxneN{#ZC1_=$eY=%Xjy+^$IHXdTWv11wg@V)Ejy5mgDA+{wrghVNIild zo{j=q%AqfOy~G4ZXZN*+DBp^+5l_9)G^O&H@|>~NobUlfDrda~;u1lCph`T!ZRZ4- z(z6??p!%7Zt80fZHjJv*?tLdv(CW~-!?Ta|@TGBz-Ie0~TrCEGBp5(e#=RC|MQQa0 z5>R8DvfYSf)qYS5uw>+I+;(_0+}Q9|O~;T?mZ&cc2Jj=TI`FWk#>2kx5Rh zGG-t=>=SD&hF>td12V48Y zph>Y{o+NJaBFXFA#xc44QJGB}fPyW+h|}5Ip{oC1613jc-Q%X!DP%SVQ64j{w6Dq8 z_h?hQQ`C*VU3;8%p1f?FHTl8Py_)B@S`iip_U*%FcKJo%jv}Cx(2M*zGILE2ZDZc) zf4rULPC9rRCq9#ExM)N7kpnI`I*4@z+l`irbUmoA3B6(-ozBrNxp@0T_5(lc{ner1 zO(HD9Wt6-921)>Fk!fo6cA1cB%kFW~B2aFv@Qma(aK2vMi^+(!i`#O2jgUm74B|69 z^(MLzS-33~#rY>*P1L`+E_lV7nGuDn@)!Y6ZJu~J~mH(?S|$ON5%HDkU^Va#_; zuu#E_hN29O=2BK%Y|7m_jf$g(B^-07qV3OhHj3L=r2y?%6|j(}aFiK?j7_0lq6o*W z$YrF#`=DkK?YCgOWBFojpU6GkA3<(zcOU9~{WJ50OuDRgFYeh}ca1c6f#Ww19!!k4 z^i{3`S=@fD?o#0i5XxqwWqx7;EDn);zfY7L=(c^6w-!8pyH7k)peIK{t?X?NLyYHo zB}Xy@7FMDHCNO>}=n*^|rFhmNtW~V1#6HU;WH4w8S(ByjI3FD>sGB=1;Re#G?ej!>sCA zkE7CXJilBs)8;<5*+^%-UP&`eJCXAuYhz*&l%nNMj5Poh4-o)|BA$0J2Z-CQnV<0S z{@Sa2|4LuP_YFj!F{ZBr(~M4F%u)qQr{@`!-F>aZ%Q21yVPAW0Tu-In z-rTP4;}ZVGWip4VUkTt{#*j0^_H3s9BPuePq>}TLhBSyAd~?h2==9a7d{sj=t^D!$ z1BcSJ|5wB7zw6&$<4MM>Sjd^}&XHouEp0HnyO@ixJk$Cc zw%Z+vk=9@U+XJapVlm1${1)<+{n98~9sUz1V8U|NwP|FgwTx*UkyCsOa_{rS%UU|t zLF(8WGKb2jCk}E4Aq105xzW_IEa>$abCFST)t>6S-^Yw8^GfGP{ka4YM#wy%{3AFL zaj?oSFtukIHgbYxmN~j)dd@~Zv4L1`O=clCF-rY-$Fw+Q#fYJh7*vg0Tw?8?zUINV9)>rw<2^24ISGTb9MbF+zm3P zIBqe%ULN@Ld0e2&I)>OVKkTr4#iP!BkK@LKbU|Pu*yVJ?0c3ItmVKUg~!3A zCeCJEEsYtXEZM5?XY!P7BF~OUR1UmGYEwLZA~+Vt@HX&G9Jmnp^|zFZ0Mvqz257%z zh8>tVrN#094(BZZt-i+UctYnf2wIkHuW?XTWk`GE%$GyZPUwLCR3TUhZi26tlY3B#x9 zxl6Mu-sMfqlhH`4fIxSEOt?GmclA&6ofB)l=wblv0b7d&s&pcZ@d;KQ_kO)RnFdVh zlJ$pcUw5IXS|i=YH}ch!Chs4@*DhCdep7pGFS?tI(=D+)z$IkhN8hBto=pOrOOcCo? z4TJ_sr68n0LkdIdosKFGRGMFGgte>Hxb>K6%tmuJpGpkKe@lOnXgO&xafcTI6)@~dt#(gU?f<2=_L zg^XoL7r~JQ;7w3t^I1K}&HByhjo%#4qFN3{DTH!RqEFUOGY_mJeoZ?mQEOhVare*@ zo94ryJEC)o`<@Hnpjl+bA1G3a&l{#kA8vBA%cJFG8TFT#hoj?KRaFtc^0Miz+*ZR5 zfwXr!ByCh%2~(9UFoGQCTUq*p<etcoaS^XHTxdU9`w?>+9|0jX?De<|Kepu=Expya&@p%mKS?nmj`Yn)t@wxJ*#F!iitiNs9N8YD;2+*8v5v*Ir;LEATMFzN&9Ya zFm{4VRg+^1wyl+04ENH`hd;#}XI5PH0i~ML8FLN_w(|Ag5gcY>+?FqXw;w+A{zoA);CV|&bL7x)&b_~Q42zOc8)xh} zM*tB+U;@BQKxF;rMQRT0?$IHA{Qvzof7i!@fCl6#o!2fcQ`(xj4Uby0ce{OUcnt+n zYj$+(xO1W8>Z?;9=@-qd#A6AQI7k)AQ5K*OAnkyhbKGLoCB(KE9Cf{?l_Hd4DyA~$ zI$~w3>A2S}zwS*vC3ITJQL8w??dFE=Mnn}X!*oe)1F3;oU%U8tmo|w#b8q-ss z(yRXSn0mJB`Qd_3r&W4{d0=)RVJ|n$+Vlsy9={a8C}O2CHH!rb%V&p@Tac;YOY{dN z=9+I_B~yn*B#$QN#=DsBn5g{t_xVC2;c1WtfZjkfM{1h(CytL8wHK1_kZG8u&yYa^8hmGsv-W^5m zbrm&7Z_eiJ6Mb3XCk}?8%n;*);Z;;X&c;p^L4ee<`~%RA9KQ3t`Vag79}PzW%uyK- zGP=P%xE+xwkjYfid9Rl04xd0Z?x{y?|FY=ltLC>M`7SIoZLhqZf@RU=w+b&@rq$x!0p22UJk}?kAkwR~j>J60lR}r`nzYixL=& zD0qQfxiduafZzYclSy1#kM*$!wA-WeNyH`J8t` zPR}vqFUVB>Q|8rnU3xmalSs)YuG}F{Rl*bkN)??-rhG4+KQqhDoz|u z`c`Ns{Pdw8P#58y!JiGFgHx`#Iy=OcGIimq7sgxb{H`Affo`s%wG75)+*&CFz{RkS zV{bfKlsq$Y_Io>m*wzs18(RI%Mf!-w*#xuQJ85sUSi8VdgZfY>=%-O0p=H$}B-<1d zRk+$K;y2;G>E86{^8z6z9eZAgBysr9^1P}%A@g3Gyc>(aQF`_l&+g~H!ScRtL-4(q z-}%^D|FQd9UN%*KpHUco$+4&DxzF^wylRZg_%YpFXkJY{vS-E8kk`#s(viZ8E>!^u zMIsntnBJ~3LAbV!82Bc5VJYluQ(QHwgOthm<3xH<_L0;VsaFclck)Y&x95S!Z3dRD z0}cXvC#(lZ}4u_`Y^C*2y)K{_y4feg9^{v?VM8I<)+6xC!khE^HL`BT;2OvDZE7_R!M>()OE? z&u(O3qk4^}%lBNR2pZlH=%#4#e!;bLO5n$EmX<&!nkF0lcFjyjU{gNvbEkmJ_E7V&TTdox!T|E zifS*0K#_zm%g}*&*p4uD5AOYYqTlyH{dnsQ0I{hBD~wu0W1K;sYW);QnN8PImY8f? zjuN-Zdi!^r3d|d`_`U}#%M>g59Mtk4|Eu`*@@I4Vr8zG$`F8V5pgVhU9!N%@0q4LWAbZ3|QWIn#Mad(1-aK?b6D&YsLPZ~59HP(B`P8yF0a4V%?fwP4tAI^UY_ zO#e(s1V%lD-mvNK$k}8}pw|B4dFCC*bX66^emp{Jkwx8(BI_J_SV@2;MtmoK@`P$&M0ZxdS%*5}ks?_)Qp81A*xAO4{;NEBrPm z8Mihqz;i?G1e!d#;BFjz{lSv^`1r6%lbqCB3DqO~b#*E-V1{fAk88tMeA7DTZc5(} zTOF$oPKtLg59anPP-4QtTU#mN^1-DdW2iv7=gO?(m}Y(HvLnOiQDlg5-#3w7lPl7N zMs{vvx!wGc1Rw4#9Hj}*IEH|<*L%R_0@k&b+^=AYlsXNHcAbS)%nU))pPT_l0m zM^;(qfiUkAhL0HFz z;le707W9hJHEQG{f-E%CBAz)0DIxQR*cN7af9`j?B)Pv&_fP8`VC@06f?Y*y&ivmY)NzwLUud7sA_XGU%WqXa-3 zPQU;fp52G6Jr8b`PNFEu+wydO$Rp1Xiw9p1^9YQQZk_BNlNAv)ESPa^i|6wp0EbbB z9l=_CLV@W;s9NEH^NhYUiVNg4V?KAu2R-BCJ@xFianrit@RU%A*$9+f1dd=xAyrQhve0$|9`cN#D}Lci%i!JwNx$^QVr>f8DS3=Y=g3G=LT5 z{=n>jdKvw!o6z|(j6Y;L>x2}C!1#r2vE@G$FVP%cs)Ebvnp&>SS8*Gj^o1%W5*EG3 zePR{I6apZbZnj=bo=bfoQLN4B@nZXq*LnT{H%+g#tya4^l4|y}#Qm|?$mi$Dh(1)< zzn4V`E^j`F@ArR~!Xb46N5Y=<1Q<^<)P1T6I&$L)6w#%EEPBV3%Zh9WXli&+X0St| z2X8WB_O5_G)Z?>@JnA8e$&a9ekiz~8C5NNF3upI*V))dR97xD~_?!XwDF8#7yflh0QF;>kYoFJALl3pw)d z{a{QqIOV~>FGit4@IXQ@ehF&=<@A!waxenMw&BlW@CTbf#lI-e;q7f_(Oa+VdgA=} z6GlQi`dogkqJf-1XT)qa8d4XcOR8s`MkJALDZt8Dt)gR?!;4I99xmB8EDQ7#Inr6$Q%=8&%&IzZ^@LN{4~Q`W%9UoQ>`og7(Pu zx(LS1(!#266*0&YiS~@byAw(~N0#9C{C(!P_dcx8voy{WU>wIy0QMp7cskqe(;9o; zF*G2n_GP8H^(}GREjIj;y6l&bDPeI&7qX$cNdH*J1ro>F%zPu`P=(LtoHbf38$a>d zoz9QvQ2_9A$0IVU_8eKuS*@Pssa1GeEZ}e(bmHYce^=4|h!Kgf&Z<81T;atxjZ=R* zcO+eYad~p!J~++sh{xXvl#}c}6}?Xe{$sxkq!-QP%p~ij6sU0^pZ#(Vs z?uegGc*$c8V#|a~Tqhn}{d5nuAM^uUjsom6L0E;6HXPwgOrq zjjM+TMD=uwj;QIre(At7p};l-HdQdQhk@uPip0#a;<>dZp&?zK*44^Ct1F-Yn^F>s zaX7ldanH;5@&UpMuN=&f9c)}adPl=ezqe>aeKU6RO}952Tgg{K5~6{zsnSv^@M>0@n-^Ax*-Pvl{Y<-fMqT2alG-=*tl8+S*+{=@QSSTz^u?|yBbK6|11YR{Xs80?3)!$O7LXFZ zEM53NLnm)wm%AJDw?2G5mNfQCuc!53kirEeS3{k9_n!((;=71I2M|QbHq%2KVH^`= zcwOmG94yuY{a{A3Ktm{DKSmXLzgwbnODn)3|2=Qn(G zdu6#PqRwy+millDkrxH@NQ~-wODzG7D@PF?`irO0E~^Z(_xR-3HoS(H3N-#aGxdUK z*{fb-@#5y$*X|t}hvx53EPug?pc;{MCFUqAmHX5xl9@8=(F?}%sHl%yIbNICeEe-# zl&Wsnm;UTlwn>(xNtVnB2^lvxR~}w=Cx}760sn#eMFwIN0}KPYF%!URYeqQEk2$D` zYK-mUz*6~P|N4?y*ZR`^bgL;1|1B~Fd@VN%Npd?4P1#xKULr?AirRW%?zpI z&>5z!`!KnOC}V;Ug>LR_CaaWJEGiZ>G<>-~-2OGsEbD{WmJ{Rq#b4YmCjfcY*Z{f^ z^Eu!l2_vaFQ!BVMi!w;}AQbCcwcxWO8K+OuWf%X<3&|7bQ_sxpdvw3%_*U;n_jV#j zZ^&?jCu911Np5QvPl{;OM@e%ex563P%;w>|-O>p!Uu<`~2{wY?V@(&hNgpuIGnh?= zOamxIy;IiaKX%h{(wi*w6PZHjra)Tj__c)>4RWP3juN)ocS&8fcXURctX+M#C!K<9 zX?Q?e77Z%h$9VFLGI+`1Qo{|4INcg4I!@;_$&3`I{Ni#51QNZ87`ostZM@Zy5{icxQhP0&L0~6No#nq1tXHdw2xWpJIkYk z&HPubWv=AeXr*(D%Knk~>oN&2;sCVH%mE^Irx;`^p&OriJv*Z;=0~50`;30==wk^$ zII;;3%ReHT4eBp@99Qq~>xw_VNqP+f5d*3!Q-~`Rj$(k8U0*z^%^-&?(6)>1fDvKh zm)4mAFOw5z@gnH&io-Sc-rx0{@p{wN?sbkHnzwKJ_U*b11nA->?!@kYaD5$?2%m21 z^8JH-7C+T}obh-V|MoM6XzX_{n%+lm3oxg(1Jg~kus(pf0D-6zBvJ;+SRIVbSmX~B zAI1THm{AsdavSp4XnSYCWxay8Gs7?P-?|!F@(G;Pd7d*~!o&HERRhUMXDvtpJN3c0 zQpzE9&m!nDL9+*NE+hUmjOwD4wRd`Y)$p*A{S|dM#MmK9{AoFltoquPws}}M2Lu>- zA;`ui`aLk`tCC_sKk)=|pYE5-dLBYDORJB4NzyxNr|F+WyV&Er^2%M|q{BOT0%;TA zEEtd}_+k5)31_JCQ7}pc7pjan#fWTwm=}}c%pj-y)LVW@pU9U!`F5$MaOdu;+j-SR zX+&Ebr4`HVfIHti{z)G(Mi9<7P2KeETcHk{jYd9MU9!FNY^jz1-EYtp_bkD5T%fI-JP-M=~ zom*P|H2G(KCm*Vu|5tGi-Gd|91XNxrWJu#Lo}~cl@|10tieBqPX;*B6Sjg#?<7Lao z@gc{6mQq~?U>m8&@856AamIBWp&>T6&)_Ja5|D+n1<9N}eU$ATgl7ac^&m-G`e(;m zO3fJ&XTel(l|8G+$qS^*V=pd^n9u39S=e0Bu{YE_y^^H-N1!X;x?MWVk4WVl>ZuEv zsK|e3<^umMrY7iy1xclO9ByY;yMZU{8C$v@sRMmy%0yuI2|26+Rt!@YHWngH^SQX5*oKEX~jG1f` zCD!;JpPyapN6!Sj&eXAUn|+{J>T*xxjNi`q6A>FA?EI%-@$bMe7P*9ED}zZ{@i@P5 ze(YPMsA;wB$>BDcCywFUA3v`x{`sviRzILPAzltMdbCqWSYH2_u}?F)6vw!R>c?dO za`rk$Qlzh43(}hfzeQ}(v%Tp_VAE!DAyq3*^K)kvI&k5vXGs=j3M9Mo{4q2hiORkR`@1qT6JPWo)FuuLK{rtOy%u?CudUorx zuskYlkI2qkTLnKMe~MQa&5@%6k-;&J;#OewLVmA@Ip@^4l|5}pYSG|E^$)p;!&WPe zf?J-9rJEjqdRz6yl`Ghif^~ z-_1(wp6gwl+TGZkL@n|Kjxl@E+}!jx*5rRbI!95Ai~aq?Oq>)bf&ztlq%gdTV>=f4 zfUl!{PfbRb1@@74ZqOBPTa&t8soQOKBigtV z3c>BNuUc|CHv>|xJqz8HPFoz0bDV27s#5?;t}+r=R;_8+h+edbYXtW+4I3arCq0UsdzRg^FhT^RIqeZ20yQGI5mop}*6J zaA=QN&vFV-v@3o-^Bv1y`re6=oENswYaxnoKJ<#JUE0l}LJ$9lqVPMiyIc{s;2oGi zsDY8#--7BzCZi-89w2%w1Ru$;vZ@x=f989hwd>3~vVU>w!pF6pi6A#(U=UO>By~NZ z9>)X49NlAbK%eYVBh@~?az4w~xxblIuGL(9^eZl4SvsWM3=*N_D8`hr>7Jzos@T5u z6U{3HR#V?V(FhsL^IKu_A#W*<%*1h_6>zkLK=?4=+J^X@UmXSYvky(^+=k6x|N0;5 zc_%nqAPJ@u>j9W?O^!zkSn26E(D2fWx);lXUge6`iaV;8f_Sg^?wXrc1GXpT6fBy{ zt@?!)S!k&QG0LUoEejTp2X7fIdb(a0_%bl{;jL(Hn(fm&YLcxTyfyblX>Go+=xX+U z6os(pmvx%ZMZfgY?lR`R&k>y49bV2iT#q=aK81^nPCzm8LG&|w=bv6F;3^IS?t=%% zhL7!l8(xo50}NivA$UKKQ~58ipogmg9sl3-p#=k&aA28w#86}kTEsL*38N22FDlH2 zMjUIBIs^HLY0?Iq81mtVOYAOsYx9F_rp#dC)ymVf;X(Ebh5{&)Ss@mvGH4R@N% z2%O+N%I`+8EPwpPv+0I@@@WG+q+k|qK*7ow_?~tmnDgwm8G29xjXo1pcy-aD;Hx`} zVdoPxpQ~#3(67#r!edziIsw3qBsAeV_pDWIti5r8g-sdOGjk;WblOaKy2Z)1V-?Bn zf|!J@I}=A`m}VT6-ON^+2hje}h{~ttq|9jGog0|*J#1yZo<5SDee>vG)~{)sW`|uq zPNx&MXWqM$AV0CveGoaF!#MyZFF5_h0$X@{>4Qp^-FO@8*+ zN2B-6pO=i3)epJd$)-#&bkQFxISLZsyP;DtYRplJU;{1aCq|E2yyoq7X!ObL;!8gb z#1_xS#+n}quRGj(kS}U(5ya8gb-$q?{Qypx5DT2w+B2X5_Pu4Zw*3@X)H&S1PFx^? ztxMm z`8u!eOuZLdK+_5Vf>{Or3jMz{fYHd{ML^z@ietFr((&SJ;VFushrCW}%-PrQ)IPAJ zDcwp8_hex_Zdx!TqnLV{nr3(AZfRotFCUd%cNNv z2ThsIP?kJ!?8n;#r_8G7CU-5(LP^`?`7?8k?0?q+CE{_N7f|2L7}J1X3F71aDwY}x zbiET6W8TjK`S;a+B;yo19lbTwkLl6%b{T+_gFHhg3PP&Uo;D#nn={4#*tWAZvmQg& zp&r`@M6Gc+uq{#G0|*Nfqe1IPFik6qt}a<0_7P(=_U(`>rQq7Fs%-O%>Jo3&Jr;qY zQUG}WD{dPqfIJ=D13F$VbQ!i~Ob&HEX>Ewl`@6)E*(}Z3KibC(^q({qU-P^ z_`-AP$GzR0lsmCO?ZD@C5-EhynZX0uj~JvGxq=dTL_*U>u#3p+!NQK&159)oiV^0* z*|D4L4kYS`K|>Po0Is1Z33DJC(=Lxke{8d*zDLyPwMDJiN;j_bHYffxuF_4|O6r&g z%30mu+w|Z%nM(N|EzsRn-W_QTJ5(+GKoFj0llTlOeA=hh|eO+cyiHW^-HhP zJAV6t*O?B!g>u3J-;XY5mEY!c|J{lN)70EyJ?toUD?=CCjg#Q)78^`=z=0D2k_^VgP#01i z&83S>vSs-}O4XQy!)s`SGnMXpl#z@#g`Wjno{&7NJ{Te?7pUNfHZ&8s*q!t_i71bw zf&sg3giX7leix4E^`^ggzN87v~PcNF0 z8jFLjX|lJqkA;PwoU!NGU1;*!HFx*L?Js2R^YkwX16l+z3q>NsTO+`?It$seonj_B zY*~s>4()ZdmCa>VyE?-lk?<>2>qLTwQNQV%KgA0o{USAvP1xSKX77H2TL}E%J2Q^N z1iTeIlz`?0Nr0LJ!ZG8S$rWD}Ns3%R5N;0_ZV-DPzm}$SmS0g);CgBM3)%kC4DPLe`td znrSG}??u&C6T@DfX!|SSzALdI42>F_oK=G5bgv{u3^(%}vdBef~%+E;MPx*lqp%%_wRv9#yamU6cFG z_r6`n90iG-4|wK2-E{&i20Uj%Bs_tvap^S3z{?PWbCTp{gpiUA-YQ5jz;xK_7+Uqh zGk}%g9C0OhQITX8_o`;w*4sV$Y}tLujyMw>Z7%}aW6u?Y!8i_46!Z%V+(Tz0m=~9m zg=DxB64Bx&LnMPvFh4u9T`@RHB<|(+E3vXI)s2r>jft{f@j4Fe?^O56S8Yj2Y>7u5 z{ZAnXPC0NGb;NT>QW0y0n}ZP}VBh1#!rFY}80)# zpA#v2Ks`oz!6yS>C;lId`S(=%L^srVBniy|mvus1$n^laEkyyN`vO|-kBG@Q#+Z4i zU@D?y91wCT;iv5J&~LtBL*FFNxD~~?4;U=Gxr}TG?+-yuMX=668EmIuYh;mo6Lf=e z!=oo+;qo79aXq#5gGU*L09INUH~#WYqp%dC z;=ga$Sn_skmbO8IgLluCE2V3^T-RZA3Xu_xZ`Dk4pYC{>Y#AVsQxz!n5lKvYVkOO2r` zAkw8Lp-E2=!-6Es`?A09oO8dk_x*9kxaW*J?hhG5NLg#Wvp#dqXOi(de_oS>ALSdr zzKFgk8m7e6YafVwLzc+V6C8E9-VmT?Ax<2z2cqdJ0aMDNtoK-#l7 zeaxhWVUcnNohEJmaODHF{Ygt*F5gFu+{H!HL^L~!E-Qt7PadgCC1XReRR39~@I)gd zyrzts^b94L3BuRRKRi8`%=n7UX0%deIPWFy{2FY?h3^AHXA%gv2OwYM zG9%%_$A!SS!M+@03L&zEzT`9ZfWK^Hj$Uzl)Ob&@$b?+@H0tIl^FpE=D3gJxgb#4o zgcL$OnnBtSIlrF(HwUePYo?yVjaaoJyg{_QMeZVhoeRC7YiL@2a1G*~xBDxM%=^go zp-6I5OpQ#n|I!Q1qdyYQMa#BE9NxhHo44I$179K!cKYa=frHdz1=ngZ%-{uL%I3lP z!F4;mn(fbDgix0IKGZdxT8(6c-jQ26v9|@FZu%JNIj~u_CovDdfF_WpR$x`ak{U2>H2`$O4JFYpQR71SstWyY=w2#> z)0=#uJCVc2UOuiOFU>t23hRfa0=RVVSfc$00sLeL?r?kLAJnUfMjUfr1V|OY1{_dk zKaQuX!z&DLh^QQRgX&1XOP$j0it^Af;q@Fj>q<9$ch>cA;yDj^->hOrq8BoR@Ah`zO#$XHk=C=9&d4i!(dchNPa)ST&E3_Rf# zc&og)Z+P?3$voxYo>L_wwNWpRT3O1U(10iZD;^XSIv;3k*BSo$_|21z`FC9*(>7rv;g}eL1TfrTAZo-t zCWwMm_em;fnn-QJ3)P3dt1E${iPAiTe(%_eM?+FIE6p3MY%+FzY_q<)lpvenAhy1a z@BDRp23Bmuw?AAkrwpPLJNt#)3ed~jIneQG*C+?xUZ?D3c>+N^EUF>H#dG>fc zbN{UnjCqa*PmH-6tbDr(Iy|nRpJeS0k_l=ZKS4!@GdU^kWQzaXdyX6}F= zdr;hghS)4WWH97PYRePuP%QNVzj`K`9TMn>Bx}|8;d+K)cZ3g|R!H{W`~c=iBS5Le z5dm1c-IC@U6=)3zt}%M=f7#8IwtL*a=b0OGy+cI#Do0!Uu_+R$+Fk_^{67PSpK1v2 zA@HF=NI#}neldf_*$kD-H!2c#TxA;2G;#+b$aS8FW%^|*%c5-T?9-ji2gqZ{S=`GH zDGDFj>{lBRSrmAC6Z9Pi_D~6cs4d=lVUQ^RViE2HngJc0R!9)j9*~IvPA@A{N_>cw zpLN>kCYvbL{h4w`^VH4b>3+}1`&_yXn0*}wTwoX9;in(AWKsR&nIh5Fw4nDNV1)_t zKz#P-ypVD3+%fpVXkA9liK7iYWv|*S@8-5~OdiL7gh(L}_1^kYHsc#9W+4b^*_7kgw`D=3214zaVH;0goN-?#j;zq$) zrNvVy={iM*^_$=m@gYIn{dy8iacX>H-95^<{TO?rsJPEmMO>#)`}SHdoqgAH-aE+& zvb7xJJU{mnJgL+P_#_6g$R$AxnKWx8)2Eslh9AWL^Ovvq`_ZskBVqyI1n*LN(07xP z%(5I@1x9&?KlBw#SWa<=HjniXK2)O=zaW~Rv=2m~mts@9mtlYmLcD>z`CXhDd|*gf z!e(xrU)rnINcRKq?ILq}iiE13TGjV%zIQkV282v@%gP*kE&E%9DH-Sl6KLGP95h1$ zOm`Sh(TTNh3OdyPw$CT_^o-T*6jz(#7RtFx&puQ&eE1ND0*R?AA`Jj8j$B}kk^(`v z)A&9IG?Q;)$SZ)_2fJyk^_jrbP6LJ=ji&(STVxda=U=k_{WB0*z+kx4`tN^S!#zfq zuP^2@E)^~yolsfU*G0q%EDg9N3<)MT+K=CSB_|X{iCBrjAvE=_GxIHu4qRmM1c)Q9<70oauQL!H$3Lj_HvsvO2h-BO z87yothU$oBj)}3pU>I5WiTVG+!Myba|GWPosI9mU0kBm!e#1aM99r}n1k}qP2&VbJ zsMUQMtD5IcA`mo!q#@8Flfy)9~<(U(G7%F3u>b=&Vep+`3AzK zmgOt{pBDebJMj7chK!&P#lLUhf3O$7ZZKyNz_re)NmaLr;$8osXUp+&+$#Jv+p{CQe`nXTRYfYnk2%mFm^; zy|_!)92{4v{8VRS(=1$aFs{3$D@IYf`LLzDlI%UTvdKCAunB4a;I4?E7&Vzne73_s z223^=C#?Hq%afWi4^anF(tI;&Nv3xPhM)K3oJy!l(fT|Uv&{&!FoG@6BOsJXVkm$` zP~rrput`G&uw@4(PoADYc4wh%=R}7KCn9s6WdPwtSL=j`dC*#Tmk@9`dDPiIDCGq@xD>$t8`OfX;9s;Rt42Z}f2U-4P=~ymdpvXx zi>6EE!;dJjXR0kbKZx4gjy%His^qv+g7mp5V@Y-x!Vf^O+U4{8DiA4uc%T^fcGTF_ zof>axlAjY6bGLNt$D-BsBe@k7Yzt+RukjNmfT%BmDQI@NF5Q9v4$Y5wDCOZ(JKJ2` z&XMV{IAe^eg)!cel}>O}9WtPw>@8Q8r(b1dGUZ8%k8XOWgPvw_UEeozeI6DJl=9XP544%HKP~Fnhf_#J7jfLW0+s{{}XvXTcN84{qg?1=U zbLrN7xV3#qK!8pXQqe{!fn6+45@NuX8fZ_&mbbx?}HcmIx!?IKy`?%8^}9Q zGIetDKoiwSa@XgBGtWeCHz;~MB8WpxZWB}!)lC@)4FoUwJ0MN;I)%pfS#vUuAwU+l zgCylKSWuLMamGv7!=Vj&hl?2(lir`&mB=ofcvyhh-7XnLtG&Z6n|&A+>M;GEP&O8n zwut6n4+YTwET$l?`GHs?VVDmq0!K7!|9)R%#tn0ume6euJl|G1+q|G~?^>)-uB%&o&`r+J-Fu+2s9hA2xp>5CE2|3qK zBu^iyb0X<^;2Y5|M8)N~m0rKgMX4WS zl|#6ASRMikVlu?@mYf;q{PAaLenhM&XizCF3GY;mh)$W;JOXl}RS_YxBUWsZui_nR z3kA>0zT%`|u;2Zddxmo6Kf>1Zl)>A*DuOMu=G5n+>#^KBb5oB0_D$3Xyz#=yCi>8c zZD-_HcxY(q1|C*lL7oj!Af`GZ2`I1Xf=VEwDCB{K4^AvaFqBL|6gfaWTN$Ezg8azN zMx!fy^=Hba2R+P7H}PDe84f3xko;*^kCFPlb8+kk$#ND(dpLinZ&eShiCJKMCVadV z;hFDKWyS=qdrP~@Piz}Bp z8T+mU4HLM6|SrY<^#r{IptMFHiXOEqQ-I!}xHCgo{^D zfuX>wCC=I9t}{g+Hi(C%q@06bv0-t8l*#!X z8Hs+w86OKa=AFUckHO=J0-^;{Q4X(eiOap7%3*`?K2=LG)RKViy|UZRwrxCrcIrke zXHUJ@zk9`XcKS9}Hdx9$oJGGtg_`j{s%MBn(w%P)6vgKL7%D98X6Ei15d2)LuOvTp zYwzAEQ7fKPClmi37h`^96+xs$Rlv@_7(oLJc<9pl!Z(2?w3b|QbhdN;f*$Jo+Cqk= zPPv@Posu!^NLjqZwj#@2J2-a8Ng6RgJV_jas3uv%t&_76e^AbyItFk$>ER!gkVEuI z)vGjmM4r8C_}r%`&Xf`JPcFlT`-Dq4Mo%eKfgJ|mmO1fr+65S@1v)%!aoDNB_9|1L zEM%Pa`i5{xYk^y#oEzMDte{xb{2oW-MJ*oWxb-@YVH5-~y;89yh@8&91L?zT(^Z}9 zR5ht*c&0m19v#%lsrdAx?UpHN>3~gWTsB_++DTx{41flofqJy{gdp-hPS~7t3cJJ( zI%$o-%!cb9p_NiuUC&H)w@(h_tZT>(ll z@k46#GpuLK%QRVf8S=FhcFq0yw%ww8wztP-h2p(B*-NS*I|Uq%-M>dV_lh8fpFsYF zrMgbA_LTyp6n15v_=9SHfT{xDtm|s`hH1olt@Q82CJs|CMXf zoObjpw($Y`SVPVnVa^Y{H7$3K=t%%51VIGGBbS1dzmq1R4n2&%OGn-!yX=T#i*-7!WLw?GO)_ zKf2dt+@~iH=+m>bTr?T}=pyKHd0?XN67zX)j=+4$`Acj{4eEZMbu8o97)GEulmWmi zqD~Mm_=!=)xB#;_aiyE-Obu<(cBr1yHqFh^entx!J3DJXV%sRIa8)H2l4!^~ZqhXEKMm;%D{R%l3G;*-LalIFE^Y>kzxUgC%RX=#-JHD?~lygN=Ouziz zy{|e^03GB{0}c;d*&J^s(5FT-au;GhxByySN9aIcLLMN4Ah$VNR17S{G&Q;MNuca2 zY`g0_f7DN^>tNdU_8PsBXn*N*gPnU59Hjb*ElXd&zCt&@EH>td<>@ZbK5A{aN5RP4 z$c+UOhfq89jLVn6r_Kh7*7UB8Jf(b$XiAze;ZbVG?WQj03M8K1vHL=zVS3z-Ibomc zikptK2kx*pl=yzR}(a=Dw?Ew zvVc*$Iq`#G)5p|js04LLb5o`0r8r6P+}UvNLxoONPxq8$m&7D9Gp@Ps)PRR*QlU(;Psj4)zlS6ZrTJAQ-+@}A|#vr z6QlUP!#W-_Wi$XS3V8wNoG3S#ekRZV;BvXP6}kXt|2!4> zHbHmLmHmUtOFvJoUI&b0E6f}M9aK_F=E!K4?yx!M`tGfcKH8Qw+W~#O4Jvr53@P9v z2X?{Z^}tkxhGM4GpfWKQ2+|qu+XHc@(1eNqntn6cz{~dhLCyRmEF1qpjk1D3H^cvF z76P#Kb?Zic#{$FjkLC;P3(PCnCADfV~XL&~~HYcI)!# zG;nlD|Li}g2RG z56t)AJ+Hx6eV8Z(sjof#9T-~!ThDzBD(qjlM0f?-`bj1-(eKg6(kOaW&4Lxa7%R* zj1U~uZXV}>Z5$Zqjy8K_xc!97_J*;x8@%5?TAY&Kb6j>A11n81cmJYqC6Mt*hn^VK zuLX*VAU|XtNj3@%^k0t}*gRV{GtRZ-97VYm=RafP(I@7Ua4T*qLJLR|Xu@EMx?-5+_#x1D>&z7TMFM-6?de)n(1ZI6 zGytSQ0&{-Ktjc`t--@F4ANHRBDu1XnSx`A4)q;0t}9yEO46oBF}kz`_M_ znSA)y@|6GNuWh0~*pG$0e)f#SKHUYMY(>n}NQ0~-iFnrWPVCd)tO!8SO(2Cr+c2bJ zE;3fwKaUbrNHW_C8%y`JH&s7M9m&qR=n^QOR`DtFScF6O7a6CoB_G?sTNJpDV+% z6bJx_`%Ea~{0v2jx5Iwqyq2z=aJ0FJZ^oj!s-WFiNAWhcXE_#X+qXX#irLmUP!DWf z2sklfH3kLN9$(}621#y5h#G0%bg-a8mlD=us|)<7i}MB;oj4tl#ndgM*m zhXc2No0dXxokvbg@LjJ#&fK6Fh_L0$Ki4V%JPgj;vgX$I&CkvMA(e{w5es2nFXfCB(Q04Yr07)}%Z? zb-_<&!5UA{9rl?Dc*sB_nBCxf9&@)4!gYe70;*~R=ds(p^apVLgrEjmdj9Zz(5U4# z+Dd<9s(j6Vsc?xLZo4n)xb9Hjvu{gsi7A0QCnMRH?6x*`JyV6Xb&$(=S{E{%#WGY+ zgSQd#CtyWurUW}f2Ru!4jO!4_{iN*VcsX^X*{&svmk?9mp3Q4DVVgL5@;! z8j{khIF#D zix4RbyY`ZQ7ep!r_IkPyWe1hM zkKIx9dt?O93rKKvjUC^%`^nDHt%Fb@6nlSeAxYe!T75+YlGx72k@9Y{D0ULqgF`?r zTas`Lkw620HqW_##*jWutfmr%6_Kw+;0Cfyp)=qCn8BLybl}EAt}uh@SZ|B;wB510 zYwe}~qU(u&s_EI0ipw!NR&te)EV;^hV{}?TY@fNv&x$qL8yc7*bAVfVhPd zU|ntf3^nKnfr4f2%^fUeiUV0a2;C-Z2TTpMwzN?C)~u_pTAmx1UNQN2;Qq85{Wq@# zeH*+qR?`r3ch8%Zkxz=U0t@S^Oi>K7GoC67212Oo4a2H&y3hxI)n;j~PpQamyohVX zguB3q*);3zu!(hope3ZRU*cXhAmr3{y3Rj&n~)z5tBWD#({G`nvxCsu@%$N8_;EWq zk0s~m0D@gt0QSfaE<`Ye@N`aC3;6DS)^jF?ahSO~b92G8=R&G>T>kqDc8@M1SNATz zxM!ujxI@+~`3l>Yb^(5`zo&X_L2J(xT(1TVS<}Rjm&*$HMIbTqvD9?r>+Mhwx$RWk zM6*f>q;K&qS|M3Sp~+`Q4w zM1|tBsEScf_7f;p%GUM$l0f)?m42?~AC0sY0y#c!3dtw4nt|{cLO1yz6R750L+EBq zsp=BvyHVH89l59)Q@4SYFOLq+^L*i*G2YdnDe(e3<)4Y7&4g#kUKnAk} zp5@Ki2x#riCV@+l0g{*yaOy-fppM+i?;FvAa41Qv)r)3kODh~UDukEE{lm&W+N*77 zCKzz^G-cWw@4G$vtz%XWB^(3=0mPN+n#UBniHSoFCN*HGR~!_{zArT2IG4bW3ZuDe zVt1VoxoW?+NMq;QgdK~o6Z|%u*wvrx-NU|Wg6zx#1S)q~FA0kuU7$^?K`Ri^n=oyS`;m%N`vbwdWU{&Th#%p3 z0PDCH+K$2z$Sz)rPN$yEyW!1mgXA><6Wx7}{d+F^Q#760_%4McWfaDsyTLv}Bc`AP z{R9gTy}Z;mTU48L@P)v52f04ON&a!*ISox5Yi-#SMxy{b6m9#Dho82?5{H5!^=5QA`OHsjD;^X)gzWizeo2-jg)uMwx9P~A~FJoymy!w zxSQ%n=}0waD#FENywJ(0*L7I_g^KmJqk6t#0yTHW%8Oq*ySAQ3&AI$%(GbBxq&0xt zpHF=Z97BjJb~9jLVO5lHjqq>34#r%8NL~CJChM%8PTU`qp`G<_YwTCh!Au6i0?cQ% zZ6RnxZaX$Ru;KWA&$TM&_$ zwfcfE%#E}{tn!e0tylaLEC027J_rRAEe87Kja`@fgIW$EK=1xfFN*$8dV|dta1Vxr zX`W0FnG{ceNUi4sOB3Ib1*^Lw=3mHJ3@H$tuoSkKhM7L#wS$;xaWo%hvDyY85b4(V z8BIb-7OZN-6yj{!>JR;KdX|J~oY+0=nt23d0g)a|BhrS(VOwSSy70rVk6if5W$EvE z@YD|-oBZpOoEyy`DhLHhFsU-dx_79V!8Yzo;qWAc8wil0R6az;z6Owh4=Pak>?4p)3G_h3 zyO^Jiz@!a;}n|Pdc}yjXY3D3wslDsWcXRTfE0~c z)q`V-bOjC@y>?@`5)s)%14m)KJi(LzLc(!-s~wJ`09KvENCb6nh9&y{K~YSb6A0bv z?F2n3g_gea*TY5w@kbv%eesoTN_Wx zHV?<*Bg$E3P3vjnvD-%ul(U-OxSk$1cH|UPKRxs4$t*7I#{#xiWCJ{2PIDlh2O&4m zmWm^Gg&|t3gf$a0fd!DumMdGT>Pj6LwHnk;uFs{5sOc+xOV|-sK)mJa@I_YS&MPAn zVx9kYKb&RYLwN8l_@vydGWn=GdJ68{Zr6DG%UNm-esk~J4)!S3x4%7JSs6HUsPXEz z(?zIX_EPS@vi^BK*orKw83x|(x11>$qn!I9c#d#O+OhSy)w3`aq5CLEDalm1sZe*dw5HhE?@o$*qVY-7plWbwFR}!5 zz1~8^i-AVek&Cq3ulXZ>ex@Wuqs4%GYA|964) zE42jz>n6r6ToCQW8NKF^80=N$cP%vZZ+R8jbOcE|5N{$M*rJE~;YY8hsV?1MTn{`E z`nkdi;j5w|$*;!deJfKR#H1b@`Pv)muJlyWMP}re=8WohwnR2Y)fW8m-~M5V6H7;F z?5$`Kj0;wrc?wRVbQ(AeYM2R)cfOmi&h0fl*|^c6rBf_NMbS~a5SrqK3M(H&8e9S7 z6;B8*ggrkexS`uCuvx7Rt4enh6YY)N?OHLg`9!cXtzdt3b)3x)&f%}^jfyCn8{b9;Tlp0R0_2i7zHYM=<71X4dx zNx6w7YaqKl@KGA3C9AytsVjL2qIl-z7G{M`){0X`FCf@mD3n3()ouH{^R36%5S$FtDVem$Y!sHz;^< z_#1ZjRrtYx&x?enm#HLC#-tK@a!TcHksSt(6J_7&XC?#P;H2=! z1u6MYT7qrRYR%Q*brfD-#AM9Tw%;@DVS4-1J*Dlk#RvXM>Kk0^26_cz9O6E}(Fx0o zCeaKi6H5=t4{_XXLE0v~I;PZw`tbSrodx55T7qA2vQ6wSsP76c3d!&N%uIlXz`Mbs zA4g;_(|WKLWN36}J)Q4sWBQbyK=FHZja3vM&!49Nd2mB7BzMdp!do@bj;r&zfjn!fpS8l|XCK1ZWD$>^)VJ zal^w&wydV&p-ryul{gEP%J=t#|Dq?>RHFE&Gzmcj(m+?tAC#y!0hV)Kdc>DSldyTJ z%m3!ug#Sq^7A^Z~MeIX06OXRtzrG0Ll3P21f8)Qc{EN;6^+H583@Ik7Ty-%F2%<}Cg5qn!O*=Qt4fSLrNxf6bvXahG;H0Kt(>y&6ecyp*|wi) zln)4Ruzky_IDC=x!}j*~7ZSS^i*Nhgp8hM*SU%A-C>$bw$Px(jB4c1>d;^{fc5bc? z3s0p)s|wiq$D5T`--@uPjMnc{$&R|kDX7rJaqQiRM#CKM0!SPpew?5JJ>d2yT%iuc zGbFq4cCMRG2I#}%sg$t;9{5Qkh0>;jHOAYHvjuxw5%-$=<0~+vF$k97X9`EqZ-6J9 z0^8H8SOt2!&r;VLA$}!|!*4<_^)IfdXyiwaX)Y6Qgn7t6Hx#(~!eL7GiKTh-ms9*d zGykqEgAu{$b%fsxq`vR{(Ma3DWRQwycoO!x|6cgFU%RmX20#?>CC>@an9=}1rk!#j zOCKg!m;FDLc!H1B0)y~hN;lT)b06luyKOprOY%M!H9tmv_I}p=@a6c=gRAKbSBUr? zWUs0K3qWgh@o#acZg+%}NVfQy^hUdZREnZRz|B&X3+AJ(0lF6sTJAG;_&VZu%h?6R ztb0>G_b(wMKpp?*F!U9F*gJeRL=iWGRXACDe1S@b1x5Tped&Xm)U&8p`8SU&0a#)e zi~kQQ=^Z=S?O$q1)*yoU=Zo_fy5Rj?ObHPNCng9Q_WfW{_~UxqM_KAxWHB_Er{Ukuuiw7c3en}ycc;0Qe7k91QCgZpI(>DU z`O$dKMM>x5Clxm^G#pb(0?2g@4INdE5tzeNh`ab!Zax_l9aeckGTmKR29LEX*BU-N zap35yeZp{YiG{nc`bl3>hJke4hIvQszY5c~Rl zgm(~xdO+n!*@SX4rf_s)Rxu#_3ZUurQqU3g`{ssYeEljPNX2E`e|`1G+@K%D#Wc3o zjYXE=svG^5@_Do#k3@6ak*)2I)?#} zLSP(@H!mv-*w?E({k@#qW;)|@)4AR*p7S=r*TB7sw1IQKww;y*V$V`YyqsUu-`c$Q z`k+RQJ4@Vs#3)v^&@{Q4>x~jBfm^nSoRBPAj`G0(rntEYSXnD1Xy4Y1V(OBaSaewi z1{C&88nyAE>MqK^kLz!c0+b6T(~dOC-a9Ts`t=>DhlpR%R9pOT4V{CP62xV~IM8{6 z3<)xwE35l(x|SLR)}<~LbT8Lm8g;c=yUSF1%Bco&Q6mRsz+C)T>3X~B#SaC;$X6*h&!PA(y01qg{Gy=dCJn|C=x1o|u$ZMd zHe5Z!@d8csHC=4gyS+X7q(oa~1}!otX|Ynr@}qrQ;G^DCXf|M+E&@wN*wkw$Fl7Qy z0&HcIg$GXJAvh*a$mI_zdV&hFizcvhFs~WRpj8hxVV;Cq^c;A}j>A#7qfH*3*k6}< z=(&>0ggf@={suL}W{w*T@~c%?LKgKr0a&&@zX8CWO^>9`WKnCbrJK`y@%Z^`q9yU? zpSS1RMmrqCl-a)>ZI@=a4&Ix(Er!g!;_m|H96qs*aK{Hc(ip{5tPOloDwy-&3@+nM z*k-)5W^wOFZJxxtC*%u*Qh`SVS%8D;$OL_nRa9%tkPrGXu2)9uEIDj z_ZpgtA>{`FY$7Bg&=xjeoM7IofPKjP5y**tUHROfY0lwg?Z57-FANph(OvN#PWP2M z_ul(px%>P(j(nYsuEi{n+nFahZp2TIXI7)BGNx5EM8ZT17@s=_~-^XBXv zvb((QA`hhAMRGX?n@K-sV}a@{u@^(N;~xP_vHB(*?G-B^=#_3ex?KK zg~ucB?wnQo!nCtK!q46aa+fZtoHO4Q>=$GN{WqDb`R_6ZI(7KHsHz}_Bj?uhFwX-E zY;6vH_``w=rs5Mr=taUEv66Eh%u$bDn#U?eXF0@~+Q1590W&bebE|9<@)RO!21y1D zU`5rXFc)d^k;*HDK7WJs1Pi7#iZi#<@*2T{ zUx+xy-kK@Wr|v!nRjg!}_AN>8wHuzbt;-zuC}JCuJ7%=M($L!CxW;-XN`(0`s2{K^ zRWd~-n$cz07!B+bsO%)65zOSOROyPVKJkv<<*M~;j(TR?0-qJ&D%VrPF}~axvN&QX zwv#Y$65_;^r9KXHrgqUE)AE9meXzB8w??5_)B5*$(D~#-y=s3>J}^H!_{!S+A@<+0 zpv(wl0u&mpYTbd5@qt}-5Iq(qd(gxh&>)V?h0U&XYhZX?Q+E7t)A~4ExTt1p*F>?g zJ4LoB(T+IH=H$JZh4A@+hcq8Ff&PdGxyZzj4*dk}ci!VdG7q)+C1Ilyl`ie40$ zxGZ@1=!$8DyJSC$E%+%HQxcj`cYh#QcYh%0JBu5+;di;X5^|r_o>wx z)KvDJk5|5Qm` zTlXDIY=6jMao9HFY$qeTE&G-#27z?9m z@uonhO$g3rd|0N+;Ojo!eFgpxFVQTYr(J#Hu)kx-&1++wtMW)|{mhu3=8oI?%+tXb zU`8qOAZWz0&`uMGfSxp{c|Ho-@_bt})db)W)>+T>G{=yE`>x0#nkw<(Ll48Q<|89# z9W~2w0er8uM@9{B;tjDvw#T?+&yFU_+!~Epg;XpVK(%-HGgaYut;J}D1N=dC+kEF+ z|19I4vMxSswAiYTPMcDBy!JR}%Yg*Feq5FHfxxM*abY$ZC*Vib29#QtCb5y)t}C`+&M}~_vAxz=jr9{$%xAp@%p_>w^BG|1>cO? zckWg=$1ikvN}RdUTeiT$nui+q6y#P_SHHMfQ4w-%MJeXg6-&utxnR^h80NOXP#wBp zWz5nEhAjB1BYHcL?@*~|R-V6oGeOFGOz-#a01KTN>byywe&8I({g#_Ao~C}C*e@U~ zR4V7eyb7|M3=F9XT6>fV!KYy0(vyF0aspFwoGA$xXM>Lv!#IBOF^drIukn{Z-BVC| zy=1P0{mm$J!>)1K!Hbdd>ryXoXw|uSMe6=wOwkIgiAQ!hGERYaUc}rud6~k$q&}dP zsYJ!bIDyrRkxGR<>;-RzM@DlyKDOjbTHKs;-1+(R<-6P&*=yPaG@wf$Y=e6nvH(+w z4PMoShya_r zqq+wHSfiP06uP{taf0WEyZB1z+PPajN*dN)cLs&Vh;H#R7u5EvQ%(oJznku-;>Z7b zfdWE`<9?s()&54NZ^0{M3<|`41LelLC!D>cv)Dy(A)m~M9h zcWXs^V?;HR-Ea6gPhGPa`f{s#7?kju!1dc&evL4UC8|iBfjVg%lDn}b1gaB%aIJ?Q z5M&`ZnRa@l!7e6Kq8H)5=IU2HZ+D_BWOLHG=WWPPZcDdftH<5>7hi0yJ`Pi7W2yb; zvUxUI7gH>hz7B&dl>;b72o5f#C&2r1f0$7cAm{9#>)MGcQ4RK0JdoQe^-Ljp{0i>* zOJ`Y`KPW8%O(>5kri#RutN?BrG(WUjPYA$}7AC-HwE~QJVvijx-%uT1sFF^JMVk}i zFO~(QjrS4~syL>wBK;Vv zMwXQbIj%ev&1CDT)gq7RuuQBfx|Pbg3var&JqbQu7josPEW6tiRH99(WEghB4Y`aV zt>8f5qz@ijty`wXK&|3CyPuouA>NK}=KVzDo?!ztk*oEW6Uynw6@JKbhMtmkeGQc0 zJAr!jq^wRQ4uU zEMdM**VQ$THQ3pmT5VT+5_PDrR7WTnC7EdEh6Dq{SBv35ROhhNJAR8<}$W|mv8 zR5HbfsG*d%`QW2sSs!zI>!lXd@G)y9?PxCb2i{NI#T4I+H+>)RV$2GfhznHdWEfpO zh2H58u)qZH%_DGji+J8sb=6@yhJOe5V}meX3rGk%De-dJ9<^zFuRSD7KkbRN z3m`{|epXd@Hw9*6tN-6DskdR65{DTMfx@gfka9>%&-;$#|2 zY}@Kg+x1`m72qWSQ^o`jQ}&(zI)(pfL79ix`3m<~qsx7sqK7;vml8WVP61mP#}6M@ zLoRDE^r0WXlvI4QZuz~2Ws20$z63&P4f+P*4dH zp)QTK`vnMR9vqmA@D@LqKEtLlT5#@C`TcUP?IXLir%=#!H1(g$2Nc=KMvWLwrWlMR zcZE*ytGkpfu{WfNyp4b3N)uW5$#33pY`%@Y6})Ap;M=P!ZDt z4lO9kcm=x~RrmYU=~Du>7iQF&s`n(vs2#ECPT$2Xd3~Ed{&W0O4Am!|DY$Y4j>gaX z5~zktVolvw7pyH>DW=yZjN(=-tzJadypV1&v~}X_xbDz)Tc*?pQ#L`hz-$=dTDutg z$H58rp6!QtLGFDKA__w!5*n7DSgK@PZ~*jbT%bpcq58O9nWp)2P<-2*l~p`v2P57@WYs1nz)aA9=iIUV{&i(IQuenr@bUqREFZ z;W9bo4_ta18}(k7&qdABC zLET12q#5SlIFizOxrZro1LuX4`APdCsh?0s+)?idRAv;Yj3wb#7kp5DaXrVBaUpO& zT2O@<@UFZvLQl%XN%)ytzN?(zXqS3YtK896&KK;wU%&O}>RU^UW6Bk&XoMf6kn!lD z2&BfpYs1Bxtrcpz0#PrZH}ygMy^5)s~&RB0;_)(apZ|%PWm0{R3^H+ z(Cdwxo?uD`@NGMnewQ7^^$y-Fz)5<2*&RE{#^d=?)-x6pIgI3BNFBJ1GwV;G#fo{* zgY3}*C~ABc=m%IaxT2i8I#xpc9&pBDX|93nd1^XBiA{gqGuBOXFMz$M56Ng^r{1J7 ze3n|;m?=_T^Ovcg2f2=KyyZj=yNjRAB?HalUuH$R0i|iw1od(jd;+X(pSNV}Y&BnVW=)v@=!3(M!IGZ8W0F+C;Jia{IG29}~9-kN{3& z@Jv-SvW6WSu1SOT(2$#R7%rhJeL)R+`&{UD-oIRVwbjp0E8offtED;QgyHD!A4`EA zm}mmkihtw0^O7QH90No~*v-INyezB(R$AR}n4lj8mSXNSn1KnPagrk%&7vO&!uAw= zV<&gh@Xf1Ab9w#|#^Lns$@<(faZXM=>_4anugBC(?JnP%dj%lPhhPd%C^6>|8cmPV!`|af1rU z=0k|l6hG65pX!Jt(7-&iY9V1mu)Y;I-Ve|;Cj`)^#RFsl3u9xwDi4;~RWf(iAl&5) z3;@kMYA*x(ygg8-pD8rS&-LNz1nhXFTI{Ahr@~VosV1*C=yAc=y=)O*4M`U#J_LF& z6oWd@f=m;yNlYY|K#T~k)h}>(B4$@nNn$A0yaacCsam1f$XO7Eyg_uVNBtB-X* z2va`&@+2kmBnl^-e${s6KqLv>D~?!$kZ%?A*k#1C5|x4IpU3}X|H&(c@Vy70iXCwm zP2~rgaGpB$EN(_qtD@Vb@W3Y?{=b4<=S@`2kDXvo7$Eo~T+AErQO0fL+XHk91{I(u zLse};^F^oI0G2SW04Hq`|FXEBf@$O5w~Ne`LLy3tfYJ_i|;&& z9{4sDiSma7Hty%PgZdseg%BUdm!2965^9iJsQ~)9C8r=sCKyqt1>5&Y(OPsM94^0&$ zGMw~3S9_;@H5RT{i@5ir<*q*(1n@*@42iq~%R@^Z3>*8Pjx0XthCP3X^Z}Yrw!y3U z2WOL?CoBlHh+lS9v>k5hW=1XNuD-C~>jluGYX85k3EX7C$y7yBe2Z@>6ocW^Riw1^ zJ_$9`J3#$izERF!38l_v-x(`bi*a_FeN`aEO+U^`&{h7whu^WX&={ zDQk%+G9+6PEeL6pWDD6cGG;8<*A$9S$r3U4ov|xI*^M$|CMxTU#5*(Q{9exI{{F7} z{+@IHdEfVSopU(&>z60NcPj2YR@OVt9F*qp;dI|VOmS=Ki%7~k-?aCe zTUzLI9=FG4@JxB{GBH-q#3-k0p+?9gIQ0&Zkv~k%+{p;WZ;Te%F1Nyj{{XwY7gXOO zD-m}sr_?q}z_VR^xG_VshoA)P+%dgAve(T~dy7-tjz(s`ye_*$5s=DW`z{GpIyJ&h zVk&HD4&VX3pi%rks}XJ{kcJu3&2&YGLe7mMfnEf>3K6d>I``Gw$~@Gz2TM)sDOOFF z@crJ}@?dfO^$Tx~puHcgZ|}(?{3u99Ww>Q4E{aNz&Kk;LcBZV3_>Qt=_jw zH@Nk~nmWml1X0*gLkN@zog9G?wX^|~&nO}_5Y9Nj)`*}|^0;yzKK&dFnkj+1&3@pY zNwxS&IB39MrC~wK5gw%Q{@qHN_G-ptD>9&zGu8b^+l;R-4~ zQ=%LDeg_~=JJJzAaubf7#xDZJ%}+mAw=uimRuUj92cvXgFfj<69X%+e6VHRbN*%DH zeNSRoy6gaBz%iToKsl}N0#VsKNaf?>ri|b@)W~7B$C8OXFaw2d1Wgjrk+N;l2TU`y ztqu&nT=)~irmx`QdKF6TGx_~Q&wfUb6CRjOlmLupXT8sf_e|fcu66d?4UgJwy|TS zYh18iD!eN%2G6aln=1f0lPn~H@3nnsxt%AvmnkW{KeHlde%@4-mQT3kXs_9RvY@4Inbtel?Bpj_KGOf_DUz@V zIEq-eGr?j7VmqdZ)$o;Zx(&*I)0XD5{GF^8W1Fk?;yUDM-!w6LHG*>W!DZezaY7uQ zIo12+|2#eQsa;7FIl2~y*sI5jc8M67p|E#a@}SCf*dJ7+bT{#~uFsqd6YEtlp1L&t zL2T%AZaU)*Fi0T4o>zregA2n8jPivMd@TX?%R+5aZ6|r->?NZ8XY8G|=!~EY|2Gqv z58Bv9DH)uXBbD-%WlHz3ut`R+2+tF#Hbm$sc9{rl`ovDkZ_A{-8hj+Y$NnzvL6h`j zQctyAOsv}7WV5~b&jntae!P5Pb!5*g=X?8Zmk=-6Qt}wmt>C~;KwjWh_74D7XF@Fu z4Gz8lsC{=F$Nt>M1n<}ufMnX*F4-~!BAHzNbQSPd7eQRd+2TOgK&t^u#D+4lI`g2F z#8#cGUE?mVrhN0N;Xy-rzw+<>im5YQM@jXxDCB1>sNnnslV_!T-$_2B#!S0m#lCzB zPnTl8pH-Sv)3qe$O}ozZ7QFQkRrAFbFG{mlWQ9D>NdO!q)Qq6J@)`1Jmn^)|nveh$ zL}K*-1qW%)S#s&UEUsUh!|k8Bo}HRq$H~Ld+_`$sRr`zY*rcry3p4=-@8>@ZyZ+th(eCVi z2mXbs;XK*l7P-oDuoG+NC@Y;~-M&ez*|!KZs@v@4;AMJKp2h zvwc#qoA6Qm#%E_F-zWC1ub{X41EWD5%Avq{hzVs{HkHx8ESos`k3?MKlb8#WQ*0-N z83zkLZ1|^EBUTMheTZcgn74z=Y@mVL(OnAs!;KuNXGpE=hO|&5-GZS;En4|X?LgqXi2bjF-L?zg84?G;M&W5# z#(^iBg3UBOFfW;;Kt}dh*q?RnuJcPFPF*(?PqD1b|4FhB>Q^T~N&4lUrcy$FUNotex>Au@%s6#jxKdT#Cq4B#MS zi9H7VWhQasJ{I)NsBb?JAxw+Q@|3ielob`sTK0dI_VMnDQ%%i#6d=u}DKe(W7&IKeVf}Y~i_J5rl3} zEA#a~sXLnfMPGF#yuE!-YubOP|AuLR)f1!^=+D^II{0Dw?B%K_pQWo$x{OAb9BEwnI z>CsjBG`(?&oaF1wMv&jKVLLRiq%V1p!X&7CP@?))B&ZQfy}7Eih@We&uR$@4KTt=f z@Kc&mK+f{S#KhMtX3qW+rX62Q8u?#;kiiy$ehOlSvZ=81gkj`5$47A9ETxKuYdW4A z`!e-&_89GK>(kLr37=S^N5g@>>gM;=QDU5b6WB?33J(zK9Qgh|*idRjUoCU4+ zMZKuM*4JbWJN1ttEOvP6)fDKk=rRZ%0MgK~5Pfu~Xd}sH){E6jD_5L`jP`zAKOPX| z7ZDg?cRS;x_te&EntueIRbO}T%yzeN2#J+HN;~tSRO!FO z=C;zQ`xh7KB-;oauKZ4xd&}X%meC9I&!-5z=7PZ>OVVKZ1J3lnooiFECvR+U*`*nV!)8XsQKjY^oC;g%>%dg+Nk#23oRU981o3MQC@75YS z1rWR$GC973E53?0+^Iqv0(RE&od(MeOE{|FfDKI{O2a2M>(q1M`R<|R?F0v;{iUDE zD~&Ho6?OK|gPN;-i4}m_1klCiMsAi-z4NL5OfE1`A%?!}@>Zhtz}!MDP0J8pJMJx; z^aiO;7#b3H6-fVuuApDW_ zwA9UoqxpN|4hd?UxBgKQpAY<$2~`weZFOjU9fZ~Jk}p%U1k~$J2Zw>vgk!gS?}PLz z^;+bo$SfFh<0u$oBOyEx(=jC6N;(z9)YZrpZqkohA}6YFV3|bkj`tnYMazu#g8+g_gb{ zdgjTDDwn;I0o#c;itTOVi6YJ!cL~kF044y53{bp7v?F5eQ#yT;kab{+lN*(78ennN zU{crF-6zm(C_y$(E{Nw%sghIjBirUlsvT|EZbH|HR@y^Yi`8_EzRd?a6>U=*GpD!yztHXqrl__wKX>21|qaE;{*`uVUA-vjKLK|G5JstVtE!LEq1raI|$Rp z36)QxB54voj+mvXe_VRR)11v4E%O&l6XcXhp4UHNNhKhia@mY2`7&*x(C5QBqdW1W z=H92l(iuaII{?z(yC$J#Z0mBGEuQYrp~sl}nnezT9U z(qhk)?yw3e>o1jarv(^)QU_zc) zvqGC3waq=BD+gBmGJZKHe~U+^p+i>w!yjXqjv5$I=Kv>lU$Z-2O4rtf!UrAend~aF z)@%ek#k#zOjO(9d>OUV=$*4`%DHXq)bnOLOuvjrg8)gfdm&O1t-^bv$CHbrvs`cWg zd^X#VNo{ATs~ICdHwpDA8fDjiD1Pgmyx-;Q#bU?x1pbV^8Ljgm&5Y^bgTn=yPuHqk zSa(Rsj8t>Ou^2`tr z1^&6E>>OIOaL(H7YVWm&faB%tF4GHegjCzLsXhRhtVY~v9_GV#80tEB&8czx5PeLv zgMQaO(^yL3`*Kn-IQ7dHx$ZV6JqM-y(tY$sE|YJTjMIQ-7TCx)pMZaXP%R*7>F?h} zLTAC{Q3k1xT11m(ZmGX;&RR+!Yp|!}a^=vK!3&~R_h-!HU9~^-*)x4WmJ2=`agF&N zU&+4gN|*Y9*|QV=d&VNDy(#-tG1^7CjCW&Jw&A`o&b2Yh=JbscMRa^g3HV2d>j55| z%17YSHju4jE42?46iKg;9QSIaPOX|d&~;Lzrj=7IB!o~?;@U1w2GXzb;O3CRZmQET zIba9JFa%KoHoo?#!l!q-ceW)z5lxedj#Fz+va+zBfNN%5EY!R0@OBWx4pI&9Gz zrMBX?{RUYDFt`dNJiIMd_DmDXvimqv^VyT2j5`7+6eXOUavL;hMJqJ$1t5Cr&;sB1 z5?gcgE!`4Du>-UUIDw`2IIelbWtXELoX(B=O5Rj%lx)(&pB3ldIZKy=pe zCTj=>;dNXX=7^7gFw^p$y$UK8ub23=u`?^uSoBFAihr|ZL?BKLvm zd>nMxwi6luP_(<%wHGPO@br1@!TDfTgcjXe2*kvdU%U%CG4qr9K|Z9Shbu%>e4g!?OMnmX}3Ew77D3KbIgYo|st{dZ(>x~m!GR=u~#`E)U% zJGM4^4d9&?K%k%>>>L_Sz021_|ADrCD?iaR|268}K&XtH7t($-AI1M&NkTsDn8AAb z<8U!&eSY8~<-qye4~mo#!yb$NTi7{Y6f%WTIQav|;IC(Lb^#+CzmvK_6i0)|B*F{r zA48sfTXL!C5}qX@klmmC8K{ALOe}xCXi2wu{pPvHrhV)CC-*gS4B_@PXvC;_+3$4aZfm6(8qZCb7P4p7%rt|A< z>t;KIUdSk$OFGLx0YUNW6+R}n0NCzQnrf}<+#oBHOPm_94)}Al4%^s%~#p32$@KBg$bLfTd$OaR)E(?F}xC6n4CrV^xi`nPDd>mw4pIn18 z4$hH>h24PzGUJ=MaUGRy6-kG8O=g}kjs88kd^LybPhtg$xiTOtH4ypqYVKYbrVqk1 zxt73cHB;H+Yt>i*c7=MH;)36X*fhdBG- zdDo!ZL#xxumRBtOkuUL7&r{X!+_BUf!;vEPcT4?!_UHSM%|p+LDco@=Hi-#5cHR$m z|A&0M0>MwqIB6UViME1Yb~yz)1uA5_3$F;#6;@PehF55lkqp~Vx74Lc3~c;fS^k6+ z?E3dc^HTFLmuptpqqh`=POX%tNJ9R=gMiiPP|{_m49fLp;k#23F-Pl0esh!;QnzB! zCWR7G`f~f$T}8G7t~{F&b4oiTmpHj(A;jWf^tmQ)zuQEv^e2rU_m=Oext%YUu)v*w`m;AKT6_BWvr2Db{+Jaz<@>+a4gOg% z_}gGXAd>r+{3_+&Dh8T#c2IZJ`w{=XJHFdZN(PRaXc*~9H$3m(Xdfd5CYj{~JQ zCj_LW#pDDxi*19ZS=5b<$>aX5CXO1gY;><#6GI!}Q>>b+??n`f<;N?z+1>Y*Jy&%r zR@EqU`v0ERFe5z z4h6R|16HFEkn^n(v~Jdqw_NpkOZ-u9-)*yST8&bZ*`FNz=TM7O+m^nbh z2Bcl$m8c5L{oukyyai~#s_!eesPxTCJI^t?o^U?2Hjz(-;ZW=$&DWOIcV$e?TkW?5 zLecZtmlO5LI+}Ol&PA*@Bc9o)f#EAcJo88wM2` zP?L+$!i*z@fkXrjI)d!9Bw#5|x6xo+$vE=#q-wx`T6LGdX6t+pE@wW%TG}Ywv2#wV z`rfr0vSFc`M#B4w{5I{qdcNt4#IpzTeoEzB2vc~CPXKNFJnBtLNGYh-C*J;hj}mUC zl&lav<1i*Yk4BJ)LSKJ2-Fy)f6+s}J_LX#ddGy0w?Jvu#b}1ux>N$X2RsbS?m%*#$ z0k)Z|w3C~)G;9j`BxOy|VD4+kIofKoe~CjvNDOR{BjaM%8DXBZV^ZUha_Xski{$#wl)tCSjmdOo@mV570Ct97!IzUFazGu28^&oR>L-nFYVrs_O}!*0?FlV`ikrw1lZ!%u3wvNx#}K5~a1k%p@jyBZiW^96TK997)%;`>t@1Tb?j^v%w~HVLOcU zc7y$^Epl`fhTl$Mi#hE;2XNK_D>g)%D%wJs@7oVWADjzj7x{14nJH(NeN*$9Z&)}j zmtKD;AdbR(=EnNR5durT4^%ojY=FK1xAAAaBV8n){m{ejR!N#Uetg1xL5I^T5xNOn zN7XCDJ0sM6-{Tffw(AKcv;(L;nr|=VMq$#Mnd=2zg`-EK4=0|GB;_>rwR-NNzy9+t ze2ASP3_@uO@XoR5TRW|73JlcFvp_yhD6bO`C*LZDO1P7djuvMY`YBe;8H=7;NnPFo zx``tC_dnF6?$-yaHEqk}GLG_oa0GlaWaD)TptEB9So5fCRY>|EVv3ZOv1A=r*3LcwPITMXb0@5c`+?IBm}>s z`-97N?)AS2J#u|Qd94kpCJSatX?9)&Jt0AO59-^--$x)#IwGf@Jf6sB-7~1AB^9=8GLaPT!uf z0(3YGJtUJ0L{nxm&?tzhM8UKbrG{mv`Z6+A9ovCK+yg0eR44%18Fmx2p}Gm|lxw%) zg+D>q&T|1i^?w}dLQ6mFZ2Haz6SXp{w*nzxK>0vzG@=?iVydSBsvtdM2xtX{y*JLl zGso~$Hxq`?1Rp`Pr;5SaO!bLp!2QcC{9i6LrNmN3S5;-6{s}W@-QygnY(+}FkjBYo zI{`j}V+gHnP5>OQEaURd3(%zJOL|z8Qj^22AK86S4@WL-^wBi8o1Igc74VnKaN|my zbzrTHzE-}#`{y#<9tf3zboU)r81Ifah$8dN1oh1xh*&~4Bu{pM8o1`{^8)uz-MvTW z>g$tl$UZU}IN|D_@!;;=+bmmfhT!CX4ppNtK(8RK9T^{^dYcvA5hEJLWC)EB|< zr8@yAzMeE~7^uB;Vh&M334>mCrQbp20+ZX$#dQA3maRaX?)!psTurbUin|;Z2k`q~ z3l+Cke$r1fXP6Q@>3CN-;eaC)GM96*LnLdX8l`qQ+p={`b+w=&DXN>+;F=_|cUp`9 zs9nTX6j|2CV;Bd4JJ}oG*cLsz)p-+E7j2e)w?R)kicuiHXXe~Dl7IMz9!xhAOdC+o z5#hV-od^jCuwuTVmXg0dWvW6}S(u_%9`;GNC!Xau2@YS+PYn54G`&~-#(ii={J?!! zC449}h9Q!|fZ_GYNGY08c6aRVZf2^Fb7*& zj<3T;AV@PrP;IFz>LQA`w@E`!Xb~0)hBVPQ4NsAH2aeS;wKok7l@jmk8nBfmae6+= z=5oezz7oAGf1ZOd52*FHAbuE8`Iw@qSeoYc8n9WL*bYACDexVq@Gip%vl2kk+pDmT zJe>I1z&Rw}?UL%N&nhan+aknu8Q~gLp><|lVZzqV;_?;9?=k-xLF{}3$G`|UYkckS z3ZCjf{thHvOUdbzNhw{9S530Ja^)xQ1pjtTxWfDtT(+ue7uS%iRbvy|pe~dCsoA(v z-U*&fJb)k#V*wdnOqNxtV3s9eWP%_|a}rlI<#yW3gOtYcVn z17L;)*qOXZJI(kz0O$jMgz#7+bXimTVm6R4vs15%2yR)O9w$9AEqPKoOs&{VMQm<= zKMP~cVutTVEMT~O82-2DnF0+U0b1!QD9}2Fp|2L%L(T2EGEc6KY9!q>;B70ZHm>*} z{({D>m_$l7=#cb9#t~^8U^}leA0c7D0tB-I%>d;14nW6%0vB5rI3ChylbtjKDK$?R z>$D5!Jm4h*Sa#^V?q~zw{Bx>@sC18m_CGu1ZtFK5jjcMc9?>1%K8Vrt1SiZ54rDPc zCUF5Pd|gb~*t?g_X*y?((C(7w77yPRv`TJ%;ukh!DW|PFxzS30!0@I8Eq|s9GLSp3 z87%I%^qNVCX#2HT4L42Go96DS@0O1=#JQZmb2qWYLrm$IoVHvU7H$ugPNa*;MI>81 z08oA+^)fc$(;znJ@W)R-nOe=Y;o?)3KzVvXJM}=+d24GiE*K1;8&Vh&pTX&w$4uiZ z6jti6ozGRESJ5>J?3>deL(+uL~=`h zsb6!tIQGp!n9@3txgWfEgd%fS!f%)ZeHzTu_mLeHn1XpRRYXGG{-q#Uf&|@RqF((% zJ^LZ)liv7aVorlY`s}}D@uJ{5tpYo746zbN7e#r_@pYt3JCj$y@JJcsL7<}{ak9|ZkB|l#dxRAGt=T;8ecD%mxlDziRl!?-KjGr7(^JJ zRD1!$zVLz2c!qE`GtGboijMnLUx9+U=T^w(ZW;)L(d@yJ^8)Iqqx+l5ISN;xQc_@^ z!3f%{_$S8wh00FB)1Z2R%xTRL6Tjhxq!f9#5xM!UFw7k=lL7~?`h-WHiU>0uW@jE~ zIKBJejXoxC33xIaW(f*cH59mdlLo~1gCK8JJe z^T*UAYb5Tu^iw>xbCq!f0hDqfrIUhvOfCei=*bNE$lq`LyNk&Sde}f1>MI!kS#hMX zZ(8gWw&^Uv`CGK{e5$$*MIO?}TeIj_M{!@3z$&BWjPai8)f*CAwu>Yq2U7Cke?eUO8Up$^vNTga4g)sxUthA)M1l$cr z!eW#BR;71DNU5#*;g&~|uis-F87uF#d$4DLO5gv+B-{db2DF?y4q)AUpu>7mG*PnJ z3W98L2yICrR@#}&4GMm9G<5NF`=x$yl9eUbIH%3(_$!ms+VHmW72?RzvMggk)Yfg<()&T8zl9U4C$bjz?i$=mI^ zKkXG8zjeLb8Li)5`tW|681U5*;HNv)@_Gz)*-^iTY0yuv<|i-iw0H`=CaAY-WD5uB zeKPlweBorfti=%40Zks7=;k|cmbB2v!GAFfe(04eA|Asv<2C|e)@;KZI&cF;KdYw1 z-Z)uKGrDuWXnkZ=jO>udWP-v&|9`usHw|`gDTogk{1hffAkBZ1A3PMzxVe*ub%)1_ zoMIk>zPbdumJ)vGI~u<1oCp%}tr|#Gl~+2Q8XhHPJ>;J<#x;irsd8M?RtIvrlnn2+ zMIukaA4?vUTC^BX?+zPP`K+n9B)i~NBIJ|LLZiyCF}?0~OpaBEW= z*GOS^$LyJ?$#y*FRx@}5>eTTM@h(5_AAG4f|I5o$qwj}!L*sxc;Q^l11Mjsql>j(g zUt*>awQZGiW!#sj!cla$tV=PaM_#Go0z+>3JM}L&@S8{m@f~tb`zWB_OGX^l1@F1b zMj6q7`rfl$ln8u1=@nM^o$GilXZd{9XS7R@iz}&ldB zEzExX6T>>;tC?7evm?Iv)NrNFbZ;u+t;vv1JIZ0D1mmjP<9MbTVd z>hIjLH%o0Yb+Ni4qJWz6Y>U6s0^+!ljE&4OrFxxM_CJhu5Ps%*{jvSMTOF7LEBXV% z0+`1fxN`Pt9SbyEBE#o!H^Jt4(8GxgtLd34<@FEhrKeU#US8w!PcwtrF5np^oU~D9 zB5NlWy5Wdj*MwnGz|q6OS=&??tl)_VNUjSZj%F=J`Cl2Y&9Stv9Q|$D?cbU-HtW*z z@%fkVQ65&Y-)0qy`o+hQQyqCRi zSJUM$nc5gPW@!q(fR!~QbKnN}pyUITq8%}I5b;wRngCAO1ICOm#tm9RhtwF7DzCv? zAsPM@z6d?7UBEksS`tI^O^)9XuhwPH8Huk)&G$CfXZ=W${GjoJOSnrpEHeETvLFCv z`^ynrOB|XL!h6xZ&>B01@({Kd&Vpnp*HC-#@y>~7i^o@ftJtiXYq`kxC_MHTij`D* z_Szz<<@p2b1ScJDMe`Y{_mki2Mj9tUGDRjd6m1t)<@Fm%K)r7W+6C`qUsTWF|{ zdsLh}a`BhLg;RT3Y}CyLK%o6?+Ufa}?75z}c9u(KxD|FL$62Zgd3YIICbkqvff_9X*&ti+)Wi`y>0K5)T(i~iJnihG%aCW67Dwe5y(0Ol zc9p{Wrlu2aF1#{B*6J@;_k`(vE!l1n{PNS*=F^eyX? zc|)Umz{!%c{fTAp1DeLnxjaZ2-i1H-WeLGS?6|20VKrN_Hq$*(lMnptZ~P&;W{5wx zPJqi=y}#yEEGGkIVlU`vq6ML9`Wdv%WE(sBK2;=t4kviBZ^|Rn($&AA}=BNQLh&!4$ZqibZJgG zwcvVNm|w(N{IqMJ5zj}K9XRv>Yj}aksz%I$}F!ROsH!X+7bcbUkfm|AXB93B6l}7F1^K+|EPu z^`<9}=jA_tIOrAE=HzImc;SUJKx~1V$NrGP12vQKAjQoB0N~Msjj8A=P*Ql@ojY@7 zP?7eFYVNK)qL=gE%9sf~Iw z7uoA>iDn8mn9jv(?0xh2Yr6F{!R|1YBltD|x9kOsZxFQ{(_0N z38T+`9?Hi-*>f2Dn;#ERx!OsC^Fukh$9vk0uUU+oDPM5PKP|uC6S_-_iPS|>={-$F zVJ*RPp@znj8WzSOysjqW%Cp?r5)F6U79!kU>`Oi$=yJ!2t14Ar7T=PRkawU(g=#rx z8OJ<{TCf{EPfdxz1(w}fdG$-)!Qr}e@C6zAVAZM4EnQoZRsuNPd~v?0Qa}Jmthf|; zDz@oSmr3R|+6&s(v*o|$HTr?BVbrM%E>qqYf~Q~drGbMnni577Ym?jcxhA(&w^GzGo_Bc@ffzt(DUK)o zSVBO@Vpf9P>C&Lqm^DY$t0z0^E%@wRJSb9m7zh~=W-J1I_x3$MD2D$$7i1#N)_aXq zHKD)|E@kd-Ktc-NSCYz^AK-iOE(1OXZ=ejE>uJVixTUn~va>CoN8|SYh9P9j+%C30bo85c)4#|og>^6HXnipyeqNU8SuZX&mq|W)U&Y-S@?pfHC=zT}&Rb<#aJ)*N@IQso)v%CLKgj zPm{9*f!mwKW=7)LFTEZ*QBFFDdXKnn`zw1MytMAjI&N(h7&*Xr_W>D>7gc7eqMdeL zuc29r4M+)iZigSIF!H%F8IxzyzMAOFoHyHNY)bujjX(S8iM>zZy~G&_60&D)x^FX| zHratoM)3WH^P=*MX@~n44;mNn3mVJ&EBhT|dk3JVZ{lP{OiXO9ScfCNMq$`7qj2iO z@*=Q7l)iXfUMlzI@zEB8xwCNVv|+Oeh7sBQ-RjzTtXw~}k2|(Y+pn}Cr`bR9F+lFX zspk>lNYOSOK(^Ah#~sVj246h0+$I5)zC>kr@9RHQeQ88v=MrIIb=~#0h&CmOPuckU zog`K>S-DUUx3OpK+Pl}-j`O-F=mv{7S;pk3HhAiKm+1)lf-JA6=1$Gz6^nazX4A57 zDl2R{T;$8o)#=p(6B__-D(NJ10#T+Y-;KYR2z&ML+b8z*Jiu_4i^J|qFK{3I>gaf}(!s%@a~NPzST0}N zV;3g_J1sxCp#!|U(@s1nK=dysyofmEh|98KJCU0 z^MfaDSL~FCJU!|x=o7^C*!07K4V;RG_d17ij3b=Xc({EuF`v8`KKJhkeDfyoHtYq0 z3yG793M)zAYq}vRI=KEF$8>kURG+#P1{4pw`s(WLqxialxMb2OSDK>TW#i|2(a%yH zJ`Ryi$m$+F90kKlw%y44HHcSaCgMpS;Nb`{T|TM~C8iH6hSnPk80LW1tUr07rOIZz z1UZ!W2ma){~W|$Mv$Tb z*;@)Xw&RZ8eiW|z09rA#qfMPEQsBiNA8@@qY4CaU@)yk)6Ted5&1B442fFOFKRIq^ z0Sg@lm4p8xM`up}zsgik47CRE#s`R$JiuB%3lDYZZc~Dc$YJWH`1uaW$iJ?=!7+COM7 zif98(G>UOlWemA!?>qL+d*u2{g^-gSKozhVcY)%8BM6GzA}HRZ~3N@uh(N{EG_qM=?Y?DmZRzQN6O zx-}?%STWe4E`RzxhWSomJ``Ha2b)4GH&c81;_oCLemkM6siL^5BY#(BKqi7R-G2%iiMAllDMNl{_`V^vG)`W|4st*cS*v9P5!pB042pGK?7(paSVJ> zPkLOnO~!*zR#3*!4}IOgy6(QRD(7*{G|l3#{{7%QEsM&4Rsr=!sdP$(7I zelR6A@DXj|d9k_cE&d}C=AOlc4Y|gDTS@WfQ(-42McmJdnvxi6x4&CZQ5d1+} zyAN{~)AvAH9GLj{+GhcUKx&zM+Q(bPTOlxh-|-KE`z%j9J+)u|w0I8;v;03;qF}bv zTeJa^C5Z2)i^vOjDx8AvS~2O8R;9MRoUmVrd3#>O!Cjw?|62NE@e56+QEBG`1Mb?` zAZA0c2s{|O+>)|2hAMIHa2WP>j@AvTSNTgCD#Q-7T^6ca+UjqxXr!8OzjKIHKHz)E zl>Nmg_JtS*KPy;XVp1N}79N46mh8F{0?M*aL|TrfG4YsB?czw8s+{}k_XmfZll)3J zPrC*f@vN3$s}C}+0b^^Wvf4YI8MJ|b^tj2Ro;IFj&VT+Khz)RHMT|ct7o#b-XoZ;F*KtxhE#A z!GxI`cOGc4Oikusru6pFEW5!5D3LR}kgVniiP_<8yK2l0lc+)lUGDd?!n zK{~|){C`K-p8??u$+b_Km4*Z9_9A|N+ac&G;EY`1zMR!%_xZQENyI|$qQWs{^3+Cd;u-WvzyP(X8t)oN4&DQ)RefDxW267^$nF{#`dQ2jJlvW=88Vg8^$5(8)R#(=Ge&#*B%TWy$qIg#tYbGs>M&04hRZEpA(TPcVRF&3O8#|}x8phaiM0Tv zsKwbYB0XWjvPmTCAXfQfw|3LA&)AMH_-d`bOmMs~n2*wWwUA5k7a+Y+^I-!YdnW0b zS^lST?=p>I6~-?Ksmhj#_49|tQ$C(}~B@gV-r1 z(R!j^pwVu)@CmI5jgZEqg0}#9A6$N<@##Ld{vLn+Yu1Nz`t;v<{!;pfrw!8sozzHu zCL7ub0$B@NZu1x@v*?YX&%%@26p#Si;Q6*&?upgsh0%cbGOgD?%s#%bIVv^~^-GRZ zaTVAl{8dk&SSioW5LyC|R~w%x^1x;;wJ4&Tp;bJ+bka>v<&!u3WAcD?gxHaEr_7Wy zBFPJ)CNh$G6D&_B$3QiN=K6O8!c9vcfmsMFrS~~4a&9&M)aquJ$Cju2(<* z=Dn~$9@*p+dYYkS+@WY@CTRLX`SJUNox;od@J_Ymw|`qjIl3`)l9bYpkJ(O%1LTOj z4>~U1gGpoO=$m_} z6_Fp2+C7wzw7HW*UuG!UKfzQ~+z(s~2xsw-szUql^cs^>6EJ&R)jdm6alSr^_Hy;& zi)hjGdo6xd5yeB~}0x8deJIGF7)8~Sp3kZrkSu)+RgdE(1blUzfU(CPml-}7HM zpa0W;!iR_qI52RBQ#Kz0Os;D;eEYHs^3m^|b|w-2x9rI;CYK{{ua}@8x;^9g!LM~G zA)93iR@KcR<}+G92Bf-CfyQDyja^Q6e39Ukb<5+aM#J0k(jb8EWYU3Oe#sY(*$c){ zo*;z~%btks^q=XF9rCMXB;vNuG)QHuIx#)=d>v=GnZhxfa_+X$f(_MnD&=KUw-}#- zA5SrQ&joq^sp4Qmn*huLsA&Mk&L62La16NVYfQy3AZSkNNBl9vc4O&>f{5F}BTCzT zxNN|-gP@w>ru>28JndwBc8c`%Mrz?IUtBEzDouDCogn_4CI&0E_5Ledab*F1j zL$asSLfl_7Kh%w&UrubDO6z)2$&YiSx*9WCsRz6Dy%)&{N0uU~=ksESR5Lgi=3`cF0Wc9M69I#+ouY(|N+`j0 zaMa_<&6i}Q;kw?;W3^&pBj+zycBvZP3Bw+={_V3{=e)ABt2B3I6tUrgHN}eq0jCs5 z4&S@B3z5VNGWL#dteAA4iJT?cqe6OYZBCh;TwgdU>-2cvy-I7COERo{W>X3Ld<>TK z3|@igZN?J#NTMC)MCigjubWcp;3PfV*iNlfOB~?3c%e^~#fp+DMiGC>S9=r(3eH}j zmIzvR2Z&70(Q-G8D8RhBTWED`7sZyjG>rsQ79c&euIMKUeWXrxoa_?6ZGBU}WM1;v z?PSs37LZi31ANzh#$_@$oYx&+kI5Du$`2MUYspH9_`ReP5G*PC+%>mDHk?H+AWi)> zPqliODU4nhPXg*xgh&$mayr%F+RkgN8$RCNHL68CUq6lLS^sh9nV*PelB`XdtDn9p z#U@C8@4hxe0BwGWB}EZ8LXU&0NfS-sFPKs>>rXJF2502g1BQslK0<(O>h&T%x}Uj> zy$cYFL7UDyEttIqv?rSnptyYKS36Q<8*w- z3E_J3s2*Ajfta3I#W990q8%@Ay{)L&;=!gpGRa4#wqE_TzvKy}+uD=b*CZ9gbgbCt zL>UK(K#Fhzl47pK;eU&ddp-u65r>`E_zMOA0?S^Jf59v^Yj?Dn?Yn0l2PuGQ0$1{- zd?2xU3)1rSB8M7UikO#qZnqE;H?0KC%c?J0K3=%Pt`yOsuOQBj*rb39!Uaa45u{gn zqM4NX6;bN#bxWb_!*wm=x@XG|D}hC-T8m>fiarxHn`$r(ceWs9O&M_x(Im*qB+mztZ+?ieawF_P zJ-hI4kWo0(oiY8y3ce11TK#4zR9%1)?G%};+xDvcI@nTGGbwn-$jiseJ4Iuy)^XEF zLOA)Dyo|heKg(i(Z99=B#K+)oW~LOv8Co~NHJ=X*jRa)l84x?|7fCgpZiSYGs$VZ5 z`F`h!*E+z3yB^5Zh?I!EU$4o&$SYuCcf~4&yZ5JTzT8pR>QP+88t`A=L5rp^JM>hJ zFa3eNM(Tyn%?2bm|A4rn>0w8W_41bc{ibrMrnrYYaadAOUInNW-X`A2-aNVU3JgMh zn5&NqY}OdXDXW&94||&05yGQh;}yXcU!Zl&ahlDFo%RPrEWlMhEyRrAH<1h%Kvy%+ zzmAB+uiNUX0Olj96z0>zSW~c(HpE8=sz*PQu)Xu0A;d?%jq)K6&ZpN7$8Qih0B-iU zx@ke0XYI&JLA*}Ux$=(w{`a4oRi*42h>*f?2M}9Svf8O^;q;^{F|O;$K=FZSNXMA= zjJ9X$EAo-EVBU(-^|D;Yt`l-~H?88sUzDWseYv32i}eEN9?M<2HZ~gd(sLU3=6h(V z2&Hs!qL$U=P3Y^Pg;nypQ^Ayxm%C8)nd08>oRwNKx4FwP_x}C%tYs;|=eD}xtRR{{ z^%w}3rvHy*VY-tnpwe`W3dVnXT6D#@p0NG6?#I+m+8N6JlRKOSgNvN93YQkl^wACNF`910N)T(q9D_Vhw>`i33!@<;m4NhOjx)T4?7fcG(j+ zZDw|D37rWLIKcKqs7@gb(R%+Mac>?E}!%eV;dtXjG0l!ote46zsvKS*Lj_Dp68F>U+4TGUNSS+ zec#vh`Mf{d`wh5yZfwd|ikoQByrm> zX}s9m{+Ul@faI$(LhpNGfM~|Yrw6Stmr@QktbLE%z|*q(vke5{x&UUXaX@b>Iav_; zX*h8|`G57)?f=75*EjPYg6qm56$5^dd()DX&43CqI^d_Q?F4snJ~Zgdh;((r;(0|n z{cLTGixBO6I*PE?Z+PGoQLAHPu2<8vb?arG0yd5u)iR!C( zAK8S%m;K+vMvkA#?cHw0x>Z$uN|Gb29-Po9OnK3*dm!?-;Dn_0Ru>ryrKGK|!c9QC zLL^A=b{(L2;kZf&I9&tBh9GfUKD@BDC5B!nyxOd&*2gFVHP^D#G3%nb2)F1t8Oi*% zpAFHMZBA@*i8LbyfrG&E`oR;w*CM;JRQoSVfM@0I@Atj(a2AxLeLZsHqJyia+|~z2 zY~@j?7wA^>e-YiuNys>i}d`90x`6W?|PD8&l%0nmy> zhpm^~6=DA-XCt;YPkfz&RI_>Dv6GP|tLv)kQ}w3Q&3!#2OG*PGBW6cubFCywwqzWfGK5;J zv8=02#Kjec|{54go-P-omI2awP zikJ4R7g5~fYuIhw@SC2@HSD)?y~Ta_NKPU3Z)~Hz{c*vWcAhv6v6<%^YjPx779rs-d!Iwe)08Q^10eqN<@HP4b2AHHW^ClT)8Dr;A zLBUGF4%@VtERYz} z1L@Yfmp+&8Qp3dA`h6wUNdZ3hXW~a)p5NFqx@Twe%QoSh7b*B$MLE_~3{P%)0V71f znUxLPuG8r^ax%hwintz4-8HzbU$oiQ8YK|E8RhgUPDohf5NboG9_j*q8v(SmT7&(n z5_XK8kx#=)5GU?Bc*`En)lX*+7=tNphrfMC&6f*(Ke|TW9?z3FwUltsHdblm0}*fn z+L2&2H6RJncDJIL{B4z!JrL$Z{+SgxH7~3Cb1gbMP}6Er*IZ$xR?|>j%FeTS>*o~a z*xX(kdUgO2GXsZeEDYrIiVy&_-^R}I(y;U!h$ zsN?hXnBlpBoE7OM8D8q9kGZp=;}LBEam&mYfHr`+AW!Ed>m|n;+BIZ{APc7l_taLD z4A))@4ffJ)YfN{G8oT-Eu}oSBLT~8EHL-0~oYgfwA0;pL;E+ zPl=t3NUu)uY*BZB2KNm>1XsXr=V#X0t@FTX66@@qBA)>h-r@vTfcRxCv;m|T5b=R9 z3@us2-nh{9>AoXQ1I|w8f9P0z)A(%>)UJX;{q?^HjM+b+E>wN`O~4x`WFiBn`H?A^ z>_fwC@79jH1p7#q00SHG3VUS2C$1Kw<&h`XYXG{h@KBcd8UX+<>=) zdu@j=TRVBnN)<+Y@D8+B>8@AJ{;270q2fBG^k~#|`FkQ~F-0car0`X}J`Ajs;AQ|V z6e(W430DOhun(~18PDDk@>>m>x{nW(p+y#CZ-$1dUU|Kd&iS(MvblP(p2s$ogp?N^ z!50_7i~-=47vLC!`*{}@!NHl_>%72$TB+Cb)R=F#E2>JXYI2tEzlm%0_mM2IkQ6OB zAa%BoZXg*vm>RUzCX=?Q*EJz#49Af0}YqS zoa6l00I~LXw2`EER@2%NJSn!pgV%%l**+I*AN`3>ly99Y>+Wjze)VFCqV&=_P?)uW zr!m6N=E>{_qSpiLAfyw-j=#S2S2N*DQ!H0JU)~NBEt$Oul)oD(=P0x&P!*-cEAE0F zW)wgx*t!-r*106@Y7%1DsnbT;aB+xw>8{HH5!EaT*KQ<2hb5RNk+Vp!e}5bC6S6+fbW zZ{8ylb)Y5UNWh(ALmYZuR-DY6awi+=KIdQCb@$o03abP=D(i%Y34y1Z1z?H2F*u{H z7r{~nlHTJ}hxj1t&=|VzcN>MmOa&uq!T)R~vfVOc%1&p{(Oai}#aJW5)lsM25(=M7 zmedtz*w~MB+qua-I3oDF-SR)|t;ON|n?=O0lgb_jTgCkbS{KAYsv`4~u=L<30h!K? z;UJ1{G>2w!&?pIMtCIuVqQmNdYoLl0yROu}7{s+9wtqspsH@F5zz{DK?2kUNCI0Qt zN&M*U(&fhT%Hj;=*wGVc`M!UCKR$n+2*6~lB}9uS{jo3xCpC({`Y%fAIA$00zYEdl zw?Nm>u$uqkGjKA-U|awzh+_r2XNf(#|4x4?KQ!Rhh(YzkA4XU2JhT$s7+94$UE`;Q>{2UQO18Wjmzh0$qFwp`3LU99@q0c%ix_mk>KP{Oj4Tvt=0q2Eaq{&{WQ3`l9 zmE?!SBi=<;H158_TB8|-GyMlEu9jXr7gRkbd9irdvQX72_H4>aT#il5XB2XLVKomC z_4^lJIG?G}%$Xn2rA|19KE`6L z!z%(PUPh{wJ=$H7(QjpS=SocCEp~^_$LHzaOga#4)+$d%mkW*ia1Ij491s$Cy{Y?d z_1?*ARkonDL8Vax|6( zvidA@1}MV{aCLkjvDzT^d9)nVbrupqG)G~(c1TQLz0=sFdabHoU&Mh~ zqB(B2P1Ik{W?xwpJw-svcE+hCEy5<-N* z?J(D6Spfr?)ee$j`70#OKJY=}1S?Je_ z=&iA~?(3;an*Q6%qE2M~$h?`%B^|mEEAUs4JOmJ)t`A}}^&T3<3`2j&xWY*gA%LJX zGFEl5^DS`@vw5~$qNpO{9^cvEc5j!x*@Mk0HBp2NGb%tL!;)UPj z0K_mbk3f1lG>@?6)8T(pUvA-%mKM#_%S$70?2)Vcqok3o)4L8O_oT)0fx2RMGBv*$ zDn&;bpW~h(4npWxAg}V#>`oy$qEI_;$URAUPUA9tzUzo>_KRw+QkS zXMmS{#_~Zr6Wfu0+{rC#2pI(B(->F@3o;HgvMeP6^k<2{y{}=PHllcOngt2R_Q=w! zsXO}pPHpQwf@{XnVbcUD*I<9z{@;iv$kXmvH!jSzqohbYtT&AP>vSBGPhokF0CwYP zG#`S2@xiSD)nL0wsPPbV%}EduD&eAGIgSDAUsNK~*rHytwDR7B-?C)vgtL=lcW7Wo z95z%s+c;8B^_lfP_dpTMz5mv1qrZzFVqn!z7I;DbqN+SVBVOX(dKCGW10n*BJ~2@+!P%T&T(0d)%-eGSmfdR|gY{q28VG$n2d)-vF|VR8VDDbD}!FY4z2 zo|;aS0OXBWT0fb(gHy+^to7j9Al*=D=mU*X2;+_ZxFBnnJX;ez*M1@WQO=R8{lm7Y zG z4~o~GQ+uH0|6}Ofi`o7mzni%a&wTr)c+%FuUw!YCCgrs{L4rDIS0P1IUo zVVVpdHAk!;u@{hrYC9z7qx#ZSb}zgSvluW9t$*{Knfms-&Eyb;#ZONB^RP^I@|m!e z<`Xtfp+(}ae@pD%=ws|ztGBagncaf|usa!eGFT@MxXl?RaGPX=q-WRW3#vYf}cOL!~#PSCLR~;LT2&u9C+wi4%3_&)Gw9rOUot6UQ5>JimjBbjh?wmm)pke=%F}}al$^Oz{mU%8(-D}(s3>Z$tjEDg_7rU z5ixP#XYLSpLyZpgY6NXgJfB1~@SK!qXU3FDvF6CH;qEU^Itynlm;7Gk$Y{Pyy|N_X z^T;+mJ@r(|)#YfE_>O*L0qR&^#E&nTbYlhA8i zqul)@`Fcr5ms1@`ogt_NE{{C7I^)li(dXKmvOS#fpqug?92!8o_|AxMjprK#iG9pS zjQljAS&-J$|MrZG@@`iu#`RupDT?=d^=0Va z_uSV1EOy>eWAOh)?cEKA**eu8VgHN-I9E6_TR~WG72QUDSicW8py$`r+}CccSE%a? z-v8&Xlv>i{ykO+%#B6)hsK@CVA2$dnCG8V{Rml_zZ3Mrr)Y40S1g#e}=Ka^9ZyL73 ziISji1;yaY@J$Xq8}x-T&1#181{vhNWny7@Ir~1ov6$T-*;Q=#;IT}M{tCns7eKb% zf&rx1#Nb!zk;jZUIs6y3>iNC29us!h?w7Ez)iWE={n>wS|LfY+CF-6}D$)|Z7L6V$ z>G2=hY^>4LtpA}NWwIW{|AM+O6i*UYbuAeQSBHzqOTC;XeirBsbtQJ!8}T#3Jm?~P zH7oX1bm8sz9^Bcj9%c4+8&8P|r;3Sf{qE}Oq5OAd5ZLEVaEx57osaI(NSc0?+rqE6gWgX&^hc`~+l^?T~I}0`zA!6#rF? zw_P3CRtbY=xv7MoTSlSg0X2(-X~w-4!Q+{yS#ycB=1n9* z_125DV_0M>6bwErpbnnOf6DwrG5pc?$-ec;Ft#+!RzhP_khiw7?Q;B`XnAs0;i+%| zm`kSOVV#QzE|>m;p9*m8K=vE(?|(YfssF{H;@@Cc{{B2U{tOdVv3pb-z|NcLZOg{A zW!oMKr`{k4o)WB^LYfhO$Qol^2JhT4mWHuB%|Zg5W%YTEE6IEE>U&y)VRpo{<6 zIrBXJr@qBb_uq>9qYC~>Icn~1J%4&-L)>mvb31U&09HJ}V!g1VOwLyr;+_O>tCSKk z#~KL_Gxwk$xoNyv|NF?u2VT9uFTuBila1z+P^4f*I2g^&Te_o_zt|YgvP0Zm^KL*Mz=2E1{b& zGw!^|nzZs%O>svD?5RF-pvyWkYAe(WxW#lr5#hb77gSdn$5>a7h{o+rRWP*+z0|9E z=@;!#+btoP70C_U696u(L0vLfk0~xN@SOR(T*caajuQU~PaE{UcM=7Fz=9k-p6`F2 zr4^x=Ek=I-N@e7xO!jxT<5C+!uS>N4;qSfL{p*GB@IwGXW zPMi7BfZf^o0kT$s zZ!8|ZiU+(;2&k6OK%JvRx(zdiL4?fNF`QsnV0v(fhcEoKbB8DO;<(0Xc7eW;)ku^~ zQT+{bzxnXTBI%nFzC(MafadHN6G;f25k%z9f!&evQ()ycB09(~!+LASg4`?A(mNVY z$F(Jfw&aLGF=>QX3Pvr`Oah1ckHOmg7a9BAG}zJ6)9GIo3W{}3Uy|kZulq}=@gykx zL%<(asI_7w;R?=E_LGYagpmsa1)8NKK*!_Y0D25NHytt;)q3~xg5)io^OoO7et}p@ zX5Eq_{2~)!BCw(o6mRWB(ne!bTCXrB<~QMLKW?X2kk_hRRzk(FG$C#>*j`aDZhR?o zL#RcZh(aR-3J0`Tz%HEu5)}STU_=O_heUyP6T-hJ`=sA(ca!q2eSjUBLaVdWhYE&1 zr!}htjkR-E?HMMa?J_ltdsH2FO^zVEqm@hQoz%9=_Z?F!xk>zj5%J z1;e)mbh5CsSH|V?kkA0fv&>^HXR5wEJP>LBG+fXTa=neZU?%y0er=eel!vxZc82ZZ zCll3q*H{YUm)CwgL}VALzr-7V)p)NkQAjdol_=k$8SXzoP;wbzW9hwPF`Q0zdt4b- z7A67w5Eb;^NA}@JbyNfUKK)Kf*PNxCcZXEq=kq93$VN&zAg!B@i8C=Ec2J~DL|_HY zv8!yTIWfqrZ+OXHOF20Bnz6WGm_mNNS!ZKICz-wgx){zf;e9cYfJqcKaOTIDXoxVB zt+TrRTm_SDRq;KKx=cE+N9Cr6?r+Z3DQeslxeaw6ME+ZfkqNM=`k_WAV{sPx3cN}` z=hK82FJ)XGXf-s5)wlQ0eQN)6b#=Jk{710L$f!r9H<~ zh{XUaa6<6PP&dPJJQ{x&#!SRKv$V+7}cWd;IG(>iWQvhht{((W{6ZK{l3?&0Stp8y7 z-@J=o6v6^lat6NFz~Y@P_?T!A)A(z$9vk_ACcmX7PcUfl}(E zsyN8r|XfJvxy-UU|G9|*&%4hL?z%wzS_>!okHf+U69M*kGARNRIdkN+9_Rk}d=7DQ)d2wgpTGVI9UMCks; zW{ph;JLZK1%ZNn)Q@p~naQ9~y)?oK@o*zR?u%h$ zHsbNAym&y5afVK%s>TW0b^w5=98{w#VM*T4Po~gb0p|0DDeAq3sU2ZLfR1>8YlUMg z6t^qp(bb=Kd*!CI*qCeR8mYu~EWWvNTYm||vd6)yQy>%H2&fmBiEBfY#pg4W?Ow!# zaAqMPubJBWkf$B`!G6Mvnv@#=7@o`7Iv*oksqAtY`ztO9hn9OsLG-DTCzd?$<>(dE~m&6xk|_c}t+^7?_r{9Q_WLn2#d_ENV0 zR}aGnHQNVmkPC%6GBeeRkHVJX8+s!RR}*+Z4cOZsJs&^qd)R?nV^v<2IxRBRRG^m8 zab5-W?!bw*(~u-~dXiKz#oxtjL-P)2feeQ7S3xGA>9f%uQ$}`6;m1r7>1|Uyjk$89 z!9;ucB4fR=>fzI0itdXLnX9JgKRNaLN2opzhMFCTLzF*3r0o<|DcCwQWqfou@D;WE z5(-^nh5rzm1`e!s$N|q7nBE;y4}tuS(C%TAo*a@V*Fe!38V-BL#P_c9j7B(IaQqZt z;5nlQY{7K)T@?Q%xJF&eG$XneFvm(dJ*&nemNLY6hx*bW%S`ngVI# zcn-uiNC;EQhsC;2B`q`=8m~PW9?2+i?A&usbZf6{`m-~iK7dMov?*xGVI&E5jXTS)PW&Ui(KZr# z_RJtjNG|-yjM7)ZuR}dNSGHC@t@ojcI>@;nXuto4;J3EiGa;}!Lp}7iisIJs+y}$e zXMUI*B;2^fu4uwZ!n&u#pHJYMGlwr%%lN8!L?wGxkm_e)Y!P3B&^D>s3WVr^&wodRWPh`Ri7y=jA zhy9=xDiKVm)tn}yCda)e_W+bPJyxJ48CBpBVWS)%`?p`WtJ18qL4_uaLO%Wknd9~B zF&j8*kJer}-Ufc8i9Rm~_ITIsLC0?4?PH5oyqM_T*i|%n_18eJ$P>R^f@VjI<*!rX zH6hLrzle8%>Ms}g-n-9{7T@wWryj9akr5Pq{a!xuL=TDxl4^}_0i={>Tzd~S zS}#&OfH4kXD^^Ww%Mv_nGM)PM`8E#U@7!d#<_Ib4>h8*RP(E0$MM;zmCLEU~6-N*me(%RcXE3yshV{$q9N!i3hv&XFonZAt`h#Zs60+4C+=} z8_P}M(A&Ep1wVkG4f#co7doO^un`|IB}mkmM`XSlyXa?IwOsfa>g#jt`!LS6oYo^F zCQj_zy>II#hT0J+ZYCbk@Z1qx)a70TLAN51SE!%~Hcoi7UByMVk@E=A1I-~i&UlB; zyByz5kyKSLMeLu@%Wi^R>N#S^1!-uWemlbfLAML>s^6%#SSf7M=m1Ga=RzZe9~ z53IxPaj#mj^N=2egMX6)>Y3tR=7;AI@G8|xd{^Nv>~zru@(G<$T?C=uk5oGzDQGMg_TtEacWfd z_gj`V8`WdtM%i-W(QcOyST+d$9)^5K)T&1}xaY%8b{@sD|06QtCxew8#`-2x(*WCe z8e5Lz1Shj|9od)%YOq@{{Rx=xC9Wp>zaN`7bbHk;S)4&=HVxiWEl?KMX3-VLO9W!0 z*fF-{MXngng$B@#-VpRxk#Y^9fuSC?UPct_2B!gz2gy%wSe%a8LO6qsu5gwtPixI` zqy`VoG) zp6R)0F;JkFNThdBhrjeMT~T_pCu&G?(}}c%wA4HLc7O#P+;d@mS^aio0xJ@BZW>F) zx9M~T&-fagt1SbR3<39igM5O6<#(8==o%?+F~8Q)aYQ-A$$%$0h6n+pxLFwX<=zXs zTvW|Z3AZ0?cQE!=l?Gs0c!6H5 zUuVb*r_koKsU9=<#rME2KV@gOV8e3^Dp^Za?-mUG;R(hws&D5d`SOED8_2T`{~bg{5kWGd@#$1KhM~$XM<+~eBx>`n z4)M-SGlj>zuD*@fSrsyJk%|je?K;2HOXz;WZquTF1X2x0mI{dS;FbSii@?YW*I+E{ zI{bsZN*gYtO~|zluQ_MFa3Ex~S*edb^Hr(3Lt0$^(D5nxC@MrCKu|_NpPTrf`Aj01 zI~m89nYWWhAwUE9irj#Y&772}WVg4(`IHxbN>x6n$ zP_+J9CjKdg6=J@4k9!gs0paGkY{hQQZ8-D}J=+xYq=@^5nSbMc& zv@?6C@W%D{-%_H-Rb0ZMgCy1nrp%u&|D#2lr>RYW8PI8n<$+~*fQ(EZ9^W>wg*5{Ou~S$2EO?Vi91202IM%G+p5)r!Z052bHvu|zhj?P^rQ zkS|^MPikzU7SuS0DH~x(aZM0_`~~a9d6ID62f_^tHd67?ivG6GPj+&S0nhibZqod1R1;G7;xp;^z=|c;_4}0er*reJ}~f;^$A` zo3g?`B?;XqPduA;T5%%1YvaYc5{H-E5U>UR%Y_0kDb15GMT8G=@$=w8+{|ppP~}=M z(poE6*)XN*k=r-{IR0XyM^#yA`GDR+d~nM64x`hnfl;B|o3~aOZ6*qML*+O|@JDYu z+4@}Wg)q}p`g~~C9qhDDo!rDD_ahQC*PJ0fDDyV%{m*F19E5E&DwIe9j<{aB{?SZt zf}P=98I8HkXUF}V?+#90)xmYugXzOz*BFLgOzQmSKF)q@{@ILo4?h$xo28z`O zTtPm_=K3N3(mKc{b=7wwzu}XW+~?1I*~OK+EkqCHq_aT|WoIs+9pie2SpwVfCM=+M zm;#D_J4Z(ergZ0uX@4=V>b+IR63a54<#`+I9PZPwHY^O;I~OQoRC7oL(EB0Vc7qq* z0Xhky;7VT@f}S0=2Pce;eNe0VlCXzsQCZ6lNi=z`**Y=M)c0E4aZl|FjkEUQ)Y_o5 zxWCef$T47LD!^NW699={1iQkig8#%gh_0a428pkbrQo2etmJ5|6)U3xqYHytS}rEO zwP)Y&$zLRli54a{+I(!lm>}JN&k{EV(Xc@L3g}Sb(d&G=vV{e?K1H23t21K@{Waf+ zhE*wN2xaGH7?|G%8a1NCGCswQjpc$6eIMX4#miAVaa+T*SiOtB>>cN8hsruG1Y;y% z6D|mwPMNr-_68YS$y>ZjXju=N+;`(RjsJ713)9lh0GF_NJHH$`OdNK`!chPl2DSOz z52D9+oHE46`e`s;#>rgWmz%W9$wvQ@nKg>x40gt*d*H$&5RmNH2DD9}O>I$J3%GuM ztc4;`3c6~th$V%+_kHi=;pca9M#POT*6df6PrvDAZyT}6PM>=dYTOTkO_CtES}@*0 zkaOTCA-lbcP3iZlrU!Y4rsXEiw9U)B@w%R&9#43tV6T94jJ$PRDUJ!A)MFCMDjD7< zod|w@18rtbaje*L5qSg$`r$;n6Bv8;h$8FN7&(ZDz*-(R^jprn>1r=imT`9%*wu-` zxPdK4$COhT=V;Ivxsg(07jAOc2T^1ZpRwk9@$qNB>dR;(RID1<6?JM9pX}u6rrRCP(TE9tl? zh@xA(K?;PqrbI$vS?R+-jU)j7iNXrH8E6IG&j4Qfbo#Sd(nE_2IfAiz{W#hNY!syU zWCI@uHP%eAZjpgNvbtTXwjA*bsg&GE+UDu3#X`a-mHSpG0;U05vIFu|IK#zbJ)4JG zmftjW-%2Xhh*B|kxvaZgH_T_fHS{~S+`kz|&1;(yh1|7A2c{Dg(X20&px0Nh$$E9C zf7?{1Qcp+QzrJ_w(|murx1+K>EE{@h zkTO1VsyxiblzT1AF6qkXl`zW@osS32d1uvLphvOalo7Nfzt}_@8FAhO9|KA>U#3K7 zS())XQ=eCk=V+R`)+K|!9@Wp3qJw?vP7<U zSbUyhm4_ML%2i7|KR&W*`ajLZ`tTSwLu`@1%w^%+a*q#qQ=1NnWyLa5P-QMMqL4AhA5 z8=1E&nR|TC^y0p*JokCTXckZj_`LQye@F3m_0qu2DiR4>24*O%kefDYyq)BBKX#EF zixLwjapW5R`PP|{EZ)cq^g<0HoC!=>ALf$*Pl7T{VenZVH1^O6lt3k>Xh+;m-chbu zIx?s?^G<#0qeT9ZBq{b2t zHvDJ>4bK@FMHw!f`Yw; za^sqdQWELsr>}z7ob9=O#KA1|;yJD=2+25wY=fw!l^Q@S8R%={w71RH(yz-!;~FWd zTPJdy=1Q9p99;O0gSBtloC60rkpEIS?yx%+?RW5l4vwk(0^K6Vgpb_-?pSrPA51g+ zXt?5=b@?zGH(lp#eyX?YL(G8LXDRo&t!3so+$;~ zsX{;3_>>s8kuYolEJmH1{5*C-<-)nmEq0+7DbK21U2lhpS4!;?o$6)~i=f8mhdO{5 zc#n;3VjQ_?n~Uy-hP_N&`?B5!{&`U8*zgw|YC&L4R=#84g@lJ!#b@TauQLltp0wCMx z1l}-H3b@|Jr3(Q2Y)fHz`)ihCqU{b~`)3W?}VfPo>Vp2h$Fz;W~24nz0+PfG=*{d*!?Hs|?M?fxCM1v=u=OaP& z6o-FU4wCil;*@8fX{nSAx;)e^E_ryWX^T+x$v0Hc5~Xt#J6%N7?U^laWDO7cwaXx1 zlM_Z}{9EDIG4$TJJz0lO5gR!({BvvSi5zIYJ9LEDnDhxHBa~BQ( zsI#{Pw_}uWeUdxPc@B9%dGpdEha9es8;of$DQ9fXIIePBobxI8`o-=vAcAs49T78v ze}Fsu3GkU?xf+0nc5|58N1p*gh(c(K;Z(9_p`S);EmPxGqrcRfLG+wo)ab+%%6G$g zD{Yz}Ps01CDzF}TNRO4|&6Y4CD`59&g6zagH5Mw18;YQJ$C-iuuK2ZFdQ+-i{Ac5p zBrz>=NgV1IP$ynVhWGPAzjl5g_Cg!n%k7_xn+nG0zxz&DHaJi<^i)~=@veN-oyK4I zJ}fv|7d)}$6^qYl7-~hc$*!8do3)|S9r{;pKmTPrTdK9yb2Kfwd82@+s|(FT0JbGF zb|701$)`Y#3XsRU*~Iji=6uI|QSsk(sQUr`D|u8Sg@va8^$n1*qCJL6_D0MwflmjH z4Wl~`AE-ATTsD6vz3COUy6cs?bcC;Mq=s{JZ9 z+Tk$uNrr6TyeMbm9s3m5sKHI?opv3Q4T9J-XJBj}i~zJMZMec{kD_>JM}*vU%?v!8 z?eWg@ZqH}d@heYfQeB7k*gM&6$i9(qXwyUMkmM`i5o~dA#PZ~@+<-mDzjx;mdb>8D zdw~$z6Mg**jquT!ib1BMVd0P6fA75-z2Hz@+yHsD`YAWP>qRsU$_TGd=B zm>gEiA{5eVYbv!A1UuNp4A;iX=qh6hBHRazgeP%ms@R;-H{(AV#IfO%5XH`yuz<~d z=}Uw4yS-LNGDFuMo;@{iN`qO@zfb4ntD;aNtJIou?Pfg7>t-MpSoub=luws{yqd`4 zBX~IT%+Wt&6j=ZFRuw$Uu9vD}CAS|dxpI0#w2o-4EQ&ir>*Xodaf7dhUHqEIe+ju$ z9*!VFvXN+sF@k}`AVdG6HzA}vS?M{*_uDMRsK>$M0K--)a)PjRm3?fW0 zo>Lo5Bs}Z)u*ofUHI+rRZ0~b>Gq1UMy8N!~D42`3cMA%TeIfS2#9f7^45@xF*{NC^ zqakC;>rQ=LZeYN+!Vit`WX0uEz&?{N%~f#=4Kp1%^ZkZ$nB!N$d8z-w$M_#O=MeZ4 zupMbH*KKGWKZ=LrUKt~x;O90)IaBAqmcHM+OuM&`aj5Bc(eB!G|7T5MEux|xx7J&1 zV}~Hx`>8)COi<9YdI2a?(kU0fJ1u#>27}{0bez^<-})n8?aLEr_h0|UcMhQ-dtd?m zR(aCqbUPl%DK+ODVL-b~p$>E&;~QmB@7W$tuTv)kwu zDHl6c^EnDoN<4OtGgAYT$>+F7i9L`oU})gKSeTCYm;EK1Jy*cF5KazOEqo(ta)VZ7 zG2rbtT79nmMhJyfKCAu$b8$icu7`DQP@sL9O}hRB!SxIsUaQx!S5a=fF; zSC0cfU&;%LeGIwd&z;4$s`124{=@E7B44cJf!%ZN6B5=9*Ls|Wr%s^O6~v@^cy4fZ zZrBd^1f7yFx#v@L^#D(Kx`yG6P&@W#dHVWJhFUaB+05Qha*asUF~9mty%X|(H;D&X z`AcA9`9G{z7XF}PiF8-e!?8I`S!f;QTGy$@btKnds*pmoS5J+iDin14b?fR;0Q${6NcEu8{Er0Vb-w+g#M;K zFwPOa=XS?iW9j4KlD3;wB~x3=xZ`#7L{q7oy}~7_;K!v%p#%FUN6 z6Gxwz z1Q)o@sdVDNmt6%~TC(%%cXBhl&N!LNss!$db`J?u7VrKGK1%!pxD6L6KO@6{Rcf*W zx&GPKyA+;6@1}09B+^p&wMb~%*U~W^yWi07QTTnd;Ick!dmJ<3zqk$9)d_$SytQHF z%tQ(;53_?P)RPBNVZvz_KCvk=dg>7zUvq1qVOZs*an;T}NyuhNVyBJsUY)(aPHLnQ ztpIQN-CmyLveRk069#yD^yb|2bHgG)h{L8+{*ICDMDQiC2A7?}FLFYX9t=XX$A?EW zK`@qqjC?3xI-7HVeKcVc_6X9_x5b>Sl-w{feGUcTuq=NdTJVTHg$O5et$d*3JkX(b zvFYmR@-Xb-vgscGhRY+DhhNEsm}SX5EnF&;ulg;O(icvP%#9|4+!rkm%L>h805xo6 z91lv;pd>tM3f309jzpR%63ztoB~brY%!ckj@4OEi=!{J`s(oWJW}?SCqHdg2=z5Xz z>}<{@NCVu~?7ogEmR-r7wpP1ozp|=un!A;wd%ZT;x@D~@Se7k@YO9*~gp{0f&}G++ z2L(=#q3tJlo&tZ`*Jm6a3}Ag1m`OR(1CW>YXw9?}Gooj@Sj^Xv=Q3h>}7NmYNNC1z{P4!p&nPMtuR8GDrb;&htj*PNf?LNMlpbaRm zIP`vSozFowqoM^pW?;)Y*8T{Z7W-7rRgCe!Rm3k`nqL-C559vQkN;AcP=ors{gCbT zjikHeUM6gKa{oF$k`D8`lIy;^RT*DxJzQI(nuSIEe6c45t1O+OoEL;eMzf+ z{8hN|12Ehe0o|Z{lWR7lJjA-9!|!TGECzG`)DA~KpX=0LeD_@W_ulNu)0dpatjk+; zEG`KsXZ%j4Vvwz1_X5D(#Cw>FW(r{IHSBl_S?lGj@M2@u@ulG9Tc8{AMS?hI~R*ikG!=u-i z!XyMDDmUIbeeS*)Xcn`}S?`i7!!}HPqtM)IxMsvY=cDCcjkRAm$Y>S5J(>#kPo(66 z#uuXPiB6MniDP0EEr`Zs{xvOp2^vb3P5(uGefwf7mVS@~EC1jjJP=GZ?S}sDsRto` zLcOaw)Uf2}k!>%%Ay1R{qK)_7wi4wicBQ-lhlMxZ|SHdgHOW>D`(Sk3M^TE^ zHR&~?qbBA;`f}mGmM*4%*MT#s2XUcXG}PDzfw88s)tNF9AV>{rUqjOAi8d2crz_ae zv3hWs%%$W%1lQlY^jajkVm6mQ*13gdPvb$Fun}Y+X0@kED){M|x}e%=Ex1y`bpgNE z0h*zpGjPGSFPKUF&GGqK>jO8+2vP241;?Wrf zQZrjvP@{CMHu)QzX~B=+sdMTKm(`!+OXm%vT8OSU6+&AzyUq7}7tPIA7u%|og7*YD zAp@wFyp$IK&OdTh+!(AOqxDNe*9J)L^XF7S!lzU}3+ z!WKLa;G;mW_W&&RXA92`JdO1s>kQjJ5Kplij76`(;CF7R+I+ovDc+Z_f}Cuy@p9Jt zSZL3jP53bk9zaf;7JNmTC?a8pSN%Wy(ur`>irTZ3;X&D9@j1czPPDy3#Bs4jEx|8} zz@R^$R#nM?M zo*Y1K`_AxdphYmyu_9MXZt6lI-4LJ+VF7dqD(yf`lU}#lz#N^Si;>;v-A=}nqnm2aiY{;oRD|2-jJ9bRC^}~_XBUW%j~AY=LAVw&NECQrUJBc&2Wx^ zMfvb{4j@O70gA-t!si7O?f%m}K)BeCxsvJoCTm#g+FV;~lX=>1sWY<`hU(DvU~V1= zVjlgA>Z~HRVmJo~55SZi4~ooMVKVf429a}0`$dzi%UH*={SN-;CTPUL|8siOND=V`-oL|DIZX z1TB^W1hD4Y5BRGbn6QBXI*mJyQi!_e zzX-6{rg=VCri;Tl_g>Bke$H*&CiHeoxdmM>rrv7Su75;PLzOdtL#S%IWeK6h$T$RyD2dNaK;G4JqWJfIb+w+FS z{av=}ZmW6f`mPi6d5S0X0I_+-Jp*t7#Lh%W+Ugw$0I;2otO|ZY2zJdHq^ui|5eyJ> zPv;T{+Bo<=*D-8cAJ6}dmjP%^9#9(OefwVe&wHD`ZjslqGUsJXeQ#==pjQ56BD(>U zi&R7ZE%Fxwt?ssAl2qmi0+!oUuQ@}{=k#=|Fo@1}f$rX`WyuGa&H8ykS7J(~LCGPz!>p34Wz3+Bv9ZUAFC#pa+FL=|aL6sEv;xXL`R zF!9&(6<}e5v;pH$#rgX$O)WLPEFZDmWvf{h_)Q?7_X#fqEK7d9vgHNE8;-PVCqkd_ z(ZM7sLfm)9g}cQQlho&(oG6`YhhF)6W1bp~fs0m$G+Idc?_QvvQQR@GawuS~yvgR+7iVUK|#_n9UsO`aDObOlJ(N)%WSw>;2(yA1P`0OpHM?Uw@^`ze3nGy0> z<#0wu(Z>K&iG8-GY*9cn7A1;0c;N0n)H^cyV`=yPGR-r~>i2vISq|}!zHd$0lTf0o zD2WnMVWHmpB=B=7-zUR>-@5Mp9_S;deg*UiK$4Jw20Ww9*jU5%=3jt4+9}|V?@235 zeSb8;+VRSSj=~$-lh5_nCB)=Vin(zs+TXMhG0+Jf=>wjd8Bs48^a?GqjL5!)a78Fflx38ykTKSf?0c9QNy#vyOlQo@^?Y34 z-+kZD>%PCgfA8m?=MNIEbI$YpZ13%O9|uTBdO83SlIm}}wpD<0v}*}Sjqr98AQd>V zaO%g!Jhnhs$@iwo4%WpjzN@r>uGa-W?VFv1qDJ&@XAYdm)=F-6y5r14WMcj=c3>4u zLc}sN58g)Dh8rMR`_HgL{dR4G4F^gOL*S3L1XCAjH=rzc>3q%@*gb zvlFR26F7%ncY8O6yjJCV;ccelJO52# z`HrS!$8at0Ya?w*IJuIuSg-T(oR2v1C+yjL3ZRpH5c`?H*iA2F7aevY3&U{EI3TG! zlf>o|qYF~Xl9;?;pTB|Zc_T4_5v3gmUwY#Gf4VKKOoZ0zNDYRD>K(PXF=CNSXzOx5 z=(GZQX4(9L;EEYmquD|2Zo8QzhAr)PBJ;e4@|*hs5lI#PrLm@~F2Z%g31_0~^rDC- ztM8QW`tu&xF|hHL3_N&1TfI{!aw|BSP<0gcN13$F6paJSqyV9bO`zwHEmu#GQ}j9u z_O`msNnFfeZ2eem6wJ@443}u}9?S1LlTo=#NH^**utkkHIFj@j&WrOER<{D9TLn3G zbSWx~hjrq1qa8FA7cahTN?W(Po^HLj{B^}O>s_y?H%ju!b~9m2kU}SdA>BZ-KfxKi z0C%1;s=7;$EpNmY>IVo^2|5ZenIhRPPHUE{B6LfJHS~TPfmZ1v)O_ADhP?FARHo6@ zqR#!c$3(AId8Jj}QuaF9_|ZB0Kw~hn;{eVHPV>`dKJPyZ4wR&7;3tHh=Uq9lQ;8~A zs7EpP{s045#!AN?`2q6%sca>5o;zUOMow==6!Jp%Lsm&|;aCxqf%7B#{x}Q8Tu=V8 zSC8QiMz9b^dW@@2>roIP@kX^Op$kt=U@Su!L2Bz=Y`B~JAuW;54+p;1Sl-&GKF%F4 zD~n&sVbK1DWl|tu!M#zPsI6OXUlCr1Fbj;@L$(j)im;z50Vt9l(3S-jK9Lm*I=2~Y zcr>G&&RPjztK11v@9^0BTDmEr;XJW=Ug5_Q@ja$y)nXZ;8kA}eOiQvdoA1xxFeE{O zK#NDQt?&atWa>XUKF$Bt@ree)Uh9mw9rxjihy;#^+$@|4S&ZvM3PPDAyLhI?=;alO zthaR&ng_VeTSA}eeflK8SA($L6D9VDy5nE{J8$a}2H3I8c1D|j3hj1sWV0csvp_O?n7U~kjF7QOipjPblw zuRLp(^6A%6*|1{^f%8A3aT|E-Q}4(NEY(*0x7bLt>#QuWM$>lod&pECB@!Cn)@Jv5 zDAMqWId<873c>P86;GbUy5Eh`yQLp>@QPv1ahVF;<-o(HY0N$LY(Wr!a&3WCeFfAK zpsU+w(Ay4$olW({;2gOEIoU}6Has8t#wKgNNc4{h$B`=~i!v&!a%@|0&oz-hXMZOW zR&9omzmkCzj8+`*aUyU;#jvV?K}!HJ6Vq^9TQPQJY!f4{Gzt+hy#gOQ_b($N&vCRJ zQ>UiQfFn?L8-eX9d^5>3wCqMD-NRtH-quA0Nn9NHYEw4x@p~)ZYL*-8z?ZXcVvJ{G z69h`q$V~*mn%acgsAyaT8~w7W{NJ$hQWorfF-W3-ktM>B%fBh%Y%xJ#Z6z6i>;Yir zxP@8S1X2n3!LrA08r5U_%0J!X*o(3<)JY}6s0tUwK1_mgu22jtSjXW`8^?5^azl+;zQP)A5sA zQc{9u*6%;+2Rh&A`#G!YK0n_j+Hgnf7&{U6=Le$!Tnb!m&`}&bDDTX)VpSr&t9EtE zG4ii6)GvFi1m@lCYNfUtMfiK_!mMWrcRoM$JopwKhoc<_f@eMLcm;o?c#H710NQoZ zWoUiz{m5FdHh%5mp~Wj7z8+drG$O*tk5i92P2E1v3&KhG8{F#hwjcB0TSy+XxFAMa ze93NkAG8+Ej9+L=rAW=~9qyIaNH^}+`r>`y@=!)~9~Pu%{x81H2aGbrO+%58#C#P7 z8ahQAwHm^R75V#~8^!2(`t=TK=BH1C?@D~{-2C{_uKy?qE`S;0avI-M zPjURs;Gy;8hne&nWmN=Rc<~jHU!HT~k{9h2OaZ7Kj)QRBh5@{?s8Dt%vPR=xANvEU zwjqHON9Rb}_Ms4$a|53%-r1K;vEHlHYr-Lz`P4Ue2^Ro{ri>|J#2AL~Mm^dyfedrn zdZMY@El=Un`DTD@s+`n0DbV$mmFw48S@Cc;skQU;r?K;9Il$vR2LXe}1p>1uTlT<$ z)NN!ciU)ZuP3Rk2jW)hK^GnlVT{}1>zk9;riRaq_tKZd>4_($40xPG~YPPxoV5D+g z8amZZ5H&sY5-r_hrtoI`*JYn>H%+%QV?o)05&d`1m!w-I3pD9w2&6{|`wbt5i-D4m zaUt#W`qtt9a!R*&8K+qf30|lvJ>phjBw!OlJ#a-v=@0lYJywSOU$`Ke|57s7wb5%y zmOL?2+ED#fjl)M};@kgceUF>~i~65mwFI&bEsZ67G=$A-$u`&k?Z4{uG)h{V9i$uw zxQXFBm|NvgV5j3fv%Em>s_%M#RRt zOA5l$7yMnRNtl@+rhKxr(8C96hg5D~KkluPWqkFQFg6&x!~|~Vw%`PfIu7MsfLs>u zpE~hw($t~!TT|7iJfoZgT`|PRcEQd!#o8mMl+x3O9vwFWer^0g5^$|6AYdAW9|32p z9-ECLiUO%wf&h|~4i{ox4JyX?7E+`WP4Al?qQJ#_ycz2Dfk9uBBG5q(M;^{D_qp`f z>K*yUSq2XGf@<+A3%md|X9)TLyC$Cf^lO!wG?g-tuhNz3;8R@Q zJ}GhdqLrhneDGXXsxvTlOz}mw@Nqzhxd%JtArD#`l(91xm}l~c_QIAOO*~2D^iqv` ziHA3;RUUbq^Gea}>F2mPd$W_XiaE!0h3e_*i-gXqz(wh9ZMg*~sgn(4HRb|c$Ch<% z!}Jtg(vHL01P;G?1|pYTAP9$y2#dU(APgn|uD>W?!-bil<$iQ@VfURTc>MF)yzBQ; z^GzNA-N51Dnli2%eU*ogEA84=9y7SFV$BzJ`QrSPb$%=*%y#NWa>2bsymXSOrfG$c z*@4lyxu_ydig@QQm~SSkZ=I8S`gC558uJxGBW9rVfKp_xO8$Ed0jHR5R zIWoI`B2?Z|Bi{aIVcx#Sk%1fGY^6Xtl7(pK0ArvGyyB4^R4OSCCx!{61N=3arcCMS zEEKh^KqqxB@S5tq`Y}`0cHzL%Qg=>>#^~YmFyHj)RG6%(0OJ)5N@MebD39v|1S~bz zS5^YXe;dA1U^s>e#;VND5$W>o>59ZKzOu%Qu^z5N%6Sf=R+mIqw-bWqV9a^CFH`1g zBUWu}twjWQ1+r2{HJTm{mbe7qgs`KxLIT|XD$e#AQ+DyXG$T8qFLYwx9);rHj6@g{ zWY?lh!Op3Mix5bE;p#E)`rn;)Gk2CL$oOBvW)u4Ve=uqI*nYos@&E@RJ~pq#?;^Oc~bmZv@f0AQWYgxre)cUDz!^tiS48lU+yg z)zJIFWtsx+hR;%MBklv)umMi)NQ7 zw<{tjC{TCS{8rGd;E|r?%7h@E$)94wix*{H-o`bn(`wlwXsDNN*N);d+l7&&^`zFz ziwHnH_H@aG#5x;m#;I5RSM-bhmmhpgl5qdH@{l^;*C};cjpJ(}o$E9FR~8NoH2Dpj z*?^xagb>S2i4bxK0HZOEj39kPV$HN$=<>by4}zqT_`Go8%uEUWbf&wdsE_l$S)nuO zJVE72@A8vOa1xR%;Mft>aDk{9I}r9npJyM%Sk^-Q)mj7^<+tmwsc@yi#E*P=yVOxz zzbbP218=K=kL{)zo{j>g4x*~OYsF|nyI&W)bL)l@DLO*&1aoCPwxV>UeB}GRtoMD- zI}BR-Nbpn9sJ5`BrTOB!TF=WbTOaG((9ayy%f73S>hdHFkztM4{D_qr$1CLSPcjcbCah`^_lx_a`u+{awk%u`LPRu=?Y++e& z`Z%zsJoH(mY-#rA3z#EuJxyKR@2k+hZnTAYT?kJcUZVF z01w~R1n>U@C(x?M9#WK|FLxs$Rc=D6sY%gF!hIasmY2AA_U%|pWz%Ub?1D~;(W@M_ zm_#0d^N%_<05IEa=Y;4Efz;Ni9Iu!c>8;PJ?Ve-Yb}XbLA@9cPk<~Dhya*esoRZKU zaa93WuSSo?#~x6f^@1nECD@$o(RpqN50db5ad8;)5-4}L(jZ7TD5I;@A!TwnJ^hKS zjHI`ZBbPWpl#kan}ITOpiD#H|z z_HopsLBEW|VrRUf&i^Ppo0v<;$3wa{U{bczQgAY@T<+iu%p^{*#TzE7m?ntXs^??E zpKsK+AKFeG&aY~ek91%5>-lB#UntJ=bj#RZg))%u1zPI+T{fPtrfhv9Sa+@L-lyhD(d({?l(2P#$(rhx>(A$)FU|33j!Ai zqzoJ_5Z+Fh5@fWDAOn#K*r_DCkv;A4MO*TH6-ydvq^PbXT~0alh;%s573FY&Le+k+ zwYFV<@}UH}L=RGm8CC)4Q}QUy)LWf$RAx?7H~yu&tZ#Gv>@BO+dl@*6p(_I0(QM&d zh)7403z<@SWbGIx?_$a+n^BDCOZV?s)tdS|(Um(|4Gll_YN-;hf_vP(evV_Rs@x+~z|`w2p3%qhkgly8%Jz&}s|3XwYcOFiZGTLfO1K-=ix$8I-#UpO^BM zE}kjkW`3g)5>2g1Y2x46dbHs8cS{4d%zrsGd6bp8*BMFWh#8~5CZ67xK0c;fqZGxl zhl7=aBZ6i>;F<^n`%(k1t! zPcXcfj1v#*6?AEMd0_Le+NVT0llxiRO|MiPedNQXd=QgglLPDazE1~VxeyWI-=XV19|7r)Cmy?yp8d796m*WtV~_1Y!;iZHwR`S*+Q z2FH9xrl-Tz>~54>Z4!^E&Ee#bv>T{@_Nvs52LPA93S|Ok(Iu@z(V%6Bc}#e9HuKzQ z)tCM1eojUEzdYsW*VF0O0pf8=^>m}y^=he+{!S^qGs|C6)$4php79=R1p!plBG?e( zZ19Q1D_A(%dUHKR4=^aB9#ogfi6j*CP#X$;-s#*gnh*5UKk-}Uni_dLEA#CEyA5?( z8fCe2Fp?nt%PtwmYZ?gTyt@?l5^UzAT}M@P1_E_|UAZS};qW-BvZR8OpZ9kXTf`R_ zFB?nRavMBa&MviHFhmfYG3q+D$=W?1^e8~uvFeKEm&YQ6r9T^4_8U>t-M|HV^6xy| z!GCco6fOu709Nw#->^8GH!8+#|NDum+g~%bFHLIaEsx=sXGTA(ah|s76n-6bvV51u z%^oNk|Kk>0_`=SHTl~k>Z{I5HvXftt+x(P3NwK#WSqnSV(yLU{=+`MlPCIa3$DtP? zBt(M2kXAblyi60;JJ9!ti$TG00Vru^2xPY%-a!Mb%kq-EfQ=1m8XfSuTS7;_N38xIa*UEE>0OpOJnTo1;1R zaW^75tgZEFW#Ik_-B+A#o;P~&ooSGo%0KfCf&|iYqzuNJ9@xDVOL$47OE`~Kt!?r* z{uQK&_le~by=^t3?;!AuBY6{%TS&7_+p!nC*m)FxOBeub0ei9Oxky#(UBCUkK|gy^ zI`ovO#?hmu!F_#gQ6eBfdNBxKb4>Y7S{aZIpe*oVuA^q;P_Z6{w27zsM>dn*YXle` zjeU$Lxc#8&BklT=gSnP24u4z_j!x||Y>xk&D)fH_C7}zn2%S z(qdegT8(reMOl}=8C-zv%K*m_4_y(0b|zP7h2b47S~I3+4k`sHK3o?H+8?soya=3? z@f-k1?dZ&9ok?__oo));t$es@ug`UTgwRrzw&-ijRSKUJkDm`XUqk0h1O=iZ_oH> zqk|k&fZ?tz&`yOw`$gIpKmj9)=~nJXkoSQAcvpT<=-yz0a?Q46(TrQpVYja92Ol4@ zN5njBmeQPpsjuY`#G#w*PL~78i4f!l)j( zn8}BACYo^j;X2u`3nAX&p_(&x8pm1f6xH7xk!O}=5ryrjWRy&+Ds~9D$6zo8G}eWd zqcgia5MzqJ-6l4z8wbMl!c~gOVQNK@<*)t;eUL5v!V#y&y$}264wUx4TJwq`yFVB; z<}KS#&_R-F7iia)BXK0aA!ZOqJwY0rIkQY2O!OKl9%b4v2)pw=CNCsP3I@}A7=H=v zu6XAh>wUFxiA#LpK_TrpVP|QH;)w2S9zq8aY#~*;N?IM27iDv(;l=Y8bME`?z!8vP zC}XS~>u@#JL3tY}Y%l*j1&omV`o4+EB_JuVCssY>NN}+cgMk!0tJe>^`rb2rQ~J`) zcN~d_dATZXBVWL2=k1{VjjKPlL6kx?+!49v+KlTyniKsC7t6@K3x!i8_B90^43xL5 zi?Vqbj;h}mWi2-T#CqP!`-;r{GE@c#lrRI6quv>Pm%+!|i~MiP2tN)MWFu2tWyft+VcZ!vlSk`O zQU+5t*QEj~%F$(Ab|DNut$8JS+atFGglLhnC^p>ZT4#gdtG{8gS*%0gB|!QC1pLq< z!yNc<{oRVjV+NYM#Cfe6J3;@P%#gIRHh&$=xE*I&q*s3b-Hc2IOyefUwmrZ#bGxxZ zEJmK#UmB+dJ?TF0?ye@KIBh!S$X2BdeEty>NKxnay7e&Cq|V3l8_aZa0~jpcW6Ud1 zAzhg*c!2IioIe6w&6nTJgqmKDV${pr2mZR|+uVp*kIp&<6aYKie6N$OM zh>9sN9BD;hjOpfU33$Ko;rZ`!4S6?)?fgUKd91}IL2o_>Yh&0we080o8Q291<$uE- zw75}h7|mm0cD(*M;lIzp{L3;%RKk^D92%W8oq;?0Fvkd}xjp?|s-495zP=lMA)Z|D zPpw#Bg@~B*&;t+`@!zM9Qt<6~$;*D&{X=Yp0O_L^_-k#p{)f-er-^lb(K~k&$>#hKo$br;m4iKxFU*P)aO>{VSsI|5rp&-_l36drqW0 z!QsETy04{K3TQG#4M=1nVMK#c!4SGczQ|pEAgouEo z6mQ&ZJ)GfE%}C`x9=-9UtDcF*L$Fso&b1Afr{j4UY5o+u-7C_9{#WtScIwMs=jW95 zu6U7lWwuq+GZon%2|KwVPQauHr^k^7Q-QCB&O13-wLDUvuN9Ped@1VipoOD_m)8}C zVSu&0Ch|MxI5$@e_g|#lCBrDNJAslj_yB$|!iSjwP#P^_$G125cKug`jKO`i5H*U7 zwn2rUO^oH<=V+BIBQ4qRB`YOS4n{chpqbd;u;*pAf5Y0o{SE7`np!!EeaZiJRGa)Z z*>Dsg9(Vq}u82n<#E<`62xDSJ*7=+mFf<|S=;$CpvY>g(65C z5pZM`P9I<+cMRZG8FP;wv5*tr5>jqWVkwhjQeT&$!#{OZI{lW;q1)c7x&K|xSlMW zGOUkrpZ{;s5!siyr;*XfDh&e)7Coe%&L8W^TZ>Q$mM`>k$1& z9`8qPC-EP&UY(NKO(YcD1L4u?`t?o0u;!ZPM0B2sckJ=L8#iuPDg?MP4}wEbsa1!0 zfsSflPg~%|*evFs+-`3b88bDAAm0~X38HSPcu|r9s3$1JKiyynV+#7qh3UFL6DD_5( zl}L3P__W7{X*k^M_yeFDabPJCuM)m1!X*IC2B1ZpHbKP6?aHb*0CrFX$QvzFP}CH% z9hVD~#?+;=21?Ku1yXf!BeiX$0cSg8cR_4vYiJE{W>`?Jl}b|K0e=x7!}F_VRW+w=wg+WN|cP#EYaAv5a!fEBsR-@ti5UGWL8K(qO zWuM^d)m3ZNinlxVimt!|LA1g{ut$=nz-wr&Q$g*GTNRpH90dIl#LI9LVZ>C8tv19& zz4$)#eS;_zZkV0n+#ABFar0&1<1B>y)Oi>B8!0Sr(99+l% z1;_RlAZUg$8!wEfJ&!LGWS|< zi(o>*W{<=Xbxmymxb)6wP`$_=|E;;y_x>e=hCO2!M?d;DK3wGS5#D&Iq$Bgmig(Qw z`pp(B1CK5Yfmz||1=rt2r!g`iYjU81UFXQko!o=*TbXQ1JLSi{q=q!!PaEwS$_UP> zUL{E-cz+MmIE^g`f;=l41ibYqB2&~e*dOB zapVEip>m4`3)=h6;pF(mTemsA)=XqboF!~GT&FXV=4Q8LOc245n@4u2My`SJ#X~@| z;s|)gv$PPv$*@o#mYoBfAM)p>`(uh4qngUK?4gF%z(+OKLE;(#!}30zNhy6R0!ucB zdC6(rc92m7Ti_2Zlak#`o9SR2ru!yRf89Sp1rTC#AiVDT)??4d1^l}F78iH(36-g} ziJIRBxlt}~!zf7{@tmnPxU)WlqNgnyt#;utNE#v1^b~J=Sc-2KqkHC1QR5Bl;BH;7 zC%qh(gmybW;xJ%YgZlrcH0E9^_!v*{F)`SoLfVC-LckUq1o6INo}>*Xw;a0~oJ4j$ z`&-V7;W+Hp60Y-f&s^lU4z;%QrxhH3$Yijplu%45mCmD^sSl)?pomK0E=GBn2RFLL z{Mua)+E^hS{&9bww7IXv9G8s4NsinSmD167V8ckJOFg0gDohrZ2=xbjlG*`MC)07UHbh_t2&hbGT=12@lvDB^AgwZglqnu zsS~GldJ$jg(h*-Q5|sE{#us7i`vh_)j^-S)Wf|8wxKkQpIBe&E8Kr<3J<6WMf&BAl zP$UE3@%u=dAv0y>K$jBP z0WPL$cPv1`zY5uw_-VovNDJlx3Q}LDXhhEf?P6LITu{HtvIowed48ncB+#um$e$wL zY!hKEPEWUI$)lU?bP;{oDSa~InB|x2S8cuVat(dM3!XnYZGlvfNq|Ru^&A4ftId_tCNJ?q9uC?h+ zTWlH_>3Sp$RXB@c3&?Lfpfee^tQw!;mNSE7n>10sf`g;K-s*p^G#k(mv%Ws;*8Nf! zSx_lMe*AnX8ukoc2;6li9Hay~s(=LRA$T`39w~_P4iSZZcX*gU(Us`hlXTvPnzjS4 z8^-WuO;!jMo+Ff>LctHi+@l<_(sHrAfcY2nC^b^FsfS1e0icj$Mz@UC|8B5-w_Zd7 zYWOhgzl9Mq&iazoZ8vw4vs~lj$w+HS)`Q`H@{e$^?TNPup134JBY`RYn}S;HryVpo zgmJ3qD4YK-)%f#|^&N&2<994 zk~0<|b?wG(et6@h3;&f8y~4ai%U@5!%AhmbX2L$6{~YKly!24QE#{%S-;X}b9{zzh!K2>{=pqiY$JtbBs+fEz_6#th-V<<7VX zoskGN%+XBE@hSqj~mY z`|6~(8lGxWgOXDv267#*TgokKEXSO;{*IdN+QUJ__AT&l;v+#zW(V#;tMXV-hyh@s z60{t`udUkL)f@2l7H)2dhzv>|&d}u`%5pZ({Ga+aA)5ecNi#)3uHzO3EVjZfkY48j zZ7!M-6y6vnj&g))UL0s9+F+z(F4>XExLX^&_rnRH63=4J6Fh{MX8?ox+&{WSYhcJc zfCz;BP^&zO-EF2p%Wr*Me&GFJ0>`(vPK;zs+Lo-?Kp;yP5?dEqI)vi?sqUZDN^E+-N?*^DR?)~6Gpy}aw+io_Ka;KD}J+0kYV7#NIDI0KfKAnQ(n~~2m zNs8!yq9qo-S2F!{o{JTiZo#u?KywMu<{|4$$%xLvhQbbiOvp9@$|1^WrBQweqa5if zNmau2Syk^#J8!+!&~=$QIDVdA!}2}O{Sr{tg)*lMdj7ERz;N2g_^gq8(Z{XMdjqS@3(4>X4Wt6i(0o`07-fZ64EeYDR)p%JL**1 zYU~j}w)G(`4NkKtoMG%V(Pz*afr6%a!*B?Y%(Bp*)V`f$Lm76rGO(#KU%i?WJehxy zWM24oPM`J*FiwNJr)6LlK;n+de@c^V3Q44t;HRW~=!qbZx z)sQbT3Y681lFp6ZM(qyxV{0jO)7>;j+(K8}P%p;)Y5opei@pfXc6ksabKirG(LFsN z?=o!nyc+#}LrS@+RI3UtKjGbIz46;ars8|WD@*ns-f=???EhygZV$ybc|*1mUf!QwIRyGD_&XA1vYoKw~PEc|^0sOtEg{>u6b1mbm)VQbW9y zPNVZ@$=g0~^`I~Y0Wt;xdD5&V+*R$_VED8I1KBc+IphyK8K~n$jTC@zHFoS~UHwGDo%&ymLT=y&I(cqelI6#|uhdGH=qlOM^nf z_BP6e4MysZrGA%u;P<3B=CVVi+8Ny%-l*NexC`*tWmIl$>ecN77=2cmFRkfY8huyW z3@RRO=|(Vhk#BZhKAd1!O>3q%`+YhJ$Q_R(|EGuW1FVu#Fm#~c5`dcGWjNB;>bEbU z5s=#@kTkS$8_?@8sx)Ukm3rOJAMx>bRjFZmT2FlFSctX>QTDkzNE4;=?9f6Sm8-@I|UA~b4c*DIH5 z{Y!ES3@G;;xZ*KjtR%d_Gxu0_EdZQcPNkoR@Q@i?Fi^ffuWI+CeWCNa+!^^y6ZyaX zI{E70CFdcWppMUHPUdskGJE#|SQ4EOswdYnz9pP)xg$S&y`HZWXl#d)s?5d>WOi}V zo;-{3O1k+ad)no zjq7I%IUtL5r^XSLro*LU=?5KT+h05M%C>#tG5&%2s0AWGUT+*RU_DGdsb4$IL#{T~ zWsXu8M`rxh%b8(iuFZc9Mv_9%dsTooqaIuMDtH)E>;me|I<>0`JC3VcZ=+g*jUD$Z zjn~7kfq^}k+;O-3^JRSvA+h8o(bM5NO5TyONAOu-x{59t64EH=*Zr05Vx6{yn#M%t3GXuZZ$klke=dCP z^?W&l`pbwEb60A*_~g{(wdTa+bmq5z6BrkRZb2klIu4G9=#NOdCPpvKrdl)LAjQ$a*1|^HxlV8sosZcFjod>WP^tRu|CC*%1Na z#F2iWAcQoPyzRoCDiC8;AiW2pn&Q%a9bpOwk^8Ae9idO~b=!G?dGDCL*Nj=CRUZUa z&dKr__bqR4~Q{pg=R6r0{*8Nbfcv(I`>zr<918nnH#kM)P$v$KV0a7lc zhA)OdM!aOgsx(PnSltcEqSbCA-Nt@hHd}3Eq&M1(8k81iaay-E%b#ewCBcn&6U)^3 zcgZM5ra26I61+WYfx!&|%?L_36-C>!dxYiWhO?-wx+~`46&xtAfAh;1Y1p^kV2@&s^zDr$5x8{TLNDD7p!320= zz$%pZGl2`)VFcXy4NVwU2p}{I6zJRl;)JB6B?Z-~_v%+@NBx}fSn&_|IzLaj?{*G- z^)75C!z%mdn?p)9Q71Y0rr1v^QlUfJraN`_TBleojKua4rVuSWQa`Yt(b_y>DnZV( zRhx)-aXO_8UFCV&U%*l>M>*7a%vV5X3ZytC=_*XgJhouhwi1rGKWtD4l2`=vc{?H& zos3xaLVN3;M5%Z6qm9-w@1K;Rhn4-Wp@)J$e@uJu@EXi)7XK0GAs2QODA9SDV*T!! zCx-CaIp^r}Nq5T)@+P1wf5MFkNs7Od)Vy3>R#|lUcEA)+ znB+6swf4FcZK00&MmKhh9*js(zC3bBLlVdhsL01+13*QasaOw|!`67%4+T9JRxVb< z|4z73dRO>MJD&XguBSilI@_zzZ^F&>+JQS>23DZT33@@Cz<`efw;~_Ddoy3P|HnHuYthw-&sRrmt;i{c}_RlF2W`+Dl@)su%*O8=onI}U~_=r>C# zu7d#iw7}W=98?0$D=iLlq73_iAZvQ6+|Q}Te=Da`a9 zxe<%dTvjbU3#3H=r8ORK5#7V~GFsJGai+ZfB>ZkDfck58J+7hYKx;l=Jy`9^Jt3VQ zU>f7Z^+D*a{oCII_nknH?(}t;xhI;vJDe4ZBt1pd1ud@nAL4&q_h9t4=a%lT_B z6)N%~A`%uyJor408|@8YMdt8ecbYTdl^Qt4LW)+}2PgfrY~zU#561d9qXC z6oAmNN;Nc01uF;5-s91UKTpLf=sMoFnAW`gcHR%>J|OqL<=bW5U8j^(RNH(3eEz@B zmcbT%dnfN;i@btQBG7D`akGIWMyV%RqP3Fz zg~I+!KZMlO)TdF8-@tkkiD2ObADC~v8xAt&Tt@+~=7Fa4FJlPEh`fLD^y`BcQC|&?n6a}aDM>B#p zmo1FE-qs3->R;bD-T%5T>iKUiYx~r|e+V!-aHRYNMXV9}7U_39TLN>EWNSoSOz;B81&cy*Gb&Zir-Yp>UM~q+}!hf5AcO07q-Y@B)u<7NeHb1()*9bEEsb<|5D% z(JE)N%s)NfpQf~yx0^}<*`nA#F+NJ!{A`Ll8kxM?OuZPM-C;k~_D|Ue$12QBf+1MdCbk|^zeFn6- zF%sEI1TvsjxU{fk?MvkVRv{Dg?R|NL{CB^Eu^0UC6v2P@S;hbEm;e6_{@=r0H5iAY zxoSh|pk6BD0xg1=^TI-Mj!}b_tBk4n;*Ys&1F1UO{WgCNHa?Cpy$*1%SBnTR0O3T@ z!WR<402!KpAtyK~%I`Toy-4c`x=Ob{UgLKC%$f3_lM|*E&%R`82qdz&MIKlRsQJYE zk@_GthTAc_vtvOXf=MRvPfH&4%SR4>NUf}_+f)umV^z>VuxXcoqMciSRMxw3@(WB` zsAF3Y^t<>b7&uvuo1UPcVvjnoHW$@20=x?%?qr-4lRV*L?6B+E9u8&#jwsADq3kqv zdLVt1kiy@bavx5B(fw7?9A6pa;g~otJK^%;%CoFg z9=qL8bUTt}GdTUDeO_Y-um9f84l*G3N3cvT)T31zp6GwuWr+yhrOe5vc3den@YG3g z0A0or^)T91(@+jrlII8yTT2ZaUH;|Z?D&VL3 zM!1!G9PzyQ#&Z9ozBW>n$Vrg^7B7exoG~U8;Sjc9CN2Ep@NM<}<{-t<&YXMoT~!qy zvK^N@wvsvhpPUfu_9>8wHstwJ?7s9E{vNA?BX;0u!P$*;-_EA)Mqo2<51Zb)8&*AM z6SioAf2b&(VSC3S3%IPGDL($^ybfMyS?j-*yH(8QD+V*414iozb_`V)T)J(H?MA)6 zlSz~BE>Ipd-8R<52$uLGX_-~8Z1vg@qqqrE9e!_qEi?Ol9x2D+@c^T~5NCSo8IBmp zHgf;bB)RnXUP0N5dH*+;XIcWogKsyVmauRZrst%Ze|{&Q9(z+5qgVF*_f|bq&N0i= zV0?V6U$VZw#r#XQg_4?Da`rVV*luf3qWuS;T(>~8fKT@@^5Ym0fLP(0urgqhXKI&m zkEt}#q*5wy?Z=k6vw}?LY`9&9N@A2hz+RfBql}s0IEe-=)N}Xc7wuv;k$@p*|7~QP z(Qx2-d6dE|6G8hZ^4yHZJ(V%4_u}L7ZO@tY`qb3cvMO_{lj2+c_e zi$R1&vwOK3{N|UeD=_5wz&{{MA3YD&ILG#BrU>1C2#W+}JYOTu)gy5q zyBGAX3O61he$qi*#SQ{Ngh8eRei7B3wi}~Dw`{kI$MU?P^At7MQ3zrkUK2+n-tIRl z{E+|Ul6Jiy_rCDp>#7KB5ylG=rWr93X&KNiMiit&Mk@HJ-_EfqLicwx%#=RM=ipb3 zT~aVA_u+V*{+9IfIGlJNcbPy7y-3u{(Njj@QmwIArG!Z-u=Xfu&=S)G)fg-3^Uwo1*K6_f4ubcxR$%an9zNqHWgviMW}s(j<2 z?&CuIux-Oc<`dQnB#CARsidv`rVn=Fxa;6Ca8b-H={N(PH+h}&a z@tcY@+u7qm?_{~;=2!_h(rOxLt{7mIni_2DOB|XW#3{cz z;b+@2qH9_`E1F*STEI#}b=T&>L@1Cg0Ni(O1}hCo$}l@j``rOgn&C|_b)w8+`>EhG zPl3*5$QS?WQ(!ghFmGfI%RV5pC}4|8^9Nqw8*JO?oH=*TY8Cw^-@LiYL9bE1 z6zipU>C;cc+P+32ef_F!NXQ=TSa=*T2RdU1#{@#f4 zm=y7Nw#oKVJwU)HvJ(;n;YLodfh8bM4Fi|_VC1p(M0`oWmR-Dl-sOlB`*6#7Oor_d(xj zmW(Lc7+Vh#q^Gq5>=YHB+qdjUi;EAcSNE%S5S!u2o||(#o?+3&|5?i1xI^eCZVxV~ zaH^k#ODd)Db{EbplG7dU zZuzw2q^u$%`?b{o48| z%Yy=~&Y0q!R+(Nl?*DyAS4O!+C}quKqJ$4rmyv9HGO4X6nAvfdC|JZ^t;@g|iE^A4$t#<7L6SSh!u z_j=Ax5&{9Ivsvyc`oEK!O+L=$9WO9j-|yDAVdxDSz*y^p%}nnav53#?jHl>uS#rDZzX>M z#{7Pu+_Y_mBdTCw?if)(7hb-SBUfD%!nC1Wz8p6b<`yYXtgGECS9JAc`KHQ|>IKm4 zOS>Qfses4E*p#F7x^az0#5rZ>5$)19a)r7wtnUOD zr|Pb$3TPZGums!^$cun<$!btE=<#6Gv(QXAkf8Fx&**IKVHJQ6q2 zDq%ASd6Djz%xv%Bm>ao^!_ z6|Y%^NYcz8v@g4s-q;~`2c!(SZw+A)wg>TIX>Ip#skeFZ+?Ce0)vD^cJ)33r>om)D zVIP6j?oA+NFVv(_7q`{?a(StqbgfY`Hj6fa9*i~HZ(Di7S2~XJgi8I=7~UMZ%iK71 zUc>6v`)g`954XX)$FbkRCukv;&aVS*pzilLkgdG!irfiXpnXOxmFV_D$YUojm7%`5 zz%T(r@4)Ur^596hhqif{dNHOb8lihj;&R~3#!8VVrR;sw{;k|W!9AVvkLrAsYxY|< z-jG#-gK`oc+$ImTk#>cqqMdhbn{G^_nr?lvH2B@f6m4enfyd>VV#Q)yDEOIOj8m*vzy5}$%LCLzi)$rj0E4@vgj zj6p?YMj2OT={bGA-}`xepXa{+c%I)M_aA(!*JtLsuJbz1<2;u4@qP<;fBi{vaSb$8 zz-N0q7TT@d+zb{F>XE&PuL0dCVDhZy4UF;@g^CC_=Kgy{DDP42IaAy^TGEfHGo*Ca zotRb%D}6LzAbkl+d*nv&9{e*8uV%hL(k7qF}-Trh04)mUxTp&PW1tu1Q z2LFdii^xK75*~;vae0%$KXiy-rlygdTXKb@Nop}*QQ-s~ z?B+z^;)(YOzd_A&KV~6nA%ki4jO!MrFFh0R-hh+$-3qQCSm^;2{@KKaltjtXCnd^{&px5wE2+lzUXUk{nGNer;c->Js(;2K#S z^uv3B_KT(#6Ctt)XFN1OGEP_`haFLl;7dUc7$b@Pgdtn0$O!ra$bmyu1g=Y3Ws0qo zRB+e+q*E`))q1Pv?MFBE^OG|i-#y)MbS{0kC zeg-H>n6>gVA-XxhbpC|}1GfVT&of%xGIgMFUC%Y-^>WqTY(Y)>=cP0S%d|KMZ83ncw>f>ypV?&ywVrkRG}J5eK|{3emI7a2$LSW?7=3Z3?Eg0%8r%wj z$-NC8E6FuPN6CioQ8gR(0v1I4O-x;y3vg3w8>`wSn_w;u1 z)?AVy`SiuMpqidddAV>y?yRMU)9>GrWY!4)3@uN|-sr%k$TT>&W4R+@ZLhjpo}!3! z+G_S~jy9_%E5`eceR&k>`Ift4&k6ZxBiH0#xcl&W;PG>5rcm-?JDde5uqSw&7^_mv z-~3@iPDrcNF-Q#TcrI3G;OCxmY*Rj00oI(ll>FQEABj(uUU=}vpX0n~SgIA>`Y-Io zyX39rPn5FQ7A=Zk8<02n3lsO_UlrGV_X1J2IxZ+6@O9dE`s9`Cd^bLRDr1%Q1*tm@ zk0(wb|D0?5Fo;XSY>ABpYsdUYg6y5}2vxqgD z1-P`I^dU;_y7?jyVOVhzzwbc!E~0<8wB^qIGau^I6us`Kc(LM67}~M_;9V>G3p@0V zJj3ji0%x#QQT5Kth>(tHjr%Ag&jrK*zZJhSznt@~7hSv0O?sy7k>_3_9=$hM>b_eg z?g|A>aSf2tS&Rm0ry;2jq~aIur_~V}5JEv%rCVLKcg!5CW^rQWGhO0OhEZQJ>{oARS-`nBHY3DY=v@#XVc&cSHF=;{W!qsf$)3T6QID; zm!(se&Y!r%)~1k8e+bsQ9MjecHjSl53cLZHXsk=Wvb)uR3W1^R;)Bq+;CAthl)9AQ zhPgrQ7lCI^Y%IGt`ea{fm@UdwR7uT-$qY0`gF15L5N1s+5D?Tk;amObii|W2aRvvH zY34{PZxeZ&E+k|YXlMo zXiVEMb~9!$4Zk(g0pH153Eg4$@S}MGqS$@-K=@Qy7ok>Wou}llb}QbG6q|#$UK+BcUgggO)v{1DDC4(fVK?u>Mc``>HJDG=RjU8T69YFK49S63 zQ81H}LW4yhyD&m{ui)vU{fok-BpcjX6{3f#zg4j2^yRetqm^E&vaenI5tXYO2F2fx zKj2*&p@LP&qAUi}Bzfc^H<^}RZ<0x1-t;S1-zOz zkP?$E@bSn7<#<3S1pHD=nlE0m#hTb_(exbBI7f-kxZ&B{n57&4XYqA*sdv0>D$5_e zM|0mzj1Ph^aR9TdS|||5X4_FYZKS&B9 zV}%&t#kUQ?$ri!W6U_aA#dK|`mL}`sH!nMZVr!fZ-Huh?;!wTzuJK(2a{c(F*}ex~ zb?>vK^=09*uQ>bB!-1)~fY9+T?DsVg$3eGRz-iaNA4aPn>UANL!9S6`gFDlL2qcS^ z+=lR9z74s4mFJXV&*pZ9=?+gR#P`j#2-USE$o)7R-9I0DBy{fzOC``awOqH_Hcu{&;sA0oT;K-?cm@Zx@ zE|(V`G+6u*!SQu;J5*fC1bq?9I@Y_Mc|sJi>U)cw+A`q{96xV_WK%h zH;x5{s#KIOHPyA`+GRyqm&A*X&L~<-f6JUIy~eM4Gmpu~XvGl6L5t?XUs!`kCoex8 z2AP$jP6G-LQq+K#B}E6Da=S|FQ~IjU(=Mg@_1RlJir#&?dymXyU*Uzxz%XqmS{zB3 zecges`$$%;@JCgT$Z0^D0p)O^T69 zzU_O$p2u8lWN}-^L;Ac-4o4br(QW2%-{3vyad=Hm!V9Q%5kbD>Au!P*Wn2;PVs@%T zB)l;b*<-Xc)S(>|EUlcOqIm7Z)aeR^$0qP6;>39tUoV$YH5=>@s%WK% z@V%R(Dm$UTg}DIodL>3QoQMSye^)icpuQgDxS?24?}3U}D>)ldb&Xm;%l>Gm;-7r& z$Z4Ih{g-5KW#&vi1C9(J!vAx)kUf)0U@B5XV&RuT@>Fofl7T zlP^upc9qT=I+)Azffm)PzEx9u7FK>_2lt50O%;S;<ry-4SM;ky?o^C5+X(JPimbB`+x%sQyvI(*>y#l;6~ESYXc zVRFDpJA&C{`~ghc4WWyH4!p6jl1r51SlUb8uow@`I1nJ*Do3VPgm_}S zZsiK=jfd1ZxP3|$Q?#@f6T7(QA67925!`flOVr{M+>{U1ki0&Ty5*+F!UiUsIW=tT!+A=v_{-YCm3h#}L6~@1D zDtZAWbRqXWZyH%5m(qv)HgK-<=APWSe3uYMJGdXzC!dUW+E3}169`UHc)a(|H5@?( zQbKkil6HLP;wNcV{`ujZckj%9>$vkRu@Yr)ldVP{T=ac)o*I`k?osb{nBTRo#qVX zQtKFR=vprzH92y2PeL7n+q`mWHC#WlDTmWzso%#i(^L{|VpnPN?&O~fhgDf1;Pv<2 zr#L4YdVxQ8I1PztlA>a>xVKOa(B=8NppS2vT19`@;m7N#45!7wrH&yLIStk|bGVd*EJ}^gv zXtB*1Pn<+TM{_g4vlhXHTD8!h@LbyX#MT2)!lFB(D?h9@kW`>$lQmE;(OX?olh5^| zobV}b>FO`RTUJjbg|0;u|H4lJ6f(%MVp=0&sSn73x!r_VedmtB{pQteA=OYTty}zD zNrN!CF~2MDI!@ZJB=8A8%n>MdFS>tzCcOfVj^>Dp&JCJ9U4(1~WzoOaMC|#wqb55K zpR!-sDAavwUJ!FCF&Qb5{5+MopKgL%>0ut&ElC_gPsxz&gGEs%8P70YX)3l9i?W3@ zdH21}>{3B--!A=)SVV?J<*BPUT)6!bNl(c z(%;%^(vIE0{B9XJfypxi)^8AXy(ZU=j9n8U4H(H?5knzEe4c@bEL7*nvuiui0d;V4 zfF51s1UGiFm5_l*@{j}+P;oFc>O9hd4pM-a)4+mkG1qL=Y+RrT8Xx`iu=aa}U~2r^ zLaCf@h(xVG_VpX=_hYOiPp+zbi|SVy&|kHYIyHPzIjNzadno(1@qKC2XB_2#(Px*{ z=f8ebwrEW5m*9zCKs4q6*a0Bz{hEM$ptHjOAvOqW`W#Sq|4~7D1>*c)X3uz6b^s8B z_iF%?*NcKoR^PQ{0vDY(lw|YgwmY2IDKai@n43P{FySoTP%8PWhu=WJT3~w0{R8)p zV7@D-)ZyYPJK!p@MHpfnh?;un2a$`oXpBE@vfq5f&^@#ivNk+|i_lzjtRD%g@=9La z+*CVt^5@rdv0|@V6vL=2tEnsA0+m^!aMgc#vhe@#WYO>%H~OvAEzTeC+Fq{>Xse zU>=>$32Jrd!xTcYj)nxZ*qOWYxxNxP5LL0dQ?)qjj=a>jyj@6M6d9QyrzE5IQ>-76q6;-On_ zT{Pvjocl$UzI#_M^MayyyvI?BB^V{d!D|OL;tZ`Zr2}Bj!=|SB$+?yY@c@^v{D{4e zWb1BO{7g?+_P$cWXN6nW$LHf91wehI1FYT*vIf5ep+L$8^#$ZzO4lc6bPV#jK{-F{ z-%@m+yy`FS4>)QVcC5oIR6(V0@>5peY4foD*H+ja$A1kj|KIqae~lXQ<^t2G2Z;_m z219&^tz+Mn5iwvWIMhPkFO3R|Uw9a`kdJ4umL!@TX;YB>eF&edX?^n0Ze@)|y*dWJ zzA=d-1+P^-k9b2k^`7$WzWVPJBafeBFOtVy95{X^o2oouY5U0f;J`6VE)Rmq;|i8- zanj#eSQr0BDsIj0Go>dofTC=||M}SiMN_5Cw@#92eO0^X2W$%tX{7)g0WjQ(?fHWv zC~Hh`=_;OE$3@3Tm#fDnswdvfOVCEt;Ob%bz8v z@b%0lnj3n`LuABIa9=`P^=(^`j+;N6?i743-?o)7WO_f^v}C=R7~LW8b8y>4BI6TC z&YUoit{`)N9!OHaJX#k#O$N+;s%elcV_W4Tdd^dVu$68R*t5f>H4o3f%m3rtF$=D?nLfyEB9jGY+f6q1qTI=isdPb*BFS7FZZY!-FI}B>T%4Jm8kGFr zt7_kEX0iH#Z?`AJgSoRse~t#i)BfyqVXiKMOp9_6>Kde0iZUdZ6zs( z@&d(vNsW*9QN9*haK1S)y3bE)Z;N&FX=e3z7!DlM#BaPp5yfqQQED(2nf)5T*OQ<; znhZ*x5*nX`gI=}wxeL=SB;`-exgckdUEHMyz7IRq^NYc}L?5DiW@@70N%@RA;As>M z9Z8A!>oTbx94Vc-Kc3H`bmDjJ$^}ch_J&FqvEKYV<5t?lZK1ESxSHU5X> z6bI5g9RfJ3XxD%v)@zlcpJo)puNNo)^l>psjmOZSI6{};&@dV#&2OluaaXCUWY|pH zo7GSBTA0C9%j=h4VHkZ(J$yGy#Y}sVcvB4hcC%DRJf*Nqjav4~!>FsPkTmX486-DiXA4@TO3PQ zGj80bl~>#MiK+%Ig(wb+zo1TMz6@7Nt_)(aWb>&z&%#7^J+ zTu^Egu^t$py}c7Npn6lVUfPsfZpuVe{_Mj#uT4>b1E#p|+sp&Ez~y<-pcDjnu%bCDPNCfBWGt;EdN0Al{sG!*OIQ?SN?vO%Gr@X6)0y#`sGN9CWf0@8Ql{++KamQ2nj+Tj4~>xnfrazFJf z1)j35vu^W&DVE-0H&&^Ze^~kHyN^uQCbidE+1xiS9==jhP&`xq|CWsV-}w7p3JBg> z!D_xnPhwv0%V-~r#k_IOz%1~-rL^eBmsLa*b{~vdvayymZ_N0_ot%12DSi0d7q*kQ zE&`P3%;YkGf0Oml`U}JBV(3@!04YP<#>V0KLyZE6&fPo)ZO!+YBEz@WeQZa!J_h!? zO}h`ORdgIoHD1*3UJjY?lL{cqN5kI?Lb4j`y_5{*ej%`l2gwQR?3A^sep2?RM%~>r z%+G2u^l6)Au3V24g;Zg#dru#*1+4F<7-i)>_sMYOcy>R z*+hiRsIDCk;W87~?7Po>^?@mXQEo<&e1Im93>#Y3A9NdAk3zLAMwcpi35@JzvAZWLRqOh2f*NVQ|+8tYu582R-`O5L%n z7T+8TsnaK?zWJT+7L?<^0AsW64T2ld;o#%ob+!4_@!aWg51wi45+U_%9vMh4^7B!8 z-`&Hr-1nT%MMBgQ95NqqyJ=rD6TB>P8C{6r4m~8n*;a+`uqU^u6gMX5BvpNE95o*F zKB$$5Ek2le=FOJm%@|I4+i+bK!hKAw&Q{R2aFe=uJ7#!zTsX^RIc$IC^TSGckE3B? ziqVJn?(8NYL*QnvVEi%BaPn!)J)k`J?L*iIU1KpgecC5s#%^?-ukW@-qwm1Wvw^I6 z`(l-IDv1kjCkD{cn9fspvMcY7l}PGe*v0@BP|@K=GoTa7)SJ+cQcT-h!kdgZT&r*Z zS2_LoFiO!7bFHDYd=Y7cwqIwwX+t9QO6NIwl#IFexnNdKVV9Dg-r8$EXfqVu` z+)Lqo`+sPNKovx%+?O6Jce++1?SI$^=Nau-hAu~+@TCTs92i+uq4)qA)|*(maB!FL z(NVfB?|LuER!v9hR6xmyc7=`AdLV21CA0mHtuIa(ToACbW3xqr`cK;%fb|*dUUz14 z6w%z=Ch3}Ec}&IcuJ5tYW9!M7ptH9Po?LFA)!?^3*L-`im`V;Hi+Fqq#d8JZwu({I zGXW~sHFP$oIWBIE=ivBAcbD`L{^-N2A6|lD6`${3DxjkWoq(8COq&JX7qpLXASSC(-qoi~{inB824K$r%nd1E z+Ps+3^ixAEftBbf*X^W0?f0h)Qn0)~8X9muyu?g2Sscxv5X0-hJ86;=5oe_$;v-zmGyEzhU9V8*{Yxdn`*&to7%OJZ(UBO?>; zEhF7Urd@bwLAJO{1IiZYtxTi*<3=iBV zV@ic&5OKuNtjHF(%2XWjv}}D`vgua(T2tj{S6j%vtecquy4AMiwtbTueNxd*&@tQd(L^o)5Bv2S)oAYdiPwlbV>J?Pu;Ep-xF5|=FWt(*af)@2!2m>$0X#K;m zTEL^ingu)b3Z)78E#gzLe7A$+klcnEkyQ~1o#OYM^ewVIIkVjImI7bBV*iRTVNr&O zGh2WS#N)~4GO=*7H7@$gYIuHNLn9@j>&|W4#X&J!KA&hwIMur_GJaea)?nDJZ9lfS|f3T0Z3uTbx6-{SaLhRi3@tJ7cB zbA-fLabvt&S(yVewHs?43!*JtG$WKa`CDvVc~QDXt*`inu&<7$alfQ2Rxa|AZuQK+ z1EMs5tsMrY5rZ(0`~>J;Gxu{npEIX)P@{&`LRUTk*$KMN^&Ii1Ayyaas#HxPKb@&Y zedtV0P0in_2FegNV6+~f0e#jTK$tMz*b0r13vkAVP)hd)bjI;zE;5b2eANejK!PVS z{1j(pkEf;K`~QTa{hhae-3$&652EJYP^>u1TU4_y1=Sj7Q>$ zeb_%|f-~@Ige@e)*f;dgUL=kHPyzH(y&h0t+QBig+WIgi_CL!qD$q7G1#Dgw({`Du zigKQ4m502jwPSRV7pv2;wg3jeGc%K-rY^n5ROE@E0qmJ~UedLG>9@Ewd3=mUGKR}o z0mTocRMHipCgOluN2?Nfi=N{;p7Ltuaamo#0WmSeSB{UeF`TSmP!CjiVpM^r_<$u} zfQw?<&VYzZ3RVSSg!=YHx-cWSN04O7r5I3gMZFVANl z1BF;T7h0Zf55B4y#gVc=h}Ge_{2k8s8%UGr>MnPsV*_twAwrHwD*C>^xGl?_#ljpc zhv7cspp%Wv{R%WQOxsD&vJnr@fM2uqAXroM(hNg_7amdec1i~x6;u8B(<4IYO>6MZ zVUvq|t)5`F?WiEL;N(bTJ0TIv4<|mB7MVbCfnDlUN5*!g7$G|!UUbW2EAV1GDgO}U z-J>F!6Y;3gv_w^j3n}bIyVSNvH&oCWSc5H!HI=maGk~7xw zDB&dKWJ8H^za5$67SA&50sru@W%*I3`T47PMamnm3|{3(`Y4=ahso6c-bE{cA2f(> z+r~kb2cNEGauUcQ3%MX=zPcfz9Jdvmq~BelqaifYj*IeQD^y+LciY$}G7#|8KuFnN zNOaZ~EdmBi|9gBhj)072cAc02od*)4ICso*k7}S!E(C^qG*YB%0)sqwbZt6Ey2|qs zc4`z%kG?b%b_b^l<|$A(D8>*UW63V~=`zwbUE&+a0N0^fwud;B8KiZnrg&;K_t+!% zz{=k=iN`_Jna2zXzXx!&Ec9oS`_j)<}2BG-8}N~@{J38 z0o$wiTu>VV0wJC5!9d`s0_8f45-Ue66mTt!m->fRjK zqO2sqB!f!&|B#%8w1C88AC~d*1u#diEg9&j&iIp}eu~+-H{z!*y|2BSZ}SjmUd%;J zIq_>x@3%Et+=R?+5nBa> z2;PvVb>2-AM|s^mKLUVC%L(+;D&Tt009g56#2}Ln92O;+5ecq+9!|tz8hX};C=MjV z8iD3YKZB9zcph=n(6S^V!y!(1XD+Q%cQD}bnB#M~j?~nOy`M!VeuX06V#rsqU_;}{ zBN&lZZ4h+qZTI?c9+W{I=rjpemFsM^v`<|k4@dfbzTmT0tlPo_UYx&6zrhA4u7c9( z1OWGnqbJY;j7ONO8!1{5yf!Iq!t;uw^##%#)7y6WUr9e%xawZ@)yU5Tv6v`f70D`_ ztAJER3T*trX?OG6m|(m}|FTupvhBV+%eK#=k&?0Vt#832ec8=yMu2>_2M^S9mpE_= zzJHale@*+n0Wu(j>ar2I5xkT?3r8Hc_kH~G8!r%KMH{xUZab9L*J&tbow&GYu9TPb z0d}ASh*%WEgAtJG2B2u&CFvT4z(&}|3WW!j_~HeOkEHO3Z`jvI*WEdr%Ne26(qQpX zfHZPc;%NL?;=0goz{|fM*tIWwfRkm8}KQI;cbe#?F_5*Y`xKWTZMWzFL z0ON~HY2J?6;(~KS&q1R{*h6akXJDUc`QTNZkgx|eG^X?PGj2;*Z|V{sJp6yXAhIV1 zzX00k73cIVleHz(&I~=(awSZc4CP22m|(r?Btk!ok3s&nL;Dd=m8?*On$QNqlfcuzIQO#kRyIxjg1X4vX0@s+BK)&g;H&~}Ax zOCsFgAZwk60v8EflbC>>XVz7K5gD+CSwNlvk2iz?Sm34M$Bez*#xC-=BSVgMY;LSS z1hGOsT?rt>{NO_L1CSa`036&Bk zx$7ei3=;Lh|5#%>F*Lbw!cOF{$|j8pnqd#|5V)+jvO8!7-T%$PU_*Bz$&v7Ouq$Bi?B^MKZPjmLANC%?I_hlvVVVC6n<&BV za6aR0gHN+Fd1V>ds;$})=^YE9X0bdWf6eqVT)4a1z z4@+D_D>@_}pLis~($3~+e--idS}RzC7%|WRL*jjcX-8dsn5oZr*ETv^*~!jn!UriC z_qE?&bj#6QbQoF0_Nk@Kguk45%Dq_4UV3ecE&>^p&2?cs$=Hr#2-XxuEwg7c;E=i{ zW_31(gOz_oXym+!%u(>tFXM@27*-%li+SVN(}PIG@K&R|)N5a=r!C;vO%q34S|jEW zhg|Yc3e3w%+8bDXq#qKJN_6cx&ZLMsu9ir%Hs`8_He-W4ag{$d47wGWA-Z| zcl<_G{uF?l1~_%_F838OasdK-V=|yirv6+CQ6(dO9L}{~y#ysJcnP|96?L%+xNASK zVdbQjPqX^5eazlwbC3?pp?{?8a-+D=^lcX3`y;t9ZL+ruLe2)dF0D+@H!aTCxXqp7 z)oFQ`k|<*J)J!`iA)<3Q?wYI>GCeGB-ErL*$f>>M*21x9R0;Vj;}T}ujRZ(+`JiR(BduTr*Ur=m+>r}48y~1{UCP0tjnGRKC z0|r45v#RxKV(X8Iv7x zu4yaMc}M3(k&iAxf0#PwokVVncf7~Wx<^w#+JEjKOCLZir9eu5VW-2}@esldLlGLD zoSa+zn~OR+N*;#6U_=)Cf03^4`$sGpPj-hx>Xu+ff%PaH zK`D`cbrWifpc>MKaL}kKT{-+tj;>P!e{Nd!Qghby0%}le$tp+pieAoDmT=|`5c_${ zu;N#^q(P~!BQFpbt2jWz5VvLi7!JlFsb_=WzZZ4T%3s*GjcDxXDJ%zu_zd|56APRQ z;@$(K8x+zNran&9FTg##>20X_{(V<1Ztl>L=UGM{&*%>Hop}&2QSqzipXS;lNff{o z;XpeTv+@^ahx@Yn&kiQ@LMkHk=lxWG+Ni})a+!}eRG=8#`ZjZa7qG}Tj8c&N(7gdF z+@Oys`;XV(yUotz9s!#RwE`Dt)r9a=CKOkcw2tByu^?Y;Y78tpU%*+oz@3Y~#=Rb;^m6Zv_jB|$yjl4k zgO`f&zAbi{OfxXy9^1@j<-8s353<^03=Ol#-K4>VI5W7hvQO zg@7z<-*;4o*>%UAMw`07*AuoM$Wg?uUu$*QrNLwcDP{~3@L@a1?Wn9uAGr&S5~uh=+a$K&nqxqI^iftgdqb zSl^`c8)sm?BK`jQ0Y6tFWo#&JcRFpSw9~609 z`TQc`dSaKAv-yb=UFe_yI5|Ww8SWR%KM(C?a_%~pc|dg6x_Yt|49Wid-BUB9x-r?9 z)2MmaJ6YIH(s0D&M&w=~9g&JBhXVbIagjn%Mh=q=C2Hls8$e1!A+^83BSgi&$(#ih z1~0sy%)P>6|70t03Fc=EpF(N?dLPA|8TYb|dq83!;0C$2`_n5|-6KtIdvCoy@%8O1 z2BsT)=oW~XO80EAkqw#U(9H$z!Q~+2R>)vGTBnwpQ#$ja zO?{1$`i1@ef1!+>e{9pmfTB`nph}0Z~k1*}S(+_6R-X z-@y*@U%}25@bUjPF%u210ct|IbP?qRz+_|7R(SL-@4}%X;JqW%Biua5^`FZ_5AQ2k z?t1@8FNWjvz0_MOa%o~>`kD`Ve)UsRz){hI_vQi0t4Z2~I2rO;BRC135Ev@R_HYr) z7ttW4{=+p%$a($Hxb2~Z1q<0VUbzs_Q$HjG#c#pMUXAqZ!D$(0*WdvFVlm%B3)r#$ zeh-Y~4fxt}_%#i{f7ea^w$@L|m^?5+WDwFN{fwn37O|wm^`6G1iixtdZ`0NE$4JTc z9KXy?xUhOYkn>ZZf;hgBF4;lOWpctPzs@o8T6Ac7t1^Num$d6f4Q;ug$KP|B&uSeG z@OH@`UYh;*qv`R{SI=gC)e1W@!{Nka%w{g{R6nzyEddPv`0Y}L%*t%$Z2-mh>}Qx2 z(PzB%buy5#+o`t`*Q@B~!R>x}S39EkE-h5z9d9aEh#{Zw9?Nu*;VUke)Xx-Ki(Y}L zKl>>IW-ZBf8IUR%HJ||qQJdavjhSB#IFhQc~AjtgbgsvA01 zm-->gTH#}=qg{0R@_Bsf4g-W=F)Nt9YC??$<2E^%`8@YGi!)Q<$nHsJbrHlClKsI< zyNOk2xI7zIXZrfRuLrsZE<~PBynRzL*+rt?1(Tgj zqcM4aOL+7H#k~UoXLr89DH@Jj1&O9H2-jlfz@vIFS;225jL)1zC-z|LfdPsHlG6Qx zS|~)8%L}Xu9-mssT8(=0QZ7$6{a}t}aewB#bCTnpKUrOWT>i(X4VYr=5`M{+@dUo6 zOu{onN+y2I0V%NDHBcG?QCk+6qL6_ONrRodPx|#29Rx>&!v|dvC#EsEXVNvYj%!9G z!aM)-Uh$)IJ<9`o&5s_9w|2PGPO>VdSB!&jU=ttxCP43&A0CiH^Uk}~kH&MoBP2G| z*y;xvc$^kjxqs`}nS>jh3L&97K)evM`4YJf8f;$3B>eB5=gte>FpM*>Dl?H~M0z#hesCkSN%Hywa^-(K+mk$@JY0n}h0w z$ignY=H41^Po>;nxlC@5GC#s&=2XEm{q`s!GhD_H&m_3Dol_jx^?;?ZO)JT8YgrubxB zb1g~bXeWS_qGicF@pLy&Mm7n@dvP_a^)jZ`UpPrSD#2o!n3`I6_Xng*mZ4|pYcS%f z@W1^hwg*fkasMrSyo6cD(>&->V7<`j-qvG`N@HEqJ18~cd)<@j*XvR=^rP~W@(lt4 zE$;clMhZ9#2e@uP42|FtbScdiOW*8md^%MKhoS0$Z-4r!J z6GYUHbauejW8JmO?@Ykw;Rpe2Pfr`Z_8w$I0)6H3(VvD#d{DVm=jM>dB-+*R^V2is zkItMjM2O0YPPv*~vZNgpoO&fsr zrrWtO&tzMQ`OAwv`o&%n(Kk#^Eb3&+d;k}(v3k=*j%{DqiXLJ56zi}w)jN93%vBxw8#{KY5m*HS za3h`cec*Q^ijN{)O2!jA@jNZ&L@5hVLr3C;f}pDFq;~-cbxn=U=^XaAADbF4EO)Oc z+OyfeIH^RtgNI>=-S9un8ni6THXMl`{jZ6g|IPoh^wMubBIFcm9%PQA?}N68Jn$}Q zK8i)*xa-%1m)))^kCV7Ljsl2QUv-w8bJ~!Un7d=bS3SCE5m*A?u8IhYdQ4$q89EL( z?s|Cd5&=4dr1Sew#lgBDaEbNssf_mg`frJbc1~)Du6gy=S#rufWbfgPX2he1WIXJcdy>7u0Mzb1gP547c!9Pfz&`+?wsx zG*hZm70MK%1Z? zs7x2%HTwq;o?>$e!UlU%xZ2I!N2Mt(Cy5*UuC6O#>hw2mW=c=vMf<()dp5jSFBdvh zzVR3438q7zFgc%q4}%&i_&eZlWm1U4F!+*;DP)%?DEL z^0oW+MbDbB#;}|?vAMwu&WcDtwOM#%q7tQ51jb7C zmUN*2Y9!+x-VZU^j|Ah92OaE3wDAdd!gI^@l79xo^wnM@$7>MHCDV4 zy4)$#jf28+>EfWzYHdV8GKIfjHec1liTIsv90=vu#+jX3v;K{mM$~L#cWZb-(pxw= z>CB&7h{0d5*>9PJ_0Vw>nQ2S9Knm3onAJ7l)e~|TKy#EWUnC7J3cXvm!39iYnZ2|o zh28k_DR*tC{4qywO0VzO9?9z+0)%X&FzDG3$iC?X6r0XwbPe$1+~X|<;cWOwsfmQ& z7->tT7kToZ7pgA%j4WSE*4%$Y^W&LvCBlXz079lXkAV^VBy6LCn}06<6AkVk0xpjV z5*`3MO?PitYWs7}tGU5<8B zLTAlv$t^x5(}nl8?XP7bg3PZ=Yd+iB(L$93Le)wK+(8csZdxy(K-c*WUX zd)wZU9LxG^`g5-Ch~X`Zh2kt-x$Ju%Wz+9VZ_%oNMJBJhrebT6YwrHrzqFmD#V208 zP3=3=FF*BN&OIJ|jfKFvsn_`O8SgkRl*q;87y_2XMwVePL6WlQxW(I!OhNJ(8_;!z z(8mGoUTXmf!38C^Y20=XKJ!+fEqw27Z_ioDm@@9l{SP=~F|73r5De!v=H?>F)>UYU zCodIkFlqXoVLueI$bK>-e#ql=XaExkTv zeOwzhOtmq1rzK#mGxP7p-v7yv0ShA%(^f?gLY<^&FgcpY9hP78{YRPl|v*n$xR z)K8Oc=Z8aiu6_EmqnjzmAmm80h&(MiQ|bTebz-))!@V7xi5Qxb43B9QhY$#|NC%d~ z?+#mB$#hji_Ve%gsIZ2d8|Cdi2ag{u@ijOaz0cqIYwkK8*!wx)v<)$y*y#)>Mm7qd zGZG6AkG1GTQQ-%{bm(y9{0QQ0&j5#>$a%x+goNq0QYq3ZzX5cl3ztBUHWu-f5n}?9 zS~X+`T6y^LXr1jDA#V}k&1Y8wEBDH^wK=go+ARW`NJ(ju=DFTVkFT=k9I-3O{8*Gd zn#yOo%CUZSG*p5|0fBWI(@ zj2q)A-cJJt*p9ly56<{BJiVs!_#EopV?`O&FCW>Q%!!*PWbZMioj(BS?hGJI*p4MT zU?+dsjnS2hr`sqXb8%Sp;2ev4u@aQ+jG7L4L+?JB9U-Hu$PotG^B;MHbc@OeetN%F zTnkEawsl9eo%znjdFa`jsq6)4NQLR5XM6sCA}clrfdbtu;A*DAMq-`qalJ7lmxj^)YMqUu8_$FTm1m{=&X& zP_TbtTwJ_N=Lvw}X{+7N4F>>vLiDfsIcu#nEtgco>WlYn!_S|-RM_|G;DHY?edoDH zEWa%2f=DLkahfEi?I?=90#c#6kPC~bIKFw#ErKAFzG_fqPgh5kdt6OFCYPbspyO!g z@_702!_C3&K}b7_$tg#{l3L$WoOrZ)n7qvSQRz(t52Vs5oRjaE(#lm;Dd6()a%#g@ z{`ko!$B9u+} zEPkHp<|$V^yq;yf_#So_NyI>h5Oi_D$VGSQVK7i&fLk50dF58(>QuD&vAFTQm;62M z%I4eN=Lh>b_4zpEa?2SY7>Y?k1J6UY9R&Qbh$uje-=LeZFt{H~&sAq%-|;B*r(7(H z%VFKD@)=p;Xy@dY=r>7j8~fz#?he442kU8r;I1adb)vx7^#)a=@UGMH{0@SRQ9mDc zXpwm&>f4`QIQ&yl)1a*M^JtKKEleZHz5vm;-GzbNOhGyS6$2FcW2~6HpyU-C3$M|N z&;wF;x+#mn3DRo$>nKx_P4Dy3YZQenij5HM@fRy0(M~xKXvs9uy00-cRCct0n~)9G z0PVq6Ll>KH1vB2%x)5$PxubDaKC>p@weA7#zh}JR;P4Pz1`$#k1(T|S5V|-|zwo7K zw4Ss2%Fu13h-+0r^1B>9lS8#xrs6*S3E|gSa+LF_fKS(wE&(2qbp_m{H_G=dqsW*I z!gNk=r+{w2{GjzXD)dzwYUij`GMkQ)?Qu`R%ZVfZ*LwBW&8J!kC1)O9-}Cg ztwr3~MwBJDM%=cPsbo(oVbqvQ6lJGSim6l*n(SFJcCBR35Hlkw3^UZYn3?PTp6=&) ze&6r&JfH6$zu$5EoK0?%fp{l{lSL>e~+H?F*zf6*)=h@8p$jFn^wl>-Jt2;sW zs~^0jHRh*ovmWr?R2jtG^AXg_Udg)%OK`$prOzIYKZ)x!(%-<|J-M%Cy=B-aY_oeK?G04;n0BXiqh@Uu7Bjd+Uc;L@IQiZ+*L` zNMp}C_a9;bIs(g@HRyFwMepG);F-ZW2bHm*^T+0SSO$K6H1O0Z+s8jXx=Ry})sNGn z11vugN=b`b_2cVm?ANnj0yBeQ$-N|aQamFBhw<-GM4e-~P( z{iy5czJdLGHx~dmY(c}ujllA-WTteJn)h};g;%=d;RqL>F-RI7>rkr8cOQQtlgGKM zU7h+ddHU?qhjVA*%o>#J_$FW=Q7@JM2ny^?pwxJ_Lb6CNgl-b^XW&Hf&swOVW|nLI z+nigo2GjkrG+Z6NxoRGe({S(?in{C~@@?e*Onx!{p8UQ9UhW=Xxt;t1scjyt+r4BffHRE~*$PWiAJLV}n@kzh)zT7}e$n9*b&PWl|s=sJBkD)!IRK4lU8O*GQr|5@_6mracJZn5o zq<1KGyh!Xcqreinw7 z10)9l&1oe$VS$c}4+_uRY;t-zzO4rkC^vTKT{P>u26V-ScjUM$5K@&1mF9ef|IK>x zKUqUT9mDMeoClbYj8seomM7oHSYaBlM7(PU2&d`%?Og^6t_@W_H>$E1U8^gDhfP=2 z4ml-Vo=_Ehk?ZLKqN8nS)~`u;=LlaXe5xDKV1@!lo*+T)^~V$xh6cFscN#ItTw8z? zl*jm!sM?$awr+=vWH)Um!M(V5D$F<5&E*v(nbY>U%+yo5zf4B4)j({UU{~2Si6duu zAdH9yjN*P6S}{fr6Gpz3|CGY4XB)L~u0W}gQbx|72{Kc$w)O*M%&_x1@jB@f)JW^7 zhx#-6fRXsK2*>m=q={qwNdoGByiEfwl)AdM+(ArO#w{zofg1AW;Cw^!L;OrvLt;ho zSE}#H&hQZFoaJwGrVAnyUJqQ49VKgG`!hc1yB!Y@3%gIQd=gNOzacV-7 zy}gwDB}n_kHBA4x@|US8`nk-kY3G^?m6zOfUaD+Fp}{u%kN62DQkm z#{qRyK7RxlG)tgQMXKp#2GYY**Lh8XGt$gP(#r^vRJ%uXrcdYb>m$?$cB_krKEE04 z>35PfdHeRIqs>Qwv6{aGIkg~@lL7AEsF$bi#&v<#@$2XMxH#fF;7cvGSK$*5*H*Gk zlBUdmmJRnfIk>8vt;<%~wybmrch765hwH&pUvM|qCs#K5fpEf&9LZJ3$n*Y$F{g{2 zV&p@VhbOi*zs{c6K(n|dS9wr-*SG5vjW@fdd+dKJ0~qEdAoM_?CFYMX*9%0CV1KR< zK_0HsC~RbTV*938#3F{yXn1KY=~=-a+r|z(>HL1^<51Eyd^vtI@Js7dUj$* z?Wa~ceXdT*fg$teYW82vD*Wl%j7=QZFVAbU|MIxkaY^K#rVqQo)GLTcZ(&X1|SQ=v1VR$hYf z4ua^i%K|XiE<#%1)}T8WvQB}#(2qGcNgZI2uDcWDiYGE8f(pec`M+jOJ!?Hp_TJAc z6m5Dwl}P_0B-u0H^xOtr2u==g^R1w9RI|b)abw_8!1{29xC_54Y+rYVBB$mV-0NES zKFo@})#1pY({*>86!b${3ZBT@Td48BgBbO0cp`=spv>Ep;RfF*`a%FyH>@g0R!75i zzmp(GBQ7m0Nt{5~k8aFQdFymtP#rwG{1f*aoDW(KLh6@S zQaZ<;Hu-(LAW{3B8h0I6p*KT;cg-;6fIPG_Sei?O(|{RU#`pLbT-4?(`T9+tVR+c} zzIH2j#rN`pieEEjR0XxtMMXZ}@nj3cf?Wk#QSlE$Oshha0pyvK9`86EyxErR=XFbP z&pYa^3yLBZyW&KwQRE>KY@COP`*APw!uj=0Sgs-OD!+*54lJ4YEK#Uw#GMfFa(uQf z$}@L>+L1WH_->ES%8mM}c_jOVwTMm}bZF3XNf+F3y(9hxAj~an!Cy?^5qgF+ag-uw zq45}R#IUQOID+@~3O=3Eej9Zg#}!f9%dozS(?|3xcNB01WLp zK(Yjrgka7H=Q~Vd>;XE_K{KD5f~U&uJqL^)<#f0nwk-e1+?@P&eqY@C3vb`QFGFFf zsQ;)$O+moGFmdc+&JOTI#|X#x4=I5Z+0G~UI&6mXtAYK2b@DZtZwAB4zUA-6%a=to zYRlTZc0Jf7vjiNY%=%&%K>c=z1w@}1rT`{s%Vtd&V0GM@UlW-j{*UIGnbCEUVpA;3h!K^* z216@17mNXxBlNE%m%iWX-r`OBx?1nB2RENezb-E$H1c~H96<+gm!b6l6tqdC+!F!b z^KX?PiY{pA&DWa0QAXd>}WA1>#N=HF%K|!!k`n zbZjNBAQf~2vlTC=3pTNjea=R=X!(6VbMabw&e*@Bs7-I)=A;0^VLuRlZT8)lhpmKf zvF-X7J8Id7$Q6d+rv~&FUR}A{K}{5~ef^yhtj88aq&Kpx8XtnK(Sm!0?TS63lVL+) zXC(f*d;ZC}9j|NI#u0ytS)fXy#du)kt+%~d^D}Q7or@O7`yGjIcOKAZR1eID`H3ku z$vt$KI=p$;i|44Q4KLTR`4ZrwM?oyvzev^Nkin{5hm$zTfaiO;t=3lpD4S}(fHU!F z0X|H*>C-bex84%}`g}TbU&oc5m`)G@R@BW?Xyh+-oD(d}d=-ncz?gAl9ZgUog-2c< z&8D{+>6icTwTo_{oo}IeK09WuTDYH;n+A^7#@PuW_U5B&#JXoEyl_v>yXUJG1&wdi zkRG<+kZ0O~HBK-hn*F5IhSj_$_FJi?&^l}DHI_n->2#k^D%H9@|M=_e_mgpc3aX+% zI%U3+eY}m6Q70XI{TDWIC7%iQ;olj+gH2wh`4z@9>;)M=7rH z!IOg!>c5bNkl(8DLB%j(7X--5t=4g3L(s3)N1sK0@xjQL$hRckdUJ3imjK!-@nxWg zX%A#fj0nywX5Ou8BdOxVNRkZP%y0@lpzq3{_n{%Cvpc5 zavD=oN?y+Rrz%upOc>V5j$1Z-yPX}|^Uq`wveOC(5eGxG7wFB1e(0p~e##hyQJ4@x z4w1P`)lF!+S`ut_GX8;ZJhgu!GF_$5=DCXg#rn~&_il=C>ToHb2ByqY59R`^(dvK+ zoeQVVaE`$wy2C$!sWEsA=X4(DV(g~MCwc#7?a zxTPO3nRc)(%}9m#g(OP|rvHuF3^#vh_ucxL5X;~)1A2VS>-#jO0MVRm&viYWl0>0X z*{MG!4}#kcfpq$B)L1ti5SbE?j}*}R0x}4&U>i}M)l7=^52aybzXmIu9QU=U{IG4X zwqLt9e#7GmYTSkT03E^JUtl0Y7ZERtF88Lw2@GuOoSxYEBB!|zkOB^?BbzJm&FfRP zn5w6a;q*L*1uHV!z&mL0JEod% z<|Cjnn&x=xYX;?$XxTzz~3H?FSh`89;8BmQVG_pbn|`>Pf~nhcx(n$>-OqiiXy89aR>AQFJX zRRW1`3{P%_f#%8Y+B7|cTCL~&V6<%}F} zezbqOvl$G^yH!-*`gUpl)!0~1)h4v9KW{u8lqpI(4F*(j)e#-aFV?%luqQ zMvgfC-7zd@OpTRht+8}n}q=(QgygG!EERyV4DEZcw=|* zY}5)R!0ASIP~U{|!Sm^yJtOE&-u+a1BNzGbjQeO%9>b4*nH1#);6Vg=xM ziUCqb7*c?qqxHl6zY2z_44fkF&{CHe^&{zY*!5KT`agR1UK#nl1-K$$hu(m>|3IcT zxZitz@ozG<5r8lY=rOD<_kVu!(8@&yEMJJsEBbuTcN$(bMJF=QF+R zYMysiUp&cuo7aXXf^+Zn)~8UCz>rbvQOZwkNE5?a48`$o17F7qoZ)Q(!eIO-&HH%A z0G26;l0n?c=3ILQyIa^l{R;PbkX#4#7my+0DxGMM8pr5o+?LyByThOAy@JA1nBide zUqmL=R+w9+WaN<^v@%g6WI}SNrhf644@Gg#)5Md$-A~Wn_}K9y_p|QhU8+^tanA69 z8NfJDcxtBHaGor`j;KMv^Pdy-VG%5g9M|OL$FKtiXGM9Wt}e=S&$!Q+aQ?84!-T`g z)o+;(ZlWvJ6CIQL-^iJziRTgY0#O%A>xpKq~f=KV(O>#tdl|%rJAG@ z30XOz*=3eDwN^F>@LP`IQb{ZqaCW+xML^)Gk|aSnmmx;5OF0DJW*`3bRicBqv#*@k zdr@-VQ&!?Cfw5bc<4g zdsEHo(!-F=A!>@fiIEfq-w#(`Y@LOu0SvL1(^1*@p|Y%^2`AmVbFUvB!j9(uKkQ)t z!4mfW`|sqQ4A=SO?d&&KH5=$we}- zN+At)q>E2~L~SE*TZ3PWPW+jrC)bSKKH*xCTJ=iwVHFBWpM;O$)G0sG@udL$G)F?V zQeL8OkXR?fZ}@@|ef;ha63de;%hMWuRK;Q@94v#adhon=k^YUz9k!BX4oZs=7PvR3 zytVbmHoqL^75t|MRQTrse`07KWkzj*h!U9@&Nc@n+3+^YKnK>J80i33%gFvvyIRK@ zZ}P9Xn%Y}u+mCptXpBGKVrr+7uvKt12bjDq{1i$EIi6I9?Zj*ZObAkpP^VEscu9y7 zVt0F6hZ}YLtD~`kz5co~e=J9g0_;f(k0TT8Ujz|#25P?A9vz4a?z~ZPr+3%( zXvYe#J?_G<#JU8?AEv?a9;1YPYKP__7WsTkXV23I(BvBbH$MS$PG%!AiK$e+28B+d z;82&o>R$DqQqh{XODc_rv%8Hmo#oEb+dgc%hD&>|m|S9REy7j=$*1lrkoTv zDp&tucA3aRjeT^4MnPHKrG&~IqhsMG1a4c)AH((Jb%7yX@)Qr+RP(kOg}fz7KpX9! zo8d3vlur6W!A|G?@HP}u5FHv5t~In-kf^VC&BgHKixBByxR~ot9MhELgS`?gGn&_) zZ%CSI9O}R9jBCwPmfHWaNLxndW8U6ZLDH%|=T)OOs=xXxjyQD;mx*SbSf9%7#<2d( zptc!}qB#(sWE?I(r)HNWyqK~dr`)^OXQBSqwz}k@ux}oJq*X4=4OLN|O_@9y%6S`=Pj(}QexGG$? z3{?W?u+a-bY2EGRwjCE?b%xII$y<;6rL1uyMZ0GQt%M0Z-n-(r~ zL&rAr$EFhb@C?q;e<}*%t95{Kjpf7$XKH{jqYEBk+0uG%d>d{$#xN2b9}8;ubMLM< zI)}@Ng_ocCetI^G7Pjz~ocTJrS6H1uzZqb9q(MOz3IjmrbtlN z{CU;7SYYbv5URn|eZi!PnYLfGd+SDruf93_U^~&J<4=J)}@z_ha082fEeL6V#lKI~gp5r}&ecy*V`0kiVKW&lvQ!BV=m*=jQS6dw3p?C(s z7!Dv^q#9iy0JJNq`{ESl8c`?NXcM52YI3G?OPvj!?i&4iW9wR9U-LTMAwh3rT8s7p zdrv!ooVX?E%OYnNzqHwsIA|pddjhnoa3oX+XD60`l%lf1^Cu1)Xmjng+{W z>eMx68`tKzYc2cvr*5v=wP+qae^ifi3fRH{0F9A`%qm~T8ulS6P>}NU8eSUDv|6GwBosFuCSPgMl-fSzGDtGCl z-H=eG%+kNGz??>~V+Z;51ke!{*5W5O$#KuXm^RY}DM~WaP`D?9FuNEW9Ts6w{$8)7 zy=!}-b4Ke~>X8y1naGDSGh>J(7*7R+1Q-F{ak!p)2tP$XN-o0NtdBCUNhOxQpx(b2 zL99F0!?w#lJ$VkV-x8*NMqVJX!d2JtLAsI( zF!eFG4#j_{9~6I*DpJv+3z$%B;!pNm*G#*wBu*{3mvIt*TorU`>O^~Yre10hc>C*9 zOaX`-m7rEPh{g&}q)1isZE1Nt`I)xdiEYEVx;3{M;Wy9SyIhX>MpbM3%nYsHlSWsGgTnF8W_66j>h3Jj}0;+ld$e71Cw}_{dl*5zViyv9uW;y8E>I7Vp5bkd^TI6paiRnl(^F?^fm!A7|_hB=_1iTIGZ^+PVc^_ z9$9uV_g%bpuT5NP1kMO#Rbv1Ipum}7h4VJ~aJI8dnfCOnbNgb;e#&(+3amcH88dyi ziA4j|k-H-wg|oagaNE6h5ZEKt`5I1NW!8nzft*INXnQf$ljGAejqk&RAK zKa9$4Wrg0Fy|m7hdXlaJ9S3!MrX0(SQx2Oy;>Gd{iQ2)1yI6TQ=Z9yG3>%lU7p14E zXS~_;eAIF4g-W~Gx3gtW0Mg(;7+&gSGANsg8ptOwvI-J$Mpxs;`}9TeAIr051s|86 z_?->%Ig_^~x7!|{!shyo1JV{N-NE!cJikAoVVpX42XSy4LC9Tpn6VP{~j z_t`Dlhf&!dtx;$5k&_S&5Yhf&GXu&W4E2mrdyezq!aO4WqmE9Fj5!??$d3IOw$yVh z4qv`0wObvjx7Dk+(Y6g zIdP44rknlBW6A;T{lB23;aww-0D-Dv#P$BvBG;4;vTba>29Q`9{C8~QI&q02Pmgep ze+v_e8J%Kuyt5xo>D_l$WlOeKPPW(Rkp?hgG?a5E^B{TQ z8KnT1Mq*v{?8Z0i&3z{JfNaYs;VJ9Tm4xENVs`AONzQ}e4{?>_5;50<77m7`WL}G3 zh;6(n(vD^w#C@6KJc7YV750M-Uf?}x(Ko5RZJH*p*yvkEWaD=R4_^(08EVHAXFJ+F z7u@wIqW<2{UrML+z-+YneU2mli6I= z2Q&8{_~u?0oc-OyGwwktv)z!^9@U@5EwDznHYWk~-_M2HhRb*d9ug$kv0sr#ks38Q zWtyAojg0$u|8-dXz_y#HEBVMZ3ay^>ip2B*N%qD`q=zB`q@UIjA2Grcq7aov_*PMH zhQ`OWvnCBvpK{u7q-8wN&d0aMTz%C@S}4(85eKn=1VlefIR7{5@c{;zr~49ullh<& zwcd6+lR>=-I;!zb#DUxiZP@8~@LnFSZ}A*av%U0AO{|A||Du7I#7?)(M8b=Bd+L+K z)>R*LLrpI*hdFd`359T=DFh;iLWOC4Qz!FHOhB4^8W&(g9&tMjOQjLmTQH zmEq(K116_t0eC>BHTKF_+oa67{r7v7=B}g{az1-?M;GC~jJ^D6vlydWuxYFRD~*a8 zm5gY1F;8=n>p@BRZ3l4K8U%}sgJ#z!zf{bbjx3!lf4Mv@mVdxq&H3)QySrl#O7Se!EcCHcWByIy{G zd9gP}Cr(6XOZeXL9!TzB_{#xI{b;gM>t+jyOPK6$T43dgrgwwzJ+1ZdLoi;xm_U5+~a~?p6&r zyS(9BT1&j06bd>>LJ{=JVPzUNzDbS|dT`^@BD|Ax#h{kAVn;-0=;lj?(>f-q4sUfd zhlWRH0EOWPM5449YBi+zLrI`YqfF4T0`$|Jom~5b^8LfRzfiq9HL6$SPp)FWt6DTZ z&(gL{>-mV2W8PkxLbSv9b(9~UtrQ{J(WhVt@Q2`<+v8?XACETzUc==w1nc4E-GjMK z?JQ2X#N3sDH<%;>?unSihqFUk@e?sRHJ6;P|GZ@7$5qm1Co!N}g3W2eaOEC@ob! z{BS@3*cF+n8J^u3HyrAFNZ9o^swK?{sGg22)B$N94x2^cZ7SvLWJNO}I!`25LP5lt_i$@rAU+0&{QZw+{L*dYD*uZblSB$g3eJ>utPad;U_ou{#j z+hYCfS(g!MtjzH_wepfWWwi_{b3t;?le^Jl@(Tej?$KI0N-of&kblV41Xo{yjr0z4 z7Kn`)o#4N*Q03ag(R8Jn7x;bdaJc_N>fDu+_fK@~mk4eA`Se=H)qIns1#SOv)ak}x zb02i35(#BxjyrkSFOcQaneA+xoMqRnQNQi(1C-U#CnLX@V5csE8>I5U!wcPqztTyg z#k9PZG^0-6U4u;+%{S$f@Uvjou8A)1fJ7C#tXboGUoi=#wih>0d(M$tm4G9NWrc*}?GFPoyHcI1Ln9T;wtH_}vKIY<5CEp~znDXW21Zq>MQ!NXnfVf0 zIF-_JMGDD#il4H{I94+dY5DBx8SkvAGuIQ_i%{+AaUz0$Xg&h8VIaEj51(&juh3@N zpmdTtaVKy0kOhX`s~MuHQN_ynUM`ZkmYN^UdER*T7F#zrEm}P0hD}wtyG|D!y0G4P z{vZB@DWJesXv9f*{x`7wy*Dwx9>1rE&2d3=96vUTW7*Fcjn3JD$rgU_lX3jX--A}_ z*P6fbi#e}~FV-METjvcYf8Uq>`g*dHcpM6YTGL6tc7oy&06GEB z^&(HN_!E^p!FEpqTOb}FR@)oGj1YB7oB}5dC+MK#^E8PocR75=t?Zuxw6w8%Gd(-5|1b-nH{JTe66^P`%TSq61HC9`g)hTF^)Dcaqzqs9D} zrBw=*1v_(p9M6`un)-H6$XmFLCb%j{o)1N&Kr(hmn+LjSuat_x=xNF1acZ1+AWxAy!!nzp4b8MpN}fP^79ZAK^Em{@8LvCQCk`3Yi&szC7lQK9r%!H-ne>FO)Rok=`-wli;?- zPx&Pl5xPtPI((jRK5{i9d4_crUlzOwbwBV!K4cz0KUwg7OTzZ0Ns}pGP&F00JPgrB z?D^zVpb=G2G&;)jj8&pvzGZQALThieH*WvAo=T?5gO|`C184LR9ev@U%sUEfZiKk` zOigl?EGz+~$J@~507@-sEC~^l0apZ8AU++RXU7J}NA3Y{xY+A+*x8zan$Xw)u5-QY z0jX(6gH%@2ThVPEHmwo|Tuf^TIPfH_b0R$1#rEU;jGObAGx+Z$Y*X@gl^Pk%&=HS? zAFc5pBu74Ne(NAm6nXY^W=lM1g`9_4iO}@%j4wE(RTFHBL7U^eEo5dLB2J{Rj*_K` z9mFkwuj|Q`sB7pp(Nl>MA2e(Z{FUS?BouBF3nf?j7>ao zv6GOk=z4K;_Y2Ub6Iliac4Hkl0+G!kP$MIpZx?wSq3C_beM$N^bzcyT#b3scdB8a! zD)(=L$b3lr$I3_8hjR6z4YQsGc{V)At*B^QojjX-P{P!O9B;36CeEtnakKD0N#Gw~ z1yrmI2sVAqs(2qxS-ytT$wBSPSGT=bQ6+xowq-Jp=`4?xUEJWHUJ&;!0!OelHkEza z*;J}?X~)^i=MqIS0jn;`tl0)U!YcG$9i-Ygu5KNA&BSbt^NP6h1Yv&-9QJaYaQ3zX zPV8CE0k(Q5g8S0k`D(Qpy0;W_bzi~wnK(?m8XpI?y9s65U@ecI#Rp&F$gj0E-IMF4 zq`iDXogy~dg{yJ=eqWjWHm-JxWb`2~;N!2+r70B##Ttl?Tp{yhkIYVW4NtP{3oy>r z!!nr!Z_S61ZZDct_@i#6RY${Gd*w(*#E4}Vk!4^kgh9dN&NJfrkoh{1O>yJuY??i53t_n?Sb2LxkZxk5JO0aU{w6gSw@7AQ=kUJnI1Q!LeQ}7Q*w{g0bdU?)I zLJt~3L&~c1^E;EMLF?MVmfZ+;A8mwxTRnsS$0yva-~29QY?5^p{RJF?{d6h4R&*_< z1<+ED21R!*I^>WS&b=(x+2wvc+9bPkZ;|BrRI9FCPvYGjR!PVf@KW)h588M41Vn$& zM*oe{-?2dk-XDs%Tm%GflyyzlV!jQSFe9`NT;oy<(oF0$0qNn+^7l<)Hk)2WYqr$o}vFlu}_ZW zUt32}*J61JP7Gs;EX5z4yrO1wOVlrZjoQ9S59x4A+ueC?@l*W;_kM8Qlbz2D^ zMnQIy72Q0|`ygO9@$l8;*Fk~hfo%^CY^P$ z)QCAsPni<(M?Rbmj@Icv6+7f6`i>Vm14x|`+5iMwf#BV5A>|h~Rr}d!=*Hiulo=N2 zTuYO5KPn!V@3MEbnI4H#w@icd05G1jzICtW(LsYAi-=~h?|bp@TImH%Z0~lM7>UHhgc-z05_e&MZ8XzfVXFWpO(nNl%6dji1B?StFaV(V1U|F7!`p1qX1 zJf5O1XEWH4!9iR@^J+-Ct?iv|Sc0OaqvPzo#H8>I$H|Hdye7BwW`BUd51(Q8&H z{(10Hh>B(mcD_XEb2D!>am&Z%)932_tkG$}1{}t$Iwj3d1M<8C+rLr$)nsNAWeY^h zXz1ms^-R_tPQ|gDW7T?k86SR(=sj{|B(lXWB?ac!LSu$~6502F{?e@`im_nzYYcf^ zb1G(%c?`%Nqxi|q;vtjOy39Rn&E7U-Q(qQw* zsSaAwN&hZ3byM4Xkk#ugF2egq*4fnd%`tH6UWvaE9$5E`x1HHgr_>Z6 zqt0&~S?DR%y)o9M_pGM-!NlnrrM(iTrRV2m1(e3msb9o-f&VV^8w_$Ug=ZkzAP$Om z`WdAJ$vycjGDfL8S{o^5%=-3Lzo&A3%rtCo3gN8i$9n=71lAif|5BfsN?|?)&UhWp z3S2?!K}6>`!IxhK1(;*y@cBJ0-iB3y_q#$k4Y z^SThex|i}L1L<_e{*7{O&{&Uu!443<1B%e`9>Gw>6o>^|y~}=iTW!`j-*>suDeJOP z+S=n~TVgG{xn>dEf|yFY@ES%RW_Jn@Hhq;ayJYFw02l(;56#`gPX~w$Y39q(o}) zViVXlv!WvL5jaP1hI&MJn#+Jt#EoT(yyH9q+(vDbqBZy`B9T-AwW^jO5_rI7qra^% zC*pB!E%wG190+wnZgJ3i9-8?{R4DEF9%kijZW>V=he-&doo16qx*Wo`V0cjBE+WL)H3GzJo&+icq(?5`_LRdcvl?EesXpw>p zlu4n29X(H&{*p&t?YeMKv4pz=YHb0B{fN<++m!Epz;G_ie~OGS-uQ3+X-2G~H}dy{ zYi#`Um}9hL&{9^mwn&8VA0iv$Ffth0GH%sLH;FQz{Nr!b+QlKgB@fEWMcD9V3HUIZ zYm!E~ z)>}UIkwsOsZhGp$I?3=!!>`Sa%_jmuEnzl^r~VUA@SWw8pm}?M04$lCzcNFV8z&ev z!Z+JNd`|I6>>WyB1s>11AB5x8Ev| zUK~e%y-;^Vzm!!-c{vFi1@dGoAXlX$9N-FF(n8Gz5*kP9}i*L{-ti8fUWAM`dqywam#L1H|AkRp`tv7 zVHkxJ0KfWwOJ^Z)V42{$(F@?&Uruf!pt&}LUGUFf+M*gBvc^fzUJT`AV~fB3yx1EN zA$Cx9a`VsqA4hd}{g_&~qaKI3V}^7Ad?FScw-zi{AOu>&H33`8SYlzfssflHwV_y>@cmt8t3NN@eKXjpY()PT>3v~ z3#(F}7#`hgZ5i+)baF}X4d9V<#Qak5m^ClOFB+V!&NJVZT6AhTes_={EJ7{?UjcBV zEsl;M?{k4qEYkb+0g%3^YO{Tp8M*Z_EM?AHRwpUp*`NmC?qtMnpLOD1^?TpBug%2V z@W_ITDN0&EM+Q{8;yr;0eyX{Tp8|y$i4lh|z=PU8hSPgnMp9nnANc)^4xG z?R`DFx16YwEsBs8L1m*b=YzmR;(Dx~FbYkhapHj6r1CJxcMEJN_ozrOhE_vi9dTAh z#D8*i+F7d2TOH|P(7_SE5sRIr7QX&3e9K&ITp7Dfjh#35%iM_<6M1k7v4T3}Fg3+| z&E9r0Pi_*-2%%y{fEfen(7Ttsi$L>q9VFbpojfiM7bBuzw+bbIc^6tHy2_W#YTzKd6DI0yH`j6x=wngR#9=TQpN50~$V1$)Y=GD9`hxj2 z8HyWqCmI4ZFmSCwk`i$eU7K_h6Pgi+sqV?5C$jIR&va1jldD&YN3Uc(@nKR0I>I&w zFV$_i_AQ}BWZ4-v5jYPnrX$!g$zs}L-@67uch9a5-TI|@}w}{?N8SE)Uzg6j*_HuO77KR@-~H} zyr!L{+a28@X!BnZ!zwV3_+@U6x(os5$nj_4nD@AQ@ROY5aMBB$*N=%mK5y?|Bwrpr zW+4u58MvU8*?LsQWlvXxHM;F2kqkckecVzm-wAn(fGa({5_^RYSP|v8ZW2Ee?{UKj z7?X+6*ay9ja3VWNc~Kkpr3P*eXr2XoMh^Vw>np`P`RFOqO(-}#WfHC5gk#+QD(XOR_hCkAaZ8Y&M5-|Aw{D!w3}^|og46yd|0y*8kESR#I_DcC0)2Z4 zx&>Y4X#S2P>;0V~3YhQ)7~Bx-n0Ys;x^8}%>;B`obOum6P$q62tPWQSu2UF4J{x`e zoh7wfM=;dJKkJ@=z#Wr;f=3z3+&xfhE?S-^2L=!(_JNVvP%!-*;w#&~KWH&B&AcuD zAPZq1SWi2`e7@MH~fu=el8G^=fd!|+h}6j}}6JZ!ly zDkX#!%xSoMzA{ef*qiUcBD!&k)~Ih&zp~>j1pg(AY$+F5y7=<;{k=T-DbDS>x*Jcj zv2nh8{vldVzlBEuRaKt#f70{*KefI8)1Uu!lJW%&?|rO#9}&-BZ=j#{QzL8`9C{)i zm1s6`(^_|shS&p+wBiO((8b)Pz?R{NYZz+|dFz~n6xJm3qRDe2hy+OYH!g$RzSWnw z1va0WZ0k*rYL;e`lZ=l&4B0a`oN=*uA~q_9=B2pmureaX|KWLu+nOfz#=w?Wkz?hoG2}y99&)cHgr7>zqR^@-EGH=mM4ZmmB#ll?F*Y?T zj~Is$4WPCwttlUCGhC8YA4){1s23_FGu+J`P91sGGcGH`SfX48HgANodMJb02TI^i zN07UbYMr1ou@wr%u33_jz<#8Fh6~>gyb##Bt&$e%;2kFUx0Zo50qFs$CcQv=3^%hMG+%|b2iu>Bv8j~shFGwSsdq8heRgrs;q!N-uUu2SL z6%Pw+!skzy1x25kKfY+?Suj0q@#*f@>Zt?WiocC`a^Z;7dA1QxA%gR10N{aviKtG@ zW4v){%T*4h@%7q<6?{4j&ocw73VohE@Xt9_Xnphb2P+2AT1Aj!xIQ`CiiNG5kVBKB zRY)6A57cw6I7600{3NIqkD;a1;#gjkc)(k=nAEH%BZ}{BIsUX00MaCWsD)Vto#$O| zOlBCXFS&}2YG5VrY9(!hc2O9jB47H2~FhaWQVKl4LErzc;{@MJ&G z?dMCt`aNLdcnaE`6otr>VOBcrP~;RKjrO{cWR^Bvt+cd1e=o^UrJpM7vKsK^W7Hy* zk_%E3V406e9iasQMc|P*=i)+;L7SQ>{>bdUU59tEKIiwxl(7H<%?eE7J>?=l1L{Cj zV4$~FN`8-pw|Mf@!%Lyoos=M{P)-5d$zJR3M7HN0%CrBh&pQyPqZGv%*c3bw{>S;~ z?Z)C8QL4HJI*H}~*Gz@CUR_;*T7_|l#KvX`_WGo2Dr-H~luGmweKKrN7;80$1w6Q} z6=`!jE{L9T(8{##3KCr1LY@O+&mc|!Z2@D@7MQt7`NfXL%8C>w(ldmQIq zo;#Br{3z(U?Uf^ySgiwZ{ZD@;V>VKu)+}_5W5Z1xaSSJOUGvOpmxeUl{BgNXTea(A zK8lYAmM0c7ElH|c zDDq^q-9Yh@>l$E@ID2h6Qs9eL?w?!2>dh-MMR^h+jLcxmAW2_=uibbXV!QK|z#`Ky ztfLuY3Y?m1*q%McIRkGihq+i*So71Tp=+ksFaP{4<)Pe|{grVqOv^6Y%Rc-4$GAKC zPvYuN!xuq3Q2-7AnDFQ~Mlc|cxj%wgRf@x$qX3^iHp6KjpwK?|sv-j>5PE+HkYdOU zm;Pb3dm_oDa-gT)zxcXi@`AiU^^)D~nNK$EIYWJ_P$2|+qvbF^Vu8w0uqmL#!1a?_ zNC246d_`e}+9y+Jq-bR0z&x1Rs8GlCH|aeONfCUAj8qS1cLi@2r8nTqsk>jc%6qzJ zJG4Y$TaDn$Kx+rke-^X<1|2-fiTZ!5_a%Wf0QfslKBOPfT@pcexSg3mnGX_Y=M8cPM(pC_%rPZQ-j?@p@@X?C~krA%nAxna|Wb zmkiFN3FgXW>^tSAe*;YXA3-39wuptdn=;$EcE~VJ1_L;_+BsJM-2_$=7~{rTRt~@T z+U3v!PuGD(=HhSlPs>!Fu6~22tdq~Z@9a&w1$c-aFI!_ol^LELf1Ua{#SD_YBsvrT z%&{hU9_X84*&~ceCh!zOxv(C)Xo{R}WZ}dp$FF2YlAn?f90dRsr8lle$x7%NG9F9$`_&GiFmwZyH90 zxDQV9VVmCCFY;!1zf;=b&!6}G}|;uR%4nKy7n8TopX0=+;V_B&Y9u-j+U^6P_r<8PHL z8^2}S>|LyX^}L>%Eb<+l1jeyWfeMNrgCt4#3&dh2!FgFHe~xgj46;9kS)9{6aBQg6 z!^5Q|{UB;&-jt^Ssv<%!_>}b~PU{hXBnt+R_{qwZRN_{=eb0R2@1Q1|@sCLmOG zzWhvGzTVrgGtqTAaX&tqIFv6{oXv7uPG~2w9Aw~KlgR|z)AY1`h*%A~N3XzcNzW%H z5`81Tyr97%UGC31Wmz2;J2CYqSwph^_wH=47U8IZo6o{UQwfk1jzyGw! zA9JCU{Q;3iz}JpJz%i%|Yz@DTQO5XFt{*3*7s0_Lb1wM%?aivyxyzn=W&@8p7#hqo z?5^JJW2*YgEnJo_n%`4ac}Rda1hsX;z?j)C)BvaENP%Akf2bE_=9usD+28Ls zXvkr)pJE^d6iYw^4 zd6lu*4SIrvH?I=5pX=B8?Z9&$q&u$63-AC?5Tw2$D_#R?fSzMv2*?7jfq`XdtnGrz z(62Jog?F{pHHk+UFO)5!kI01kruP`v8)m>==$fE4Bol=ncHap&IilxH$uCy6922zh zUAHBeR-7M{60K&A8mgu@T>CcgzOu5yfm>8_1r2Ehu|(QnJ`4!hp#4RiKrQ_OLmNt& zzs=JLc-F&+lGu=T0s;uQVQz|z2e`y^3|kGkG`H^Ww_g0fzvknux+^6=omI~;1v|EV z*e}95@KD}vnmFN2P6v{&a4bG>@f}?%&4j1KZ}JS0`^?^KOj7;huv2H_+xPXoHk^GbYO*IPQeH`C zxqZ6G`N?iJhowb`C;T!6A4MPdn?%w#by$cE9wrb!iK;=glVZTjt`5=U&`|UXzb3`U zZIhjksEr!iPUC?4oXx$gGNqJ)gSr__0kM2NaZ~RML7}Iuh(IkN|4KoD=Yp>M?t{&r zA(q2o!5(On9?f_FUfr0b0<=%Yy6T$L=<*c-len$V1@`6YQGNFH)peprIJ#E^4wa-= zt~-Y49a3eJ(Td;$?nHT~rJ&zz0)WwM5Rmvi>6bm}n^fIxptQo>d>Oh&$vmYkm3~8K%#3rfgFs4p?ldP2iZ@I+78017VyZQ)+ z6hHrquL;ruW1oq%?-WQQh^=DB0O3NLVL9Q4HV1a61W1qm#lIl`#n1ll|Nj58g4b|H zU}q12Bj^I}gIfEVVOj8fFwVQSvFXN}zBwHZ_iv1VVB(A$rAx5kkBJ78vT%^~pno4|=S_ye{+=0K?&GO;6fbiwJPwjYmRW#8} zx6~2}utAnJw9U9*<65g%0|)tyZTwm-1*|w>oqScm|z>1hm5 zpNKkz?dza$;|RBB%N*@TfAvako*$A{UFux+QfU8cK{D+YuJC?ri;_qt`hy`$D=nV* zpX|lT3M_Ppo!Vp!e?bElFKL_{VDUPT`!!M9z((*I<&;=p9xP`qkkJMrmotvWD%~>{ z{Wr`|J&r7+aF`F69IGMrGZd|Ng~&=WLsp_`K2m_Rrl+ffPNM3hG^s7;*4=9`#1t&Q z7fLJc`?@z~WVp`yI^CFaS^lKVKDjH$uX<+?1WXU(M^V+pH7PpU9h#*R=N;n>DB{?h zvys!j7l~*6J`L7#@XJS@SGDI$kkxph{0i`^aBcAb--9AkpQ37!tLa66jOjb`3`=aZ zfy;B=Ds)=MYyjt4Wb66r)98&$CRN7@x+~3je~#Ec;aqgO<0wFoHa!A%7c@~oJjQ9N zy#ip)Vd!!7IZ}dJf7q^CMI{>Qj=?c6tKkbfGUi6tXQnfpp8oab+e$`;veEl}Jifq* z3T>CC2tZ55AuTE!&jq!VJG(-4U6P9ObnR02>i%UTx9N8OW7QQO-Z{JSHP7$+mRjg+ zA2*1ig{Q3<&^IrF&3n;9(;y^nwc~|pgnQK=SjNv>rlqyl)avhgzs(U@qo};ORKI>z zqj=B79c_dbK{7RsKs#Rwkl)BwAdhmCozkQPiM8c%(S3*sEOS8KoQEI}XblbyVy9mu z+7vtXy}e<7UV8J8YQ>bS)%RagEQJf;$w};{CIHDX?^-)aQ)gVJ6WZ-~vGr||k~gbZ zDBACtieRr`U%n2l@at2jHg?^RkxxnG-BG8haR-0ck01>n!1}sW(bePd`=HpavqS6o z?I0riaGOR{-Y%Sxr zANy^+C-rcBLtdxM-UqJ(iVL46wB9*N<%3b7NJv$kCE?qiyOwA_iaaiS>oPR9K57e8<_Ve&9IYPrM{|3BQS14>3cJ=(V*8V zSRIIP4Lls8{qB8DRiX^8PNOjMhP>!5%K#3>|Cp;qrGqy*+QWFpezx6C*?AXwO8eSQ zlrnf1isZEODTHjSVrgwbFDE~E1_mp?u&kE!_qpkNwI25@L&6L+MEllc=wD}%+k*fU zL0AV$vKp&aIH5VMa-n*pP}J27`R_xC{^+J|pn=weKI+61kYCb`-kz?zIE*x$n#e5#E2 z>y!_xR^#-!EEb3$|M#MS9Jova+o0bmBqN3mk|W=G%EZd7?boy*yuf(ROKjV5T1>;( zLt}s{^9qbbO^&j7d$_{5Bx+UqqRJaj6dG)zsBaXe$MMebBck4&%c{W-G~db}Hj{>? z0I3qw0Y5;NAyZ$0i?(j63T6?)L}r5$WjA6?j&d{}Y%B7Y#TSMI`TGk52c9~WqHFa} z0Rk@Gu4WH^&vS1k0FKo)s_OUbIh0`fw*&V~6&AQ*axjdccdwfzh};1@O&d+!cDzJus<=jK>@Gn}pGMoao`P zEKAx-%Vcbw=dsvALU4$9koIeVJ#W(@Qk}f{mb;x#TF`Bjc)C!PU)S82R`@=2wPkdZ zF3=Glm=hFh&y}p_p1hVX^lb2|){JYurP@NaVakkQ%1HXagcHzUTiswORxqt_p|rGi zR5XBt#+a%vzxR!_8}>YV1xPkE)fb@R$Lq{dUq z(X{YQmKcH2%YJ3zMk{5+vme1bF%)6_G@)-}E|N8cSVJ2k2(*232}Va#3Ph^YcAUM~ zeK+dOj@sDsW~e+8{%52J49WjY(8|HVB9Oi9QYhP@H)*!3 z88NA17Hv)UDGe~n&M)Y=u@@r}8EG(l{V%iTN22zQ_F58mccki>A$_;`U|C1;Ur?Ze zr9j_g)Il-ylXG7?0|`!JIt}&_@|ziyY7a~CUh#7U!E09YRs$mE#5G>s;oxi(gLhAF zr*r6CWNI;zD-0D2;c5up82ExnQLR)i0ku@$x0E-$9NA{=1^xwyNF#(J#Cip`{yefEoUSrmunK zsl*8a!-1wAH|0KRvLBfkMg|C@fpRKSe$4P@y30vUBVpL5AVWWSoXdk_F`i8zEvdi= zm!UA}ZB{LWLBdrvS8JHdzrQ7PKtw*nvJ9j3tt?scoT~TTZqfZH>{%WqG7Sq@!>9*! zd}#%DB9OG0PkKPo*Pgv?;|)<){m9g3s5hH! zrd|bBuJUiY-4-A72|IgM_Tw%uDovw>ChN@Rq^`A zkIkC`yQyG+@4=wDHk*~-&s*$SL}jSZ#Bv$hk0I|^?sN7dDJF0 zf&aS9J)h$yoR~j3^dOd^CGg!&!z}xfWm91qxphs)`t7hOw^?C>zv}mwUn|E8Ve>RG7;Mi6t(1ZJw(@9$GKSyIN-#*tK_$p8o> zvoRl(<11TaXiJ6C;YzsDrcNy6M$mZ%|hqm6CMz|+*S zilL!7_rl@>PhVr`k6u^TyP;N7uDk5?@h`|+3x2_Bxya*{A&wjjUuHQRyv>l8|W^!f>6M^3z_#ycN$FP))cI z*Q1){(JacL4)v0jU15R#p&EV=>UGu!M|$cXnr8Wqs#Ql^)=*1!@=s{wMe1%?UP?{| zmcO58#o#|(MdVRJi!=FJDIm+Htn$uNrg%=%{(@qeZHF0ezUN-d{1f~kEIeGWKwBm3 z$EfwlxX@oET$$)TLzV@wlRG2+<{;X|pdfQwFgat-1l39&+wYUH!1Z&ZAOOPlG-&Gz zpZ!x{rF_6ae@x5m%iRI{8@ zXn9X1t9(dMz(iSDQB{V0-0B8beY*0yFY~$WA`(KPnJt*|7wB05L#BY zuH^&T+nm{QlJ>nGe^fK`h{t=s_*82#rxE7?S81&mRv)QV1BTYAKBF) zs6{gQaK`Kk&;;6TDxcGtd-2DNAKCwB_UO{gI>tK6bcf%k+VyTL97FS)I{d1e`~d|z&=Jtlc~V( zO@c^qgn4-i?xs`*-NhZ^et$S7uoz0ZM0d#BeDS+HU({UwP(k{ifTWRxy-A@QAxt0i z2M`Mn6a@ejp|hO`#m&i9Udo-fKX=q$d5;=@fJ9=(!RCWX^UtN92|WfY9Nh(W{&lU( z$kV0zXE@EKio|8ZhRU?okk&DgT|4JR`7_^CE?jHr42e#5{8hzNWf6ve6QfLo%zj-n zD8$|hRGR{w(IMRNT{*$~!mC~Dy`P-=bmVe9J$+1qrQ*0PGR%E%s76o)=5=Kn`McEI z8?AEyhql2`p`)hVTs5KdF^9&-agF24f*iM{E|Ec@GEoxAZORYWI4nvFdv9ZA5&X-t z)4f^sF&T5luKwzN!+Bf>lkeEc7z{f;z|Tk;z9+64(;wRoU0r+fLALPIb@6&S?g03gzCpu+`Sn2pVR@g+day5M^_UOIQl)#ma- zmsl?&=BKkGN7Kj`i|cxcTbuB1(4gbdhsRNMgtb4<@qpiico(N0fti2IKRY|^)tc)y z-t;KU*DD0K>H9LJ604RMWKN? zn5dk?4q~e-8mM!QUeq4w@pVjHRgvNEKolbCI{r_4U^VMSJh)RDXF>Dqc?0s?DGy4e52=_u>4d% zAKKmhCD|xS^Y`cf=s5q`fcZ+8*KNS(o@3uNRmM*o#`7sE*VAHBUVPs=`6a^o*vRx+ zWAZMGWfkXZ=P2@74qQ(h`?F}EFu-3F0Hl$Gd**M~KJqm96yK!qXg*4LwW4cITi041 z*g%O#X=bmisb_iS{(X)9S3|mW7u?=yLJRn*9}3bSrQjlIDval}!!c3`CWi{t8ec-u zZHLP+f!dF9HE-5t@PwOsgbM+~s49F=k5xUkhGxm^Eeb#|ud>i1*_~b(WNY_qWV*UXSar1kJ*x78ucgV1CqjEbL5937G5tO1p$2d9&MXe@C0pn1P=512;M^>xf ztu8}#5(It=8m?`1-W7@Mxll8V9}i?Sf9e%aXQ$c+J&fHc?UBv(>lGPf)YQ&E!*fYz z*hOIx+acN#?dm}{{Zp2z=_$caB`TsmswUq*%4x2?!N-RFDT#Cb4<3>KhQARHL3&4_ zIR=e?g?Wvs$K(J$ZdYUJB|!I1f7WlZf7aFNT8-=C>4>U=^Sz>4hZe)F*7LPq86M(C zi-SRFf51fsw_2g2PC@n++0i`I9|VJY`UwpP*fob`;d*HNM>vX;F&0txVP>PzOkVPO8Te zR>Q}D1ludJ?Z3^QlloB5fz35=9=LBnr%w&BC}#vHm4XksGfFVzOO|`0ANX;G&mMw@VmxBvDZuv z6@N>roLCF4Id_52bANT`fNrTpclXEFg_bmb^!OOa|G>n(B?K}b4{f_`w8HR?O$pTu zOYuOha*2zgpSn19d>I`1PY7Q>t{)roDAqQA5O*$gobKojp^N+p;wHX|F~yCK=$28e zV?|&5U80MG7*%!z{$Am$#hD(CQ}6iqI0l_yM-iw?fd8_HB`(K^1+MCfU7T;1DS{4W zb3p2Eir?eMxOsE3tXoC$dQVUUYU|{Ix%gSGRgHpo7lRf~{0lrW5E~@Je4gM6fucqW zia^aqiieV;&Y`;FS!1GQNW>A z;s?~fKo{RZ)lsGZ{ctdtC~QMl`jFT1=}B`bjbEc`Fc^QFx=w6iB7-0%_prV8d(ply zn`l*ZgJ6SFveXs(ouyPFlmZ+vuq~%T=E<<=Es$DhV-|w||2wPn922b2xJnCQE8Yfu z53rTD5L%4M)B<89g#}JM)e8EOtY^4qW;g1OV;u+N}SFK%^V?U zz#HqW!^++0LNbxn6t;2z8|W{|vmX+uL{x?0s$1#uj1oJGaK&ynBnrTxwW$<`MjD*` zrYA(;sKqWNYpQ;CM~`)UO+0GaN@^xRroMpA{t9wyZ637< zc)7t%M}UotwgPg%B2Aj0&lOZa_%TEF0#lTPsmCzy*xUxBemISJ5#OtLu-`<5X4#5Y z@P1d5&0T@!-TT_3*E5_m`^N$0FFVZ8y)zjWg9QWKz>=)tzk@?{0JV5_W3m+PY$SjV zM5qhL))Es&n1Llc=YqEN52BUMrwj-G)tz$?Yh%NEkY~rRY4PhF%ZB@onB2jVN(j`4 z1ezOZ9I!RF%mqd^Bdcl)=WG!jvO6oMduD^=zmGD)>$1ILFr_|GbD_EVSL|`73tIIL zcAY0AFaIm-Fb7<;Jyf?3e1T!}&H3=v?@S}O7k>~UGo4r7yT!Flt6MyP9@St^z4>xn z=?DBtS?SS2*3-*Y4sd9gvj9*jz`5`c^g$QikH z0F;!PL8w83%vAzw4D-g($`P?`79%?;3Amkv=TqO`C-K-{Bp+(i@6p$?%vSIYOn&QQ zyHfv&;MWHkN5i}}Ok>tnx>So!1%Ib+FByQ%N}?0t4^K((6BSY?B?nN#cVXwS844w3Pq($moz zK1oL%_E+!ieCY9*&0PVdnz$Z=E@9sG2?L`ljc_wk`~!)p#lDtNm7}@%ig8q;Xz6t-xK?zqY8x0loH|HE6fw%N%#mK<;lK8rn&=p zOa8%*-y{Dija39zekZu2QJMH2(0<#vYdJr*W;vJiR%>0yq3=|GsFUdJ3(oJ3sA)R6 z4P1J$IQ?iq!cvQWU(j8`oPHX#*Amc&rh=Y@^>-wzhNKHVAqBNC4ujhD_@5H+u|*w! zg09?`xYh0V;bP|kVKG$IbQ+jP)4^6^))ePv*TS^*&Lwyb>qoY8G!}l*SXX|q%VGAB zvzwbtRy6V|sWT1I4T8^orQ5(gCX_Azp;CtX9X=MF-3Z*qc3Y;1CAPwd9R}7WoUhvD;(guHzV=T=)KOjFoFb`b}8tO4P^}+ zWFJR+Fl+qs4(gd5sHB8%M_rhCL`NXc?9Jydadxyzw-_(7y*d>2X$)deq5nEdW!iQp zt53kZwj|T@)bzWLO1|jVz~H__(Hu6$H^6?<#w?*2rX3V_@$Eah!uLAbMofb?MsT#W zWap$$?!LloRpE9So9^{j{4b)MN9^bT{zsuj_OVjXRfg;&crSRITsMx}7r+om0Z>6i zeM`lG$7;0$=4`PH{{Rniz?#iwyD1U_-;eT7o2c(hUvZbF?`-<^q}V3u=oqJ~&BGt3 z^*Sk<&-*U(Yic5APmoK~AblzL?BYwpg!vfZ*O6kb-=f>IFTJSAc~}VaxT0L)ZUT~Z zoN0{728Cgk|10d}Z<82V@|{p)xke-7nmJdW+{kYNX>y$9_J8f#c~Z1sH&{FbWV8wM z`ZJwDxD8-Ne+s5>r)UoObC?R>LX7*d{#9-Lq0lepT)jJgEgepOljwZnnv&$DMEEix z2=T{MCz-;qXPE<>g>hmjQxX(J=5cCZl=#szaDH0(IN(M-2($8>MaRMX; z>0lgGW!u%oQnpvIF@nfatOID5!`rYrZ{;>%+FUyD0Cl*T#yrCEf@GLxEIt~k6E#8X z$URd>)sD1CB=85w%H!QCYPc?UR!CgnvbfmZUzy)dufA5?{n6M3LcxMrAW)OXfpsAX zhJcsc%DO#kwiEAyyACc#6PcQc-U)B_dinI3h=e@DG_zs%4-NLz+gx_<1`T*l^F%rq zkzFZba@MqckT!C*AsEGj1hM4Gqdv?Wv zP}v_1)Q!I&e>~cdAdF`{BYc;BhA1-;1eg3K_?UmK;xPF$V9i^!P0U}|3YA==V;so> z)m^bI_Y%$9lO&5&D);>9wwjK=aL+3SMJ6K}0;z-qckY3vJvd#u8u2w0;yR?%)L#>* zHtSQ>9+sj}z%?s;xUnjv}$%@XN@&mChL5++1weeh$CXNZDr5BMj%6Gbxz z1-*?lmU9X8wG;J5r!8{L)u}0a^Fm|J@v}pHDmmeMtT?h&X&g(SLzo8hE;5DTPNSuw zxu6L&!-E!gH%#Hmh<1(58kdbz;oBJL%FezIQ*Xa=aopE?A`q!9zJsQ-%|{>vYB5oO z0N4O;jGQ3PEzIcA(Mj4@Gv;02K(`Af-OH0N?Mjs^dRu=-_0zp0Dd7jt7uK#{f;QU- zkcscM4mMZUB)fRdd2C>-OmR# zX-rrMT#^S^-=HZZ^$!$6C(nM3!x5-uey-@fu|=G2 z6JORewZu9tj<0@Pj^HWb{@RL~Z70wyiCdO&Z9o?S0MqQ($(2M9&b0+Gg+mE!<=FaV zpcbPu0l;$XJn55&I9QQo07X92%Sw?PZL@lF*XB?&l|whO>d<-@@l&rIi~zAi)Y zN6Euv{-z`D9ReHj;n_1HL);<7s=UJ1C8aT*-CtgfZ^u4wP}KK?A8{L#>VUH~|8I_G ze6UtA7gGzof)~i#2_B#s1|%W+VkuQK`E!9h9YzYp^`|@hF=Ou^r2S8jI%|5z33?||nO;BnuAZ1J@S8^{ z!!I0ZdQ{XO#ZWYp-<9xfgtU=+hp%W3f%xXG)lV|fnY*nxbruv{oV7`-I(wY=yp0(y z?fljp3k2bKq69Ai7Sq$9GYvMYMUn7b5h(!DSP&*pIM(8Ok(G$mHUOdeYqObLNol4C zK~mPkq^1di&+OVU57z|7y_tm@?HYGldw4a+iDcYe@ zG#}17Rt552Fkx;1fk;oV37GbtTPT1_{82<+?E_(;dkQ#ND-w{i8FGTe5viq9Z2SG!i+kmJRPy7exp-o& z=9!9Q4g$3k_2<0~GlQ^+LX(I8d&M9@8WnI&JK!b1A&$a?{1{`bs+_Cp2dDfp8$~EU$UgoO<%7| zaJ0e;b3JkSxEsw@%RBmMIR=`1hS+%AGGjtBY(IABH;Dt=XFYax?n#UGOj>bAnCM>; zPS5?u?=Kw0z5pef>9TFW;RuVS4T5?;*i^)!~SX3MZP*d2Dv zXm@!7`Tbs}qDO{8H{bCbk%qGXPOz5(9q3~T7cuO?8F-FOYeg#J_rgO7wD3+-WTos5 zx-Bbs+IjiCN5r~vB_?9s!#w+FVcw~1Ljeb8`%yFHgK)azPh(I`AmMDFyW!yCa6=f{ z&G}BFXskmUbvm!gclPHEig(i4w<9%9YVSpU{dCdv6RvZsxMRG)^*#+2q-n~8E5jGkVe;ZXDN3<(n-JQaL@H&+x0f@YZ)48u*}T!~J@)CR z>8`Koi&@?LvN?Msgf-G;kBAoNJ`w3pgriUx&}~4?k3I*n%}tgT6cY)MS~+fX!Nt6U zatxy6=)0z&u?yrot~P4nMiFOE+!Oowj7NV5Gwh#D)B_Rbf9#^0VI1xpbtB$|htYS> z#I^|I;?oJmtY_geTcMAY2`ZO$g^st(k?-Du0RvIjMuMs7$TJ8SA<_jp{(10huPh2d z`EH~&rUepLL&0LLADwace1+TP6mIfytiXBz_(b8vdD1Vq(d}9YQ_(Io-aV81;B z4w+hj1TGe64}1b=F5C}Yp`O694W=U8KCDp7`V*?elHue`+9P>N@0Wvls><~O=vM>iOu%$K;*MSS~xrU z89`M0;aW#+*#ODXqM6fs9w3_Xyw5cawd~XU69ovGK!Uj&^w3x8(sN}B81HKcgW@e#Ok2c-_MuBjSke& zDJ|LI*&z*5H*?(gRi`@bwb1BZ9^`zzTPaF+cMGX42s(_O@9T6^fp?ff&{dp@Pj01|nPk*$jli!39chu3w2NagNxj34V> zs#PCXofmFiwgJ=InIBXEanJ7tDZ0&UPJ8AJvJkBN&FD`RFQetfGHp6SZZnXws2BKo zwDZ>5k?-fAzBx)OG0yPpIO_!Tcy?_U=3&oN+v}CHF4eDu`1fTEzv+{aV1eACr3590 z--FeI`JHJ!&9DSLJKr-OOST(ZZ8%prIj!6B>bOu`v0Iyb_mS&4!sg0f?{^;>{+b5D z+XqdmSY{V!)}fPg4U-$t#Q`_-MR{Nb_9o*B1EDP+7|4L|4c{*tXc z4UV*pj@}p~(N@KDIo!*<&O?&a^QR2gWPG%vZ}HpU2L0 zsh9>gpBp$r`6aY2pDo>dnZS zs{|RGTV&Bw**@1H&&0tY%c}bYnI{*Hhh49Xa3#ClQW<41Yc zsy7C$!GvD>LWbRWyIX8v{G)YYu=}6{AN2^EIeX>l9Qy=PaQSZz{@-$Ib|-DJCC3a8 zUYdBVr+=IOMURr;n2}N~Vuz@Es8fKu!?hRF!5jP_xfS|eIdb+nM<5tYfyaQ?(plA6*4@K?=XjqQ0sDW(JE;v0+bSiNhT!tvfI3-=R5 znmiVCEBJc;0g6gDse*`LHjX6`&cKe~6|rq+2g(iXlJn#Se(w)uW+zYkZ{`-(G>*J} z$fVl{q$U?dA&PgAYh^iD5^umck!z4p_@x^FhRbcwV|=n~+YjP2QVl~v-uU&rM?vzP z4kxb9h`-pcAubw<3d&5r%%e~U)vnDfG;Jru)%L;HORcQ-)&*D)WNr zFB2<*CjI-5q&zJPoT#{S+U)m-XNYOr5XL=d=j>CLc{93t7lP;eCL=?7{WYn<1|#18 zcDX!7$;Y(a9>je(7XZYRUBc+Zk;QPwNVb8}k@2B{0sJo$tikk@u>?0g6WvmmjPsf! z#RDd5T*y38cqU--b6LdR(V~)NvFfTXpMa+gzvi59wD)!+c*S9y6@3!~dEmqH=%aW2 z6JxNfOWph@*=0|ARBy#RuY8;NA+P1q6e*k_CQ~|gvw_}VFN)3FW4eJzW2v7t#x&2^Mo%HU=&un3lex*oK6K=-GsX1n-a7o1UPx+Le77jDUlw7xo&Ek`8JM zO5hDvZzIPWDK^#mm;=qCYQqzp6O+h~SEinpCI4>uMfwPaOC4FV_B0&^on2;P&U65C zPOriUv16tObJk|m`zfuIsk4106{dm1y2lo@`9A!}6q(^#*RcQV1?&c%{KdBSm?c7* zqcNat+Xk$R)-RMOWaI2cu#Q^Ho?#vLRqg*%GLUHX{6XIF@55ui!HtLoM+8Of25?$M z1XTJADxJ4mBbQ-^S|{-@a%9Kutv2xK=Zrje$A7*y2s_@DrWz}6OuSXGXg6Zl0h81Y z%ta@Qq`h?xxBGO*Ff7uy9;t)8mf1mm~ znDy$ph123=2uE76g9ZNy!GS`HB~kDa%M3Wr*ER*%dWXSZAHw9av&vpS$FhpW>-LBR z^BDtKEB5|vv=nWF0RsznfdG2aN#q(r5?%)jz)L7kS~||TxTzwN`ZWAZ?XYTX{o5-U z5nkE1EqF~zRIh|tI8430)9$gG;PE#HWFW;*fLxguA)7g30%-EyB6K`#I8^+yERdSQ zV_Q44*zm5-Q}w{lC0BHTG%O$vyBz#ZhSI!O*j!DO3tJfD`%v07i(R<=Oj|9og6QPsta#J zL=Qn-=YseqaH@5qfjHA^k5|)|#g=m)h)(bRS=Z}VHf!7Uw3wsw3_)ri7qg5Fib6g> zVDJrUwPqGQL7={$cE%bu=bmiw4=!6a4P6&VJJvDMUKT@9t+dfw;{^$}s6Jm3SR_#5 z_+PVQbUdZnZf!692NM<;fgK<|*!zfF9B7 zVSx;YXWlLepfHl~yYW*-V|Q8mij0e-_}AA$yo7QP*pVbfr~iDI1b7!FIf#rqZkUi?JF^&kDwf;4&(`FRvXy|uo zSFDuL>$|F+Sq_Jqq@ydEzO%mrQL?F7)M^<`9n}ALO+@$fZvt&JKr3dRM-w ziMgC?ZSu*QFJ5ZUBmi#!y{d@SEWtYH(+Rej1Ue|64Tb-K=JL=#dU&76~dqD)< z@5qvhV?;sICBTMOk}JCsNlp7PHv-TiQmtp9C5Un`=TfSj(Dbxbj+T(OL*Bjx53E)+ zsJr<7i<9LR6o<2(rAU422X(1izq3n@x!`*@7ec^7LEE(oc|waXUI&Wo*p6W}5d!Ytw+L~s*VmorhRV)7#vDqNFj0P_ z8m<(kWlf4PQJ?D*!-}SBPL&S6N5AFwA}3#q92$7+L?v?Dm}xGprUeoe2LWhJ!$q&T|vz z1C<6dcflV_rE=zWm9n4C+p2Axr~70~ze#hfk81l1>+|+pdvaQ^o&Zhu#4T!Ca{vcS zYdXr30SI{yxX)A>+C!JIsG6^nfH46Lp{%2&OtWRq}j(CVCN_|<)v zG@@*f^1v^HK&$0BatGlm2vs}CKoYQTlI#RsZ8|z!IBH@(3pDB2HDUojKmM4WKB=E` zzu{c3O2kW1dDS$vPNz@7mP~#?&bJ2!Z$|S`Z$`(WPHxdX zEtaT$?8rTjO0%&9f@Yh{eYGD+jrWc6J3pz~O%f%csg}GQgM_+<@VPZT^NODiIc1@9=`VgXm}(FBtG zIU-9WdNwE?Rsm7)=*HT5%~Mmk=`)KjvjPKajN?6$LT=croHAF40x8(DZK%lh7v7K1 z5=x*(qd-ede5$KSlKtFt4?J=Rr1Lj(ic&=9(ZiP$*ZUkREVBK|SPARTE@m5MIEY&O z(Nu|ZXcdQaK=>Pdg5dQ|hnh-&v>juwiM%{ZbA=r3oK`{|zvB!3_Ec`mA`H@Wh zr33u>Fo9Zt7lO{+1_l8w=#S>pGSEhyz+^MEq(Ug@F8vkcag~BY`>(w<)1)(G3hT0b zqZ}+P1X5F?3U2IHc@7GM8;r+b|F!TRh}GM)XQ#n8?yfy$eu|>BvKTT;Y4VHXk4rQD zMk((%RvZC~``YBo!s|YHH!^Kxb3%HA<4ZHg`L>$wn1bn%s6*fkN}2M3 zKjmOi?BiC`oltZMK$qtk4tu<%6Os-1FI=fR=kllZO_`U)m->m z45%F&-K;N%P=D1JnLaJ;UI`d8bR1=z`*1Kh4m*!Y*R*&YdtAnhzCJ_#6wWeN04{P2 zFw8Cf&GE@+HVt+-%k!PWk}U%%nkW11-yELEN&Y#+njI63Gf8Wv1t;%cs&d}IKL8w2yKfXxZKx+t^;GU^F0Mjwy@oIm=D z#A`iQ#@~798zVPx`jusvum9fYHXnhrwqkNwS;))X$(6hOCHhY?9l-U?|MV zQ$)rJ`z3BC(-H1Q3uSg49?j#9ZguI4T@Bh3#ptxs3EB|dFceibIiMrpz3WSvG&Sxc zym-zG*~JN7<1L!zD(O+g*i9Gy(0S%vp$B1M8b1RMk--SSWF$y7XxEVUP}E#^;jv%G z?B*QrUXFIop5b*4tdV=TzD`oDh_!5{z!_f7a(umt__JT|6^@+*nmY;Fr_+%Jse!eB z+z#$U^O~I9JGxtEssHnDb+>Xil5Dd#Zn3LAD!uNNKJRe&55Gt4;!Y>7KOlCD4@OaI zQNKK9RKVjcWr1myFebRsyMWY~8VW;()a~x!HCX4_Lxe?{zobb!&$LSACo6nlR6_m+Amv&5rmfg-Ye}#p^QG=T~=Z z^=_*hnB%#?Q+5Mz9~fvj$M%UW9lfLK|DX^_Ilb(UQ`xSvKn8`_Zorb*aG( z=&959Ek*AqRuo^hsWRq#MdZRp!a|^$9<}KAo}F4zf*;Sm)!10LaN&}_QCeFNLc4KU zGuChKK_|<7mK>$h3zj=l$1X&1pl+h5OC(6`>zW8SN_*M&^a;E0pImO2m&_5kq0cDc zj@j=ooeD=@NY+XAaUWPv?lxi1M{umFs2nAIorm{q=aJ)JO`@@Bz?J7Z{>`y>Y@NW8 zREPP|%u|5)l4gO9Cf0VXuFMANw9ObmYcKumFkv^(|1mib^ec9_0GpqybKrabo`lmS zRZBD77UW4H1EkDRj;v6Ki)o#NVXFT72 zWOnE`Z`q5H>c&W03}z(dW5TTNC6$L?1V1|4*S_iINo0QqzjOsfeFe(1ED0{gX}Ws} z^P(xS1Y6atsG4+|*m3x9x=U@9I9(=vu`WtKcE8wzPXVu*_fPKLOSb;T^=AnD{tciB zOrV`3PXG|A+bnVNE29K50u?#T>?pD1ulp-fj4>(M5s=D-d)#lX9qblYpA;xxH`^g$ zw)aOv7fY%h7G#4@`;dhJe`aHUGBVh?_+2092++O~F83VKUJrG%808sq^GMuP`}BZM z^7<#&=mW#EH#BAsmVtSz7%z$6`QR%RA3tkGG$v zQxC^2PRApgq=gMopHu*!YUgVjy8W132i}-nq-Elj?-U@?`wHuNfhH7#*;T@UGC>#= zCST}y>&=tW!o!mn_Z#q?Sc1_`*7)Uv+S`?=w<4d>`D_{&ZZB&H|Zifx`x zId>F&VlWt+hfG8pI{%#Q(Ku)|f4yQO-8-J^_sa%Qs&{4{gsK>Q>?l|R-{a0by83I) zw{u36cxFh)fMe!VNXF~Oj344zosquBO8IlTw;hYAd5X48@U#tFD=WJqwZ^GY*ifncF)kG~y`*> z<6E0=&fRHfqEeujU zRdOH+Y|+dLHr7Fiypy7h7IE9%jCyO+on~0JI2u5)F<)_8aETqfUc*_#*-85yPMt_25KnDGMc6ox?P zO?ZoX)(qO|&PR|JGf5+k_R|d}r_!x|uYFbP_#kL!DqlY#)~$BVg7sNJwmakXkLn&i z_!Q7b@1uq=y3h>+F<7V3ucQ5~*hep$pv*>G1)M?U9k|gtnX_WNWP(-OU9>JtM#(f< z1lIcgLNwpzo8N`y8N2{C5*bVj0N>NGT_?Tu_!*xBW)|SOZiae2*&;bE@NiN?RAWP1 z)H*3G_U7}fH9w53iD7-_LNgXcK|=SNXrcz67+h^BbGDMrq~{JMcHsI-3VeJSc|G2J zsLSkP!&3rueg)@_B@`Aww~(`vgaI1A`koMwE0O)teN$1AXx~`L>WO!k@g5D(w~>X0 z<#*_dLS>f3R9jT$!IpE|vZ3O^oE{s)o1M#*6CyaJD} z%b3E|Y^~>#KVUuHOHXv$?(~Ov+Mh9jdh3e&K4tQ}IGwnU^sUe(Wv4Qv<+@mE+P-AAQCP&*6?1}_NRvO9oJYBpJd@^Uh(r%%4;ZC21Ans2AlO5tO&lVK!-Mt zP)%sO8N}~V4C&DmrM`fUhlFA#lRhNr2Lp`q;)p~sgn$*m7?lrTL@sv%8&1t?FL zo>pve`&!)0q24s5={u*G$f|$e3ObUFPBc-Zpv0DE{3* z-vWv_@T5c9Gb89B-mDI)%6@BHHq4y5Q)g0Udu3?zDvR&NC5!9PUY|OVDV@Gw!B%Tm}4Y-+YWbsnfKY2xut>522A{#;ZxQ5U{BwIlAlgIm=Xh7%0Tbd z?61yz!J299YPKIW!>?CcE2^?aeb3H|)V8-B`xrYB$9cq@$Qffg*UA3*sns?MmH@=8 z)mju%=2Ced@D^%f7Ajlw4e7@X6= zRA<`hFCA*(BiVXP&`J!cP0G=VtoHuxj0p?>?A45?`!Q$Q+SX@ZayxCG6aIqjx`ZSA zsD_n+JcCE50fbqnl6%bH0HQi~P!;vUxUNnqj53_8$qE#-d%oO^tIM)VzbNjZ}pysl&OPLcTZbdWnX#NunIoJeE*3VC=2y(p!JxD2ulW)tD`IS!UA3EZg zu1TCuFZC8T(nW;TfwOuO+%_?kx7l19B^t(3Ey4#;eDdU85oxZ3?wrx+dmyFS9U-;7 zZ#Vh)Y4u=gbBo4{6U$BxVtQ8F3ICf*Y!(XQkjO*f6SD(gI1qG4-TR} z9s2+BT3{{y-@JBhnkE4dQY>lo8Z04i*Q!rF0pBqC_!X45lk7P0dqcfDV?qDyvc^pN z=CokF8u@KY0cT-cx1jF}t97Yx;1aqj@sJI2?}j=Tg_^=QsCU**W8<`OOqJ1vcb1gH zG-ta*n$@N6Q?;fjwTA4QA6Y75-x`G;Q}9M*U?p*cb#+(;*9!{wJ4xX};h$}>JzG2; z5|51w;p11Jt8rM746jb3@QDydg|Z^*(cLWr+x^Z6=L|?fQs=E~4PWh8v#L zkV6~J0EnrmhZ@s0CAKL`j-W2%nj*g&=YNtF_@&>QR<05eZF%A0F&W20Ls&u*4)Vj( zgwK!$XhL?x(oP^$u*6nCMs}n4Y5HEow)dJl-L00{N(G9OWoh_}CkouTp37tYx+?ux z5TEAkQ!v|`NytDEa0Sq{>aRtV?+FwGdQy8>Da>vPeUGWs=CQIh-PZ%MKbQ7cBjG4| z`v(?ZrYM0j7nQzeNpD{wb7qk=NwB-Dl~V!bXQeS;F*Y?OZDquewT+M~)~Y?nZYV@~ z+eE7Q8JYX=GOeN;v5*xGmN(ouO9oT=cOVQ!5f%?C2?E5U%p3KRaqDte+0b+Mw4VR| z`>N5MRQ&l7sO#{N}} zk~sGXfkAq9i5x+NS!rTC#L&k8N4r zij+JkFgl3h#S&(!VU_R6y1}o5F51&mK%2(K!}f{g&O=6Rfs~d5I4kY9Y20s9{5!|$tpaF^nJ&kb~*m_=?HuP(KO7J}{x_-et%q6(;POE1e z`dUNo5T}wrg>RIYxpnQGrsb3vBovSW^wPT+XUBD~FQI0Wd-`%rqYH&*v!er0F+lFK zqjX=5I`go*qtIBixMWelofnVKAuExP9~duKqSVnFP2Z<j6IOV0bT92uH{-fJ~7wfG35$KbI=rZdMWo(mkt`tA1m)!@+^7(=$8B4#!^) zL=FnRPF7y~j~x2H9Gp-JgZC-;oWqPoRO9yzFQWQv8$};>(ha!H;B15z%M0CiYR(6j z4r6=+PS5MiDqISBackr6)o)JA{4>aGN&)1xMHA`%`BT}2{4ufQYr+4R?Vp;O; z{=3myR1LqO;)OiGW^s|>4Ay_V0J1;h*<7VWZ(M3NTDA;Qf?bI((-D3ajjLTX4-fJO z6O!wh=;wePK8ABH5DN#jof^?@+zgt;+vHrf7S08=Kd`M_%xfET|Cl2GMD+uw(_H@X zrjz2;LDfTBV3e0DhI)YpqWZ^JU3OrL4#9SWvwVhd4B1%ae!|_4-dU_RSwBH?cDm%Ab5g+KMTJ{&><5mbllyRo|3P?G!@4NSd3r9O zH$4ER@YL6rNI2~{+Ps;C2!%$DsHb|>r}Wv9_9Ri$SB!cU8(`?oE=JWahB$)NzyQq0Ob zTMX_%2$N8;1zt{i0d~BWa1r;oC(H75ZJx7r=A^WUNrAU-_q5rRa-5 z(64cEN4402Gl6&F38}9M4`VM@bTYpmR z-hwPH`HUDbt#l&FVaDPobH|^@BW5_pBuya(e|GYBMsk48GHH}{P!x(GH70@k zzrYxnRjY=k9U9u9tIr8PXpWwG8Jfn0?}F}ZJolcg@*2#`l2e$Ax&8jC@?S9Tx#7>? z1pO9PPI@+QH`sSdMPx@x8DvEEPOGx#5xoMLSR49u$h4-3GlobOZkf8x%MUd7wOxa* zGSMCYhnKgmlTjzK7G%+FYjUw&Xtr6ulyU zmsQrCOgBrk;^3T8zs!koA0swUa#%vaieTU&kRng7!`-QLr96fY5f6t;1X&UD8pg;4 zPq1|&heI{EH9p*5Pw)~;Hu`vZo3ZMj*W>+=rbrrv&*CURf6L;{s)GPX8;!Ez9)f`AX#wPq{uefMf&Lmr%&cVa z!$52*L0eVNu3&#pg~LDyZ~bR|g`H!{-H$({$s`p^_D|o;SmAzKy#nN*3gDB>jQuH8 z6~>o|@R0&>)Sa_BO9k<|@-8)=wpnjpt#0P1jI(C1`@UZnJRVw6HFvD#&g6Fl@}pr4 z4OIo#+1Eh>QNKMa!7#C!`C^Y*ZIio8%k!^C4Psx`)d?aHP8sNS6cjYjQxMA^4wwYl z<5nPpiX2Q#eV*f$>FPPN)QhDKqs~QQUZtUTk2r}mow*+CLe*_%cXYf#K+>eZdawjf z=pBbW$lztuC6GOR_O>#MC&y{1BCTDSLr9PFtZy$ra6RP6*6+KDZj+V;e1X*|1-zn$ z2BtiFDhkvbuk@Whhl|7j%mrGCFa;9r3_% zLbD(}OL2HZ+~RYS7Wk|ZdW2}oJ8Clid-!Cf)`HH}MkCb?)unT3Jwk5#`s3dUTbx6H zf7e)t_DY+|+h__X?>ue-V*x2&dnhGj^g521GDGb{@_XwFmkk#OJ+_}{Fskq(eUV99 zYHhfbqdB@fSaKpGO07clD6@_#72vnKR*<#d`wNmCB;o61B*l%4d2~MyYi@D`;EF2F zhw1Z^5p8S!SxGbE>mPD!(NgMu!9o{SI15HkuPRzm{qw zQ*ix1tVp4R19oIlePJ{J^0;bOS|7|;!VGl)CDxQ=1AuDX!#BrOgsIO zpJ!4Gtny7mPX_qmg&1tK1Mv5fS#X~u!})^j`cGsXe_l};YC)uWT{siWF>5b!-zU*f zTFw}nx=6z9t|G_D~L}xjiE_gr|SI3iRn=d=fZQYIY>?Fn!2Az z(;R+1tDXObKmXh16|s-mCsyA>I%Z;@Zn;vbhAI0%4{mL+fd@r73>&ld3aX@<@%4nM zI_TWUbiTuO9$=qV<9~x-77jT_e()%aXeRvQ!NdIQH#5WTZ`@mx-kD+oF>*rb;*LI+ zmz?ibfT5t&^FS~-VwVw(B@CFc1>sELV5&h7^%~Ss$QD!Xu_#(M-Vj%@;Du3BE&3us z5-XO!mm)Ynmu;a?mapmC?n>eNFTqG)D|>Xfa3~XeZ`bf~sNyTwq&pv3-`pzU9aU`T z?&{31uKOvqv(QS#>A`G^%Ew@(h|{N)$tEOMnlz3oA7n=2ttA#97$2mMll2nrlo_+- zl7yx!D@Cuz+3gF6T2dABeyQ8tU#eVUhre>m;;@TTrxba734GhH*^b4vkfZBpJfY9{X4EJSW)Ua^p;HPd!DK152etg7pO;nGL$E@ zfZ}N%LuXv8t@46cgQ_h`z;?djDL29RSgb*bdA!LzZZ1c0@exG${1{puJh34#G<&!- z+JS5ez=v&kkut(a!+g^EUWa-XpG>J;pEsy0twZNB7AuaeG+`68jetc+xIWMbDb`_9@E7oN!MAQO<2uWfGA}x1;)5b z#u98k`z~|)8T?%9vJyuu^A(jLs!4n0%jrGS1jekG$h$~U5bccPufuU+W!7qiE>Lc+ zaY5+ufp#8pEC0}v?4v;?Tsm5R1I%7dAK*6Gr~Ow zAE#}9;Od>yTNTOU)|aLmx6f2Kzq?G-X>&vcMPRi6mwik;Lnf&FDx~u$R2y5dX4|)J z`Oq?>cvg2uCtL8C5iwqi_4R|h2n&{UA7$$Dguv2-*HYT$cb$)(CZC^Gh8!t(fi-)H zBP3IhNL_UmJal>C0MXp|O)Q!uolT_6mVD0AcN^(f^)mO0I5EBbiIDh}XUA;I$^7-0 z%Ge2uNll;{!1RUeN;_z>_x_;V@t(ZY6V49x7gl(v5{H#DYbR6|gAS0C52AKFvhE5k7(90N8iT%&$Il{r3ScKQB~wKHr4tN~OP-$He9H0m&zq z^uOgBOP`!P$n#0_rc(qTk*m~}o(>-TaH#N-Bo01|@2!rV>JXlEf*1wR)nBF^Q{x{- zir;_huYNtu+BU?t7q3@I@#lT4F*+wJDGeH5kbe_RV7&zek+wMVALeoO065fg=D#%E zz?Tu;;cHMdq0?J^{~-3EoVR9PO%MiRe;-bM?@!3(IXWe#eMY4Vqx3SxRmfM=SenEB z7JJX7^LO$hkRJiWz#kYzNOWU|T+oxR0~Vs_Cl?uI|Gc8%j85eBgH69o4y^m|PA7et z{C%`Dndc@wLKj37(m)u3qy0Dm%k3f2U##AnBw|#d)uxm0YP5AX?W^2Aqnnm@eLKy~ z_Td^vW5Im4*?K8Cp?3rTX#od+4_??)o%6zOQ!Gu?G0bnQlr!4qmFv+n+=K_O;Q*?9 z_z;SnW`<1a5Dfo_=F?TJ=sVCSQcWCX{rYu5?qp9{mRM%zoi6b%DL3ZNKdK*t1*krST!x$5V6b@UZ)Whu601>Y>{x$v z3(oy-uBHrB56#%%Hh?-Yz=`${fvb=xT3tLkR=qW0Ru`PqP+=vW9>~G_byA>Zlvy{2 zak-p&6>bATCWaoTs|`#`4-&e;gEnuS))NVexrmyXy+3QLBk_8>J!`-;U6j@GqNWM+ z=EvJbc^{iq|AHM9ID_qy2Fxm~`9k1^iN^ec_{+Jci8(OJ2Di?EZAH?;e2GaH&hl=V ze6H{ch^h1S)SmQP)J;`so*h4Glz8)C#80DTE24TLO&r|s7(LZV8Z()>dpA0+%B^X9 z#$#8n(uJ~PX{AwXESq%%;kIBce6*AQJR7ct50)L>3rs<-9+^s0B=bcBn~Hjr8min5 zIsg1n8LHPXp-PBeRF1|R-8}9oaiQD5kR5T%=@ffp_)GFA82Lfc%*Yn_WQHPR2+b3! zU**AF2`+aB>%`{f=H*wfo-1XE1j-8Ebb7(c{1R}7hl5w-+B-xN;6DJs;W}r=4lK|Y z=NdS#_^0UD>J;X<-R$|(!)JS*u;26TuUP*cWucJD8U~(_{b3M@yMY7*)r#y=2UcXl zWQr#6sxhT(-lp|@s9e+bw_`1mPG1S-UWZt2|3XjTySHZiDb6smYLt``iK{Dpq>Fqr zKPzdMXV}dUcsKDqYv0fcm@dkE&LyW|?)≻v=h$qqKdnFnM-yXkrOe8}yBZLKG`Bxjac6LL4Xnz(=q)}u)^OaCmyL>+J+n#kWI)KOz;mz4+eyV)7lcG)uiMlx>H6>>Jg%eT^t6C6ngzS8 z3TFRGk_dnoYNCI4F~hyglsC6RC)Ud=={M)H!r~2se$}7M_Y~4zI8A>BM(A=1e5!|H z3(iSF_;@X22&6~sy7~~c&$g;Sdm!q7RYx*K>JA~^cvFUtlTYbVYXU_bLdL1II}@-o z2Z-TG<*Q9xdU zRxNk30b||LCGm6 zN-OayJV)Xe73X{9BCRQiKy7&94Y&zgsJgR_#lD!5@Sbp;md4~jyO31&spEKz>> zHnT=u{a24CerDg468;O~0Ty5*9>)P%6*R1D)~AXgxJGw~d3LanqjO9046lxB=hnBG zULE0#du@F8T-jKil0|K5XJoMOazWEHO{xJH**APljf7dMmL zx5zrCJz{)sOV=)-qAQH|*)9I`58T&d+-M_@;+Nmn-j?*9psIu7o7Hp1S6l=I<8{HMYau2sIqt91K~YBj{FJe4*5T}v*T(nt85kjc4=Drj=Km!v zsBXa$Qjm~6ex`@PwQunVgab*(K2JSvGB!%yt_{cSdA#yNf!W! zr8OU@XWqQ5Odh4=fkpg)bKH7_mew_au*C1{K)6zBHD78nzc2KLYsE!Jp7mPVyJw^< zO(M&qry^aqTRJ}}V<`Y!@VFK0gq`D~yo9RP$4EF351{y#!MM6ecH!D?ayieRHjf)d zwR3i-FN!${4t6nbGYc<7m>~E*fn1&$xZE0W4PL;81KEpWld@yl>Q7FGmE@HVyR3Qn z`?4}#PUs(e?G>n>BR91FsNx0RW~*hn@-r;@3KipCp`emo8fpclS(ArrAISTF1h?h@Pw z!Ezjc3@R}pyn zLD3zby7T^AWlrfj<`!QbvGDtxFRw>fLC>kq00oPuS%TOsqiT=S?IB}*ZEtEFRG3*n z(*HQH-4F;Cl!Mqq3PmVvM6-rkRnAuxRl>XzB$uet0gP@>$W_&?!gTBQ@heYUy{$75 zU}%_!9*J7s4d=qn*g)SIFE-YwMb&*6SZRmGzr*190i|pN60uSYP%wn9d7_f?)uG=5 zH$J!X1(KC7OgehT`mEmfT9-|>;Z|WzOn(_+bW)gl<3A<)o71>u_K<%N4(Zq*N|h7N z@ux^`&gTYyT0H$N^9Mu~`=-C_wQT>I?5uA9*Jc9n@?)9Q59AC6H|P>ZI)>f=#Z}P$ z-|8)hm*%RQ3Se(d=w1|^{7o)h?fdJtF-7`hBY1X3mMgVow)q<3HwHcgs(ljR?6Ld- zzaP#R+qK$25z)f%RzE`q+O5x*Y9haE7im~R5=p{hL$7~->@APyoJyqA4lxwfG_%|eURCzQlbfj(KSLpC^Ix8aF zAGnWo+;&QJ6qaHm2Z)HT7NS3HV0{4G6b&CTT*rgZRF9rlg>^;Eni#i-om?oUq*SaP zudtfcCzT(apKDV-99H%7qO!7bjm%?o9+J!g-u*xfBiV5&kinycWmsKf|A>dY3~7bR z6tKLtH?03bOnPtQxRj}mbTF8lfvny84IIVx3#c{WpM(n**nAE(&*oaP+Y+h|6m^h0 zU@&%iLFiIa#s^1!ef?`KO9~m#Kmv*|ya(8e1(nUUUG)KEqOD>crBJ)CB%yTev}f0y z11-)?Nu{;HHvIhYPJwDJQ%>bwie3!4C}pquMg!v4rO+2y}SY0q%sQ9p2E{P=p3-$uWuTcSQ0jIc#s1a_4MtHB&c(HWeC7 zJcBHpvJPn+Z+79ZxmlLB6Er{*xyaxE@5`bE?@+Io4xr}FP{!(D$)y6p)pE5WSN4?p zFom^68-av@m}~JDJ_%oPo^X>j@B{P`ssZ$s@KNOUxjh2DyAPfN286U}lKNgQ&jP@k zkKF^zlEZ~-p;DT#-W=)b>)nq^1D*afTWF)(Go5W{w%#&TVO&{YsA!*|)c)!@TpU`s zgDrd*&t;aZrw*tb@sD(+UKwIMW^u6r*7vDg+Ah|25{Hi;RkD6{cvKU^^3ue%uzK)RqolU$5m@{Pv%9upN9f z9#nkc`jWFOXqp0cqDuYHtuSzg0>DMETFtI=Zw1|JPeP)o$aejWF7@7xy^2({C0l4D zhfMtMf{20!%=!TJrhDKn9tgdkhmQ|X9^P7GD-IxR3C?+3pB)@-F*kVhRbF6_{UB(^ zf((xJHrUUS@NpakO#_yxba-eR^%%EprK<{(ZK)K-Q$?&%0dx}&%VBB!%p}8&G?IG! zKZyFRjWt&YH6X2ay&@>4+Y%>GZ>oV&b4@qMFh8*aeUQ`L6o2-HW3%Gx{3kOhD%gpx z{V2kG2zIu#Yd5<)1wOoy-H#Tq_E7p|1p5b*QRg}mEOW$WE$|p;ZvQK_8^12I&4#ug zbDrm5A4NeSDYSgou4<~14szEw2X|nNx*x2_68KcWsFsautOnPY3kKyCP9(W|f~cF1 z#8G*p$QB|or)7<&_5|YL8~X{k7)nmX2JdtK zC8Xe-`WQR9;mBEmd%CgOlH{TZgjB3kfvj9oAO?m@wABl$)2Na z?jvHLzRSU|GxWx_9bf)^o>VO; zrFVF!#nP>(gTQ?7cE&km5(>H>NGnAz70`!KyNT}F5Gxi9AoPG$iAv!HPf$n;jG<(} zd6n=$SI4nzGZRzn4M=;V0yUQ)sPJIhUU8&-Xk1^ABlEMe5NCz8L2f8}HwtnFb-*({ z9m*5B=PN^f;JzxGuRriuHY)p;87^W)8KCZE)WBspf&l7p_IgVZV+bkQuIO9j<{)Ah09k+{dj6S^ zYTj5!>ohNuqF1Hqk#dQN#pjk}1Rz7&dC3|(vW3sv2L5Bw-FV3GG3p8#K<+05)T#fP4&mC!p8wS@QB?XT1v4y$lzecSM* z=k43fKzJ2=&NPh`$0Y-7M9XL<@;GVx{jG-wO0cg5}C)K=q5OAAh(*yu8REL~E zY3+bVsCJC`2Uup^08$&vq;}rEIg&_HF^|n~Yo7kgNx8J1!JFlpUIrQ&Nqun$#%)jP z?{Vz1uonmofU|xXPgB`ziSEr>c!d{%+O`3{6FAiy%BX~gtv`p=3P25wR2ow)H|?bF z%5Tj2V!zgh?M;nxo-U4iZ*F5f!!<)6z*5u`LE;^9v{1N-Sj`Ok?JsHy^Xxl4DIyXl zpZ7*Q%f*7*4B-mO*gs)U2Cq4ICt2DG!1$l4ur%-RxpNqV$PTx(}R)v!Mu^e1BIB(9pX~)@6+|_=P)+Ed9VCvjZB0a zGi+Ery#;HQGR#2|N!6iS0c(&W6#|0fE)Fc=>$PPhtUBpA3U)4Pn5c)^w#0!WPKd%o zSFo@m-@2pY5;748o+DEzSg`owqlBr zNgFu?#*q($$ens2Xa!G9=}eh3Ap*)P!@vU{JPU(d2Ofw{`^)is8Wz&5tGWI`#s|RW zx4==G0uXH_JTGqji8Xe0&ANIZo-qs_z8nc%lg#z4IxBK;g*8rypSZ1q*-?x~a{9IQ14u8aLS6;sx&0C^o-0%L!5F!V}UJu1ug!i>9w#nW}AiDfHy z!R=Ge>Lk;S*Y{h_+b<+|&-?E^HoqBI`;*uoD|#S5ngCY&TL2;ytUucs?1x+QLJ(}+ zg0>h>(N@1NqJiV>Qo$0E3*K4L1TS{(ZbhS9FHQZix9Z(7R>P5SLi2qm3WE=Zv_IP) zeR9dVXr5|u&2Rmx@kp5cf1=(!R`B3M8}xq=O0wiS#s_2^>i4v-L(@%N3nG>mUh0v^ zj-;WB?0%lUT6;%(Z&JNqTDd&<+JM5<(r8NqXZdEO2-KsaR6q;>oHvM=xOZW27_aA9 z;%U3F}0@thdSzeMoP@I;PCH+SEc@8>-Zsp?VN2qttt1b;?159G0J_`tkCE zy8ev8;-?1^laJrRJpzZ~4H`1V#O13`bRcPGfJ`l)XU=BrgREA|l-D%f4IHo2qyxF_Q{SE`($lG{LtwS_nEyefeb)!}s-cOIWXt$l4h{cX{^bJ`NF!8p zSaS(T$O|P8FG=G(-%3_06Dz;>A^GqZD@VA^4jVSTi#%0wb7tk)mzr+5gPI~Ys5;~x zN*v5iJsxykPbPc}EupJQu;;XEXh^Xpd$+L_-1}9dcvtZ1&*DMz3*P~I*?Re}U;o17 zo{EG5yIqwJ!t=>&lWI^<{4en;D6G#q5(?J8fsvu%aCsO-o|-l#v-hNk{W#xns$*V+ zy^+;5{^zMlBKN(tkLi!M-zhWqA+VgzJ)i4po(61gx!?LBc1CEUS7^+B+dLLE*>{~uk%1cjm-g6p1GyA<4317Y!5d1P( zJNbO+ewFL8aZJib>dx`>d(z z=I#4#Ta3{=+plRY_@@Phdgi9S&rYY7FSRBy&-3ByKr$x_u@*uy~@rq&@64FWpP*7_eX@E#!e|I`;F7)OBZ18=%;PVG;avP~OVsw3}&T8aCtxJ>4| zio|c4``N)D86x~1AW8~Y@b5u4O?R21ds;^p3my~*-)Au?SB{R?x0;kOyf$>6Debw@ zrr6`I>dm{j=LPVgYMKOaGprg=%*J!*7fF3X+?|MJ-vcrnem5N;WjWewxtlmHd1{=f zx)iPS!CidcrUENV#%E^w@&90w{~g^1Ds$VwtFqr)n_YPWJ}#dep83X8Cs3pn^Q2y^ ze$9PTe5oO+LOv~B`w&O*!hBSxO79;f-0|{VB@hqAfV;7vJ{L&YqBqY`I_RmJT6_Yw zaKXj;`e`jwJBx&?*~U4aI0zT2mA^%v42E5rnniaUg$y-Jlj_3hBnx!PO{wMhB}tRgE!>2DZfIS*XEm zcj#G2I*r)yC1>t~&a3`U${5{&mPX8Wlh=oS-;AmE`~D+C$2f^1Eae5>0zn_4zCg-x zsv)<$ua+#_P?^4PODl4bwYJGQPUk=^lZ&hYLM3PgSG&sy!4aCl%rS1`9|RR7kX;wR zDX<9t0Ajr4pwjd3BNf}EZylwTJtd>Xw_a;>eil%^Yau5?oTm2jYYegjR+6%x5_*Mf zm9Oz5o`Ig4Bc1_XM~r3DjGP0UyDR~C1`r3e6+l6w@wDr{)MJeP*|P(2>dQc2{j4|D zydh*jV`0J2`IUGM5xoBpa+viFjoR#hX_M%!3ExLfl<^sA#!(spb6V;^S=AHHvwaez)ebMXxMeF{xN zgL=hbCTQ3q+Do!mTCQR2Tf((W+kW=?@w``0qCXlr1V4LA5XrijSm(NofuCf9ZUIoM zdfbJF(O>^TkS(Z3sI1s7F2>=#8d{`q&@aaRVzM2nBhPgS6~9{Mk<<4l=}rFh`S!NR zDeZc0v9Om0s`7kt1n zW?t9ofu7xI=7u3m*y~8E5Jz$CJ}}{21UAp6&2zPK1GL6ZtQ`L498c5@c6nvN&J^Lo zvhP&a_Z&Ki1|ECi2zcC_B%q5^ZRxLKtH-Su`n4_EwCvQFE}) zy?0(W>RNji0Hkt9giC>Q;t;Um59Aaqu(M{@$h>udSYPA-T6lKg?6;oFYb+8CACL8N z30^Q1^6gI;yDoeV!Q@~Fu{%uB)X5WF`Et-yT_Di?bT~4xQrA)2>{b$~-+ZAe`H5!D zSy0}h?|GdP@JGr4DJC)j*)-jUOvctp;i7fXMD^Hkjmc0e&tNQxLu+QeWKwOVHRqF2 z$-UaRWoN-7M^_sQlGgq*l*RL-$P@LdK=GyiA77l5&mkqq%LmmjOOv@@-(DP}r6%PJdK zIjvW^8@v`4CXNb+#XE)J1oR||b`WG$EZX!pC_Cya}?i zes3Z}89n|-$uG%S)=GFKv3!&62MSAv))$) z#V|M?L4Lo`Q{L4j(WVcl#4&0TmdP9eZ9iVciYPoW8oPee=j}@bVju3@8laU>IJAD8 z1e)P{b>dOt*7{?BaklfE#YI$-bO25ee26U0bpzA;aAdJC1l3b+c6yUSEY7+jdcEcA|9CE| zU!!+G9F&6Xa;d5&HnxMAhdDQ=OiR0L^0!=$r!PchtsZ_? zx1FfLg`}B&1K%kCRFt}S4fkd1`1jTN?y_hyym zftMHq8kV51x-=G$Tdv(Py}5zOnj#j7@}DU18#%Bc|4{JlcYd!acnu zZrb@NExs6OF$f!XpUd=u(eJQOxMTJ+elGg2;<2GQl7i#WhX=PGPEVqsi`DQ6M+$?U z2^vj$eDrdZCm(DWs#+L|O82@jGwdr_^rme2O z&S-sNMZC{YkIca`1nu;g_IuBs(_aeWbal!X+;@`;0;EQw=t(_*9Br(|DNDy(T*{j5 zU$92_7Wg}xy9{ty*Jp?gQ&s(WzQsZDKSP-O7O!L!PGl-NeH#Dy&ZsbU5Oms~O^+0? z0VjimkB%lVj{E@N+rHO9Cf(8^ZIc4pTY`P0T+hEMs;%vDm;>I=p=0H#Tmo6w(^0@p z626VXa-|!Rz`k6DP(A7rjA7fAOn(w4Mrrp0O}pY&taj$@4us zb9)(a4wdOa{{hII1D8Uu@Nq5@rI0W>Pk*9ttY1> zd96%tX+OJ{V+T~m(tj?JRgHOCtur^$butZn)`u$V*$|AY$`oY=cM+NGJ;us(0b*%_~sk-G#Z~+1Rf!kw#J%0FP zB53NPYQF}l zXHjgHywWy2wUXqSo;Mkd0H_#(5so7i+JX?)Y;JVPpq8ZSvdP$B99jFZW7Wi?lK7_V zYAlzALX^G*bImpOeQ(3N!Os$_=NyS54a50XB{=7ekITXCOUpDid z#0kH)N3szt0W#;QY+z}D2D`Im2fK-18=fDs zc8pXS)=GS&X~dOo3{tU@-93!>|IZgHc?&)Th%2S64Cb`k6 zcIUL(3(pork`mVo;D9nkGEj8?HO9g^ZOJf&yc-&b+^HxWOw3=w+>~i+qdz`$R({_;Qtz z#d7{(=G_qbv+Aus=$UHb=9{+;BY@nO5T<08n;XaVK_$smz|}?fKW46qRMQ~>%b}vt z-0i2JMJfI213k0C`8nBMB0VP~g;~cfPfWQ7CinVa5zZ--~D|< zu?$YJYnxP)hrf|>jlREcakszy*rGX-o3@DtNuGzFNd1g{Vqwm#({nIC^EukUd=Dg+7*7Xo z{PAMIf)8U1&rH({$OU*6LY~17kf*p|WUgZNiHEjt;Okiz5($nctEIto7h#MKKI+{- zUg~Z{9jwRihjqH;t>f*ny^Rc|#!1C@O({Exj*W3mCh}e!<*7O+Rfb;H&_O%q@pBvj zV&_!e{5JJM}@6ZHAs7p#Y`RbGR$>9tcJCPIDemYvG zSc^0-H219ZtA1?$G=59Af1HW-%xVKm6PK@A^JUE7#6pit*F9*xhv$}S%{8-&=GR>< zn|pQHanCrPj+U1>gYYp8Vy9gIwHVeSAR&F8utif}$HH&vW?%_iaD*sa1E?4B0hT0` zoBHg;jj{R#*WR3?LD&VC~%Z7Z_XpwlA`+nx=VFCQh|xsBA9RA^ki-5>I%Vv)Mu%g^zq+C%*{_v1zMm z#w6v70WLe`=(vx^J+h!->2ER{K?`HqMau79MCynAqJr4_H43_^4!dK-eHm7wH-ahritSGBlE+JCc$?1y`zStjTHU%vq-E1QB*wS0az1riP;--(D0JZ zhPoFqg(JwMGhBUwW)_TfB#CJs*(|PUjbBkVBF4Srt`;+2Kc_z*ETfF@MMJ@xZB_Q* zyqnXMiF1`MoA!FcgvDI#yeHY*pSu)VqhBfBzW@j~qEJ&7NrWj{t=DeQ6b>H9L>-&4| z`}=x5e>~6q$3=C$=A839&g1y3@6Y?)adzlg_`Xl93H#kywJz<`xwjG?AJ6(&Oukn@ zWCjUxw?)J}eee!;qkdPrD(!o>A;!1aM6*C;&rx$~q&3D1+>%2>)MQLAyjzeR{oy?; zyFzcY_QQjbxmylTyFXvOka%2sb6Kr#){&hzvu@vbu>SD-gR4M{~rzi|H&czzxMBl4G8nF%rn4p_GV&34bzP9ICeWLC1JVJ zux)tlL_Pp|&r6;>>E-P;ZYiFWb#gkr)c=ac-8A)O()<^4iZiV3+!UTp0lndHbUFZd z8Q~>e0Du8-2@jRW7MN#xdpUY%Jlm0QSxtx45cZ;~ECGE+40RoxSN?wR(w3m6<6!Ra zO02;vRt_T(^a2zD!HLV-J%K*SoXLtxBQ1o5lENO=-MllBJR;@rPU(9M?4c3`gh?oXL=w&%8P#opL}Y>S;e z@<{^I{t+CX5qKKh#0yxLFS~&w_8n=Rm_lF}=qxi6~*4b_II7Y~IXZQNN z&)+4dd!|mcsiyFKI?XY?>KcmTa&WMI2UmJfb|<+146e7?-X>O77iLWwI*Dc4hmY3L z3mKpi&HoK{VbFZ0cxD?dN+`n$E)AqT*&=-|0O&1>&*FLY3f+T}q zEQm|x9Yyiwg>bCFWktEN@H37T)TG)d#MOX@sUZNthrj2f_kn-GidVEwg>%iPk zjeO%f<88Yy9DCX69U^en8nw3gzbVzDKY=2yc?d`b9z$nw-$iY}iu*0eLCvOB`1uZT z{522 zmR;uwl|B6lM>k=Q>jE+4&2M3SmN$*E;K)*=J7+ftKMnH2WHkhZM$z^iSTod*I;2t; z=vSn2^k=5SZAw;I!Ui)l-ktx1ZD{@XzscF8(H=yU0US^uOynGbd)u$A6O%d%d7TOY zT<-XgrlNr%`LN>6N6u$!Z9ieQ!Nu>+8)Lt_ckkTdr4KU;IcH zjpSQuP7egD+BZ3;|ASJ*QY$E^YSN7#03R^YV8Oz*AZo*nE!#Yt`!1MRgk-$$x_|Rk zn_}SA^QOkC1~ZpWa3Xs-#_6?f0xtJC1BT z@A`#y!qgANy#e0P7}RtcJFA9~!7^XJ`!{L=&EaUUEYAr5nVqtb#^N~go>*9*XP$JvDBRUm&qH03y-Bf99q1ZXJgy_y;A4 zrGmN+plM5kwnpV(x;W9?{a=BPceuP8+Ja9Au@F`1P?gMc86Sb_E7ZEZK?1bri zL=^Os1a`1yxYClerH|+=EC@c+tW@wGbftl{6Qn2jVb=oKTcEb+a>@5M{&F@4H`xoE zOhC(W@56=6evqhcYE%iqh&1Zp5B!6&W?@oRg!Q6|72N7}IR{-*IQJ+DAeq+06F?-y zNdYRt>pO^E$laz!bIxeQ0LwjinIsTwDqojjq7X>F^myc+Md}I9`&5HjbzAo@KQ-en zUfjTgzS;f345l7ST2G zkL9P!WZ9rT=H3?{BHs*!5K&Oc5{O}K`w&AHCBa$&ZK7AhtES%OQJh(pwakNG+R{6u zeAN#LH$RMf4hDdA*CxRm&5Pa5>TQQ$&B6}c4eaLFeFl|uXDO}7;7|R z9SE#XIOPZg147>5f{OCCD2t}bCBq{Qg8W8PMz3FP#=aht8~xd>P>9-{tGm_QH*=VI zni)bJZZjN}p%`tdk zG2uZovRxZSckPW5%h$_z=NVBxTePkVGy%vENep4&6jR_aW+AwRaIPag)UjnKZ+28U zU*>M)75xxm4E;+}Ff^qls^6SSBkcBevv+(Z6P7g@lM{ zgV*p10r62y4#S-#$admACG44lw(gciFsJZ!PJ@>25vyJW#OUzrBNfG&`$djkx+#R5gZ@h>f#hMSv)BZ(bmLyA z2@@Mc1LIv*`~#LYP+3U*nKw^8gL1FlviAwy6_eJCzT2$%=|I9o;r0!rDc3mcV}M2i z+A^0oK0rYPY9l5p}-2FjiLD!VYm$r2pE;1h1$E4CV3xBu1T0AR~ zFcx{!dOdDqy3xrClPugBSb^2^@mfl9tS7B_|3b}Y^zjVep?(AOdG~L(E6UOjn;X>| z>u$8h9>!9^PB?{zwZDLLSilr^W|BcXd77T13H>Q*zqyf1|*#|w~g?}<;T0jb9R z+LLC!xz6Wb$jRk@SVgvEgI)6xYO+QE)t-(0!YhtB;cGpmg|Zp%*0RU+6;bd^mtW-5 z-m4-#Y7>Vm?{1{(Xf;3WYWGazf--l)##m{%5=2@e(@g7Feddk_m87JGsB$=^HDbi} z&hmu85Z$z7F~IEPC&S9hW4`IwkDdqPMr8y1%zOZ9*v6_D3Gw|6pu z(oBPC3OIZYjo`9;Ei4 z_pQtRIrl}oOuWKa+Ud(hVf8I5+}*#fDc8`i2luuR6uvFih}pZ;XOA{Fw@*AS`57du zV=J$sxA^Hm5(+?XJ=i>Ogj->s$=(Dq^JsWHBt+1HBSJB)C33KH>1@&Bt&4Mgo)awq z3)0#q?CL&{m|$n`g<3rf_HPh5t)9~4JW@xOX~&6qdX3Lni8HM#D}&g>-Jkj2jE?k} ztH;X$(ch0MJIyrtqpIi)lK^+(oNlFobOK%kMHu4{!V77|Y%%$4YX8bXTI}-xTe3~+ z-2EiiFWpA18%hp=$&5I-9}xxa9k{Q$j1^~DU>=&Rt<-1xg6nS#hXZ-fY;Q+LAGLzt zv}7X*tzom{E+(5UfRR1M=h1iDW-Y4pd#9YPB72Pd7=f2z};-?{~>t zbFy;OH((UrsBjUwG!1b&J)}ULwkhZUN$gL^St0t%9L&>YIu1i>NF$ zx7xrky{hvOc{X}sd}=3Qcno)20BxqtPynO~1D1QKqGDmI?Ox-% zphrP26M;oZ#qaXJpn}4796Ts@ezkgNZZ<73x{dOW1=Ke0U;`ex20fzyH|2mxI?;Ai@c+bfnsSmr$Z@P^`{Q7>`&+i20`wAq4eN9;}JOI@xN$OGpSP4tvr})A)bM^;* z^P^c$Y#TjCygyx;ic8IX)cJdtP-^Ca`|(^J1@P5rLQN6a5K0E+C6IyXp_3W=nesoo zDQ)#)1;rtcPS0)o5#hCu{VeG0@CGA32l-RgY43GE$V9xJb{;ZeX>t|)m>%>xh5-1= z0JxvoLM6qUfOsK2MqhPp%(n00AA^A8UawaTZ+De@%AHN{5PElIq{e2Kh&A>O2EJYg z!WATM7b_ABC+HFp9rAfHwa*aNffzL$9QO=-k7I%i5antx%Cpz(lO~a-*D;1H~Td#!syC~`!{sBc+bS>CP^-!Okl>tQa z1CQ0_v5W=R4Exs;+(U$wVlpNg}3p{~y^&W3|05 zS9Z)?P&GzRvE#7iM0j^NBI3b0g-k%gjXPuYcokL}lHC<^BI!?OH)6Z ze#D~w&AGl=&|z%;)7dpXD|z}rvh^MrgZK=4RnlVRzZy0FORMI8;&-elIFCK+=Mj%Z zgeF11=(`tJ`Cf;z5v%%EPuZ`&xlP@D78Vg1mVM(wQ`_TnH2_m;WJ!pWT?jnYHgfg~Do8d}zu&n^;NdR-Q zWw_q9@ugqQKy?PI@O5K+t+UXj)>MqeDW8i?ANNWx<^R0}lCmtQTh7uLfGHb6S?aO- zS}2ewktK{k%&XaVnVRfER&*0Fp8V>Dvi{dt{TEJ;bGC*w>fJh8v`81+_35B!sKJ1Q zu7JjC>n=VM4iVQJzyO$MO^D@$e5-YA46tgQ)#On%b{RAJBQYEEs>}-g ztpWFoa*WVj#>?xvp! z?xr^dyF3$)DtI3sud=0-c$lT$|$Y(hKL#c|D# zAs;X~Q_g*Wp=$EtYOp*@oQvT2@t%9B4E?hDJ2!K;N2^@n+d*_US^B#H%u5cJ5!ZxE z(M{M)A~UAiOQuaf(iT?WhmTT%% z2A-ZDOk5<^R)+)0;8rFXP=DNGJ2~@_vcl|&X%peLEZsRDjzj0~hednpQp>{sh*4{Y z#&R=rxnDf~>kQ}Z4V?8tR~fkCJ+1|~_8Z=lN>CKUCR5&AVu9VbaQP{z!Tr+nkjf`x zo}#IJj)644WH)2k?{5^fPCex>{urABLLJyjtvw=SXct_82+uyTd0aDP<9g2-PEs!v z4sW=cC9$ul;_a#Y6MpSFQ38Ri=qfzVUVH*;RF4cHnR8tb>4zt3!7%_SVBprMa z58zXO%1;MAdwQgP-pt^ST>x!NHH)Z3Cz|jSYDAU5Yx^P~4 zQ3vuUIv)mvn*(4d=RF9{nG~+nFuS&vm(!?2_%!oT9;|saC#^vyrfk0V+1QWIA>k&2 zpCk*C)i!V}4Ss$1^HI>&IULpso&wb5n}*Fbxsnw2M^L5mCEmcju7?-@G&?FPYY#@{ zh&SF<^r*_vN)|Y&Ve~ZXYoWzeGr9JV-)jHQ<~vb5opm7r!NxN{XS+JolulVt2q?q* zPr7993qSs?;gZJ0;Ny2@SACKgTMu{d9rcXNen+B!soESzR!;KeC7HlHi;>{ktv@^) zQr(NRUCbrWHZ>{ITw#yZyBtfX3&k|$rHuw4dG^*yNmo}H-W&%!AKQv|Kh}?Pl&FsYU`*mh zR=8kR)57k$k9~HFXQ*A@TDNBQ-i*WF4xZ0FbCUV^(Ajat1l_+@q(QLTjYAMu$}?yL zGXyP@oJ-MjSN^lej$YqgNrW*75EmCiuhE%M$Wr<>TjarGylbCC{gtA)&sQTYSz}|( z!Ze%DdcmidrFnR1I3IA-v~iDtImM`77&t-`&=1M;!1hdh4owR}AR=4PppkjllODe{ z0aa@>cWVbCI!yx<+(#OBG7a7^U2z+M9PLhfXwO|h&dRYU-aFyrc_LEL_mH}}k@TQYT4Lijm00%H)Tmw$ujVPXm%SHkBDJ{%?+#QJNy#$-BHiLqp z11@t-aAB589$_><^m(3~NtXXUo0FmDeeK;Ra<`AEwrLmG1(tOuZ8-2d!kW}Sig*BX z`WyY%u{ej9rw`WRFzFZHuNX9Jg`xA|tn1f>y$(A181Wu35Y@wddRG3l*G0U<;6W_Zt*APF#a*mL+yFR7A<`NRCTlPl?$S=F;$+jY9jfjm${2E#zvT zcMoiC5Bnj=Mj5o`926`k7PTv2Qa$kiNgh4TM?n6~5U%;*nSy#O^4FbcP& zjBp%4aV|kp(5%umK}0bP%s*}V5RRZ@|3TgIfx*-I{458r&;j>F9|Z*wnf6#%hX|&( ztFW)rcn`qPh9g;YZq%-3M+TjY?lmn|DqgcyY_-`l_~&8Z&#c5yw?A90RfY#SlAOff zBp`HS6pkF5VL2i_mXx)i{tEV)xUD**2^Xt`gy%;OC$30UyKLmQj*uc8uU;Qx3TKcw zUR+b&4Pd=<Zw12~sZJF6`ta1%gO?suf z${i!4!_{mr)e6)Wil<5EJ6w1v_p$RS7o&`M`$dmovm z^0mQp$0L)wlwIN*= zfC$xxe~)1|!a+cnrrS!CSJfzfa&B9_VAkc1chzqO9|&IxQIB*8w=_Z>(%6xFe7vk8 zbX=I<*#Av%8m9|2odYC|XZV1nav*e}LCvJ@<0@gtK+V1|eKgqob%kAy6LyCl-#roZ z^2d<_W*S$nzaX4KIc#ivPU^tI`(lt|m4Gd#C%+K6TN8}>`yNrtSQj=pH2)RLoB|0% zr^(s_uyb6&7c>Fk98p2ezIlvfH95J?YAeh1#)@x0*Mr8cH{HW*c<@75<*iOhaCLBG zzUzxJA6ZTKE9n;**$ifG^ER;lARS8JTsatM@vZKBC<%s-O8?^o6r`iL@!7wt>JLJi3Y|_3hRCeIYmCV{r7cc!8 zZXv?^IBo;6SIdI_L4j!FY=*vJ%dI)_9d*wF5%avcxuNeb(dsTA9_nR2QrGtTva5*u z6By21XnGF?L@W!SMKt@cy%7VHBux=faE}6AFb+KQE8d-0VOE7b>=!+GJ(`{-0V^CR zppJO=1gC|Yw5}R2L~i+Br8*ifcXx^pwdvenuu$Tyu}6W~xQvIuEUP7jo9K?EE{6X# zB7e_8+MK|8I@K!?wz&whZ8_3VQ+OkaFa)Lx)_MAyoJ^oibBJ=k6=v*56PVe0h8fj( zHK50!l>~;~7myKueagCPhfEsk?28wIKb3)T5^UvKpSGi222G3PY<5EQHc})@;&|?( zuF=ycI^({#TdVVp4+U@~(1^P(52#3ouuPFV%ZIsoz{kx49ZPTpb$kdS@|o4+&AS&H z$+4`9v-Avk)KwAa<2-clSaoAf>hmwkcIj$T>8FKcx5ryfoo{eNM0SA>5`ICi4H+qu z&Lb(*=!Fza)V&fNEYk%L`9I<`B+SXGQTi;)~#l&#s zziG4#Q+bq!;{?GiYNkkntU0XgB)u296pB@Vw0rDer|TG~%XMJTJA9 zyNjmpaoiCwT*;#|rUneEmv2gY7EDz7NF+rh-8Kuom!5o9iq`^OA24tU%XoW0fE`Z# z2PHI%6PRWVXIc6v))f@Fx0hYGz*tZX*Pp)H8X+;=DH882r=D%h&Bdb;`(hcSd5E%R z62c}TXyA%&J%hJ{y~uF^Sscw;N#?&;12!2E9^^!D&B0e@Bb`Va`+>W7 zjM@;lYx#ehuloB25orRCr3Qmt#b3$Gj)F#s9cLsOhG)EgRTIU2?FoH}o4Zw(wb|7% zajGq~xa5e0w9tk-agY+0ZkCfqSPuJGd*SMcka!$lxFdAzto|$)aFNyrL z6dh0Y1Atp1B5co5;a>U%1!9wlLMtEaTa}e>e=ajjkF@~>_i!xp%l4>IUz_e%U-~+YBU!ptSv@R@c_mjObrW?9C6Y4tAbaOkO!zlzj2hIm zf4#^FG>-t+36FjC-=%MKB8)B)-&K zc+3ePe9q$Rh`kIuor|X4sibBc?yrUm2CH+nA2={{A@2I!+)aZcaWcA@iXN>LmQV^; zoP^xKVIbO7Ea=|j%Bx)j(~x#>zt|CD_7)zojCYqD041F$IJ2$6Sjk_>!FGJ~-EHZ} zX!S&%n%0$b$rEIQUH6e)v7WplGPP$Omrl^A1v@ra%-*>%r&T5+<{a&V%buU}P%{c_Phadtnp8pCwO{N2hVP*(-=i&uj9A)Sy1wJVU2%S_jR*!TP3VlGg_mhZ2 z7@R2Z2^5g0h1MqmGJPgz7#o-sZYga>o|PYv$F!`VrvChxO1M6=@-wYZB z6JQ$kuZzDfx)({59=gk?gE#w0w%6C!*V8}$@?~wdkV=+MvaE}aUEEon_>ZW~D4oMv zs49p~PD141VOq%t8pwf8+=`*;PVB&Q?L(_>a1=%scDU_=@?EslU#V!xb+(6;l5PUk z8QgDBf*+MZx)%OHt?e;Kaf^o`e~>I1y)${=`j%J0wUtVkvu1gofzxOQ|K&UU+8 zQMqZi^UzQ7eu#Pl(791>{1AQ5B483Yn|_y?*d-6c1!dHg6bVbv{+aDK@kgof;Y6ubuqIx%d!|(J*Ivu+{IVna&{< zR!$n3E{=xPa6F+#D=-Z58$x!FodZkJ+PqBi`J9m+HXkfkjsPJZblmx`u@+|;x(SHi z=LN>cV8_^Zz7HlOypGp(2U0?sD{Md=8Xs!azwdq7=Cha_-Kb6lbqg8lwPP; z>04TQQkQ(>uRre9Sod~+zKixcZJyo=o(^jfx$Io90cUjZ&qn^(K6`wvP2E_88J+S0saGpKiBZW_)OHyM0!N%vjz(*S{qd=I^k zm{%b%AK?FW0MUkf9WSOFNzmHZ21q_*F--3cvH;f&DE*i^ekEB7Mn4$Ok+Go%WbIyg zd9vrT`qFk$dDUl;+tvI(kGQ^53XL@Zu_maXcZ*x?gTBig2G(mZ2}L-AU6=$r>G;xK zUS-4ay}t|^9CCQ3!?PIf))*>@E3WXdQHDjyAX9UdcfDe{_u$4x;NDXMJL(HPs_m?q z-nlVSZf(KSO2ffIQ+a!>`~wd;PpOh|z>jnj{~iv{Auq%Q|3^a0z^)_5LGHY%jKnCW z_vDOOOgeK_F_H$p>Cm>olCAZF2gSj(wxP~RuO6w$x7&-OH+2czmi&Zc9$9A`K*X$X zu|F|ykY7>S$*^Oi#{P?c@QyaDq@m}bPwEfqRm$VMY&YMk5>;K)JLz{k7Z;DMOW_h_ z+QT_IKUBfv7PlSeWk6G`MoDPo$X95YCV&D7F7ucntnR#E3XY=fo3Db5bIk6T3Huf! zGTV;)5Lc7^KyiygRK<}*!wo*SpUP7Is!IWBDpy1SX~Ic>R@pXS zubHln#i~6tzuVeT&*$aKDiw<^G}YReRJ2#>j4xhKRo%RZJ9}cHnuo?^L(FTd+(+It zK>DCy$TJQe*ziY&Yn;QC1GO{za5|YQ%mN--w_*jbgb6J%L7@GNNn!8pp8_Y^E4cnl zCG*d*fkov>Ty2p42uRl-C&K4aey7eK(dDHIor zXJ!#wdPff|c?~~i3B3L_YM)M8dAmH{b4%qn{cT8v8SVGTOZ1(&ax={$?)U#b`CY`Y zLC-lMoX|-GWQYlSh$wOpC3)+d{EosX>Gq^cPp~6~yHlJV@wWNOC3(v9_+oT5`F*u9 z3ZRmviUIuJYvQpW3Qq#>hwB9XzyZNVR~2^;;4~Ao2lH*QrLk_$z#2UwGwmuJUEdDa zw~!RZcn_m*EwF!>75ijd(hL?W7cq{r*noR7;3C)!fyfG0U0o+f3TASs^Ef1VTiFtP-RV78Uk zu=tibY?8@L{{wPyuT$aW4|48u%*z*|ID5go{-zp^)jUD^tJP*akdbxk zi+Zo{z5htY;ggx4@qcbrKRexL`69yMe8*lAErA@}VC2lTo~0W=KhHYzoI?ZomV5pi zS87WDajXma`#d5dOl5#UqgR0DU$=cUEajJ-Zd@F6TLH;FFkB=>Pd$d z%|YuUw7KU0$2SuT4#t1*&6J{N&-|~-`(9oJ*$*8D1X?>jRz|WTc#pWkp>(w9{1C?= z^zmQ6y4!ryfG^|kgS*1roehZ{DmMv2px1H`OU;6oRbdWj-*)M_jq_PogM2S2y(-n~h!Z0O z;wKTb8F)Nb*sZM)Q34jJ1(&6~<8}4hKt>HirnxBgO5JmF2Z@Qftw*272&>wy9K)sK zm}iKvPGJskcuz33OKFSCWM!wT%!;Pb=`VMgy@zY>Hg;*0IZZ1a)OOd`=(5=vaJoBB z2NLdW0Vo+uaNI*c(Q0YUv^NKWoqiFywT}=r(1uIELEjovVF&GaGkdo%~m0t`aHNZ9>lErBTigXBM`f6eJFW2rBIJ=Et zFUPJ^K-~C|_7AEj7)v+Ffi+fn3NsyK?hXJz^7i;62G~QLu`Kxxt^^0k`ZGyEB(DLV zp%2bM>JXM%NLdaif(jq=-Z$*uP~0aIssDl7UrFIAKLGb%_r5W?Z-(m85pr@m-_`LM z*Z9%x8TqBs69Z``LZQ(u z(dS!Vy1%Wx>l!E;sxD~TIdw4v4j|#dT1NnzXpMkZ-?Rq`7UN@Q!Mb$ax2}D^wYW|X z?e}~)`C$5KQ;5Uygz|`)kei41$X=e}BwE6<5wSw6ov@$!5%(?5~PFpNs>(?x0f>=NN0Uii!OC(qndFR`I=c^pu@-(;uY7wlXwme zM-Br19>hmYMVQG`s-OWEjvSK{5?Q^iYMq>I2uVgiBb_!;e)PQ(H0i3Q) z^mU@xgcHcuGitnNP!K7rQFe%HA4!|NY={ml;V+?wNY@n83$8Buy{}EM+40Ng_(7M_ zFpYZ`-FZ=<#G?xXvyuQN=J^1!4l|cht1ffA z_n96Y2-6Rp6yLAF;*itkjFP$q4HWRSup_I{k)Q&dd4L}4%YFf%% zn@O@Xeg^wPTZ50`BVF>k>2oV#zVV{HqPn+_3kny%JAX)1mg@j;>Jz+b?5{;c3!*{5 zII!kW$h;N_WC%aj8Pg^W86Y&|&jF%cEt5+bxYI1BDf7ok2ubgpZ@r`b(YY@&w=mai zq{J7)UW{l~UVj7}g9po)a~f)L!iGaxQmH}t}gYjg6z+>kTRBO2T}_`Mi&sh(9!(JFdAkPStS+aFqs~fLJQwI zdq#~V{pC@})hBc2SsyA}{k5n3#9pnc#mOlhPizw8Y$@H?Xb0ZP2w2peAmU1RLum0z zS@`ylYp=YGrF^fliEb!3=W#6j^)0u$Wc;JL29ZjYx?BSdJN$3f8})cU@jJe2KY^(4 zIWk=&97u%n6g59ObG1##$noL%aIONlF#jwYe)1Z~{i` zZ_1(m&=*eKqqbpxY!&+tHP3l!C61*TH;FI)tJk>uL^LasECMr^+B!_HxaeW3L!{}>OlQtz{L zsu~a8`FZ1Lula4yiH3L5bXTQ|mvX;;6b$1)X>>!7nBBF2frE3{n; z$ci=PoQrkLaQU_(6n;VdUi)yrs*_~Ofp1;9YH{60FGNaih+AXEr={UUV2LnL(*be_ z7SIV{u>K@j?moC8npWvC2_TVY;3ZckYtOB5i=aKjFK%|piEWfCQ@^1Wx9Jit0m~dC z*QW54C~au2;R*)XXN{TglZMyb%ntAj>B!JJ&MfgdIgpfXwFiEDi3C?r3+~eWm^*5! zG)K#mLU+q9F@~ZeoZ7p}EZ?a=6>(@IYyznWpcm+(U%$56T2Eo5u*|(D>=qp>-nn&{ zvQi4{45oArOYc`_mTvfp(&FPhAT^PIf(BRKn{yNy#@3l;VS{laHtZWrMR~)V6~UpX z>tOzg!KfTeexxgD=~7dCdse^E`m+D``GyB#Z$3fyA?g5nY@Xf30^p#v9yXp;$Jjs* zmw+AH333CEhpGWhv}7>KF(5>p=_q-0;p@u_3CFX#TespnlE5|x6ju)^wG5&WB1WMXtrGAwHJqm_=Im4vV0%yvgqI+aN*51%SjaKx<1nb*-Yfs z8Z_$u+C}&VRS!4!VECXwHGByq5PU6vsFJbd7{C{NSd zx?qSkwq3)=y0qjipETDhRt_xqJ%mEsXr=uNC&Lxvc*BU3N6T8m>5-NSxav21$klAu zXZ}7TM*!~WZn^1flZYGNL^K{hx&M;(-x6v6kBip3H<;juxl$Om={jUcl5NadU5KXl z%2DuFdOPF~cvZbC_b%%@*(=2w-)~p|OOzJLu6Dc=eKL)f+de!JGL41xfhf2VE0PUt zTY`J6FRzrqk1ymXv=wcMzff=We7mRow?#un<(cD~g?_yHGVQ>5A;m65w}50UQJYLH z=VVhxWvJS6RHFDij{A)CjR>3GLd2+W1=vyra2=4z_Gah7{WL~QyZKTIrp0_m7WHa* z;gPz2^|v+kMf9s-m9+dt`H+*j;>TQf+iN^F-hlq&OQ~IDJp6PL_(&Oj1G{@{@lWpFowQ4y zZ7H=l`A~6VL)=N;2Qu{wG&)INVqBUeW`KNw!oV=6xoUI0YBK=3T^HS)Gj@*_oq8EF`lv&O za)DKK?+NZ?2%=Fxpl=WwR^^irIF)==PC;u%P)?9v92n%@U!<8r1SHD!PzX8+1A17v z5r~Fu=RwUSTX88D@lWXeEnb-g#;1ebaC6&|ri-?DTs|2(py$gF7^mh5aYO%4?k*49 zQW6*oXh)AOA;MT@Bs&5AONSWbZS&Vmmd)F%zP<>@helbvH=Dj#%BN-@E_VeLcK-8^ z#%eHJs|&}#z$XY;^2mr4*JHTR2>*v&%UbZNON|nI-;ij>a+=60Q>W@(9BWbSkP3d_ z?ry)qE+XQP;Ht1Kw4K5{fP=L<81DhnNr%)#7Oi6#Pb&F1u*x$RVomA$G+Rql-VKCS zy)c-`(!d$ezUyY{>Q>2l+M9J^4?;~NxVl~jM9YD-V+daw`C%SCd^2Diw36ztx?F7B zBHu7dm|diL`sba9&wqU0n(xwfYt)wN4ZPjMK$#4rn6QTXI@LxGTMrkfEDhg6CxI&D zl6VWonx2_@kc_rw|_x(WJU|^LvU}aIhK*c zGS_`C`gAeW7gEA?N*p}feYoMVH~$tPmDNq;HzZgS)9$Q9_=b5Y(*)&UB$w#*EKDXr zHn6DueI)-z^@2rux3=HK#3X_5_qW_FMbErPRGxqnTngAGn`1$m+I-6!WKK?mS5T82 zmU;u^PEF?IbD%!;2HWAB(M7`NOq$euuFI;5-7$UPa!+(k(U;w?^61ie4*YJ$)p_lk z*dl7p+ioqNd5a!xp<~hr;)EWo1fmNY1wIeb7F(Q$h)kPs*rcBg99#mPFn-O0LFAl>y~+%GWSFqbX=4(SnC!X5>9EBmy+yqz zFgAXSYCWu=t&ppS_w^C}+t-?d+t4?=!C_Ya$Itp0mE~2DCV)(D*hyXwDH3{IdN(5j*I&oC3?Tjju*y<<-$b9PU z(^j6X$c}pjt_~0Vww-w|>+4xL{PT#P=fIRQ%4!IFWj|Qp3aB~&=%hyA6WJemw;QeC z9qkxJTY-+WWn8L9birr8n9bteiNWkEk!joU^aBUuJ!MS|WJCWJ((1T!SkCp+ARV)> zgPTBtgL%n(6D*G2dKwOK8w^_!sx(FK9BZ*E-W=grXJOM>ao?9s0y$CQZEzAF;A(GB zm?uc7ATFo|1@X2b!Q_SwCM9=im1-^K)9uAlJG*>BBh}2}hd<|*<-ca!iYq-U0_V_r z*X2Ba!=96$)#%t8b|g>4#Rcu9Hc1#9k3|u}>E*JUpxgq`~F{1k01e*6z`C=ONYeuzBbyoJ&ga49NUy5pAeU%KD*^fK<<_M=Aw`DV~Chzvx)Ij|RIf`W#K zQ#h!?nXGZZ@oo~aR4NC*#1?@hrwm20#wdK5WIogW?PUAGYaQ&o;D@$x@%Q#$l_%Xk zdFbnOY+HmPqzx(;|F^ggHw1ANUI9F1AP*1(v0+brSHfzf; zJ~LMlk#kH}Q4GKh78MiI5cR-Dh2hKOt#I~?OU?WKr6p?Z+pLErcBjV_^Dpx0>fFRP zp^)}K(u^8{7T~>yLYz}Db=XA6KRg+$WC^-}LbH@D3ta00E4e|9mw7Rqdllc69sH8= zr#4z#7Aa$#de!bkup3=xD5?FBHQ-T8sab7Z$&-_w^qHKnrp0%f8K3=a@l|lu%{F!z zEYCFVQ0$G~X#a^+zjtNiq7w>B#cW^ClVSNg|SyeEDIhZ>qIKAztyVefY9N8|1P zo{{2zWANDc|G}Tp)GgquV%SZvW;-O_;21*fO=%P#G+~@ye)+oZa)_`&RhYh+SHDgLT1c%ocBD?IlJBJyd{%04%at5?0>kHnl%X$7V4x5C7 z&jFG+By*KcL=+f~0CE5-A)8GA&Cs_Jt{bR)UVjF$(vdj;siAowvLWp6poF*&2FEU$ z0|1PZQc#N+VGOezK$Ze0ixQ0Eo+sBo*ZS}8_!3bZ=rp^)`zzaM#0;F+3aBa1xp`H7 zGKW&DuP&75ZSr@|_qpqyzRlj{PA~oo({Iri3!{JGR|!hwuiS*>)R6=?gnii1kwd_(FbQsd>=vbjnBt(y4g^ zyZku_cfg|;Vme^|+J-L-g2J_AGiJ@dn)Cp`pWi{*m~EC=rX;m41CW!vZvQ=hn2AeT z5sP-K?a!5`hs0O}8K@sheEsa^3tugN7Y8|&NZjY8Oe_^V3#>Ik$%t|!p^(p5{$-xX zX}joe7fiA=JsRHERvYa~-aC`ynxvWU7F?Ynb$CHk=A#Nb zj=-4~mwy&Pl^x*$3rK?)gc}OjU3E~vzRBr;zd^d*dScy)R1(_{_^<4JPS}qZL{d(S zdeDC*{#dd!9JhQr+0IjW)+zJyU6d){W^|t>GhG?a%Zox9X?`zhDI(s}a`gGpe4pU;+EM37W4!O2yGW(=^V>0w4m<&> zZ~sI7W~cVbZ$>%u0q|au_|Z2+Oc2iPyPZv(~D3AB$aq}3$+bV$yGVdC`!9ck|EIsv#ZLh;g#owf z97~7o&bbSbSsGS=$V(v7b3M=Pu8KWPp$zS~tC#sS?CI3GBWZ5mUT=S3s7bkyq-@rOEM6B~<~#RdP_yVv^vL{3uS4yUO46_GYfp|WbMHCS zX`jr+2CImJrJ8+;{c*^Ii$vX92Dp-G7_V$V!9sk>k)5h5EV>o*KBKPPMrwU`8eR>v zO|Ltz9--GvaUOJB-=DevLVcuxr5Fg}@;3AFh~ge=BZg6^;1_`T3M@Sh`9j66@YBN% zL+v(+u+2lUd{fcv>nDj;jl%`07sRR^b^A1K6vP@uRUW82x_gby&$1)6ekIouV?l+i zsvQcf*yVA0+i*)mQ27pW>~MF^x%}V~O9}HHFF(a$+ZfSzi|>?=Zr4Ab`$_H6c@gj` zF#l#vo&<3;`f1#)i2`ipYLZ@(Q5$wk08RplbzrE*YE>^;Swnf)_uF&%o@=sC!{cj2 z-5-n}Qm{bV)qkUKDlU;)rzth06*<;bwlQEZy#YiqN1dhjci4!Jhg=?l{rgNNd69}1 z9mn>%Oq|6{Y8NH@T+_Jaen_lsf2z|R;%V|xIAVZez4ishE_T3B5p{LkgLx59|KUCz zrx2oMaEJ3uismQ?tvbo`>nrzSn+4mrGW$8bU(Q|>>&p(4)kM1OaZ zcyh{@SLlj9_Sx3G1}Jv)ahk<%09J0CaGFE?Z6raR)EJ8$Db12FBo;x$gjtza&e(qLx@=^#jso?yh zxQ%hA?!d?mh{FjpbW#3ts8sAgqDl$6jNs*PhRotk#Ya0HEO|`}ziPPkV6&jPo~@2W zY_YqFs1IP<5JN$ff0E@hMQU|w-pY1@oapk5w5OsxU2>(XjAo7R_#@o)KxmJCDP8SR zz1i9&kBmcRZ@O$SjxW|noTA2{o}WvJ%Nglz^&iitZj8O1YnzUWE!jwT9Vna5S9r2I z#d+eXzw2VCMXPV>v&gHucgO{#**he9pg*)auMjcw#aDXEHa`;1xN`$h{3GF2cr{Mea{KASFJqkIPVQIX%0WK^EDL0i_WeTn=UC1m zGbkU2rH|-R$J!bxTfApBR{QGobl>!kJ=|1(E6cLB(j#VHtJi?_R-T8(qSunik@)`} z|17Tk2QOJ9LwX&?dtrc0qEAx8v*Me)=3IS$^b2X#pIr#D7T2h<4ydq{uS|;dHkZ?K zs&qXT8lRtsOFQBucm3N{ZgTZk?%&B^JtryWMJA=&N%T{b z+7rs%_^Q=-wk}nBQ;3IMqSJ`6ZGL~zWtSc&rN~_l0S?)TmmWL>b@S9Nf;K8R2_g9$ zr{35QuB0=_t2rb8@vN=`(@Aswk{^-S_8dwK;oyzp$+P2UEsk&5(XhvyN2k(2Mr>F{ z_0K%SlCk|!fY8DsyQI7Q6K+bkiSQ6#lSH?V5DpjbC@lEXe|uHKcbjUu%5a}J)UqB{ z=_+Clp5Q`fO1FiD$@qTi`V?(nv51k>1mN9UGTminoI?Au0@)r<3DV6nVe`>Hj2b(Y zdrI}?d!4)4HRkV6@(o~YyCS?Izga@uaIbA@&Y$N^Fnb}!x)kD%^8aFOML?#8fsfK| zGV_`Z-=>viI{Zz1t9DUl2q27muO78<&pG(r_j>%x$o*V0+(;wtMgPi+f~TAm;_gd| zB-(l~dzo^PH2V(!!H8Ln*s?AXj&Yt6H3=ePY*i{>v#e`~gIxKRg~|^$fslU7x=$4B z>3#MsQMe@QuJFSKp|lgmsg&8DU{oe^bxab%vnhnP{lEfJIW#HKB z>&ZI08A_4+o&jNt2&kW~EHGS{pAlCoM}&Eb@s7;I$u2a23@S}TXlj9(SzBATw_h*K z+Sohsw&|_E(<8Xbhp>|~sE9PDyjhGm2IC}$fpqH~NjRS#dpnTzKzy2nU1*4&E)2fr zquy@3?5jPSpywr4tK_il(2n?hXK)u%5aTFjEAj|@x+>Lw#ges;t)JC8y=$!ON7y)l z>&c>2&V$lA9gRJ`Nx{W7Tb{jMbk=$1bR*VoKc)s;RWh*+S3~YBpJ=Dnt(el&XNeSA zKc8WHy|L!CK+$INI z1V5Cve$FXXj$I<-;Jgn*8JB?krlu_8r9E#R{L#>L*TYG+?|F?;<7?XWf?M0WugL`R zteo#jPxv#`d>Pii0d~=J1sF2PFZvU3KHFybjjf9p)?5}QHm85#qo$$rcdqKUU)poa z<#TxU^-Y6!zG0)t3|R_WQW=foZNaj4LX*&CfcM95eEYAieq;xb7HpZ7?hFr`ok^x# z!ejfMAz>$dtQwQHv}I+D-bjtCG`+@SEniWY;qHD=#)DUhWQC>sOS7fTX}~?p-G&;` zv$}D&ZyLrLUz=5}dD--eo@1q$5$+x^RBE)nKF{fKwygAK!O9mCEzggq4CP=@0Qx}f z>;!f)u@kf_Y@+gZXEEIvnTZ96c9(;R3PZ7s<}Y$~VRW=sD!uD_UG_EE*c%!4N!NRO z_jOtPQARhv=0s4g;9vyHDwml^7lE6Q=D!88>YSDwE(<8LYWrh65i`D`bmnvNrCVDc zowrcF%5{rp%RAhYn`}Q=LXeN@nNa{LF>*~fc{zopLD)>sEP1w&qamA;@UuTos?JgB zJExGlRk*+*_@vItUs-9#_G?I_WO*E0j^2KBux8RzRlLA4 zh-qu4R+22kYo~KyY;9$|+_XfY+x3KBe>(?BM-{21r%%VIe$ux31?UoZeTmqkqjJNf zNUdZ3y5z+E4Qc$xxV_LMiu-Cxo*d`dI?$a99YRG0%d1z^yQjs6e{v4A+wn6KJ^faO z0+Oxo^t>{S2ujo0r^hul#s#!$#thZYnQ!4KhcXYfD2tVSZK3_|>{=ta9cs*Il zf_5qO`sS^cJlFXp*MXBO;9Z%Vs#2)+5;sBvpnrU<^PC)=+}k&SohF?vAU|ml5Ly}ja4PFw$!%~*@C(iexkdk)>PZ^$U(pba(OYh z&FhhNmL5TYLVxDH7c1(7_*jnk>ht?Th1sRgq)_i#Po0rf*HT{oD-7uvWrMMDDZ)KV ziX!mRcFkFv9;H8daDiB1`ReGvq`6!~d-2C2-dBNvDfC}mIUXG;n6D%;J(o~w)!2_P z7?+nTv{f@Rwo)sJR@uZAu}{Tg-_F`y++QGER4Zzsfbx1jdrSJ9VT<-{=f8r{oeX@0 zf2p3;ZV$2eZ(B`HfBd-N+O51#j*OR2(N??5@C1a*h(jcC;szwWZCY%AAQE0MHG_~Xb8SNX zzU6I7*yr}ZzVP?Oi=D@$@8tRXv4YBpIvtOS?Bv9ey#13+4j`7SBLrzgw+s@QoSo6Z zj~4r=J^b2!zJBxdYxz?=$NMYpVgDAobbc;@^R>JehwKYO1xFa_DF5c^O$eTaV=E1? zP3Qc#PWC;mEu542oOQfvm`d7x*UZ;g*V5>vjM0eLY4YE-65PMR8UI(Yi}cUZSs%%X zLiZb6?quXMAr*-{&elWp8PM9%eu(7k*FSiC$s*^Fdu_sZH0-|JA(Q32vrk_hc>4OF zIwR`;C>w12SJ~hvyon=mdIHaupfF|W_?4(4M4TjSP21(C^!|rak(0wTb2PZNUTohy zRi|TK5YuV#k?ZwCkw^b5DXHOn$;eLJ%1^ovz*GK#iDLOx z^0#d{4w<{R=kbjzI4RihD(_PgHM_~n7Uy11Cj!}r-~j+>@v@qe$@VRD3UQ+yOQcx` z+cU5+-_|oalQvW^H(u6h&R(;E?=(-9?k?`9*lPe0pojLaWKeMvrk$t==RWT8YK+4) z-(XF4sU9A&33zpE)js=zzvkx$EBbsF?7o%K^iNfs*&WKKb(4bZoM9_)Sk9moSX&3l z==d*YDd$160;x@NZdKd~xvoAF+Yrck^5_C7?3%)>vR0Nmo6#fic2VUoW#5a79YtA6 zysB{h-a=bvYA79)$*k5HX!_6GI=7wScsmdqn@PmF%m{g1pLqkJSzwq^x5i0z^()-tc218Fi|Ji^j@hrmyYYgcaA+ zh_X;%ftS!vr~FaEVo3D+kh1;u2iWAiu~q_NHdo)gm%VG)Sa%L+ zW}_39h7tEmuFAI6?}aCwE&b)!tn_q>tFm__O^wYbn~|pVY>Dyz4~aIe|CDO`pZu4X z&I_Ny`x7#`|4wFv2T^*@G4`YMb{x}a31+eE0pyLb#Y>S|LEx)seMBPCj@TxRQ#4UT zt`DDL#Ksw$`cwOPyPqEI_u9BTXuUVj-A+)=)L%FH^~Y96<^9-qn$WkBe*+8D^r|`U zyY^)I$y&=N9xgs4KF{W}htiH{E$IX$N2&b1*%AiA4H8fz9Bl^$(c##>B!@ zydJ5;3L48Ie{c#e0ACdrfms*equKJ9lH8BP1M$r*XadcZkf_@LqXYak4DlJZG&h>k z?EsFz)O4K_1cP>bEuX7Gx5FDm&ccM!;tXQpNrDNbM$lsh!3Gc73|rD>mU4%^g=N0L zw3!1}GbVN99~hTmP2;#g7r95fB0Ua&FKAu)>?jz$&w(r07lSR51ldz*UDTGFWkwp; zc>fQ~57%%fJk<4DZ{X%dan+8Tv_2@Tn+@09wETyg)9FvX|9`j0i;u>bx~&pBHJ;S zI6|rTwLHkWMo`I`bzE_!M_K)I04(x%K%d3y7{Ox8?_Xj(opwYX+B!_|D=0|YwnO24 z1c^U`V?k=YKl43)RUAB$FVydTOcpv0kQlncjkS0bGOM*iB4zW4V7~jzg*vM_qx^fcx5EU`T>bBeUJ`XYr8|#9YC}j+ctV&`quM>?;n;9m=s;9a+!8H zUZ|3A^t`G1Zl$OtIKv%!fE|ECy=O%gx}}^OFmGSX{XGm7x&yy;BK^?scn1}x6Jld? z!JXy5)z*!pBIXwytyd&}O2wCDNtP1Ya*qE!QIYr>%dJ<=v*6tPP&@F<^5wixzs~|< z&X(U)(i{a|3KYJ5{&`~4=l)dox>;`~wL;|9pV;uFo0LSZ@QW@9DeuT4d5`H^g}d2R zNN33%sqT|3>%A75l=O~jY9tk;0lR}s??+TtfqRVU1!fpQ_eaSZODix>EQwRGo$u)wC@l`We77h~NFPPm!bT|e|i{|BBec8VDf zZ|GYyGm-qO9a!{MqxXhr(=Hc2YH}Dxaz;5?+GSCjG6fBr>+kcn2=yqAx}OW?is1Uw zV@3db`zOsOG%H&7Q)Y9>5~U^r01uPUmuA?1CNn3Kq}ye!ytjA_)EZX>h{fI=;(jUF zd@(<0Lxv`8S`0q68ctFYs>wK)O+C!e?K<%Bov!&PDo(O)4>;`Kec5 zU%g)0F`IFbYoY_2MWM;staDRvWP-qZM-yQs-@NDg4&!rB65F+gK#$2i&!3~WFv8M4 z>$tqe+Rql{Kxx(s%SG|P$4ms@W-C8BI=GOdbw8={V6W47Hn^c)E4}32WlNQG*MAW$ zO47&J^7-(dM-hlb&%3GvLx+RpFU|FVPItNs^;fIE1VUjZjRY8JbB%uu=ItB~$yN^I zKL6Uz^NS^ylh{at%xhm3bK}JiO4!!u|4tjUG(u)(!(yuc`Wt<%sAimJMvl7%j2N;U zxMd5?od81W=>u0p& znkrl36e{qOB@AY#rNPEQmmAMeqxIV{^inwKRL=|zzZo2CK=9&J<-G6+qIcz5;X z#3T{A5phlH1~(3xp@C=uXpjkF+GaPsuafJIN6 zT83+HR(LUX4lf=OIGdw+!l($N@oW2MOVvR4v&vPK5rvL2N}=I;a9-&cqp7mn7{cG6JJXFK&lE#owHiq+^%uJubvQT>lgj6FW@#G3bUD8T)Aj9;}M<2%cjSxL(oW zoBq+H*&?g}jZxH5W_aYxe0sY>-FMeB&Y3e3(kX7&MEcgGj;7yGt5EjzBYJWE1{Mh% zqI0>-^mS)S#S7}&H7oKdfdu}S%4qotL^?Q(Et)>dradZJ!P^46|iD~Lux%>!skpNa*9k;%`QA%%(8X)YB1p}*x2Vf z1NWv`^ybU<$HaA`YEJR|T^WN+qTAQNehf(i^9(qUe3m2xh;R-)ZVF}+IL)L>{S*Go z)3MB9#Ga9UoJ^fz2GrfmtP4BP%1NKsTA(&L?61qwrXBl|p{(}Src;*8R)DC1$9FhT zJ!xMEtJ`q{f8O3}zO$h@`-9R9w@vF5ZRJ%E#t{D=#Dnod7ihMc@$QxToS1!d)&S=^O!WZx3X@52&ihmPRDu_S^@ z#m^+ujz6+WW83I?wFZQCvt`W^FBk^d!DU-PJ&YBinss`Mnq(RcTiGMe z=-4Fr`s$saNSRv^B6H6u&#{QUG#CEQzu`j}BvRyU~C6D3rZ=yvyvCEor3E&23w|`flF8 z2Y3s>X1Ulh=m7TnIb&bI8U(>rRA}dz4>N$I`5h2C!N9i-^xzgg4^hgn6TB#&C2Uz1 z=c$R-66A6FiCwZ%6*P+IN5Le2`zXivSBT>V15X?ihCc7=O}bOH@^RT}uZHCTY~?Su z7_6NyYb(LE3RzLV+pQBEP1gSor8y%>w|yETY*@HCND7LIXIwhlZ1WE>1o-pHPYC+uLzQQfMah z7Wck=zn=`Fo1s=5X)xv5=}ovX2g0BmLMsv3lkw?yDrQ;~zH;ZlCcnkai#ywJGBxFZ zNbRRA>I98WP7TGgA5nb*Yr(^0sCEG_Z6^pBsWL&@zM##_h|2`_RA_+l# z)hWPc2mW*?6oIepg{Ek%khp?UD>tA3J#!qcfTNjo-X`F{1i)NRpe?9&i zAoi;8pk=_?vm6-$7w2E289#OafAwF64sVawbexZRt+k8m$0@g*-8bfsmnAp4?0ana zD30c2e`EN@#Rvtny?(wAA`9EZZ(?ZVe|?jGP9&$OTuU0-XS>)f(O)w`>hC4}`tXo{ zSAIG0-K=Z;md}$=5hZxmmuDbR-3}yQ&rr%U0d^;t?i8$Ry@>?;R?GGnzpMW)ktKx5QKHiG*Dg3w_^BNM) z!lLhpn>pFd(g(-szasT^1@!4yA0g>@B=7BM48CZo`l4U{B=1AVtt@lWnQ>x7h%RN8DFZ_t5p0%nI-DqmcX)^Pz znMzNnOvb%yjTgTtsLue#7QCT1DF!!7#@3*TEEj@3r^K=Q*R*)DWs_#1C#vA9vrk*m zAg<3yzvGF$SYt$#=bJb0Fb_pu`>(t9Q{{|)be_yPrk8_kpghGQ8ZgzmM8zVzQ&#+M zu+Fn}IYkueIUiJcc!kP%kXiPyw#YyHU|Ewv?OF>DLx!&|T5L?C)Xjm*^7@PxWQ0PI zBeRUlS%%5D*~hYhpA&L!YnDuSox_A^71eBR_BK_XKG40*y45K)LPfj)E^A0p=qG^x ziQMAPQdX1(Z*N@_rhlHGfn1~FB4-2%hV{5`meWK!%m)J{)PCkf`~ne zx}0>Pmg`I_byd#m$k6xIKe5VHZqN$2fNr zMK}k2BoPLFw3{t$iV82&9oIQjyQW2vaq2%XDLQNu1|_EY(PLNqw3o-pO3Kjfs($%n zTpGvrr+Iz4u)SH}_1m{sqlp_KpvMwOVBFi_?nmW9Nxw}Z)YkuyvCBk~^kJ0p=bm>U z;)EZ!=>$p(?DEtcR3MVca0c%UHMx|ZLGXu%?|nU|k~eD=tXPmG?wa`KP+PrEc0o{_ zuWNZtP{iKf5ORr{An@*KX7FfigYQs|49DpHM?7Emf5!7|$5Dt-W@DfvgXe+J!dRKc z8b!JdlbhV;LLa7_8Ku22a7dkS$=YKo`7-SYpU!Kio;0f<&_>11d9b%iKM6^MK)#%? zkyHN&u-TOOje z-L>oF0v9+CLt9Jf(>Wv?5Jc2Q^xdF}V5>dBa^O5CN}(fGv%*%LSC69V!}=*A21Syc zA8l23>uvPgzL#6VbL(bm#_O&V#8AW^!v3RgijZIC&Ff>C8@Vb*nv6C-+Op()Ic3(P zrlGM;NU&3)KU68{{-sU3KD@hjNFJEe`~HF1>Bxdg!iT3%;fAX-EpK~^wo_&+NOaSb zXgvLN|9ET*Wn*=J{1|c5n2EHQ^sSoTzI`iA2G29;9}nN)3ee?EnAUu*;B-!ZznYWF z8V*$oXFyRvB@tE;@tucKSi_jO22fraEooD+qK%3CGgH^ISMbzE8{S%$ef4;4fR)Bq$$xg|GEuZ*TV+ zFL>rCEc@IIr=(eMFoPS^f zs$Aes&CFl`^)MN=BeRafEfvm4LG!*X{n-ab0WoQ7T^=k;|GXy)c z4j3&QP+pkO^%~8Ts?6~E)phw^kFvxe{__%>v-g)v!7CC;tq&EkoO03?tSoUDg#Q3e z*-1Z`l@gek6nvJB)=`5;gl^WnAF>YC(ADfUyO3tCrF{Ol)Ok6-k&_}{wzQDG8-JxR zr69=}ELop%r-${kAK0{tuv8~v(JXmlTP(0RM;6*Yt>^#QfT{pwu35+g$7lh(hD*NLPKk!Bne-BLrqUkEo**kpVy zfWIgo{Hfj0on;=TXDuhv>ztb3SN97|z8&_ynfTeI=jFWDx1CB?Z_THEP@*2j1UwY+ zTMrl>-~T1Ws2x`|Gl`CPq#6n0fv>v<&p?afEN*rU|JzD6ma?4i6)LsYM%2GMP)f<` zLXUFUR2-sS-dQe895^QiK1LO6PG&(?&9kAf^V^+?j6x<^KCMR}?)IQeq2FG8m3 zv&mf8&7dluOcIq^+}RBKmGE^1lS-IRq*gzh*L`j0AIH13Rf)5UTNf38lc^SlLd?a~ z{|aPl75L~3P8x9pOFxPmUs?}EBJI9Hu3dN&5Hn#8uh^3o|pB1 z73UdveV$QtoWHxGW^ciU!0|sF?)HB<+yf%W-W1d?5&cBw^pCCnT<09)S+_zpLwHmvjQ|xG(H{Q))LS%L)sOTtC@e|N1`o zS6I4RFgpcXox+JEs6&M)(3ve=KAnQ>ZA8seCd|=R6SxHj8_F=GBN_kWT_{)9wUA$( zGXj*0<#(_W061xWP#)qBleYTSTTUkH1=Tdx4g2kkIr&_m!b!|Uh;QE~U?mYEsxv?> zF_@NQ)rA@`guj)SE*Y-jz5mn}2CRLYo;v>7(j(J+M?vZo2*3iv>EAi%5@6kLPyFhp zjJwhBEfhg-smvU5k_mR{z-Bn;x_sO6zIXYAOdm<;71fKVuvf**Zs{apO*FxPM8S>W zo3_!)Y}$fxbesNo!%DB4w@1^?Etn{#Esh2yz1_Mk$GD)PVymN_{k=&3O%Vr&QDGxw zis3<>VM{r-UZW+XdyEhx1Y>+CvT zX;B6NT5B&^)*V=eIu^^_ za`*ErTe?9>1p19)v8aG1WG@0K(6KL?17ytna{?f<+QW@blTX$&3h!mwE!xd^1vd~3 znAa-Pbz|z+8I z?5h{^hG@DO0T*TtHi+Fl%#(nAYZhF}DUU}tBQ_lkE7bbt_XOPwH$tzz<;rx@9GrTA z;nuKh!Q8-id$eN_B?!*G&^$Ucu`PlueuQ-d9eb}_>kB&b61726hm(bdd*{Tu@>TL| zM^Y=!0`943D=_6^{Qk2v2SG5D zJ61@oE9*-?$@W2TESyWo(#gcuU`27V>r474yOmn9t@QPlIvuDvomrBZXp9%wS{vb1 zh1lW;yV(lhmUt?mKyZZ=2be#cXaBgz`iM@7-il!RS_sK;kg#9yqqoEG!Hz8c{SiXd z8!cTIPmIc(H{6X&qR>Bq6Q*#K^J)F4_!j5{U`TMm(|xx+MM*^ zQJvHENg3DY#UgNbC(v6E#`c#_sLus&c^Lr%lmsnHQGg0~1JzJVv%H;O( zuH9g$(;QQp_rfiC26@V`0rmD@aX&dntfoAi-hUISn)+xlnPcwuK7I0`3{ZOFuaa6T zXV#Ch*7{Lq?}B6a4-A}Thpai*V_%Ll&OPuo{HAZU*W;Q_r<#1sq>AU;YcDNk@BB@t zfI#pq65Wv;Lk4|djN$T*IZiN3`DEIA9~1e3!loCm1U_sztRrM8CnOp!q9mWNUU3nI zuN%3W)M~jtn6mK*B;+4XvuM85Fw1i2^jl;*E&fbYhS?~t)yR$)vK=^y4WzN$C@PX#&8qr zwiq?VYPD?7RG0Ur{Edn;JQ9pI(#j0GP52m+-%`sw$LKR&vbDJZ0|4LdpjkXgIJzxA zRlz>vUL+q+K>4^ovfmBmg9X_rg*3uXONKu%(DE{`j}f1yDj zpw(lOGyPrv2q$U)NGc0MgO^-}m& zMqSYK_3cd-wbUEBus!KUX3#eK!zvNGxPSVfoBcRXah?jmE?>t|e`DBr@VG9+Q?>@)prt((Eh+QK7)tGf-L`X5k3i5N@gw zNfybPJ(phO6q02&plGQTD4OH`DzV31ZhQ4xmAKj=N#-7-kn_0BIO=QIRMH=F^G-5b z1qw?77|v&Kx(3a9gwy{NY#(OWL^msg;~6a_Y+-1H%eB4m7XM8iZyQShLha za=WKxAc2Y3GB$tgLlhXk)O<^+{yHH(p?L3!+oG?{^LM+uUj!y$`i(hkCw6>|mK@6{ zpDjw(7ZFBE+HXdl`hRE6YWRZP`+wUWIU^ew4%f=vGlH@O8S z5B*z#RH~BFwp1>c63bY;2~trpCqYANyg- zQbxyt!2TGl)TY~nqsUYd`^FfJ5qsOc9?oP;(}_C#!bH&D`bKhQF}o{X9Y1VbQII_& zqh4|EamGKj%eSpRGvt&CSSb?yE}5~Mhk%Au~>-e03x72{_u>#jHoWp?g4(Ge4KwO!eMKwn z3~lDs*zO#V8wOVmdc&4CsIggr<;bJVVaRp7vXmC?N`DNt>R#T# z_vW`sQJG034J%kM>AShiApfcJtGS2W*}Ee7<{SvWy)W_C^`!^X!bt$2pa<2l#eq47pZ zc1yNlGVX$vR?$|eSl%8l<@3ocC5BYozq!i}VN1yLOLzzm8Ws>LUD+8zgJ7!ghJK3? zy@MNQzVtJ$AA17Q--sF)-QS*aJyIfL>Q0)?>IhLAEGi$C=r8=F98-B1)nNN@o;6Fl zBHOj)OrI>|fFk^d2?gus?j7xKjPIth>T~<;?M|F2k$2-k2mS>sJa>(C5gF^m3X?AR zC%}lhSc!g2%|pVzF@4SLzm*+*e2O(xGk0&OS3BPDw0*EJ%7XwHWs!}F^2w(GZvjSV2 z?vY$0483fV7o*EfZjalw{}_}A*Txs0bBRCsNh+Bp5pXRQ|KPBoa!CjokZMujpZj?b z%4s@;9o!78ZMgQyoWY;Ma_!$;ozLF?Se`937+Kl%;cY{jjHtOYs|Xc_9!{hVsnrZ! z&lFA>V}WrZoRb>}qtwmf$c+onJ}x?*)2&E*YYlON7PCz*?5KP1JH<~H-O)$y%tz^{ zM-bjXCWm8tA|){NDX74(YBf0s+xhe>%2!_j*V>b_R;(ngksmCV*klzM!3HPS@#Zyg z9SeJIDmIoRW*kUQ@4ji?u^Mwuqj{jo^x>iy-gBZW`|6;L*EP9KmSSUf4@>@;gU=8H zu4Fh`1$ZdR%oaBv)(pP2dbXXOgk>7Hs9rtqVxTQ_}*16|f`X6c?E9D$Jjsofuy@)iTC~nU2@&sEk8D zbCOS2U&}zlm;ONWC@Pn-4j!ykU9n<`vN1N}RV{dW?jm`UM&Yor4V437$ghiwlQy-| zQmK-gLWZnNo32tXIZoJL+Kohkc#kRU4FnfM2TU{|0NjQ{PVxeMyx1JGol~SinH|+m z0V?zh$Y7ZQiE6(Zr-zSi=pEL`I-p42obkkFW0TGv)k`*khFOg}_1%xEr9LiS8+_{R zoLp7>(fdy)`&V%ADdRCmN1beyoMSt(r|zfq0_x_?PD+tB1l?>{WoA3#0lt5kjprmZ ztA4%d5lh_8_Q-V_4bbF0-FPWnQ|rf)pifvBO*GEK?b5r~hM^ZvJE~C!al-mIWLpti za@Fd+pWYjgu7d%S!m!ZkotcwuYgTd@D((u09Zx08i1U3q$vaM_U#5U7XvV_VO<%%u z>yZ?^MG;&FUgW2+xfA_su0fq)P(RR(#u{C+RZi-OG(G;Nx1iq7Dw z@GJdg#1UL|d5blk`YTqCv}xLN&;_X-S<%)P%zYAlZ?df?>!_H*yQ`aKH=Q_y-u{Xy z|3XGZ{E$J0LpRN_wLHSWh0TT zwR?xlR*a|x?^-DSgE?F7Fe9H<)J;Z|W7}5c32y7T(ZS1GYkc zkxP~n^sM_bw;i)I9$SNA)PJ`L$Dhh@Z%J^Hnn4W#9*GR83VDUg&BKG%s9fxQAdl>E z?Q^1XSeX9Mp_K@s;>u^uAB)c)Qn_&md-0!bat-1~JcRk^0IsIL1D8OSH4ZEWmK)jF zgjKtx)9;_mD65>(vpx3k{B{-3xZ;Ou&mu6~l3!=&$5Pf^6)(6ATj+Gn@Dl=yJq?}5 zIyHF@k4(3@KK}Jr^LyWwZ)NVMGSc=dUl(|>Z`+v*sb_AEt&rh?1z_JQ3C5hK>_BRt z2vd)da_5Rjyw-E5c8Vj0=9#P$l*mw7&rb>Vw&aW!wHBQhYAvE>!Fal5IkL^1EvdG) zUMh&{oDs!cN|8jQ+fiW#Go2P5+;L8SXuhI4Vc6zX%%a6ewqGd{bR2@-|V2>EW!z4GQP}e;z4^r3N5FFcNwlh~d^H;-J=2*(X{C&S~Uza(# z&(844r~QuZxHl^#JehvIdwDPG5OIb?|EtWL8oN9~{aRDLCB&yQ^PH-zNRGqYsk++8 z#r@}NWwu)~_PO6lrChMz@wSAd50Ym8?_`+>J&)GAGqOODMWLE^Q*JfyW_*5B9AeXY zGd57;TiasNj?6UFAS^hd$?y((6d&>jE2?3a zNCLjbzuk+yRlDPr>Cw6YW8G49^QWDP*__8a>g#W(d@M-1`FPI>rkIhWz+e^H`mcNO z{{qy0y?*%rW4-Xy3_YZ%RXZLp4dA&6n|Kr(;vIf`vW2u|8SXIgM9=Z*NgFA{(Ik@* zo(xN^#~L@){4O3kh3^smFBNA=)*!5)A?Y=P1up-CY8&N`p?!JlO*9%e`#L2OC*E|B zicN;rYq_wEFV%mG&EtgwQu%sUNR=gAHYez`hrdp$1J;J*R7kB$7B zVkCwn`D4AU=a2n2+$!o{AWIX*Crw3YzGU@2FMt` zvYRcT-i52ITP6d$)b59DT&CuM?{l_;O7DwCo}ZKX;uCrK?9qqEMWLk;kai*jyN!#0eN*bSL z%DK3BCce?TbL5#?F-SgOxYMjjp?Iz<<$9~P@&t!}3R<48k7~OaaHCbv{hXo8836u# z5v}~E( z5C%NR{oxRl_>o#Y{QNJ!1ZdwZ%d;ROM?HWpna+3b@92$p(=c-?Uy^-(_@lz$)uLe9 zGX>d2S+Szw#&OK1nnC?HxouTQlqq9p`jgJ0S2>eCJa#U|wp!!%?4Rd9>SQS2J%-Cw z3kz4M(lc1qoXX04KI?mgh4oaC`taeMFy^K< zThb8S{1`SjjlGN0jP7C!bF#Gwy0Gp11Y<^KA=S!7eMV$*@ziu*(Nv$0=5ktZ_9n+W zFOm+Owk}!Qk+NDl-S@+H{OJ06ozGVzNuc372QErFt!hrm#$seQoUtyOt*cP z1Lj`ZZskNf(SZS%ckB57&m=@cWMzJU-aExyD_i^2JGuqe)KVDFV+dl{WbN2Epw`EGnD@t2~FkpP`?E zAct@Skz_$OZjxVyI9es19VXZpIGMXQPgvS<`|N1)d2zS<=IcG7=MT@OytA3eMK{mG ztDksds|^!#XC2C@_)W+$VQ(K_!S;0I@}zC%5MN5BaZh}D!1GS4iWa&1(e@`XKe338 z_Te!s^C>b+3mkm@E*IOrWaRkXtSVH{2(Mgv>@VNV_D+#sRCe!cDesy|k=+a<8%fi9 zpz2xe^UBD!f^m>-SS!t35gPrj*&btV&Xsy=Hj+z%i^UID;R~>~V#;g*8Dee!{^%KQ zoL5i3a7cR_UX<-t6k@mdwx^x>=L++bxAfJC_AvPuPtJ6F-?aCI+QKWMgW3{j=NLl{ z-X>~5e@2|t>Oi{OXVl)HImPK9DG<~HRiqsH)JsLZM~C}&+w-43RL}~bp}B21#D5uwBygam85M$?ts_AJJD*(H-=z@7 zd`!>F4x>>3jTCP3Du4Urn1PEjwamed_37sa{X<^2hGrK!xP$p~G9a;&<3yvZ> zE1__F&Nf*yxX*wJCs_(HZ9gD9{YWLpJhuC`rXSks=kh78Pc7@-tvgkL`xl0q*N?h9 z30g1olh}zOR6r6=`uHhkMS6R)+5l0>(g1?MBG0>CRSSl@dD2T~+v+nc;vOAC=}JI1 zqF=l)5tCYMQ?NLy>pbP$JMeMx>`^<5OLq3D887T_@rZ`-yIQ z{VCc9pI^a_eLnw-z4wf2vR&6jgMf4*AiYEZMWrcK0SSmw%Oj_Q2~r3>ZQYdn@4LV?;kaEbKf3T1X_;oKpzqRfBs_P!HgCGv@5zfnJh{R;mOtj`JeZNce?sc;iTzS{5|6{6O}64Bm==&?9Py!M zvt=1TiHq6l2+YT;TSgD^+*AI{ zm#uvGqwrB48Wuf?02skXt3?Q=NbjkbQ1gx^$vK0`joQqKCUcE_{z?KzLO5@U)^ZBQ zH~;M4f%gJay<@Kb^Oxg%HdD?KEbtycSNPmwy@qEkm9fA7G3KTak4ek8?uQ8b(I?~H z6UrSKmMPD)))SR%v$UKVlow#JOO8L{!3S@CH~6+=t1^h;B^Kr#ZB65y@W)*x;kq5l z5ohnu$_?~ge*NNzg&s~Z`1=1{ljIR ztBdIbNSnbf=I>9s&a2A+CzpFCpdvFJS@#KmO-TS5x_JjHq^8#CN^6`wlYA!m;tR?1 znCmuXe-)qiKYMT2_kp`Q0p*9(R{HkvP*u=YgVh3k#R**lq%q{fd z$zv2=m9YwoF&?Ip<6Ug|VYgn!v{Lrc;K!9$mVo_gka-4VVYw^*hCTT(k6`N45zI5O zz1SWB0{I0(78?=@QiTJEY(g{P&M%f(Kbmu`9v?2tlE42Ex#xAOqhv$ny()(}lYPe| z&I7%P5YsQrr;eKd)j3PXNw46jR`A)smb*m9H+FYC@^JAhJ(G~ldVE#KaPer*9mX`L zl}x_d!}g5phd38;GT5FkSdf8==-+wHP5`l1!2?9EyM5nTvvE;{d3al(-AP%x%qAD=#-t#s}7sqrkH>R6iYg7el zejeQKhLTGS8o3v=r(fzQNB-Y1T;b!Q8Q-=SiDkKZs>&hdTF;N~i~mIY4o3eL>^A`L zQYFtlF8@x9aM%WiMWf}!`g-B$N6z_ma}l))Q<2B_{mKNVWEanTblS!78_kq-WDC2q z2v{!ybO@b}^ctQ%#ymOi4xfkG8VMca%}%<9x1tK`9>3HU)1rvm7oPv8V`zdn%sc^( zXeX9j&z7Zq%h}QR7%+y@6q5a`&S=y(T18POMn0G$xF4+9k}nGUbNsNvmKP z_ir3VhYy{al5O1_xT>8M*A|$}dhmwOyf z$|^F9l=N}VTG;=#8^-Y#S=Yh@HoIrjK==jHk93EvVxUfcEZ^i+LrnH{V-{BGkodzl3D&k;v#u7=?o1mXp3J_~( zKZgXWA=jQRk+H4Sba~0%-wHCVmsl!0RU6-3?-eSYWn;oxx;P-k^YSJ$y9{#ygR2mD~ z96#{5Q3>n&7dGKgv3bWBe%;9X*a#cx(-mKV&K(QzX3F5rO4-9Q zp~*J_I~e&uH8VY_t+CbL%}n@NsLp8c`E%o`3g#hcRqwBIfOrceYIqKVi3H&DAQYs+ zv#vdlp0=Q$ZXVIGE{s0Fld?4|7BCa!xOt;M>~oKN?bUZ3S9pU9r~mkqEwduj5MnGq5t zQPfbIq<3uVn@%AI($A0oRh4B<^b#F#+I99%`oj=uX8I%eDx5wI-sU^g4nl&BCQn5% z_b$vAybjhI8o-G?X&n7kBhb-w@@aC+^#aokdj5ykmDBFI2fK%!QhJ>9fy&>1zY}AC zYruVwipbh2Gkii!&kg+SDQdp3Y(P(rtnSq)Ir1vMDbv=@-_&hTR{Al=EUfSX&o^nN z1aMXv|8!R0#(u(4?S~UcqMgSWsPDJjO4T6GD*qt1%BcF}%t;5L9m79WVS2CH^;1FR zb}Ls`6{8>LgZms+Lj{;k|4+KB!94Urn$19JghF9t1R2P!oLPThYDBGX{CPT8q@_dx zBYjV~a3~=Vo*8G|wfEboM zQmBN5l=PDpocA_D3m%^z{XmX(5SRO5XJS0PpG^-{1~^=Ea~+npdQ^J!tzu4GEu2ea zia5Lt`u;4~e7@DB;pzY^^&I-nZ%KV^oi<6aXpPar#hc1a$BuddyiVy${A0k7RDzuU zgr=TFZ<`Ny6PUWUPym-!#+K&>+W^?O@*t)FAXIWJ2i&N3Gseo)niOT48h!vzF5`VEIEyo@?1?jvx@!0d zia;_+aAvvT%h{S1k27tjg}blvz3q<}KhKlH3<6A3mvJ+w0Z)qlx&WXOIYt{!;VyC! z3hnRWc|NYu$Lmz?5k%8ovwMGPiSyc4HJTqweu$u=@OO-j8lVd51^RvnHLoeUocOVr z0vBQ;$JX%b5oP9Vw~LTeSb8e#Iil#4XYu zOvUssJRfv+MmjX}iPy>5KR^0qPRntug}H(i3I7a=SAg#7W&w^z(`cuj(WYl^ zcHCd6$*EaTYpn2X>(LyX_9A`&l>4!oOj$5M%ah=`@70~xWLrH!{jnWIGNh2dV{{=i z6U3AwbI%fJA_R-x$7=0TfN3$@*+Yx!1x#K*Cf+;YLHGlxjoq`*{PsPt8)1$4c_Zk$W7+HYA`MFn}?T6B3 zqz~Sgh?q&K);hQA&J2NvKz=j)#|WJPsxE)61C3RAtkp%1uUIb7?u?uYqzBT9qtCEP zv7W<89=nM%AJ%1EsReh|ZZMiS^ED-lxz5f?J;{C9GjmdP2@f5~W=by5fl-Y63z|Iz zxzcL>qFJOFIo3_HUUr@!54lE$sHwg6wod)fH=?N8*jOd#wCA$k!bm6uUea^AJumetqV-=Q>?tR%-fm|3KhtD z=X+&7(B|XVKQfM$`qyg8&y5Yi%%Q<5>Kh+|25q5anhKp4GQ_XWyc%;`Y5&+jU9frC z-dWi7D|blwKuPv>E8Wp(URDahGjuw_^%YGM2uB|Qutnjrs#$@UQ{X{!!i{4-FBA0U zWnXm$IGZkU{o|7R4=C+!{XSHRu^?namJg*A^o5GJ`mqtzxx8!7R*PEJiZ3@c$LZbo ze@MT0#z*Co3wqg}0V>ftN0Mh8`!#!UDTrU)Q8C7=P zJefk`#k3Mp`t3g13$E6FILAaz>C6sSl-B)J6CZ~ov*%3qJ7ww9|AqnYsSGSR4xAI9 zn`Pm^DqvdA!{spxXzGs?=9w^ZjjDdj7+x#Fx#RQa*De=jt%C%sUHuK?Id&jJy9+?` zML?$z;BUaEV1Ip4$M7&EW!ZMWcc^3RC$PW#AHnC)@qoq&GzMr=D*I0g%nzr=N;A(< z?e8~L9oWG_8<$jOB5g>ohpt}g%sribljCZFMugs8QDYEW#eqH31E~5QA{yY`Hj?0A z$Hg%ZHbI4esoIXm6uQoqYnt#H|BG9CBSAm$GN16T1Ji(gMOuv;D;v&u1$;}8@_sYH z(8f*?C}@^+#U~EX<-7=9+7kdE`cDWEEPc5I%QaRc1= zOlj&;@|~Aw#u5p1{%%pFN?E@@-ZnCe92Y`{Eec6bC<8q)&}(l;x)iK z@I~Xc@qKXjCA0ZHN!?)_V+JUlG%qO!E11vE;9>Jwf#oE!z-2+Mztxr+EloW**5@Hh zLno>Cuagp(p21v>1u~I-3;Lh_hz`l+s5L6v{0uR_@%fQuA|cadr%^Nh2zK%Cj^=k0 z7R>UW$DeqX@#_)beL-&YI05uL!~4)pBUD5mQtk`nu~i%H9@aybP`{+{DZhv-s<^6f zMxR?MarVU?zMd+8m?1%iST-MlaS?DX>$s^2f(U1d z11_U71_;X)H#>FbQP~o6iZn!8w5>=-BefvD?$Uj_kdd_5`k3@$+Hs23z=#Gq?0HP} z=@7>S0+b16@P9Wd^CakGDBCWWy2^fYCYfE@lE%b5f^W*wuK`E_oIQ!4 zX|UsZ@*&Rk&oAw2u`V?bCkP(Q=saMZo`L|8)Hb-TQWiTjhoDAe8wu4zc}Jm}KL=X% z{Qavu7Ej+iODAfJ-#2mfp;q_b-1z)XD0HVeLOhwxyUsWiZA%^Q&Ym5k+c!dAC^goH zHRnGS^tgD`v*nX66}!)#zu}b8W6Gbt;t0hyGqV_QkD?Fu<5J4cfqkq)2t%2k0ZUmB6RTV54g33li zdPw%&$Y20Y3w(`c=AmmqRq8BY*aabM2fc=shBrPOJJ2OR7AuDl>sI`Z$?M7ePM>k2 zR^tn=ZmQ|^z0PUpDUJ?$XTDa_bDMp%<~NEh6a{WgOZFE8{fIkuIobA+Fku1EhR(8u z$|jA~paOd$mYj2cPVd!AVn}%hEt?6gR~rXY=wqJuY>{K6C{UlyWZ*6%x)$DPnwx~9}Cm5bIm zCH!ISbyV$Da2uJ2yTEn`6yn~4>AZs7X`>>pp9rN&(nnbtYOyDv zh2C3!u7Ap?5;-*pIN!}Tt$2+xMezu`#=Y75c2zXwpie}D>nskyZ+lv9ISvp6RurVc z-)&u@4aXHH$$GITunDnpcW#`}zuo3z?Of``8>udHU+Rtcp9-+ILE7kD@DAi`3VRMT zaGc4EMw%c<_Y>>=6%LN5F|FoV=Bh(NrzQ=HWnUn!qXg5W|R_mJ_YH`y3jLsk~Y^gZ}kC}vZcxGG<`Dv z{-dlR-PEY!2i6}%+#7?q?Ne=L0V9D;4U_;r+5`r60VHO8)uF{RS{JXsqy_2w;pUiH7W zaFKLj(aQk50k(Y{@NP!+y0`(Lr)Crj^~Z2O`QDC9)=5XhA*^*AOSQGS{Z5oP zWe$6hy)eK|#?ig27Kq${_)Fm!&6myh7Q6);+FIxU+_|!ZE2u;TO`67&J?K^T0qSmC zRAgAgu7iJ`CYI2vt20VroGs2fsV*)yD{+TbRR*2{F8xg)tJ2$}uulRXq)Xt($n>H* z_Ppd?_<1Q69T@DEK_)xmL09=h64i|aIKG*pX!`L9BPGCa$2dvZzxH-JY6Y7=VGQtIWEnm%OizgEk zkyp(Gt$uu#_;U*ogo~50)JQzDBCnT3Kcjx-KFK}Bfh=0RUzXSA=CQHeejAqNcQUL-j zj^GyEy^Vb*jr1k$^f}U3ZZQ;YpU95inG<35a4c6V?`<4lZhFi(0#sBU#1{Q zM<7*0xH3{MV_9v?l5?-yVCzwskiyzSa%}959y^F}`61UnkK;7+VltlBk0_H>SBviT ztXQV!Y~Ej?IEoC49$I<7R?TxAKsE2UOrp);fuDkpT=6Davrf;&6I}sFtO+HK;{T+0nVx`^v zYim(yVHNZA*uEFdFEZa*2{v@jmxX&DdDjh45xb6P&i?M3Q=&SqDber)9>PStP9543 zvZ7djQN1`;?;@LS_@KEXInA&9l*|c{31?@^eYZ{D;OEm;R=#b2v!0zzJy_%~Fr-%R zT$1|s3Q9TeBdEg<0I>WS_ApLihGopnyh#ns6uzfu@Q8{_UDp-!Q6??^A^7GnU(LC$ zf5})2^4*(t_- z$s|{5O0Y`lF1T}Yj4o41;=r)^vh`g6ujzJDnM8K|tkd!_w1-iRcR;J&#`DusIE|wp z*F>&zT)FlZI9!O=h&G-zXPQ9Yqz6A0z8*vEaGMN2+-GnzuM7f#`^)zByT2?cSywg& zM@)4WkW@!Bq%^oj1E&lNnLfsFE(jBRRP!rBB zWHS$_fsDsu>{D!y9taakk(>nb19W3o9*{oe({Ize*#^hf2fExVd@-qe!cM33#~GfW zs07>Ufd}@Mu;}B9z)df{U=|0IekX{#REfepoh~T=}7Qx#W7H2xB3)n=p8vrlI z3FT5NJP{6Q8x9z&j}gev6!ufiag>--Lb4Yj+h=YoE`K>@wGuCwg?sJKwd|Q$i*w%9 zz=eM>HDCk7T4BXNA2tFswq>W}^#F-E;30%2v-eDtx41s|Mc9jZH;8)aCtc`ZyZ1-U z-UCv*2l(u?d#XXWM+m}A^H50X>Yd!w8hQ;BObNdE%QEN}8je{)z3OMP% zRXYfqznIsp%LK5XD@L9a67V;AKu*Yy8C$Q{RUIs$G`CD>r~|3u zhq(>?g0_7r#uqIoF2Z1VB~~#LB*c;dDK9)O+MkRTTanSjKZUAa2O}?z;uYp7Gp1J` zV7{J`mbw)u1jD)`sZOA*9lAyW8}W84wX2OLJ1{dG&vc$Qq0tQxuwhB{}^4p{{ zem+P>I?lObd^*JQg;;TKRdYG{tVYw-FvhFT!LvVz3nZr50bovn=Uu)60oRLP6OR4i z9h;Bs0bGi-&^RPH6)UBS-HpMW9}QWG5q%d}qDLd=S_fuc`6xZFmLMjVV>qxnl0R*^ zF9-cb9EL>`sHedGRSsR#1kYWeCa1+7h<%4{R&VQHckI_kPH*F~gIZi&Qxx0Mz2bT# zMlDh;uPa}%3VUfFOmK}hWi=AK5D8ss?NrHr^bCH!Eh&6*?S;O##xB7f4kYCE?lEv9xLs%B6)aMW6fF5IEPDjN;rZpNpKpa zK^ANQ9C&$WV4!ZX=h%9ngvP@-7dz34=3-h02ePt^tg8YguA0@(9{i)!Ivu6v-=($D z{rIEQR|Q`7RV+EH3k>iMIJ-5trlcD3FxW;m6U2c{LI2u;E3!21k&sU)6#@o#h)z^~>;Lj)QcIM(r2Ks#tX@H|wAAC9Dy zg!Z;79PB7|EG(a>a~Qsuf9d6w>C$5dS~?P^Tfm4wXc>oQ49xaNTHyn?hz(-rfUr73;12{=h|1_AcJb3TcdXFq$s5z}4YKlTm?mSv z^>yCzc9!K2437S~BorKdnDvz4vzfJ>IG0C7(&MNW85_0xo$+;@hbbK=&8_{W*5Qh? zZx;8KavbLZ0(AJmnWePw4uU9D?9@K43+(5HCVtb`-{(c)RMVS{zNRKotDQra`lVezxe`OiN<#g9H2~xms^BY#X)Gu5|+onoSvA zmu8$~az*j74Wc8cp>eYpMN5uFJ5$naj?eMZ|0HQPTbBBKS?&nA_9`qR?HkZti-_%I zAEOCnCjbRK@94u5;qG=nRBg2X8jIb(ocG!Nvt^G`(xK7oqlU-$s~_cB!bAiA1cIfm zj`m-Gi(z}N8mnWX`eKiahprhJUY;L9z^6StIvf03hadU>MhWuvrFaIWr+z!~h*Np= znDiCYQf9@MiNtZoNqsFdBQ3SNAH?##E?t)rd9&{aFUSIA4*-f!i(VxE9A=e*%(#Jv ze=oXmc}vLCbzHHzrNyqt%u21c>UI-Hu7$Gj);BbjhJaLlvk(?N!^{)yAdRQ&2pH1j z7HO`O*qwdk#>(`WU@NW`peQT>t5-Bvw}7S-lOg8uftn@Z_dzit?|nqfaNf(QCFLTf z5Ey`VzlQ>(&#-3-Ohn5*`?6fBK}BSD>W5XLt)2#)3bb*wX-|B7&%#J3$s%(q)O@oC zC|S#6d!B=H$5I*C9spLG>Jr1*`)$_#Map-B>{K1GYftlYlBX*)Z`67i1qHvWJaV0b z0>(xr4X5M8Z-hd6gY@pn=Im6Lt-7VSHKQwRczfH#Q!5=kv|#`Brz7z`pHsgqDtLT0 zlvuICcOUk?AL-&>?i+FCb))AxHd6q$HEU|7JbYCC%Wh zpd|`6xgD2=TlY9H`m4@BbxKor7pm~)Q(qzJYi-vB>&ceg#}T=JnVpnjJ2x$HftTV5=9-+&P%dOGhKmC?>)~0 zM3eu+-hwOdpZ2y4&`pAc$i_7czs5rc7fDRfUzpJI<6R1YGM>GZ!^o#UFyH}W^z zp7mY-Gcu@H4LTXm+b4x>o!rX-UB{B&XihihTC7hm+}7>}*-E?NzhPO9GdQXR9@?j} z_5tj`J2YgPu?j|VaE6rG8Au6)uQgHJL%3S?44g3)QI?NW)1rTx$z>~fXBSAxngA0X z2_k>sLWBERn#DlGj$@m#X-Gf#ZuTd(8dcCkb=+$_lzF0Z`RZJc{MTBKNY7kfVXoX` zF_rtR<>W5$61)JPCy{=VRfP1*7L1OdwXKU$h81QZSG#Q6X6oGI3Nvr6-?m3F3Re5C zFWQ;ln>}xS;_Hv?0_8(689kQ?pn1FrxkG2=L2Uy(S(u-$R8*#srr(WmAoIlU#%i)A zsnz(Zi9Sc=`g@$tx!U_a=Nx&ciGqdVc<3RZSi4Rmuto1~dJ?}0W zYZ*^=9QcQafRqUEFrxiV(-jJJn+&aZHL;cHLfg>5hj?H4VWZY43NALa_k9340OXA7 zzX&PQ`#gKQ)hHeZvVLkII|!+o)Q2`k$wiNETwe_@}yTc}-8KXtL;MWno< z4{SGwetU?@Mq3q_zpG%pDS(CtJ)68Bafe|Q3u5$E?&R*xy>nCY2CMetEj3b?X72HByDKzkN{qUE^2ZJSGEtS|A#@tsn=`!vUBP((JO4#7 zyJ^5~`GHa4REK?*(%^$v~*S}UpXis&>7Sy_R{?&k03Igw<1?s!q3a=#-%qaS1)EIC`_98!ijd_=!X{R@xZ|HbC4EL}LR1SNdqp9X!92ek8zjmVF z$WP3eZ>xLY(5dEOQNOqamp^wHN2D}L-MaLcX|AQJKZn`ES7_Pk(I7icEM`Huw6%yZd^i>HE2nN0XE-5QWOwEm!po<-Cd@*E5qX*+Q!lUHho?nlMkS z=heJvkLmX#LtIs6s%6i(o%ZnV`zivm&-zK7C!Rpa-Xd&5wC<(W_M7?M;-$EN49g_m zyNdhNto9wb^HwFT!wt1IQM1yukM)qrfe&oI_q%}D#Wz~7si{$+Wo?A_6$3ovRz4uG zjs!g9E&pg05{F)aL7xTl9gZaDk)*LfSP|w8C>8u8|7h&)K(>Jov1OPypmg7uGC-s0&m#s8~E(AtE+3xi33;Cc+1js5J!{)KqB}&&l+ki5~MTV0&+YhRs~Zj2Dn1p zW+TR$Z@r(%_CcW)P`2gI@Fp&l{=L|@*MB}B`aYl>%aaSyDpNy-b{&j%g)UGXd(qTF zrWaH)zscwKZC-t#h7uegAP}m1M-@?QSiP~|Rp8-6#96i|n;$dSkArs|1?g@DPT&AO zSp_VvY8%b3T+en!MHO3(dLLG^GdEL;(~`DSBm^MIzq6$<&!c0Y{bUhJjESFb&rIT< zk;Dboy)QAf7E6DeXTJq;L@XR%IQ|F@l&LV;KvsmET}EDF%Y9J=@ zd+ph5Haf z_2+lvA}akw=L}t4KBKTzYNLZ9hPGdwE(><=dacZ3XpY=(U+AROz$F(4KMP~i2)@W^ z#(c!uGpc?sq6yy@>M>etcsta*@xd;gRokr^M6tpim3Ds=68g+hH#Fzx~Dh z2G(yL+XK!O50YH#;l#6Oc@Xxa}%i9=`C=f?+B3CqNJ-DCS{TOWF-4yL=6oW#%sP30qbKZ&b}|LBY0 z=wpwPc)eJU{)YK#rXV=>6NzWBvXF}XC$Woxp~szzhfV}144fSL7W7j;`18vFE$t$< zU9>wGVl4>#m?tP7P`xvVwM-1_nx-VvzVX(P!PQJ!v>*DWEoxYxzKix36 zPmohl?;_Uy9_ZWq0zSS0mK)e`s_pQ!Hk)66esflm-Va4qG7Ue)D)?{5FVEcQ+CSFf zir?do_WylT;Uaqax3+rw={6zNw}-iMWE72mV6R^n@OHn{Y7?T%v00)QgmJ_rrQQ-SRX zUEQ{Zn8)s>72T*|!(RUwwIA!!`;|^Pk*{i5Y-Lhc zUk8(s$up97N+EZ7N%NNH`idvxJO!;4ST7C3+K#^+xK%U_oLCwfIyAV}QqP`6J}32U zPFwdX90BcznsSgsj9#yO*PZgDIiLM^-&C6gt349cGC2WT;F$rA^H+r(UM7EVO21E& zZ2UtRd(VW+*)<**M~E`@lpgFd35QqJaGx|^spIpomIVeoa4)%Y9R;Gq!xbcy8YgLdidRA)6mcFz=WiO zU-ul_(*Rb-ei*oAr@B;`UaUg4;PQ@VgP;iv%;3^>zJv_+7oB@RPqytWT3+*sWB)j8 z^v2~>@h6?@8iU-ud{V^P#ED zEY&_2ri7!5`apFZ9l+@}FLhcfdwc}FMNvyDD4B}#JXX{ogSz^%utf{4bHrSf zug-JD++Avy3qMM=c1vQv6|B30nFh(ZD@M8ub4>3!NRujtf>P%li3eYNVqSg<4WAh+ z8&_Wp)wOEvdX7x1MRiB*oC*J*XAmgZ$hqX$vwz*hTVwTR_*QO)=D&qudSSWK( z7@f{~g$)c9j-A9c6vX}d8z$F#ZC*8o(}hZ;FJ={nKW%BwdskrWz13%HBUc*frW)#C z`r~;Jej^67B_x3TMGdrIN}B&*3mD?2p2}J-7TQ_YWT(c;9(!GW+dVEot>5B83)u5~ z6oNm2{1~tU56_4#Aeno#0Y{-&!8}j4j*Txx^7VnS@Tk-ILUs65jbL@WV(x&n_f_@j z{@Fiv)(M(2YDwZb<=I>2;Pp)i|FJT||FJUOSn@0hQXF2l1zt6abuU(Diaq($ze|RO z8hD`o3yMDMA1?e@BR<$yoXK~xzrVSTtMRW;=RR9QuB_ujR$UfZh)K|w`Ue5++@KZU z7z7$g(%{b2$~#yeRk+mmNtuMZ%dtQRFXpjJPqm5c*OL7mU1^wjLHS(8=O^U0kPcXI za>$=Up~Lt!u-CcesE+Gr7%oh801e0Z&VOwRundW|c^Z(nD%L4FTi1NJ@{7&l!Uv5} zt7h0ESRaigV{3vK4rR)jfdOn{dn~Yln$V%m6haVsYIS}NM-wxJQY)8717xB-_K!8! z;Nf4KmM4oM(`>8UUbpcbnXP@MeEP1-=^HjFKbY2JkbT$nWD6uxP!h|r3Ph||Is!gL zTG=x@7dTVd7re_lSWo3DKJ17q+FIK}CU;FS%Zk@dT>5$C_8JfnI|Lv)B3gWJyJ0M-qLc-8q$DmMT`W(ns#e>I?U%F@zV#<~Y4HKpKQ2$6Zz3qA& zO{I4z^j6xU118}&?VU)s;1kP{Zl&vo-yn#YAn5|~61#y7=^=n-%gw1?`VaKXPx@N` zyr2T86tzA+(zn9zM<89MVzTYzS5cFjvI2#T{KmlseUmdFOK{ zy)bK+l2ILneYyJ1%F{B69o`R=L)XEnybcbx0GnTo5y<}hoJ~YG;BPmBGN6B#5_Rh- zuEsAFKjk0cqBD7occKu#u|Es5$aBQ=$~*h`L$c0gwS+!N#1P|rxrv(!~nzl?6_@;#)t`%iN*vK>>k7kbw;#Ov}#4fLP1G&goLWKX98IunGWYxUFf6ETnMm(hIG?}dcEb4;kAoA&`}`dBw!H+qPxkijm}PEh zLr-(h430TM{gFeQIDqAifV`>rUZha@c;YH1s+aEK>A~CCa3YQ?{rzlS+s*e2D$)Zs zH~t}|000HpV_?1=aFCuMn;#E6_d~w@s0>n*WDg5Tag6U$#0=TdLU47?82s7Mxwl)g zZ|l9DmkOQwq4M)`gzC40&{vdtkArU-a``p5-cU%NP)JvkEg%a{QFfQuC*xzxUEmgh zTx@&}>yo5bVs%Esd7Fp)ek#3aZRolm`Sntj13bU(->^auyD*#Z%hqh(XAFKOxS-;$0hnntC*yFm6YU`G8I)(?I=dcV zqmG(h-vB{)d(@39*jtKPe`0-LF&(#Il&+fp)lG~P1_yU38``hAKF`3i$LGXYwFFND zk3U=AmdwBkt;R|ie7Rp3URfNuQEAXdhpuSW+u>kNFPl;&>PEkoE}u zYpx#{qJts1meoH$(6oEcC~=XSg2$~dg&*glb)Qt-{`2Iv5pqX^_An-%F2_pfl48G| zQ=_?;zO2S5^&XoFFnEgNwBY(#5Q!Lq{|pjbG(E1iSG1<2?DK!_eg60N|FswV-}CYB z`S|x9`1c<8_a6B79{9hp2bOdR>Bd~tr7U3ilMOz$?wbfdJ~VTR_S|8KUtqyc%@pRF zd+f7sSv{P78#-JW?MuDZka?fv)Nn9Z=5N^XW7e!u3qx5L;uOcqAq^u!@CW6>wZe<3;1 zu@ed!jm7rZCVvUN%wy*zVTUf}ef2Qmfx#c?uHpEN5dkD2=1Jq!TFmbYw7q#AI;&CD z?}n8FY$b=d3%kx=EPdtliO2R#q+`G4bejj1xnP(rKE?Ezyx|9kjwK|{vR#5rUP|TM zS~zJ)Gz66K{uKOj?FcLs)4OVe}`HlzG+$0^EBTH_FRDKf_J{BfMqSXPr=w`C9 zb{oz8_du|(a~kj03zu)<7rVZy<`v354I68L-Kx9hfR_HnLM?a`drGuV7g4O`kGsvS z*uFD*A@y^yabcwFtJshyC)XsQkCY;#-wWp=LBp>3r*GaB6FhKRdx$B5?~sFspaT9d zdge}`;<3B8LQ^G+`dfyxnb#t|%{Z6U<_&%mTg*c!9}*mnJ8G?1`ug_Cr3H9j4Ke`> z$9G0VdeZTYk%8h>1(yrIOq|;28FX9XxgDr9fRwK@;se345N5EiRH8izzQ@hA|G3y; znaF%IA2X6b$0N4ralAO|DbtrSEEiD94{=wm-1KxQ4Rj=aP1D+!_2bGj8IfUr`3D(t z8+Nc|6stU1iE-8s$FC6*s}zkI*>aZ;x#S#FcVlWHdQ+A^hd(DiE8!aMF>L$p9&^)L zv>1?3g~x4_HPRw7etppLFC*=BBt8=}Sd!*QdGjHX)QjedosbEpXNFVo>DxBnvibW$ z);yo1v@Bb*?;>-h9!9)TnFLiyeL2sR7Y{C`p}u+s?1K%qeraQ@n2YCv?=v&prx{s<}?)SD~_l(D?*S00*wPwQbd z&_+;bSt2wM*ZQgLKj(!rYcWebdcSAxmxY9UE4rDtU->|pl+c}oY3N&X>@l2x6VspS z7!U772?ii%zgcd{);U^OEjeA`-J?`$zW3(te;MVh4?P(Xgp_ z=7pFAlvt16`3pkupD1-5qAtrB*{!Xca^&I3?Fu?#kTEdF+L zvk_5Sl6t%Ji655DmXRb!u*BWrKyPS_=?9KpA|wwOtsFtL)eUlTGV@43+-(rrF?Fe~ zXyOfc@^(zcV%?Khuio?8=XCacm7BobA;63US8=>XhPA5*xDKPZlf zxmH`q8N8=9c+kl)Cry7w_AiXxF88F|$*Zf1_={Q0(3w>*Z6Ek5R`GXFR14vx3; zaIGl?E{x$sHN9%Ra_MD~qBI|uT-bL@X5f>_je|`iAsg$Ypd6Psv{sH@Ze<0(T@%-E zlI){1(aP+8hDY*&P5Q^GR;$I2V(&b&AAFU|3LG@v8-QpH1z8_niv^T0({p5fy}Z;i z0Vsl~?mKi9k%A=5AOb6NxgW7h{Q&p4n1`Da{ zt_RMVCuDe5t(zzeyW!)EH6b(wk=ha}jIo=fe>*8-Dn3`OXzMrnH*8oZX0{~xq@-N< zSKo)MI^zQ#(I=oYVXSQJgJrd6)zpfNHf+cRm$T+MVNs&0kD>k4x#4+K?@8^~M^8Vw zaof7el1Iz!`0i6NS7U8KPR-z*YJWZQ@^G@g*sET^yi)Bxw5R>vKEa1HM$u2yj2Rgb z-?iuawtZznv@6AWJ=t@s;8s|G#7F0dgxL<3OY5;0;ahc@$#YUzZU|vTeb{DWxJ(x@Q+t2iEBw4PL9B=d z`h0N=O}1pj>-jnsZY}x3J-3*e(EE<>+di&;_FOfQw_xg_cXF6!pcr|$9o+jju)W^w zkGNmPRv`2*#KEb^7CaYnjvq&|whTjTA9b zq8XY3TvKt6%j3?*pEHQ#-8ANEM#gcT7wfL!3&`i$%)7lnLaip7#jX07RS^xi<9cKH z+s4@9--+5Qw_3KHS*^U*0=lvKrNX0D{8Cgc)cHf4pGv=dIpCylLzomU07HGcu-dZ* z9)jcxFhI^Qy%%1E{Ol9k7rCYL(ym6skjL&l$A+uQEbtu~le3tC!4#YbQ-HQnyPY_k ztbFht-BE2_wCgpWQU0QI*c(BM%eUdzo^%dUP-}H$LMmJH1+BD!GQ4NT&8<9u>i@f; z-0TXsl#r}Kd}&}jRK_^OJOcTXg&YTdcNx<>1q2*0WifVpla_pk6;2>hiL1iKtPuF5#=< zb)2^(=G6V|qqW%CYTo+jWAm>53xc_reDU#L`^t8KSwPO0?I-|_To?h_r$b=x3&~P7 z*=eQO;Pjq)0}j=B>6Rd*6|LP*E^MWrez{)lviorrJtXQV0nY3hYET;;N#J*!s*73S-7CFmWn3W zM$M3?alFd+BZOXfih)JHCZ|99_=uyIV}p;Sz{b%x!U|eZce^vlUYCC3r8*(!d5m%a zY8)BjZYwWcR?63Dcl()~_$42paW;Y7%^vRJe4Yc8D1XyCI}d&W{1KjV%>W| zddvN=ohw5uQqS{3b4D(-FG(JL3$_K9W1^{qUYy9X0Z+PmXfS2c`w@Ze^U^z z4KXdx+*{NoJTGIsK7O0xFh8+m$jkNnh#%+JY92(Sx%85Ww>6@JqpoPoo$ySX8UFa= z{SR$VzaY8G5&CAYi@rsNG1R^r8_>Kr(?i%wn9_IF)0{0WncK>?^QY!b&9Vwt$BFUI z(fg=KaMiSV@Qtw#u4qA_rxRFhP7~pW3cnT~Kg@ASlIym1d|yzcEbwXxe5?AUiqsED zudwB>5{RuiAL({huO5GX+?sbMS9@4K@KooX*!M`j+uf|kTc9{_2smLBe}I>69Dcp} zc64%#ew((p@y)Df7IIJb`@v7J&LY_Fw8dEM=yM2OBR*u77xmZ7y_-MHbIk-F>m?hS ziJEqQHu)Y)tu7UWPO9|clV=F=(W(pXc7Y|PK2gD_iw~?1G!O4N-1~W_1J`~|JFj_L2DUA0IiVK*;mEAZn2PAE zhim!d58;fg$M@K$9W`m%3%g6GBs6s?sbZwEtAlp%G2PX#?kjBY-L56%Yk_(!;_MIP zE~XNM4~@J=W%LkQu5@59}6#6L^)LKh4XP z)XaP7m%^oyR~e`Rwc8Pfi`{e+z*9Tk?D%1u(2tmvJOgDvjaF+{kPW`z&=Qz#yVqnu z#4OioN%fBG&83zYj(H?x2@I$7b8z1h-moVaida8ZF$-PN96G!HkaAn+KL8>_fUFyM z5rA*wC#JH0`$5z5Hg7REsS^7AOaG(2_ljz24WmUvM=6mGQWX#t5RhIYC;}p42We4? zG-)ajAP{OmS`-8nX-ZX^N=Hb9KtR|MDbghg0g)0(1VVr$p0&@ocieGa?$bGAoQM4o zo**Ru`qujXf6n>MS=-AQr6OFso}5b`WEGJ~aZ0*g2b~Uto%9xqlMviovEza6=LJD3 z`fY`Ir_99;-QQ=CDp}=;yy8jgZe1?E6ji?c(j<&4IibE8bOB4+;(BnwwuT7eYfs;c;_%5x3b=imQmG^Vo)}Aal}~KMIg3%_f=ak zp|oJ%t8nL8L)_HoyD+JqyDGpF4V6K7lGzclkvUp0PFuxhBv8%)k$f zCf|3Jlh5^v?QJF&#ts~Ee%X!^pVnRB<3`Gxyk4LDBwO$3@F9uf)R}ywzbGnaNCI&- z@F#Q@O!s6s5vR=zU&D^#=$cMYnxH{gLAO6BIZ-93z9hNYyw;bV1N_Qg5GT`|3dMds zJa(^gm~519TUFE~q7+1q^mrKeoMSCcjrE4j&#;zTBZ%q^z!-iHst+!_+@6zM1W&e^ zagi}#fTpQz%k0w9L-|B@UGPB#ayDFDidc5=gay*W8Oo#+c6uSmS>Km3L-vs(?V+*f zFF7Z4RScRcfS&1WuPce(ML{SY~H#a$~h#htE57;-M{S!WaV@sa6($wzw5CS z0*_U#eRLkowUz#K{H=!0+)tJ|z-&y%O^v}dN1;6~1f-aq!OQpd+ZGQ8+KC1e%6~!i znOkm~8+hLX2S*acpDn%CDX1+JVOdl4uH#Yo2gAJCWG~s+_QiOM4-Jls2pm2Y!Kni(!C%h32z>q3}C zdyogsb`icZIAU0la7kjk5r&sNf#HTnHd>PE1^Smf=39sT8tPTkF8z3}d(y20YiwTo zt#%o3G728=u>ehV9Yc^D1*fM2Lg$CWRq`d@9`e4JSw$%e-b?5)#)0c&4Sk|QYVKeW z{z|awC*Ds|Y3hPWdq+X}>FR?Bzfvh!0SR!d;fArzAO1AwXyJW7`5?LP{K4liWi2bK zmtb?V59j2OC|WSqnbBVhS3udkJfl}^d}gWJpTmxeKl@v^C5y;>2L(hQl5ld_ zb)T5JU1Gk0C3W{YIT;$fM3*~odgYfJ@GWjS&a9ovDl#L6c7JOIF?A;4eSRE(gRcX7RB4_A`{=NA$oKOLM!eMn=rqpN$z7 z9IPA*`M%CGPlz#DfqjwZG_zBD-?{63n)|(1X5qEIYdaq!Ozvjo05hBap0tb+b+8w8 zCMBJtX0vdI4h^!yvho*|{JlA|6092cQz3#(+&F=U$+76#$j_+m7o8)_UVb1;$KJ(72d`-aIL5Vcu8ELTV4oQIC z@&4rxua~d=zMSrNO$B9JQf|eag#AXOE^q;IoT^{z@9e5wUlx81PWE}K_~LgD>7q03 z+tJz7OXKx*vX?)8Oy`$zmM+UYp1Jc`&grX^ViR9)y)ylI8{<*U^g?UaJ4Sdz^R{8d z+4*wPbM2+9ny*Q$&0vs4G8LS?4OlD*jd%H0%;SztXx_eBa`%M6uff>w2{(QVd3=Z4 zk$&*%p6O`3b4SU&zu|X|HeZ8t`d$NcCWGWrd0=lt9k>3_ee_}}X|pwuKg90DeNT}{ zFLf+#8gI0}5`!xhWXh27vw!eCXw5N)pT!}>=@}(M;g=^jq=4bZfo@Rbf*Pxq%>{E$ zzRvh9uzKgc^4MqtM1IHsHPS62uWG6CK*rg}S%JSHvKPZ2C&f^m3=x0{*f-@xyV>|v zK@T>TF)qJ;{eQvI{|4s%Z+Ux>!1=+n)5r0hNGft3HtrZ+XJRv)=vgigp-~ct7PVS zw16o`&|2TR91MH#hw+U+7%;4y{B3m{9I5lv;%>W|oFtcWqEJ{XMrb5Yo3VRvm+|pX zgC>$&U5?p!$^S=>AndhR&>O-7 zcORuRBjj9$Ra=U`b)dc!;Ual_dhU(P>)@{>b{P`edI`t{ZN_>m8N=&$WCnJ=m)p`u zcjR~5R7UR4{AZ}cSAKVM$5nL?lNA#I^eznn)?zA?N<|IwB4Hxd)0!wP*p@hD3F&}U z*jYXrxe-o1=(UhfpIR*KXI+dnnJ+j{CTQjrelS)Gl-!&@;>U$z%uxX~`)9-vrs1NH zY(H>1mS+CjhlUxUK50E2w#QnVylII>INsBnfUBz3?0H+pt*{uH*kZ%WT7RqG zBId>DS8`|ULY^a?6G$I$HroIY+~TkZtJ)<2gB@V_dCr!pvra|4Wbb%>~0#q+@X)xZFJK&YgpvVQ(}jP&*($PD-aU6m|7ynWv{at=`^ zjk|CaDInTbc4h`Zb{gv=5r>&V2iE>BBL%fE0$H`BvO*Od{1`Ay_|~p-B;M#8U||0U z7jZBdLWvBLq6?r>R+bhyc5yN_zKmvY;x2nFfiS!FFKF-9Og9}ti{uuu#%`Hsve|76=4l-us_ zy)_<8tl+e`Xz|^PT*^!@<;QGz0kF}w%Or4p(R%QhF{Wi~mA_(>&-b0m?xhbd3k`MR zZWkM@26;yeLBi(0xc&heK?d4{F-|ngBrPWskM32y6Jlj?SG-y&>t`;)1Q9a=`gqr^3)Cybg??l8i<=?^vZb90|>lW_2vHtK4mH)^F z?dmTPJ~*c6%P>vrfFPukT5n<1S0nE@EACvP%u4)Ty_tf4&H6RaKRIkWCyrby!+3^X`Y8!`60GE%$IE;vD%4A zvVaU!JNGFpm>V}wJe`XQzIn{^EDRefefxQ53W1VD3m79@X~~20kYXB5qLwpjCeH9# zCasUlTS=<%#J|0d>0&))Yr4<7&kV9TXt$;cRT9r$eDyFoQ+VOWH$m^$;v5|SmKsB^ z1@!27z$S=$l`XMup#r4%MF7&;9V3tAt7vOwSX^4VC-A4^TdhUz1rp&us3>95CWGibEd1w664?r0)|QyHfA ztVw*z>Vju_w9X0rtA{w!Ax1yn!d1VajbIph>?mA{`tj%N@Rf-j`@x<1%R{$aAI}@@ z9KL8XoUfM6vGPkX8Nv_5(gpTjV?eQCQumK-hIn85={B5wrP-m_$muYKYzLKuU&!v0D_$PlsJF|qn7Js-8t8}N^6?YpV z$rcI-1I(immnC;zP}0|qpSvGw^3(*icrCl^w$tZhCV*O8Omg{q1vu#E*)!iYM{K!^ z1nFPO4ntaoTgz#WNC@Ol$gi#SvZ2&+$@aD@aY>Mujj@e9td^iTtM5!cnsHuNmK2ne z_-;kUZyCxGB^1Z>T^}&B*E=I4e_s6*G3nbkuG$rXupv_mG0WcS^WN*!N_hPzC~8Rm zvql-~a)}q$s_0Y=JdTlx5r`8HSU8=2x6Ko*?8w*MS9)3z?)`y&!)+Iktx=eVs1ajq z_HnJ9DfYKc--9_fu;vxSbkF{WGVqN?PF6kKxRQY-1GcsTfmfSb3);4d zLl|C=nK^c{KY|(rT>F93LiThcR!P}-gIa2jXl;e?5TYd*rX+_LzElp7b%B+LQC}L6 z{s#~`Pipib(_}l8L;F*vCoO}LF!DuT^)%rWF{>*bR7w{C#&V<)8~R~?L1mRO`Y`+k zaK2H|228&K$bUhWmv&HMu!8f{h@|kv5YYgU*_k_i-X&3BP&D|uBjDw(bQn}llUf9) zT1dk#t>Y}zIz{H@)V@eV%*GTy9Tnbi+);}U9D;ZO1F^7-+Y|8fw5|Sp+#%+*P#lg^ z`-19l6PRu{D>b7#e_JZ0SxYo-ns9^<)#VeUUz*UIfpk}Q@Itj}o2|${LKUYvwwpfN z4(jfbe5zVscS`P>k}Wyw+^wtHAg(gbctsDs1D61&QW1D0FatVe)qZz2Yw^cgz0C4) z8G+I_Pux)S+uaNM^eKmrudbnsJrfsB42L@)_utt z^Wi&w>q-Yta0rRa3mu9J0ifM_L&v7E#|*uC6Q1;iS&4N1=b>q3JX}D|oFJ>ax}%jE zf^2v|+B$*iJ3VUCT>Ou+Coi8`;cg+7Fo$AWZyrKSk1Dd`CpgrYXB21G*;yk~0F?s> z>gCGWQea9rAf;Og1b|s!*|VM60IjbcT#qKz2NsK)F_=a5H6F5gv!-nMOcG}>&+Vz( z($v19(&ktlu^%B!0N1cPCItkKB)Z4YgT|P5$O4G$qXP3kN=4oDGvCO# zMa6k>Vm{qVdRAf+wwVRgrXp8M$ZTW}gj&As+uVX0Ie4UZ{bZEjLoWp{vCj(FXH!Kk z3{Bt@DKhN)N(?>%F|#sV*W}P@#VeiJ8yAiR{pNfm8@yTqmf5ZS$QZyWXaRw5hYU00 zB+4nTN3eQ5q|zJ=zVx2I|B)Hq%JVsL93E^pHWU~ z&(>U=9KM+JhSJ`C4RwW|S-Y^5W8~%U6Z-C*y{;UHFPa)M-N{SiOM}PKEIADJN6@!t z(#a6BoVL}|SGf`L7A61mo)^T6n5Q{IW#!JiKC4#1w2| zq0U$+o&#LIE3K#EMo$*$O%4AE-ZM(uIA~vGOie^C9H9b7cgUi^SFQ_fEw8SpFaBgZ zmt!5(m3?09Zc@JNG!`Lz6*Ee>sHs?^W(O!S`$PSF<~=HL7$G;adGig zi*vNGUIMKGU@S`05L7kbsvccgN}@-%F<4ZV1*dIY>aCHtEx%v*K8%6`M?9_r3@Tt- zk>LkHLbo5F)qvJ{MUj2Fop6M4(91Sqm7(Q_NqA%-Dpjbp>H_)NSkCtbLz5ufSR@ZN zeD%`;;6zHZNz*htMVcds5?t{b1_pSuXLDBG&k{39H^U)IG!Oj) zHG zA!szufAWgu^@Pr2dbt45wYf1xYy8|fkQ%g^md0 za4nE~EGV{#ao%aXs-u7{n)3yfBSz;fcbQ2A6~`@F+Sv_ zDPx_cl>o-pCVDTD1`J9p(m1BaCyS3GBl(Ov6U z2U;*0tHU&$e9#45lm=o2SQf6Q8kgPD;LAG5fjHMnm<_U$8ky8=WM)Vzor|j52s+QN zSw5BBoh4#bSg2060=Jo?%LUU5U8Amuh(c?CJevv~6CtMQmyjB3JtZ zm{-5WN&2I?fQT-As^a2_XYjO=i+$Pg&84yOS>lm% zMoK#%wm8RcMJ}0FL`?F`iyXMAZ;m5$zCH?(Wx>lHa$=vDP@zTkx3r=2_w5=rR%i}g z_fOr)(4NMsCpz*~MmP5CF@aVk)EoomV`Q7VU>!eX52-T>{GRu{J~R+qtd8$`M8c=I zG*Wn~;O2^l9v5HVy4C#Z>eP-v{2XPRp_qM0)wYNI2gveOVpvi!bU4l8!=f$wINVY4 z!)o-ftKXxqi~PrQ=hO{mxU~1S;v==>sJwyIj1}tE=cABbya3FV)R^Htp+|8V;J^4r zpSx3CxqsPU>N2>%k36OSgakdr0QgE-Mgf_R;aVM7jb)$o)2!E$@9hat4L1#AeY4yb zc9fW5x;9gN7FwtUJrE(iY0Yhljk|-(NzL|KsgEDH%IyfQtX0KpTNuJ6r{bgoETmdT z*}9A>lKzh%jWZVGH3nXSH~FqtJUk0C#>?yATN$QAdL zb~;vc&y7k?iBww|$}K%b9Bw+e5I25QE>4E#gteib>1vYpea{{Zyd^n@8dQwfpx|2| zWPBD=ts20-z;C*z>d7YFxd^}Cahl9;+%b4-JjOCHR5;L)-bkr$xzzqWx_Xzc#3Zs3_^;?AVsm|VEMESjXteMUiVSr$vVq+_eyCE zk@Nso#dZjitvCtbr9uq>fbE%Gw5)yGo$IQP&q+MFhaZY@8fWuUCp?6x!FePjxwEUC zj}7JqXyWH8F`~MRspsrI%%4-RTK0fIDN9No6up8BAMA1@eTDQrb#VwTOnL5fil=wc z7F%|=&QyDZf2X!?DBXP;#A~M(e=4vK;)lp69LUhmV-?2hWYa@s$n9Omh?e3S_9y+-+@Irk*BGFlO!??8-U zl4fh_TOIcs*m_nmf;&k#!6@@7Vj2{Ct@b?wO5MB&;Qld++`qgYM?7QGJ+;S&qN2bZ z#`}W8-w6 zU>(#Kr?BgI-nD(yH2E1hPb-4=&tdJLX%%p%boeh^D_m_64~CJk^aV9)Yqk83ys_YP zKm5;%a6vG)f@PyIV~RSpxYEBV(?fg#w<>f>j2p^XyD?kj`0yaj&(Vy^q1emuW&WHv z2M#Vaxt--ie?{{dNMYVApwml)k(2WGLk|P|LvxV27YHylR0eG5a6%p-hu}9D$p-M7 zteWjCldmJ*<8YmDQE%Vsmw!s}p*3-0Kqn|j!2}?rnC7F8KS(}DgVVW!RUUlooC{HH zDr&V;YXInrDN^HSMI=P(F15N(E9uQJ*5&!hP7ie9cd}L~(<5Xvv^Iy=q=8Egxp&S4 zA$9@uzIya}AmD>jQo&3AqfXC+smqQ)G`L2VDF++=qxi#46dJpOX!I_SVf7 z6h*#)9lW{hOyBfKrHT>C8p6{dGFL(Ezm1BlRZu-QVab!;e zcL9A1`IG!vL7}aw$Xc)D#1qfw9qyT)Pr05A_N?f~uqCE|P2|3543sGX$hl~@e!#sM zfh1t(j|S&nnKC=8AEhzuwo2EGqxahNYF~8+gMKTQ7&n%7DZY*y-1@akFBM}7K#*OQ z<-L>|f5Xm}p{TlYNQcH>ka6(vZUc>z4~s>fV%6 za^M8C?BFc84cglX4+0Ebx&>UQLHI1TzV%|7V!!0NP;-O`b_q?4WG1h~+vo?<-B<+{ z3XCn%QyQu-MChy8)YM0V5{nrP*PyAxMi*s<`HX+-cobK~4Jv}Rvn~;Czo#aDF|S+I z6ArgeYMDUt@lc~U<4y5WzxJKd;OaHMqA7E8GwyfnB<5F7W)8?)mq}8P;!-oO{fSS* z@M3)2wR)l(x=DykyGOM%t7F5}7tyC%^Oo{%ek{9scE;@P!$$-9J3Pz9t*`wxQ#Ov< z`ev2q8~hr=ax2#8Z^hM;LXsEGP^bD<%}l8%+vbnd`U+76AJP1!vUpQA#59OOP3e51dD?-|IQ@} zsrAq17YA0u6gWa^oZZ0Y<*0{RBj%RefI&>wZ7LYa7XPIn%LwF4XN&tgMWEHwZ9~F8 z<$E1>7lpSd8(~GujaQ2}(l@$(8rj--Y4n<|tQ9Uv$D_rk)8!OFJfr4RJo9TLyFci}a!UtCk zXyB*VvEhW>7rQ6As(_G={9Sfm56~wDJn;Y1tN35P;{PMBC;&bLQ@yS-2&60sRuZks z#v=*0vKN0zw;l32JQT+MCTP3i=+PI=BhK-&M5hU09to+@At`{r3oAvo_1*JFOce7z5t-j-8GbO-Vs#JM zmnLSFd%k;e`)$x>eHo%e;@KU;cr#MCy#BK8%!?12;Bk$-6z z2q^(A3*bLHi%=$i>c1wuoulUzB!JVq5S74AZ4?FOmo7k#5oor&8S|4Sio5rmleb2O?Q#Ym5-MrCG+RV zhy+pM z>Z4|_oWKR)uztXKD|1az8WNDETsQ)r!5NhPbPmqFbsJ^Q6@*ttpH+{fyE5Dt9RZk8 zxH1WMR|tA*!W)grw3WmCBVm=(C6@99chPyOV)UGx(*P=Y7KKCcG$z)YL0KrS9~Es6 z1r*Kxsm*zd($O}&_vz1@&JY1c!E!sg?gV{`Tb2YaXnRQf2hG#XW`D-tjWphz#xo z_LgLDyFlUY{z=YMq?s0QJM#KPuw|*}JfXb2k3mR&Cs1_meC|Z<7qddB2qVOb7LgG~ zB8UPuU3c71%urU`YQev3uG%R5IlUv!vBd&lQ(&b*Ei&Pl8FHV{yI+6o(IG+sS7d`7 zsY~&M%~MXFxqdc=bYnac5W+Ovqw+Aa#J0T~O~Y^dPf9_lL$q zjV|PM!QrRc!V$Y`f$DUBMhMNuisl4(eKMT@$}b#zjH`a)Hh=%z9ceG63&0v;{{jtSS9p#>M4 zFtn%-<9frGW_M>*)Qey|*XCYElR$@M5AIKF?YHCUCyzTTWgb7Ca^YHh*<;RPYtZe< zGn)_eUcLHZcsXV+?QUe^sLD-UBkp9uK8{?RLzLPEmP){#Y6!&0qEAhmqcM$FMy&(S z1_k;_ATNT|bKaF9BJ_6}z5lgra1?>9&<9FPDFQ&KT}9gqZ={f=zI1P%G3U8@h}@&Z zICLK6@l`&K13kFbBwB=dx|Q0Nwv=~O9 zog&UL!n_USBGP0(o1`Y{;Q=7g$DsU3x-0(UUf)IAh1Zdd3o>piul3-k7?IWRQ&izn zqPqbZ?|KsT>e2Fs?qFY2hg*?%F7661FRURw9i%=0SS}&aT7YknIbf85L}||T*YS1e z1hFRW8(d!!)y>>)^aQ3gb=lHeIK}OUXf@^%KrKxmunI6NJKL%6FQ}-L8Qnbsq^!&(>H!%jJUa8&dBlf}xCJk=(-KxIIqa|voKb0CnsV8fxVfiXY=b3!Z8BX2KX^8`e8Yok_FcSGQ$KNN1rMhrM7+5aVn;<^9 zoIpZN*i6-m;tCrmoFH%IV*Io zGfAk75vlm;gWK|*f+peDSFVIQFM-T~=1m01(}dYmjA}3rFFYI;RO{+;>5N8@kBH4% z^gl{@X-4u-dPc;7ere`})k;P~7`5bO#?-;NJF<^_j~(Sh&cpMkgXIgRGc*76@tT3u z=K-G-F~&0W1`ZHsrVcd5Z+K1}Te&qxcB|h6`51!LVc(0o@z}TssxXPCfJC9vZ`i6NRe{Z8J$fDXOrot}rtxK?H zzG?A=t8J>0-lFA+Qf()?8=Byz0GlNUv#0oCqvg9_LEZKH*Lz?1FMHZW2G>1w^eC>q z^goBoj8N%gaL@eT@(`eBBzBqBuHr#~uNy9ZesW$8)X^i|vpU(nxq+HhXQ%mM8xPjH zfui4Q2fQE49Q&`)9N7}kgsvW6I%4A+K)0cuS6_X){ffWF>a<-ZFlS`{~)C6 zw#si=_b0{>y$auD{oi_5`SRFf3qmxTjBp6IJ*itXrhmNahS1k_tByaOf*p@29vO=@ zaLX`KEyL_or1;TSv$h&1iVcrDG*eWfeHDq(xAsqR;ehyRWtbZbh>%#UF=AV20E1yFqIcKkU~-oP*;X(7hq^pP2`6PlZ7@|2EEwl)vwsrt zM)noh!BByx1!g&*mDz5<)nyC+q!2Mci6luBqmqKR=d?tBsGO>keqwnf?1@={Mg&Ez zQO(aZs^5{SMl?)~JM8BmCmP%4+{Mx2aNByXGWe5|**mVew`Qfmf6_sxeT(AK0WVJr#i0nB#{C z&l=CuLXypl$w6udihcX4Elh#a4sufd9o%UTtGCwI4)G;g`vb0AX3XoKSk-79h5;0L z7S7)`cCV}OnE84D?c?!^bTXGR*J%@PVyOCpnNE2r7#4Wto8W6U)=LOY8*No`W9LN! z4@;;5(}TfApTPibc!~+AFuI-9iI$oX>*o0Q^JgMX5wi`8q1RA-d}G+A%f&NGNapizVh9+20X_0gQ!*r-sh#v$4)K4?cH7z5+3IQG#_Xe&l9=jW&Fn~uz`5kGg&5{DffKHo+o!X(Z&rSt z#?$@lvim>oK0{}h-y0BfjBwFMcY$%m;cY_7!)f-&tRWKaNUWF)590iChN9QlyzzBS zUBaCdPyE1Fq!2w0@o_|=m8gj1O{OpTMFsQ-?9KMVn)=zG9Q|SOi+rz#bw%dlOr742 zo>Or0B@)EdD3Rgh)KRsg_6%oN%{}OkmyyS2O5M0T-v0mCoBSWjN%>zw*8lq-|7+;{ qf9vtzdi+-d|JA^MHSk{z{8t12)xdu>@Lvu5R|Ef7*8s=gng0Rnv!zV{ literal 183022 zcmeFZcT`ht_cj=$DTs(jFHwph9qBa`=>j56sUn?#NDnnA0@4Kp6r^_{M0y9Q5s_Y{ z_g)fefDq>RzQ3=`tZ%I;Yu3yk?-RH~R>&#)&c64)u50get|qS*K({qjHB>u3x`FO>v8yl9rl|j+UB+hMtL?m7eiFBMlAf zL-2hLPA+aPdKO**-UkBg54axu-3b8+Dd~;tH|~&;-Fd)3!|>q$_T%aYi0V2?D^VpO z0UPKV6#*d?!Br;+44fx1!M`4$|9TKyBP1dwA-#Tsj2!qt^=;5K0z$%TM1+6|M8Ky5 zf&YVusEF^}7gi*p*0UmIbG!TKZG851cBS$*nrB034v|;x?{1LM($O<8az5bVe#j## zCN3ce~9o<`(AQ z@aXvD6nl35*DeAO;lIrSp8svw|G_RQz^-dVM1(}7f9)c;<^v2uDk9?h!X$ST^+>JU zsM#L9y?$3IKD)f_2D`{JG|embAu?JH(M3+oU(^0&+5a`e-u*wa?7tiKKkS+U-6SLc zzC1!I5Cn8y5Xqf&?Vs__9Q@-0|Mir$5Anu$<5O=QVQv#K{{|p1c{c&)-6x^b{bWZ(Mtq+U8@k4+1 zswd0)_J^h4Mo*r4KwssuI&z=#_jri52APd=Bwm5WAqe+7^E1B4yF2Z3PdX1ja-3Z-1Jf1c~S{-PErP2J!+|_mi5bAwRKCejg|+aRhQ4E zwwCo8lDWT%u>bx|@AvYnu~tcd=GcQomv_F7UyENarJ=-5biRF|aL`Tp!a6_DI#SJW zf=m~SW89ufuV|mpy{;7G@VOY0WYbl;&G)P1+uRBoRPT{^Hgvo?&WXC0$@^(7{`gkl z2CA>7E;BsULY$@}!&&4CnrQaT6B63`o(i4vNExbHbOQOUb>RzlxMGdZn zj}Bo|Hv@{`>j_t&c~RdtSD*n7Tv6*w3wtqas&41_<&IvI2EA(5axCxHI+ya-r!*od ze6ODwJKjnBC_Nl~9_dw8^usNKSCj2Myab%&7U(UQ-Bctt+PglUm}#<(gJQTkT4>#f z+cfrWT!9Sfi6ZkC1*2G*Ff<3x@0mKz?}h6PR6Cb5T6CMG3wSl12UV@cPwZumH?5-> zKVjzk+3He-Bng;f=f)1l8<*q)y(Fl%jknT;Y6Tz&66)<9f^dpZ{D%0I;4kX9cEXcxW`wZ z064paEyfl9I0|{bNrw;P<>`@j*DW6k4X;AdpM83bznl@bhA#eE3IQJC@_MXP}61H#7n(=(z#5V;}Z@K?QWcnCMr6d#^ zKK|>|ruc+%AL597JI3~CHhkbi`WZ{*c=I#Vd8ZPO?{ua%NxmgA0ybx%Am*LdQLCrvHBWyDSp#x)YON!rNsKMuF4g)7;F`fO)L#x zTC3&}t$acGz@+CsRg9{6d8Qy#SaLfxGNSGenW1S+nUB!TC&6zhHRp!m%f~@sO*I23 zVGCFeN`2(E?4#Zq4a-*2!hFBvq^_+?3NPELuWWm#@}}S$NXvO}_}J2Zy`?z@fhgEO7-(}{$Pqt7TV%sa zpm%0wGz01;LRpA?yWQP=A2gdTHfJX0Rd-OP+HLPznntycv@f2FWw4MPBA%i2izts6 zf|5|%&wRa!GWU^(W$n1NU#_RnFpS=Go(LuI4)QYsC&fA|7;$HFf+?6`z1rp7J`<1p zxGnpVYrq`^`hlKTnEB)kR)ZQJeU^I zj#EIpe$Y2>N;Wz?Ro5(+Kkk`!Xc(1n<9QwVY=QR(wQSK<220Vz(2S;}2KM46f6;ED z#&f=F%dc(G+kau7K3$z1B->aIC>Fh)&w|?K7puu+7LjsfkG72%Kwp85w>!NgW{aEs zDI<=_d5iYtQJyJeQPIUmOHD?OH&froZ9cb3T=U^+O=gz|V=zh&-m3XAwxqx4Tc=*KR{P*vra(oS3T^H|>D)rVv$$+j`1M%Iy?V*!eb zRiR6bqwMjzPs1Oyyh?Yf@asIk0wE03D#w^<*3y0XHsan4E5B}6=K%X}nx33r?5HE} z$GVdce6%Wo?4DFk>+spyy+=p;MIx<&_FogdW60zP=vV^-PWDjiQ}s^H<)PDQr%+&m zE?b-JXUZfk8~aJ0GW(|$7?KjF(nEN4zHgs zP5K?~;jON@2P>LiL-fphR58Mddg+_&Ty63sV>Dj4j(m|KQSk^(d+<;J>jnD@suU6wTn8w zvAP%^sy)Xk#xuqho-0uM1zg!|KB}y5uHNl|JK@Vf-jWcZ-C2zQE^%K&8PnpVI}8uh zQJCAjGqN(y{GMqrx0oUk3$;b4vRCPJS9J8sL`~N9Y&DwW7kyfNik`AxTQUyrjY5|D zUGv?gn?glTZ6$esmNi^~@N%5z#=>%6T7##~LJ8v&NTBO>ND}|s-tj9fB25+JqK>6v z5meUtpJ&j9%__e6VMz-SwJN^MopDQ1(UkgaNE1wE*n`1eJraM)hK<`J?%K;idz*X$ zX%q8hP|&*)bUaUqo=Wap1aNXbNR=)aO&VG^3Egc;|MqK&7P}jdPMc0XALCau9TkYX zQ>761ncw)`qP5ZdYYskNKF3NeyN%!Rk01r^yn`{4(*{<%91W?A55Ax9^7(SwnTcyF z-LZ>bu~->bux@E|y)h}CZ^gd6$2@L$FZUbI9rvMhwI3h;-~#RTaSW(c%NKv(s2T9_ z5Ij9#5$&&i0gig<{+{-gNSxETd@o*p;R;k1h1-1U^TaEu)bmJB#q;xjVfg=s<^N;; zGMumDe32K+XN-sNh|?5tdCG>ms&vFECH<{JQsQ8?X7<0xLgnAc!UOZDd;KZ;>q$+# zZ$(pla4Y)$3q#`&3pFFkl74_D&z)aQ+SKc{oNFv-7Fta5o!x5P7Y?4y1 z!qJ#vaN6LVRvg3A1NTHma3;2z;j0@Rqi%dswKj-91Ya7C`EjJ-;ye! zB@^Id?Z*UM_z-KL0*t>pX{|V4@VyVDKR!AsoA?p*U}QJ%UH{Qy{+V+5PYTVZDsNTM zhyAzpbyi&|9q6WX<3yWJhJ7jItv|+s+P0t@6kK=+oqX7L{tu6G1CTSUtsfAV>=0+m zYy_HmJ!BXa^?A&T=Yi|+?T2oPG#g@pltN2QYEGiN>P|dD?^z*JNP#cEM?_0K5)&;h z;P$S8gr+kOR;=dAN7_x;76LzlJ11Tn8zqyb_IgCoX?pFJp*Ydq1HYRxHp7%|5?XvF z)*tblHkZ>hQC=-B$zqH)guCkfoTk+JqrYnFe+9xGrfiz6_&{wV{`i30A~(e-u}3MJ z7WNoAi?|L8+BCxiK_N9Zf%)PtcP;gFDwFVNF(wxy^R`wWd)D>c=qu3q_R4&Obj)~R ziH5Aky$T~HpSkbW89;&oWJ)iyTqwntLAv~NRwRYWb>y}49b<7xBJ@7 zo6N^s9?#oTrX*ZUex9|>UzE5j)uHffPQeAXU4hK$&L%Q9Yb$h@$e+h5rK}%_ZO~d`PDw94E!STx z2GJVbcZ$u3l^JSz>%qM_&T;6RnZ)ks=kbnlSe&$oU2Obw&h{$Ts+oP1tXsZAUfWGax&Ai> zw}_QG+I6jZsd%97c^Q=G#;;ePSJO7fXO0+{aYy-s(P5VqJb_t$YqVFo9absB>~^f` zUAU~_aZ|JToWaiT`U_O#RxI(u(L~@ zsebbXM^^gB5id?VKdZ*ZD^SLO+7v+0)+;AEp2FfR@chuHSD+s$Kzxfo-u0b@FL@Qg z7qkHB!g_aT=-7VdB4pRc^6*d5h2aG5#0i^%zeC~CYlHtY3PE#Tfw~hQx3`_r;5A~D zxEpXx?sQo@%&sBR<+5xa?=L4fwHJ=7@x%HaUwZV-Ue*`E8&CcoDN^?iv~%~r?<4MY ziejLJ`9M8KLwfhkQmBN{aMh(nPieK5`rhuB5c)iZXKI>f{)^&>QZwwKYUHv(bMk}C zB;P;(=HG7lpoy1{!kBc4jHhe#i_YksEg<iv}l z|MP}D6pq=-JRXFzznJpD1wN3YaM;eoY_#sjtw7nPbnj;< zO-{FPCB~(cL3Sz>t51C)QXWg0Z{?I??D52fEyY2<3Ty!# zE=Z#H5cyRiA=Rwi%R9Ma>&qW3Sj*54D!Gk~wM(&9jDF+sLoC8N_fgfGnf~c?Ey)2z z6JtLfauwhG1cma1$Y_t@%5>jDc4>oUNe@o96p`-X#%Y(sOb&m)(Nvb>fw28&XiX&< z3ao7OIBkhN=jV4zQ_tArRXuuGt6%>RKsqh$J>j$X`nv4%Kn4b?&)rh)4_4gt%QLC@ zQ2XY)%qM%AwM>>XtOvkZ&6fbF<{PSmLY=5#M6Xu0S*j>VDh^wl0zaM-yu8&E{n+V ze5ULQq#MSfrgd?*n4-D~BAddby1YZV{M0Gq_FI=^nBcIwD*a(U@1Vgl3&XGVho+yF zSUI+C8{T-u3fG)#2%D4Q(0XC?d;(ivs-baV=x%5@CHs!hd@)1Grt_&^#OQvW zgoKh%_H-t~Q^J~?K7(bVY|U`^7#cw|7_GL(Y~ycmO7%O2zk)>gr$^!-StW8q$Vtyt zcXqa0gc`{_ZXfzuOhbU9swrCbDQ1oJI8(TM05(3i)ZI>7^SRXdLNgY}xFt`gaQFR;KZ}h<*M5Ixe>widw#~%0c_}{8tJ4V{6sxCcJpXzfQZAIxlS-8-1V#b6!ojZ@CCs)q?HaX|-9V2+n_SbUhS znOz00JIaD{nhfz3jPUHCHYIPgHxs>+;L)vSxo+0olIIlvBxt83`eLF)>fw-)tbl64 z$d;F&`#lL`q0G-k46e#kdG(_O8lvbS)uJ6jN&Gz+TlSC=9J_@nR3T{+5YRmYHW z6O-MfiC!J+o6Quy$Hl+&pQ#nT|8!sR@mhfPhukFo!_}&A^tK^RzJfyboz!OI*$~2M ze0Wov`MJF>h<{6}gz>u2r0Vd^Y|alyawQG1NenA>7ZtT0xmZ7>%aR`1*BiG?4-oUQ zgn0hdflKeGE0FTHE6~kd`H7-grJlphW6XTme5SIk&@(n{;`&bt> z`8=(9A_a=AKIW^Tnv(CiO$_cVvGBq`G{T>a$d6ou=e2vpZ_Y5TrHEBCn|w`Eqd|YX z0(JT3UV$dyKnSS|@LXOz9_Xu-xLy285>&Ts!GQiugN?|2xa$!D_($q#T+FzHUwFs# zB;TTA-kq-dDhx7)6;LE)c~8U~ie&*3I`wJ_wInlZC6Gjd{wOJ%5xgvkkki*n0N~Y! zsmWE+e3)gp*RH-w%farRJyoOCf`B*PaNC=e9Y5UQyT{yAxWXl;4ycaf6>Op!{aN^_Qa?k8;8u0G5L@ROzp>~NR)B9tz-u5P3vc*=g73npd~_l z=LywNH+;!d@|#g6=J;Qh%fEiN3dpUHQri%!s~J$Y-TXS+@8$824+6?a?;Dl< zarE0Odh6^QT2X7v{?)h;3vLsik}^#z?Q@?==xA+zI2vsf_Aytg_t|{Nk#H`|-^xt= z7r8Ovg|gNPPPhqv?n;Rdw-|ucRemBEf_An#?58c-u3!3uUxAd6@~){n$Bt!23Gy#T z1T<&L$s5JyV17{l+Y=*)_rN?m9BUP__ZOq4qugLMJHI+2x2@1~z|0=(bZlFwnDof7 zBBXsP>+J?mJs$W-7aJpp*O%*HgqJ+^#IS!ObO_dWN8Q?!G*s4@y41nP$i(K9l3{jUHv3Yu1!_76oyXxgf!}_*?*HsJkW4%SAhH>A-pyfiwYVATlSv)M| zaQaQ7N4vJ2bUGh(>J3?V={hCk`G-z*108;?*;lzfaUsDS2}i$^`G<&_S|<4 z@*2*CF$*WYA5g$bx!YnxviGFy>Utn(YD*i(B98iNiXVThfb#=v(T?H)DuZp?CmE3a zL`!?eOw38EGOk2ZnvGYca-A~v931Y9v%WT|%Flp@riZM8!>#dRoYkTE8SySB*ZT*A znMIDJWJl-3-i>rxQt!X|a!3X;4^-$@J1A@^e&uTWJ$X&c3=}CtWm2hfH#y{ZmSaxk zB(y1EHp%y;JJVD>i@y3;wKYqpo?Pb}+z+;uOuxOd>}1=={1@?KimRc1g6~B=6Ucq} zgA5PQow3-Q2kx@2KNTf{XiaDnIt4?T#9=b|2j36M(TL21yq?_GzdhI!? zr=oZNKQ)MW*Tz){UU~wm1W)7r8ANsEsL}Y}3J8#g|HhSH(EiQBN_Lu;YP`oK?;6V= zd8;C)${_0{L7oMB?n1-D=c$L4{3sj_h&6$!j;YJ|Ab@+f^mzpuo=V}%SA0C$XnwH4 z5}uroc@BGyNzv*N9+ll+t>5(k>Z5N{BQ^FW@D{cZw21Cj#h&gs`{dCl@3;^BZ9Brx zqSD?De4YP-L>542;le^x2q5K zj?x^%U;cth;Av2A4>Pf<&K~$xzgwO7g}j4_ixQomX)T4+g;=ik2A37%8WS2sb0?SZ z&qzi6ZFA<=3#+E2lZI#U0m)}-s}>%ZFldx3p_w*R_T!TWQ4~xxlCuMvj`v(D-@r^e zdIN#4;6%H23Ye%D+?HRym1dBl6Z_WXLmhT6m%dK)L#ECLE{TzV_}tq4$!Q9B?fS3R z>F;*V-%-$MhTA7ulkHqI!o#YTlAo)xxn%hn|02{MFTVoaH=V~_cf}N5fuLJIzXY0` zUV*ML;T)gq?3^d#xsU0`c~_{r5p}$t0|?e9K`%K8`gJGfTJ+>Oo3-aLl8pnG0TS48 z+Tvw7L$I@S^?vKQGCYRDjArO0c)1Y#tDp+VSap!?EW8(Hzuol?G-aOJliqH0pY!eB z^#qCZ$nTRgw{>`TcrCFP;XsaC*XAYkk#^$F`S>4OeVE$E`EmVSzY`D3qBl$BP={s6 zzXb=NEKnA5#<+UymPX6Ed^Xmx^jOzAkvs5Vqcq0&NzmxLV6)#Xd$P9*h1IIoq}2|O zC^cg~-u$Y+J}Gr`?SuE-4yw(_9{EQ@VRa44GB-Z&>c`$dAas^wBuydy=c5gq&SDz1 zugq`6i#4hc*~5>-HpMfy+fIQm<8ITpcR#?|67 z^yaxWDgBS0Ec^2IO&-2Tm{qFJGx_SY#S{L*_RjUyZ4>20g%5l1Q8C}vZsPfcsLbNq#{ zU4WCZK-Dk2b5`hn#>v%{ZfUEqh!!7;WTw2RRt`0^VR1;^Vo@TwZp0TWwN~ZjN?`V5 zSPX&s@;32cg2G!+w6{I_Jq-(rCfaFc+`zeetAT$?r^O({*=5$rA9Abl<*%$usIuai zP16_akGvk{ymjW2WuLdoJhWd-R--G`A}3JNqTjL}{nphzF=@#K;9XoD{O@8oOzEh+ z0@0Ku4dALE=QohK{P2I#Xh^z<_}}z(?>~`1|Fv19x-ISTI=BgNf05#a7hi84@D~0{ zdMANI1H9Tkvfx5KhU1&}M}wd$i(d;*y*MsM;+_~ndq22X|Ne4LwjP_OI}T_J3UjL3g)77!|k-Rv;>vTo#jCT z<+!??GzW%_tE8e`R^@}gg$sS#iTfa`KQG5MK{_~>c4BjtLt?k|( zw%T`UhMRA@d^~gurd~e+OEvdd&%r*KW*#|=8Jy@@xW65HSXd!!g)YnD(u%6j4TVfe zxvra44@51Z=OYC9uK@gm z0(p+ni7cn4aN|tVNn*Pg`$_Q#4euocTyDZo&4L?M?QHi3{%0cdu&tE`Qi+7Dhu{t8 zU06Q>zr_8d7t)7#7-ZGh6KDbBSABTXIbFqDMlD|ew5wniwtT5}%oQFy3M&0lfO)qveroa`W^T>#f z^yhO)I;!r!#F`zKGoEfzgFj^)&tk0=XlduXW}az&lMttq6zyYMqIP4=c*AIPIk^eQ zOIv_k-Cawbg*B1k{%-U2J)a~O3pD9zRc!H++{oLpmD>7-zAD>Slc3*PF!gCgi~vzImV&r0}0sGyZS(3O|?b{*=fYv}+p?$L-tYs`qwN zfte@+Je0P-`3C&WK&`{?#uezZ3ZVQ18~dHq?4IV*FpTU{;(!)ER2+7AHSSgYm$pnY ztlRmAia_;UGG9Xys>=+@HK)GN&4lfJuPH*o9X9}K9ALb6ET{eERDvko^nz02l|CN@Btj+QF*1vuN&AY%R+#|cBxC+SW zAK+{RXeD*ADzY+HAO(NRi>0{#>2>44&V}yT3Vgp3FABGxf)B6*=a7lXS+ov`YtLYb zT@_#-LOd@1Ri|S-SL)R8An_YdLADj~Fmm=z*^aT}Wx73JK-}fY{IbUX^jf7I1(F3X z23}l&zVCEUev(KTnaaNcy$1;16-db6^>_T=5FTr?#X495!a8 zNjNWmSj)aZ@n(@bi+;R$dVa%n=iR{N(b1nDLV#wsYU!xZcahwALjS%5s@z39X z=Hnk9_{RtSZ~8z})G|CYh#ecHH@u1#ub2Nqbh}}O&m=l|dh+|dML#q}z%fTFtv+h` zM`3pZ>ZI+HN_QnBr7jEJ+&Fmtlyt_RCqMGKh2x5PPK=gJBe5Z2F~<=N2Hwd6S>_7f zadGw9XWYtRDt@AOnQ5BdzaTR3TlfdEKs23^k|}_8STx=J#cO!AsEnoblIlFy=U1wC zY}WahC=zF$k{V}kn;110#;B=|-W%on^M`0qewD9~$bS1p1`nn-O!lE~;S?;yFitn| z>HG=UUXF}!Kff|GGv`#v6twLEY!~Vg9 z6rz`_J2Oth*6H`TDCJ6NB+OJj)*R@NbBXx-6v$1uLvH#(U^l0s<_p(58%pN*@3vm- z_ufgrlNqi0$rgM48GXEUtQgscTmpp!5bp{&t`>ermVMbYtpHh8hwsoOihp5J2MyPMhGl7saswt%t{qVXj~+Us)` zj_5wwMx}jk1&#J#Qu%km(memm-T@>vS`g)m)x_VQhRNWm^liA3Vfb-ZTwjaX>$5+r zv5mqL(SxKl|K@S9%(wC~@OER&t?9OTbY$9n*-F$&YwcNDK;7(NWiN}jHIgBV()r&Z z0YhPW68wD#&#t@3ud5(_;(b~$Zmzpqe_bR_2(@~2d{)O=MX-DHjGZ^9LCDq$sJBpQ z0gceck=w1TRTtn0VVH>aAkguV3;kJ|*GiXRk{7-BoM_U=g=^F%3>9?g%)<4 z5jrJUy=LsSP2H%uM1JVE{X~{1Pcq8-_3xqtb3>MbGeM&%E<)mSH$$yR^HNqn$F&Q` zjW@Is!G}8{^?` zYi;XckzIbnJ{TWA&5rN`8o7)^0$fMsg-v0^QP1$m-$=}?C}A! z1C7bgJYuR>E!Z#tHS!iSY|4DKK$Ynr2mP%fq5-(N88;EdMHp=R6fx$#ka zJr#5wnR>uRb89QVf2^llHwzad-2%O=@Jgs@VL>0+1YuU1KiS7uIqO~maGVEp{;a-n zpBbyvVJXkrgdt9EDP@(!_pmRXN)nUEsau^zBG4(?b;ChSFuz}zRsVRqgx_}V{VaIk zvLvNR9PgS#Ki2J^#15f61N_J%Z+Lh$~b?x0gC!cj)h1yAHEo&g9i zA8=Ip+a~yQ+RQ*7y3em7B8cIk=%BsD{zq@FuQdRjy=7`k;W01KlwpwCGCq#hxZN8! zBBic@(_~jdHZ7E|Kon|u@WFh4pxOhJh&l&JE4EqX(Lpg~0wKlQ-%XvI^;9Z(X*u!x zim@K%ase~|nq{5F7B1Py3y0_=OmSz-;(s5ISg%^M-}CA6H^zyv0yJ1DX&+C_g?&mo zIKcD3-uBG1h8 zyGRI{AC3iFmM0QDa6M>n$~7WDx4?l4=Cn1Xt>hJI3r%|Y{`4Ndzt?d8NhN1h$<3W; z_V^K)DvGE~RY9&oWLx3RLyEzWkl2UN<+NSnrIa>H`T?)v8TowMq8ZVOrf8EXY+`~U z{)X@od+c@Q3;t77>&z#F9&K!7CVu)zLp})w(_L~6!M#}GMGz{D#*B?sSpY1Juhc#G&Ge0n$)5u(*a5LM z2zmt1z)sp>l1+LENFUuTfwdU(nztAx{GHYtCK|Sm6!V#`j>3da;AScm`s0hB&-)7C ztWU-Rt>{r!20-L@H3X`VfG;slq3w4GvalsxjPP)TC%@91|E6qJg0c3uH@cEiy9%sB z|34IdN#6&UVvt>Z)ZHnxthdptefWfO)uBGtME)Oq#(WY;($upp7N73Q07K~`X@g-{ zEdb667i^b#eCCR*8-nrw*Mc1#R73?=@?7v(GiSPHQF|KjYtX zjM%6LF&Ldg)S)A9pquO|HD^h`fvW9{5ZF-@e5JRM$C!Rfc)WSu2pcNck@?Pocr0f)MPEXk)(CH^ z^J+Tn<#Z(=Joo@fj{uvA4l%^~eWPg4+g8GadeGO$Y@Fi`GmVK>iaw(cs~FMZmj{lDJ= zganp^wtoA(spZO8TY%9OC_xdOi*I&qJ(4VG!Ny&I9uzLdA&>`73qin@ay?+I?iI+Q zcK)2%fMBb&nhqE5N&%l#nz#aK6d(<;K)B{;Y|lI)s|9p%VqfE3pkBsd+F^Ne8>mGM zJC=+E0sX2y0_eiacZe&{5rG1W*VEzSbao@h3C%HQrrhALTZEyt*LZH;kgX#NZxlW^m&kMbhh zktIv!tP`EA;89Pl!Jt0)YqS)Lfl#+v-IjdP0r-cx#r_XC{R(>R8~%w;LQ9%EqCG;H ziU~fi3_6#}qOANzUNS8sSFpFo6nHkh_LExjs88pXDs%-h#r3oS)7s4fX`8ZkrPXHn z1Y5b0f3FezQaohyJCLGrDssI zM4R&Q_n}gKTSW8?BI`e@j{*nuEtvuU=#UsT0^i;`y#hVfRsaL(@Gi)>bN(7+W$|d> z9NT;a8lpCy7Q;yVh&z)El)$&R!jB%05}fq?y=6uKF(#c=H%gxS`A~Yx{QdUm3@NeG z-2g{6zRYSZ_97b>_{}^Ustm;}$`_X@LH7VxR$i=JbkSM%F3G!hpp{`)vr#xxNETRg zF;Q>a4W_E%*Iht$`J4McHZm7?)b2}>eD>vXg_I>&@(Sej=@?n+sY~n7P#dy@ai()f zH8o9(;Pk*3am#f07ZZW=L2iKmH^G6P3AGRw=X~09i3K` zjC^NhXnz$!f@jYrkT~Lz`@{GSx!{NY#CKlp@Ei;&lYx~NxeKtX}yRwvlxJW_)_Qar)f;4y=pbuekM z7)CJWl_eQ|sP@hay0Y875ktT-_HBv5|H4yaN^Y~FVf@-Z?t6g-xX$BoXAC<0nQJCgMru51tn)1tA zjy=NS&29(69jC53jD0AgnxJ3?mZBbpP0yLq+88%G>>mEJ1Lrv-?3lkDbRSmO;SITY z6!oRBx&CWb^P^YFu2n(1X+?U6cmDJX$(8t9UZz#=N+_f~tDJE0;+=*?$Ju;;=6kJNF8NuL^jMJdNaBdRjSa4+%U1N1EHr7YD=-8o;m5!*GPdT>SD9zt}5~F zrLH_FVd8tDFEiZ2h;B`aZOm8A>$+!9#&#&WVikmsA;vq+RB zJyvBuETiS&(A3daayK~LXI;z2MmgSP%kc3QY0O$YWyd_FSzwx4ZTy}qbGadhP(wzYG<@^xeWMhatCTA8U+EiL8?R&KS*;= zf(T`g)>+3ka#}1eqYqE^Xp@=>hzwUW+tKkpL#_*h1@doW!uKTnXGEux6nDP&Vz&=HZh)As;`XvJpI^ z%;LnQN-LTYsI+BKa$xCz07~Zz)Lr+lKxq?`49Kt+dhCcQdcA8pj%EwufbUB%EnqeC z&%E<`x;wBdkPsjmOz8ECsY3iU_=y3*PSPReE**Lu#ma|$FD6H>oQ&_`%L%oiR?WKo zJU;BDU+uP~+J-cutUhOQTU-X;8+c8Z?sJ_>>7z$7tWKApfS929s@N}uW|rnNPP8Z9 z@uC$!?P@w5ayig`Al?Camgm?jkP(JvynLZ(k85oAkmHo9$*FhEo+IU+$H2CwcF<@G zC%zN=X3Ff5OT&0R?E>MzCwWK85hkE9r`3WG4PW|!bt!ZbAB4HfY5Jcf%`KKiKjwKy zBm8}xsJtH@87u>?3>wL3NV&WM)d1zbV#bk^)R)R|$J?scMW*2x!)(A}4Pb_p%ph1{ z2<~+p)=6vN{0<J5 zmjzx@HgWw{m3V1inD1#o&}9ibTuj2T0?l@jvNAr`U8cF}BUJ{B=SyDNpm+VYd{gI*VBkM31f zgmEz(`+!2h#XHi=t5=}gFRMN`&X(#T_L@%Uj~I?!e5YHP!g+#j7Samqe<3ly zCEn_SK^!$akN2QS$~O8EIQxO8RWY+@Q-RfIVOByLvmd?ZWl0YU(M01@ZSR;-K9X_i zsCrsx^)~us%iZtNH6Odb7@JgR>e%2DS;YjEdEWV^i)F>iP)%{X=>pG?*bY8o9&g>R zYsJoj(Z4eQf6ZD0z)@fqz*05{XFgPoRvp~xP1CAtWG;>Wj(^O!(z<9EAPrCw8G!$` zij8g7ly5eeBch!LSn_V%DPR3y2%?U~8TY|y@$zOCO>-=0($uu+Ux82V-_ghrZ_fh+ zz2$){W=4=bio$d;Jmm965HmU`BCsrs#2Y4|;v^~Q4E_?ACF>>k_NWyEWB)Pt03Dl# za01_Q7A=XY3#o332NuUQ=5@y{k&qAzrbS8$nCT)u*QzYQ=u_`E^GChV>~Fq^J0jKt zb)+MSwTUPt82FWG&)q>a(OVTNY+-83Nw)0wLN+5oXNAkN$sr4-i2I(FVSYBry7v;+ z&X*LA^?BE{|juf3T50Z{Z`*g%z+e|@QPQom8YZ`0cV6>TGHdcd?l^DXVbNhF)!d^%x zn~F~Hj%!~<55Nskdx8ESU8jTnU7ChgQO)20T)4&?_uZtR^gF`KhHx(sU~ApK<@HP7`3-8g0NTqLgf}JY_Ilp8bDe% zj!_f2$TbSb4)k_g@nQ)jMQ`@>tYbsa6Dl(SobcAlsdZwarDT8#R(S~-p8DJ30h*<6 zwJ%lkrmJgTS2-8cbIO+R0=L-TGMP%)o+aH#6Rvl%a!@D{m)@Am9KWWy{OsIw+o@aB zM4>;qIBtD=qjr~I4S-q#vXm~xsD*h#nB5O2a&FVc-se7&0T?3K)}QS-t8I&Xbku@( zcpld6wXAi%sgqyx9Lv%dA4Jhk$YoiO^)|{m^BP(_6vL}DK4Cl*Kv~r5&q#guv;kU? zEoZJ3+}J8#(9Gkac9Pk`nNw3=s~;E7zsrtV>VP0XFZ)Wc{vEA!ud7^{0_29OE^pU< zydO!?O+3&e8B|8eu!&N*-gd707zmLIw+CK1#iAbEv`aG7}cm&BAqCI0>=Ly+( z%uH{?mBl5aruRpBd7`ouen9A0gPXG&}H)7+?Zbblgtzv+djZOxhPgsbyXTzj^v!-dQx6R%`zLo9$MJph6vw}*qs0yr}ltMSEN0(GgMTO!!WYP$K;G^8$2a?tiGkYg+JCaWNZA|@<_ zVoD>sxvRO?O!c-%@>B1H1O;xtw+c70t!TgAAMzC6Cw7nP_g5O5VrRNI2(E=is`V>M zU3|tttFcI5DCJJa!nBmVN?m>I?=C8cNa%;;dw#d1PFbth=I^!$Y|BFi`9&j+Oid}o zhgUf%8GJn0a=*P>C(IS@7C}|}3AVRhgGDV-hTbp799R^*h0>HZaQqy~Hh8Mc2J|z2 zMDoepIwDwYd5DcEfwoe5!bz21CQr$Kisu-4*{ID;QOiH1Huq!ejqk2GUAK)ejvM_hkLN36~p-%LtY4MoHHD-9pK9Cpu|3nlB*u_w`WzypGiyp+g5bqT(4sjTZ%iF|% z$eq2tr!Hy!yXgo;b4(g?l$Zt6R~=#)R~y;MuJy5q{uz8nK95>~@kswcvgCc=f_Upl zF}}TZ0)>?Xf3o!euI1Jnu!gn0iH?RDy2B~zweHAYKkS);UrGqL9o~%d18q9|*F7Y7 zS2CwMI__-D36C^c6_%*6=49J?F57m(I_yV_EH4%&#>YAX?9qj|2M|gzZ65B^In_*u zs{f6>_l|11-PT1zQIIMs(g_NPQbg%e63~w(BA`;FMFpgZG?7jy(wl&QA`p}+Eg-#y zA|N8YNfSa8kRk~s1W4k2{r1{xt#9r7#~JtDv(Gu>u0Jvur2fkL&iTw|K6B2t-tCm< z42PW(z!%CvGT0L((LpCQaSfl=W7-Sr)%~sqM{4Z4sUO7PQB$th8VJxtFINGlumR^= z>+d-_XFuycglb9yKOeIdi{M*5J$n8%iGE=_^VUKW^wzt(}l zN0F6As`u}6SP!6_nz;NWlJJtw%PdL_>A#0f6N zbPfykbknB%R~_O@$AWuR*>q0P5O+Bfe?g9EM_^X&kNkq*W#EKd>WM^Fz6lI=H1r&ZI5XC?qnRa8gy-<6X zkn7__`DVOj+n>Ku|4EncsEJ@sLJeV%JZDe&3_X;G1YRcCw=1-&;_bf1UmR;PTACWr zK4=KWp|0&2RH_jWr>`wab2=Qb72*kB&7=xippMl-ujq@PzTn@HMZ zhoD|;Eq@yf`{ySmz>_W47OWDx^i($Pww~>MG9+gdYXDolptYX$$w0c$) z1Otx*S`qxIy52FrAnTlf;>qE~L#OQ9_RI^T5Iv0g10DNDGT_B@HSf%w##LXdQnj`T9i zv-(I(E@3w`TxxZr#36BqUC7-}lly%LU1v~%M>~dGn?O)NIH90@k z(q}~2qigUaVFJ3M4GngQ&HVla<#b92xS7L4re99c5C+@Mo6Hl#vgxtH{$q4s9@DlA z5`p(fzEX;uv_Mk$Qy1ZM@NYP2_!`)b+fKyIX;-Sw5?Jb1z&!4F7&1s43bZqkq%G>r z#%>&>bq9D{{qT%EXBH`IeQ15`P!Rbsy!0333kd91Uaoyzp~JC9^zF867Q9+oBv+&yvAVe>+2namXYXM(^Hg+?)Pd=wqW5WH+Q^)hKL2U_ z8(-0+&|;8W1g!!{P2C4I^WBMYr)$lXp5K*aw$QQ@wq?@weff0YZi{=!M6jAGwyqsY z<3yz$^3u1_PIF$HQTi$B?k#RxKdZz7S%~@3(HwNa--K zW04fdL(mcC2ZmHkfQKNr+`Tco(*Oh11R~W>?O4eSYK^$aWx-?xvNH8p&YM-08bP0bm?8(i1k0#e8uw09x5_f&6+ib1Dhj^ML1V@eY zd>oX!tlH`dJ;raiSOvd_M7v~IsKqdU%Dq;Ke+ewqgD!Mi05FE=ovj4EJtN z9=}P1HK79@kIN;p%m;><)Q?Ly$0n-1nX3NOzl|0a=XPIv? z;#UoFN+N}TBndVoP#=AWo(Y%O=?9*vl7dP@}lKaQ9LR(cy zm<4clq}zvL)RX$Iho&1r-c!g?+xjiy`utwoOjh4$xmw9O`!mFrUciN4kb6yFaT_uq zET5*}aUII0v94rO=lnuq{(AO(?{+h@#FyRm9cfoYc6-(8%e!MBTdvbw*j_?ic=*p( zM>o9nj%;-7X0d;bg%|B{|8yS2*u!Jfhzhywf)^VtE<2P;x?Z$+P@GWFpvJiD2*240 z>%z^)ShL{o$grB4Bv^d6EDL{k;$}~UP~d#YaiBvwsrQa-LikS1$0Cna%iL6KX(_OL z4*#AP7i8EIw?luBnM^ZI)+Z>G5K0zB(<4>o`Q!83FC9f}y^-A2} zVK!dY-&mmK5a?af?e4Ff1k1Dfbe1(O0k$3J+B?t+^qHMLW8B354{( zn_{J;S4AljKQPga{g}ugsB>f9=b|(3lgJ86MxW`q zv_1;2OcA@0EshG`buk8&lNYM@#x{BT4zCh|KHJ*}9%;^FZRjy%$66pNYhO#6OGkS> zy>+&+K7wAv?S|85;Bt|go`N8+n9`1-b|_*F^rax?lpJYSA;49woNdZJZ!V$QoqW9P z)%+FCUCx%8W>$%IdUv_OB^@L6i(lwG{W2jkfky&L4ncq8UL<~=;ttd$!<*Kx4z2-gg)i5%y?#-(ob-8%1E!oUy-l476@TPeG ze|v`}PW)zPJHfL}kt_GUM?pomd zm1&%{rFrqmmYnX;&uPFcY7aI&Vtuo$pr$dR>x9|Es|oS5oT_2FL2=C)Q&LVnsu$?@fK8F?QqOmhw|D_7nkxEqT|iCi!(#Sk3tT%Z)x(o3+d#V z4LvdDrVSRbjDxeOvYsyVEQYc;Zd=dY+>`0|WPBAD*A>gUKhvvrIbnR#NL`Ti(8{8V zp_BETgikW}=-t?3yy0|vCyAV}cQ{u6c2w1Tc#Rapbw$tK{F&}O?IpcSHi)eBcRK`v-W;Y(ed5R8x-dG zl+du8!(+bz%$tlirS%tNgXzH5?b3zhPL{S~4|Fe&`h}^VBEx~-5m2PIJOEs-CL#w6 zHoM0LuEl2pEJ+Z9?KilV@i0{F>V)3I)T7rKZZ$w}OeT>}kR*@`(35k{jRlpiKGbU# zGhI%lG$ul69$(t`{aAL_tmpsQzspB-aDC5J`a{*C@%Vi!0d&`o(a&PuR^MeaZJr|8 zB)+z_wSmx1;``h+qV4$4&7Y4WV@jXvOX$|E54fjfdZCNJ-A%<{)8!Kn_#`oSPQbJ# zlI%e4CZbQ8d<02A7dzyzNYk!(_&9g)-VBlInC-p)sG_=vO>9Th0A24p-zTNbcWx9Z3;zk%L0&^eD51Dv|{zx!CzvfIeQ^ zXaXjQ?C=z9abbZzu4_D$xY|{D{6_p#j|$v;Py8QhTeLjsHQu}9tLlsS8zQ6fF7=0Y z#fe}2d}Dkb`QZcCM!=Ow0a`b^Uyu_bMYW)%rf8(n!;CA!_tb(}v*)_JvRNn@a{hcZ zO$06UxniTbz_cmVBGgoGAIG}Fi-w^{cBuMX(hrAA!kx+S9>R8yk~@r8rjP`D|IwOro!aZQvwN0G?&x94LXeL!f&GY+lBH{VZG*AbR~5Cb8&hGLGQsRE?8YvzrUQoHmST^oP6U;U_I zA77P>`~o^*kAj-AMGQNYWu9cQXiL5P+On=8;_gN0dM~pCyQ0#!?5XnYMf9^}0w&Zo z;EJoSKxVo$-?6-_?N93;hyAefIz_XLIs$`Q1t{ZpiK$-iG>hc>r|M5=@~G!ND8FQI z2ru&!IAzEX=_9s~_69o`oZgYJ%YO^~`>s zeu`mr@*I$EhQEkQnlF1atS)E(63E8OosPTFk9U3k%h|7iJUmo3Xw*FYg9OEIv_rwq zw$j0Lpv1}Lm}ik*yZCbLC|N*hRu?B2i@%07#soh}Yif1)DStQyb$I1_9F4fZUf`F# z{Do;(!T3S`ixY+7IxFsRiESC?zc(XL1JGj!kGcMdu1&jI?1Mslm#lp%b)ua1^W3QpVCWzSrX+KdkozxLVJ^0@C2O=pq>(S1Vx zN<=QM|87}xXTyfew{3fyZtqVXIlDia)(}QLr;HwNoQiF|LdmSLQt&MHVZP&^C>5KZ|2g!+P=Z#k z77cM=)-U5AIR*mKYn{LWQ{_nphzk7IcdqWf@?*WOM_MZ%Tlk-V%$)C3G#Xgq+$H-H zrC-I9_-c3hg?sp4d%vw6ZSZDU0P`9koOq|Zq%@G6N+@*1GDp--ii}IQ_KSZKmI%9m ze7Unv$8`89*TUzt!R34Lh{JYucorl87y-t}4ZwR`V;!WjjFKIIAm5*`{FkXlBlNrW}Ye4KOUQv6?Kt_UYQP9P?|wMN>1?!x~Bt%DK|8IwBjr z5HD44bP52g=SlMf83SQG-sjaXb}ACrYZvV4<(lnRK0B{((EzmVe@ndhys=^NQ3xG` z39=vC4z0Q|qElxrKjAa5R8?`shPo4{wAThVKM?MO2@I=V&hN=zQGTLuI-1OpV0KU? zXm*&a?`)*bADI{=;B&j@^r3wJ74z?=kG!8}rD*Rw#(fD@D_P#;4C>Do9Tih-JwEXO zV>a`|{l>g~y}=E|Ik9<(9dzmW9QMQj*|Gp%{9u&a#!+mIokhvB5O|G{;^lzPM*}*u z8LYY8B-WrD)>e+K&@t)K4bhgSx?0~eb>Ym_?~kb5p_vrp%mtA{)u9y0`IZ5XPuPcy z@8Pqi3a6=vbk!U0<=!WMRu|&O>Hr>U0-8X+hGUP3(0b)*Q(&$i<}8zXXC&P7 z4kAEC%$Zm)EPrSKU7=pC82#S(K6K*DvWVTtl?O?$*%N{zZqKM5m~t+nPmnrs2@P)@ z@hi+cHQv#zd5=G4-?#d??gKf3xPIsq1F>a18!jxsaW(Gn_R9Z>{qPD5B;7PbwramD z-dcCN3E@#?f&_9dA@MCioXDYZK(=uGnWDDFFaegt3BXWh6jf1O@Rg=Jm7SEbfZ<-f z^&^{aqIKT;)H(0O^(yUWEX6miAgg`dp8X#1<1jf3AC*PU$Z&M^U7PBpT<5H-jFC$yYHFVv_SlJIa3$aJo+^7->Dd1A%!%P4HZs!0L^oT1|_;RYdbu{Vq{d% zS%K?P7MeAo5=U;iy3DZ6Kz`mol=J9_c(*Hkkwfw(4W^@tr)5>g>hSMs($MfO_ zt07v-#5EHMx}LB6z+Bh0AD|*(Q$*>y55kAq=}V9HLk4E!b^t%SJ=u+lK(dmh@ziY- zJVEN6NTrAwX4{Q9*rQ>mUv>f2X%HE74$ss4%TuWHI(5#170Fu*9$gmm`}>Ew<2F)<6n z5~Qy$?H><8Txb$vJTK(lD@2Z6b#5h{B$&2ka}7OlWoR#0lPX5czMZvX_Ic*28pm~I zTejMk1*aq+Ynj-AV)j9rkv7VFkhe#$4I>N5JKB#vY#1ircMr=x5_rZA{~9|Nzs^;g z%~>6>_*FCQ0w?PHst4nK)A93kjsIEff{r8W`FatmZ~?q7c?nOVzHObc`yo(jo1Q)@ z)hpOwgt$qzo@9@N@%luXTs6_z;8l8wWJ%`Wg|=t%0)}*1^R7JfxKE5GPlVF?qSDU~ zO~Axl*asp3bFb9$EF0JRcC;EUuDk%L#EGBhhOp;*V7PaEtKkuX!ebjhttS>$w9E?P zKgp;(k1-CR=db8y^R!C!W#2Fgs6{cNg(OdXook2i9DYH{3Lt-E`|sqiX9w`Y6hNoc zrL|7KUyvyD+gi8eiiv5VQZ4oU?>uCQT(Atqh{?Yos6h@SRtrTAYLO=D>jlbUQ^zSA z9~2f3Z$Hny-cDouhf(j^N2fhL|5*PQ+fvI3w$z=++(8e1>n+1-VdqB!XzttNxEcZ- z1#e$zY~W%K6XxuTctp8z5r%bb)_)AMSaLzgJ55X67pw*No>EjIt&(*HHK$b72z zyOn{9h+A)c`K0*O%ml4t{NvLf3Eojp=9z$8$$C70Yr{o&R{d~bj+1LFB^XvsL*VLi zVef9`8GE!N%}GkyjsC9TF3@P*=O)}M&c@Gja+@a6Z$}@#FxmZ5GACxVE83n`=oqcj zXigl`+rg5>k7K*W**N zYx+s)SY&t8JjZHX1~U3#%qRa|+d7TXPp@q*UXNpApm)EEWNsDnSsLu-owb%4zgWZl z#7g<6bL?qbo@9R8qegMxII=aLfHdWppEiZb8$~*eA?rhYkk3?QaZ;u|dk2D%?{r-k z83oZB9i4 zSDM1)eWqiJcWF~HZRrvx3jZAD$bOzuWHq@kQCfVAUB0nSjUlseKg4o!-6uwsxm1VY z^YknFL&g;7=Sj~)4aX`c3`4W7E97&MdO94>op+b-7ev4WEk{X^KR2dq z(SRl350Y4kxzp2wSZ!C?MU|#2+?^Nr_dPDq)hXQ%n<<347BRiK1A(;B)%YfEh#e`N zKe{{7aOMW=Mv{Wewf&-_um<53eaPd0bA)P8j^$EYpH#|qy_>oI&NpqDkWByM*VrFO zyhp!A+VEUtLoF7)MCMA?enxF3o!cA|4HQ*}UJRNl%Dsax4G!pMZ4>(<=Rx|??*QlQs|({-ET044s;&>3sgO{|GWKWRe<(e$5!Fti#t}?aMl>#$s2kykOXv2It8RkPtLES)(2InR;s@svY)qpL1rv%`t}9_OFo0BGYz>4 zeA_1jz-{HQ`7g|#&OmLjF2-6EJ!pF0pd8c77yLt9u7z<$<7g|$-h?~j52?>uPJ(VP zHXv7rZ9Nsh4ucTv$s!D0&}rn#CVLW*U2M}P>#NO^o_v2bNPss!vhzSNp8)DnM&wz~ zjjZWM=wSu&DoV6o+jU@d+S+~~^pG93B#@}NY7G>v!I#0W_||nAgj3JsH@nqrx*B7q zS!K5LY7_fjdoJoKUer>Pm|LN#)JoE-><;13;=FT@<|bfEe)m<0*qY|+yRvcdxTjHDM2zQtbJB7LN?8-W`>68}^TCLsDLnz3| zAVxB}1a+%Loavcf9C&J9MXSp&o3IefZjU}rDOQzU6mab@nU08-pT(P9ah`*5sn{m+ z>sACV=K15NHmnlcv#xcGlCBHDyE8QV6W-mUzv_$E{Oq%DA%uEK6PDfLPnhS0?Av2| zHh3GgBR8s1oIgxWx2Cm(6NtlO&BaPI<5AyOSeCQNyGt-hve^Xef*LxU49_)iy)W{q z_i^Fp?4LK=L-CMr3-$%NI|ue3;LK#2MZV~ZUIxrlx~iBUv#_h{fqGhZQZ~(lc=7h( z0hb%91&)5|&^{dLI=;>A$EAv!QTlzKT|*FUK@2e+n~r0f&<(|KKZ)m{;mHPsy*$vl zwIN%=+4u2d5i@Kw^T%~JPW4_f{5y${a0gFZ;B5^Ai;v2+rY61d{@vL>Ai{$OZ<1le zD@0sF8$%@NAs;Gs9*Q#Fdi8|x+*CDMbTRnAwM-EzE$PABtj!IhRE(zNDZ=T1DTLJr=8Oh*+IAiSgmr<&SbR3i&8> z5T*`&H)bLhDu35&{-}fbETPn6RXKp`W{d}CUc6bz{>0qhl?|MOKDcgc0le08c?8EQ zv*Y6d;kY9gw=S<2Us2*mX;`raR33W#f)wKs6MMfP4u%Z5$WHCwLY|!dc`Brj%DFNP zjUmI!NDttr9H;Y6II?rRn5}Z*2-yo;{+To@n7SX2&?jF6nhs3@h7xFrTsdyYP^NpR z6(Fu{t32K-uJh^3CyfJs4~bGd!-qizpWUAiKo*P`T~=WmVKyz+{*$vpXAj-KZM}8~ zg3(?}6t(>};Jw_37-m32V2DSn8bH?J%?!@2?u%AB=CNrrryL@5p#nO_y=}XD<-$~6 zMXUkkC`cybX@7U8@FdRuf+UT0&J)>ODZJy`;gF!*mFcXAn*153468Bkr=~{HXES7< ze{%ZnE4^L`>cSO4er^+d=*J(aYj^&>9ZgJQRIlavSX^tE*zFsus;sqaP8`erl>5$R zh^Dvfwl(viJvdx3AV8`33_lQef=q!$lt=={uJIZ}hNOd}Bsiuvsw{pvK`yMAoy zA5-RZgyRcqeMa4Xq}>ns(eFi7CJaUduoh?+ky4B-bnF6?#NT;82U6GEl!V>=+#Pfw z5V38N4)FFAw??YFA;UY6!qg5B|HqnG6%?GT+x1@5{5`m z-7{DRYDtl&&;vjl?#Dogji9-a>Sz~~AI3+rx2PZXQgd&edgLxXGt*ogIrKhMMkNL? zA?_rXMSZ7vtx>O@L%ZIY3cFHp; zAj*6&#PDMrTr(c3&_wkhyThJxccr&LerhpSvZ2 z@yOGt1MgZOFUdofMPC0i7?Xf@Ws43si@h}{Tbq36XvZegll)=Fm@Z9h$fO;Og1aF1 z&t5DJ(FhRa$qq2Zgm3iC%d}w$i z&>)c}-v#wTx3o$tj>4wgfzzI>23h8lR31f3@J|OYxipC1qJ#+jN%)Fe1+bK+!0jLW z{9pbi=fNN;G6zAr^EG45-Jdz*aHCHZ%Tm9^GkHLAf%%pSs@dAokO!g?W#xKUE^;!O0r}+wAW{Fw_o{{%nYAjkHjb+D zpjS<9`I?&ed40h5Rx3=JO}$vayQ!}sGPu(B($hV^TFnirecqBw?h^^m(6&kbuq%?R zD@StxwBKTf=bamCfN(=vuNL!4(o1JKVQc>BEyL%!XKe+a$%Ha4Co7*|^WKmyTb=v` z;g;{yYq{#^va^4h*(~<{1N|5`2>33)49Fp|xf55{Q51CEfykjfLe?uvT1G#I>iWqDqd;hsV{ z&CYU}p9>HN$)Vmqu4-w?7Z7d3c${U^q!+85m&A`V3iv4D%WuyJ z)oS#LiA&5l00Nx+gEUjNno-x~dJ6UiDc@HH5<2=>ed66yQTrz??+gY^fQKcwKb&LhO?w zYsI)^1G>zm=13}!>_4qr=e1Ag=`u*3SQP_$&OgS3_;s#Vv3=qf>NYHB|L>833rUh}GRD$&$ ztvOcKKT+uFsATdnnMc(4$vq#TFcLpUC=K-O4up3x=#mf9{Gwv$GM+r?VlVUYqLe+| znRABi-fU|TLX^OgT0p2?1>5SzBxc2R6o4X7anRk;v|!NjGKgzskDyzWrEN5p#k*u| zl@zW&!>O@Mni1I@tnmxHHL;vFqlswsq54}pm05pEBD}UoH z&zE~ID!lQ{Bf|&^?+QQPYk5CXZXarxv!3+ep-DSE|8eEXrJAnY@BJU@6uEBqtCm~q z3SRxZep6RuOBWOo>|ShjCFLI~>Mx>s5u{ih0*lGzRknv2d_}QI7vKsbmwT=Q)_3Zp z#kdPqLNl*3Q09x%>5KC)Ow>ygXY365>4bBCtIcrf~TuixB6NuOL(e8o$y)Z16L^t-n#d+!N(`8f} ztGCGrRf^~J4TVT+&;7Q$HcnjIiI<(INV5+YZ$Z&M>^SU6ktitoyMI{d>>;H6Py=e6ncbl;@^ z^4x!T?tl77Ys=pZ&;Oh+Dq2In$HbIAX^L%P_3OFS5bcpxa#G87`S~%W0g3g-^K@Gs zOUKF{+tt>`zs=yf{4svqjQ=>2H>QQ17zzn^NU&B8tojA9HQ)XOIVd3l2>fb>I*TMO zwicsPk@pk#voISyAVDOtTKwle`@PrxyB_3HIYS4~_3!DQcu5kowuJ*A3T=<(rX<6^ z#oa|te((7`2vD8zXHu?8#UDn=pWPtkw-C`#_P0s7DcuLO$|p+8-u=NST=x~NM5(Cp z-;l*VI!*MfzEHG!dB)O3NlLI6`Z(~I2FfT2!4BdjF|s5k60xJ^;}4XeZh#}XgTcS2 z3LuAev72Nb0L{8gm-~q8US=rq~?W&h62f72Xl`CxRJ!OzUO-scd_`ZlB zU=%7IVLmlw0Ff<-vxNxWlrTH$9h&{#-$?WayhD0l)S3;vNfbqmzP{$1OakGRNJ;xV zksjRW;R{wS5IdhbeUur#peD~k)lrQSM;xwx7S2L=AL@3j+u#j)&|)wg#j){txi+Qk z0M{%d_h{&g?%5M;$4m^bv0pvLP=p?4KklUac7->sQbb?I%)HNI_+7hOlv!{f0l{S* zG>YW-)|8>=p5M;g#Em^v*p$-6dS<(+3>`(S_u@ypC*Yitsv0FeLEAc`J4H*rwqrs* zED<`{@@AbZm3_8E2!Cek(ZmgQd|%H=^6l~6K*i3w>9T}^K2=#q;-->O^##*un90W# zB-3UP4cUjF9orn}ibbB7nI6c9fln>fCVslqD6Hcl<4t=i(I%#ZP(=(T*he%Yk=Q{t zi@P&3Kk_t%XSF^*}?4o^Cxl8lqqE(ZOaPnmB z9@ve8eA<$63ny|Tc-k52e=>@l?0_DFoprK4H|he%o)~|0Z6Lo~dP5|XdyGHL-mZ)acDH9(tsAvpslGqXAfC~;7MA>Qd1Z2mT8twUSE^EK1DM; z#hv)z4ywBeG@#n<>6$+lbk?y|bnH>&vl=}D|GLRi_lPbwRrjNvm$e4Hf5J1mB*dFB zV0l?dSMXImTLF6bjq%CwT>+U<157A!aoMoeN7!sJ$uii?qlX*ftrjd2_+2yL2Vm8l z2wWkm`i|H^)|nY?>8Xh&|JN_J6_1J>VV>dB9Rz+c_H8R~A~A!MQJ(*8WKM$~{e_V9 zW1_VO4~^1}Aq(XBL|IT?A6}h$c5b7)yX19GLMDo z^*9DX{x(KEN!A(N<%{LQKdf?_*g7RMJ*6vWkg&RXNn;nSPRS19@mZW-OiL+QwH9a{ zDSguIxb)hv&WDaR|7rntTfw5PPzhbT0)YB{7;d=-eac6>XAgE7Ki-nZ@ng7KEMo~S z_j!}|<0Tn|U~YELjO`itwCCa<-iE!Nlvk=utF}?*8y9o78U}fOHyXpfZMMOgcJ+Zk zi>x%Hs`1uLY$iZGsh6gHe_~e7`OHV0dwZ7;16gkJ!{+{vd{Es3N=($Dc0SwJ8z!4M z7Fkd9=310#->e5SKpz}e>Hfrg{1~0s2nO~T#ucDlm#rg$S7^3YINz-X)4rwA!;%xK z)-E(TjjCGy04SiSuS_Yo$E9pS4?caH9vB8nIb)Z$r`Slrw=K%SFjY!gE5Fa^4N`Kp z(1aXw|GDmtubNNC&iL?Zd5O~=<#D~`2T}eO?2AdjFgHI90x?{8VnhwA$)Hh#bG_K; zM)s5GZ}YAp|z;Z!dD`gGyD?MF<^X zokofC;9{7Q_yNnU|A2LWpB~7$0G4;RzBiUwRj?u`P+Mp5Y9svHq$2YYYj_sz5nSU5 zgVC?e2r94hqBnVWoZ_C%0RE}|1{p#=|{qcDzo#)p-VtwiF5ryACLM3({fZ7Ms1te%#B7llM zVBK8?XM1t*W}{xiPkCnh5L#hcbxGJmRR}}54Uzm zYi{$0@z3u?8uK6w{h;NVfguQmpdk#us_p`s8y!JbbamzkJ(A}8(`H+>11Ch&4S{Am z&@GW*Tx6{b2dn|zT6^BToL1D?^CGykdl12I3#Kd3&bf1I>Uf2aH}l-nYd1$1YF~bs zy0sF_f8wZ?W`H-th9W^qEdG1oMYHh`2WoG8r)u1C+~g4aRFLGHkLv<89%m(HfT(pY zp`GcuJX^aVqIIK!DtSa$us(;~25db5rCyOTJvF#!olk&vnvSd~ z3~LuTR!MsFF-|8((1&tWa(6-e=KhKD{U1Yqf;dpD-;4MKsRwwkU*e}S$&;uGwCR{% z5VxsckPOq=ClC|ChD9J?aZr?SE+eJrjf&aLo^5K@d6{T=nG^kVL!flzoZ{INvODNL zimP$UM2$t6snM&#rlzF)H0>**(aU%s4;^+T-9-T>yW)@Eqq1N@={v}jgulXXQX?_! z$b|tYRUD%L!n_g0|I_!C*k}+GBLmb$mD!pKomx!~1}m*ZW?)UaT|xIvUGSzRm|?28 zk98mn`^I;dTgq}Ur zmY5vs;G7ew^&gV@gA^hkn58oaVD6-f!Locn9pY9{Cz7y_gB~n9l9zJWOC=&m%iu@C zy5@HDUhe??uv@GHO3FGcLXfGQ8fUn#B}Hsqp+t2DrjC%aC_CBssl6vxPzy|jR^ zEn#GxcNJC_nf30L|18jvTF^?qYkRLa=BVBYKgdx}grw6vC*1z7yhTqs_IuZ1^-uFv zwTqCd5;tJF|_x?GphXhzg8 zRfOMz27((UYcHUW`J^qWlp#ACg9=skDn%NCRgUcR=CFVMX7ZFV=I+Y{&_&|Fbd@Wz z-HJiqgr39sxDavwh$j)s(h7CltC%+x+|*o;t28LesXYE+i1nxd6qFBiZ3AtcwQZqha87nHKwaE;59B z{esW~vfa`Cous*{dwRVu)udLO#`ZZ2HbR^3%11DCLtVL1qU{@{7$%q(m#a(QD-vO_ zaJcjpCu`3+6L5*tiDi$PR5UTa&|KYGXSFmzl}pe)*;GDpPhC~eL+v~rb~CoJm#YiG zT^jWfj6}T0eQ)9OH|QIBXm^F2Qr>IxiWu%`UBSgmMPZ@|*f!eMHc|;eVj(a+uDmor z7khhA&3ugU@x`*zH>pM;SK_J~ll$g*+HqItSvF|{ZjoreN{*)J;hjlA``<1c&Go#` z;@Bo$zy?rp_u~m+x6>lR-*ipH=NAGxpzO49J?;Ipz;( zj&|rD3XNIsVY>-Sm#`M}w)S5#mXX-o0djylhH3$_SQ9KfqIvux=?r!^S>dYr_u%_+ zVXDudZ@A@GcN5WTykgH(6=5as^iQMm9SJNOmPhEqjg5eA8S+yd%>%D7=p4E-Ox6Wz zydflE6|%P~@r-0eFzCRXxYq=_LEWpb7VfWZ(bT<~oj_Yfn@4!=m*C#*-w2Udl%6?z zg=_J{iRwp{t*;nOg(J?)E2$BZ5RWGmqgAD_^Cz`gf`o=2?P`X^{&Ddwdqq-M9I&LH zohYYCC;5{C&fxPrx`*pWN5wnoJ)9hw@nk5!}@27uIJ`(Xxgvmq<>6wbvf=d9g{%6GDjyTyW>f?;< z&N{LjC6e{6F))po7A5HDz86Iw5|Wgxv#Rp^JYCJ(W;!Y_8oNoM`TG(eB94+!+Jwl+ zGA|0#H_N-T`Yc=pJK@wqs`xT}?XH`y4~;-8K|Ogm6DpL8%h(gMS&f&=q5 z9KmfDm3Eg}*Gl0heDpeD=e7)@IsQ! zL}|HGLOs+N+|tgskU_t_dKf2JDb!&pvfW%pmybc~bjfp#kMd)y8IMsVj-!r^shNsT zZz$ z_Bu`X_qp>*YFg^Bjz(kziocnYKxgT~e#3$SH#~36?Zr3@8-j?2az3RGmeFYM5mr)y z?EYsR9ZdYc5O5&Jc<2b6@SRck`Z0EDVs%PArqDU$h@Dg(9`W0H{ zO%kCVv<%lLmuwASZcAfu-8gE4;ugA3{O_cbL?Ound`htcPUQW~XZ*12dIH!OoOgPAyP+vkz=evs3lpF~1;d+yB9E?ZE%_ng1O*FYlb;ufeeOzj9pGmk&jd zz1%=lf$mGBu(f0d!vZKZm~R=>G>^vIpiTr;qRXG7ym4S_*3=X znn#HMYN73J`0l{yem_ELatu{5PY&|wp*Bn-uN2X(0YUda(MiU$ldOb6sxCt;W}Q!Z z3tY_Wzwx9ll>^x;j`ya70EPL^byEw=^nan79OyDhDF8pMb-{^@P2m1WBVO$uq`$js zzH06@k>$^o`7tDNm{%Dgv{y28I|sL}N<2U?+kG5B|hfi5z{UgQfH8aka1CY@Me4WIRKY zuxK2`vtWzA$+){lvCtdMx@|aCCOH-H-6DEIM!Tpthq<8Dl+qBi9NJnp4|>C;))3f2 zyp=Va%N-|AUg_xJ9Fe`e0}*tNCC%Y+0KXk?hCgncD5NARyguV2e?sd^SSnj-srs2( zw3`MU1JIge8}Dp(4DQRp0k)j%@1cgUijid#LqT*TfJ`E!33h^D!V{x_O=s zHy;b#y3=g)3v$12o+7v#lKTtt(Y%k^R0w9Luc4zc5c0`VpHIIacaeNu*r^&WMcKOp zM%%*4R+Ijrbo`kfGy_U)JYSDfvYJ`BqSozGdhl!SShDX|MD}UDb`W7HeTIw6WLXf@ zHisMp#e@=x107JY79YK6aqymG-f&8Kd-?@jb08^@ukR|bx!S^t&Bl4AKtFKUTy|H^7hWFN)O0_(D zuMFPIqMY8g9$GKwo!&Da-LSv$IL7&fmStRAsOLzqTv739-ii348}};4OVOgt^*u*A zblUtD1XF`R>`{5`@4ci8?5vioUszYAeD z`4Y&M?6I3s%9?qN187P#EPMGQ=7GZ52ENqPmvouZZXOIH!El)KPvjIGRc9SeJdWBg z^F{q_-;~|`bKmR+3gbsRl(j5;J4Y%VS5|%5hjKo zQi7cs0Y)AH;=_3B3QWb=1IOya8UK%|wdr-KwuXK$Y6{N5^nB$tvTnq<~6VDkof)Gm@&%K@rWh5%zDSxFKbcHWg*X%~Dq|=qukS)Qufp`cxJ0450$)%r`D<`_) zftKV2a*+d`-;h}BYt;Jm1Yl{!*Omg4HW}OpZNdY4zf0R$gHD+De(PV@?Aj{E)5-X( zMN+e6R$2a01xk6Rm%UsrPdFcX#dHxU0Rlrnbv1@O7(fHtUdw+}$NkKdd_=Z}!$ zJFPOl2K<<&($`%TAbg;Ur@kj%yw0qA9qu0UPD<@gmgqk*L&u@R0bsG%cp@|9l zS4nlTAju);0}YHa=K9~>0J6r|f9sz8Li}5E?O%WMe`qL;p`%gDhmL23 ze5+?nfTZ1=bP@n4sqP#53$Q{hV%%$qBO=(tz3XEudTF2lOmob6vRt0PHPNDz-qq!IQ-su0&FCqS|V#)1*&)x95 z2U6rU6p=dkH{>-Y2mT2CvNEZ$hS{iU#3u4dWv(!R-@lXAJ|!*nWnmXchhyvix4@mh ziYlxs)*d+CV-mc;9By*i_ZnQ zwqK;WI2AG?n>=2M){pww@dv89mQ<7%S46B0+XZ>OQ+Cm8HoeCkM783u2Ag0u2DoL2 zUY!Z{>4&zO2>SwoX^*b8SjR^^ZPTMi$-8uN2mXX?&m8j!3oq9V!%jX#MdA{F+{Lvf zv3TbVu@#1K#GGds@xdID4%Hz4ea3P{6jxI)8>Ltt<$3WTq&`MoBz?Vw3fj!Nk8H?y zxEphzhqvrllM_=3DLXsfCa9FIAIJ06m$oY4QT1yQi(E72h=z<|WlXEvo35ld3>P0gma=6WLzP_N;qYWXevZj6-huxW9Ygl71P zZ`^K{cHU&FQf=UQ(Xq_?3iGWEhpuA?PvR9A7%=K>Nq8(`lDOp%mSSz_a*HusB$8}R z>sB$@IB5bgg2cspc{FZHoy2Sy$i=d6Oj|^a?$-vE(jPl7>k^K zF!R&DsB1%`Io1p_6I&yL48#C2(qpKgeAJzKeWi|mFI1JeF ztcPEICR`qunW#D0<;j~>TY8c}V)M$aM~t_W%=_LM7++&8jp%JFSUgQ@kswGm!STbZ zHKG>cr(cSvBPs^GJo)li_))|b^}I%c0pHxav)=U-?)z!)H=4V;#GRN z;|)u)e8oO@mB^gn|jruha2c}EOa+Cb-{t=Af zoz}d;ZziCdFVrZ#oHoOKz3p(xf+ZF46qSSXq0b``T`@LM#S1wdUs_9i6TDHI9J49r zt!R3|{yn((_U5zr9eM7}-GkgOY6g(lUmObp+;)8AyIZv-C$BJ0!UKNh*5X|l2S=g2 z*oH3MmTuwcJ09IbJ-rB^b@hr+1hTH*41pC2#6K~appAxl#mkT5jmn=Q#0wTz4rPW2 zy}Q57)VxGnbV}<7HjsLj!2-hI`p;~(44bez*qNz6qRV5GYIbt>=1Y}+Nb$jtvD~v> z2LHo<>ezs&I#TUDr=sGg#A#t=Uq&Y}W*Su7QFn1%2f_%@K-8a|i2jeAm;-hq#232p z8UmPxjdTWo^(^M=X-hV60KP%7sFFYk?4KEUfNO85h$wSE|P2)uAL+(;fgGK zYtMMaG*otdH+y2mAHA@64E~qh^NfK_+>-3=t@(aGopB8F)pATtsjS|^a-u>JYdi&H zYf`hb$1l{l;Ds`qi!gTaOS9$iO1`D9Ym;h0VhSLnO~^WL?RO zUbIoo0qvq2 ztYQ0Pqjr-cX9CFmOrbDb*v)_>LloY93s%_E5X0uB)KrNMBN{xed>sEHFTKQ`bw%v+9rbier&Wzw;=rWaQ5AXSh;>D>7BZ+wyS z?*UtG7S@Cz+$?^-Z+X_<#AL*%N4#reA9}JKncN zf?Uzfai8B_wa(JM%W;7DxV`}!N6#6&_OqR*lW4MHFSgXyczh|V#FSO=rc$}zcStan z$K{IRatz%+W6UauG2qs!!H@ynNh3Zn^(b0Pq(!@{Lg*bS-R5^+fAHtDgCT$wvA*>f zg2tRLmXfH@r!0d~cAYZ4L2_tL`H-sVa@yxb6~}fY-ty(YCUs`frH-3Jm_~pXZrO$^ z<(tT7-y40w?`>Q?1dZceR{3 zrLw$C=o@2IqyT=-{HX^1w-5!xrPuyRZD=_XrTSl~t?L5D0Nh!+f958zZnTIOoL$xCWr0-NF+@xM zaA-Cm*Z$VOWxQ^EGe)rcl?&ylHu-ao_D#SbvjInAvR_lj;BU9v0crIB)W&NoZ>v{- zdqWL&H=q%=tAv_}Ax$EN?(SeVB!O;1*`L$xX0!j@f1f}4Kbq73n_cjK`8`-dX!IVZ zQZIB0{qIHk`f|_cZqt!UPoR0jx{JWj(*m2gG#9cs?nNorc*Wp7OxntSYTOoZEue|< z3~Z=TS>#X4AHe%lWIhU}PB70KMNYo({yI^TpI>>_a;$tR@}zGN4aTMZoEpiF3~;YW zI*4HG1~u3VpNuQV7i<590{-j1czJH>;KiD($oi%A4Cr)t@|W@jn=z*6G+#d4A>6%x zCvjKt<1Bg0Ev@pAE?5~|%3!i*h*L&s!e;(rWwdF6j`moSkh)6v=;&{VTQ4*kwEIYe z3PAdEMa@!{{f2O1GF5qjIKsX}pki6nKG<0)=9;^FB3nY8{OR;{ZX#Z7+&0|S@-PO( zAOMBkqKpm3t~MvGW?K}O6AWI`oH&H9$J-c0ws5-PnD*Ghv; zd5m`=g&u%9pUa6PheQMn3@0J5lrjWI@x}7D?1C3L6VIF2Jd)=Lq(FZTKp8^Eu!SSd za8|GB!+U+z2G5g2663_X5J56gb(dIxL=EF4w;jf6=K4q*>bp=%FCG?#i7NioJ0^I| z|J9?D*-g;?{D>HN3qf?DikF+W*}b%yhf}$6&GYtg`ClhHpY!Wj88-)7D8@o2Cf+0; zxzq}u6h19@Gw)xrKG9Y>VBeSN%Bp0UF!LN*!NGCV3zdMig5#hL|98BDyfL*&VOjkF zyy}NP49_;-Qt_Ec*01cCWmfN$>tJ<q=I;<`Pe+sipxScL>8^0hdv|)4&Ft5@4Av>#GuD z4)a=JL{Q|LObf*;<<y_q-$Rf>=>b3*e&L5S4mIhSA?nK2Q31zIL)H zOAxn8xj|*Uryc|SK+!Erv@|V`wy<*U%N17LEMrEpA&bmil$82>*UxP7IsYU zdTdT{&@DEXiy4Q`cMY_QdS$mE9yZ8tq>Trr24A=Ztq_^69VTQrI{V^Anms&XYnp$w z#CdWXX;!@-Z33Xl7r}X~42LO4++gm-dW!pC*yB#X4>UW`VIIW`_MY6l@7|E(HlJrS z;w8Tq2TK_j%PEHI=x4sLueo!`G0yHaRHR9gq=LQ#`EOPn{S%^hyE23 zKuCn0C2A0k@#mH8b0;1jHLS{-BpI8xI*LGnAet=zip24-Os;qSL}1 z^sgPaa?;f#RF^Sx5!khu&OL=vd}L6gPGiWyK9P8>Dvq+;hZ*25H3+7yx?0N}ChL35Obkbk#vW_CCK95%sv5 z*MH8?$L93y7-ys+VBAVHz*(le2Ihue&pWJpZEj&6qGb5Hf3ypn7f=rLSms`kZ(q0c z_r`Agv~#z{nKzz0EbX7Tc|NYoCEa9vK)d6)FYA4+@2mi%L)0g4Z!+Vr{f6AFsQLyv zcqIw#5#3ymEgxDnx~4x;%}l-JLoc?iNG!q@SADBVZ8;h@H_!Ey?4-!KyKw)xIuzJ4 z_yR&QVA7r4X9%m5eS>+@o;&*Fc78^)c(swYZ5!F(C-S%z7pk5?fIkHW%v<&5 zu(UJ6NtF4OI)a^NbO(wecWbmEZVS`ZO0ya%L(9!QXY%u_-@Gef{~W9NxwIue(Ru~4 z=1_QodAgNOs8Ja#-T7Se<0J3Q$Plqj3!dy+I*7G8&^k1Wc)aMu5}v>=8MWbfP!3IR zAP?S9$CMKp^L6Io(8tNdZ{pR-32)`ZnLdVGeh~&OU&2bAO1z)Oq*HeatELf zipx(~WHO(u|7@;x$@ucHCSwj?bdw?U?muj!p=~HwfGpwgI^j(Y0o9lsZ-36`(P(#@ zKtk3djfY(}w_|Doq3ag(DaY4{CMDRiiV8e?ZRkaEmEzLpq8({FnO{#`-OxasWaVvb z)=CpX&0l*zi2k_{f6kpxJJp!3!#6SrerA`;OSmFc$=55@{`38CPTC^oUb4wv=$h;s z&x4c8-GnrQ;=1@vsFLRe`JM%nT)9R`9q23R9AV99?YucHO?J_15OtVYe;Pvs))D?o zG}-Rt#d(0N?T!Q@u%y2YG^8;&pTxiyF~1>`vZ$qmLmofMV<|$CE)Lslu2l2TnzCe& z$&5N$HltQHQ4lBp#~SsJC0x)!@F@?wEr5fYKYzBzlU17d+QfO7B-wHZQ(kl$K~ z-7^OZfFQ=Ye-}6o(4x9F!GHk#*co}(@IRDLS^mdH5=MUhJc5JA`+C$&E}UVQMO>!}8vgzJoKbhaT-q9v8`a?r3UQ)DsZt zfWPh$$9@cvJ_#U6d5RavT}e#ate|4;$wSS$VgK;^le{pbP=9OSbn_X+sa*O9$Rq&g zs+v)ISJz<#9*MuS<%OmJ1f7lsxMUsZcYSUA6etIu`(u$rd(=vT>H)aZ2)KOxFg%fCLG9cBXYK&?|9wezPZSd~zH}9>Ar9pRL^n(?KeA`P ztW$T%fI}q4=gXhRZ>iN7bNUHM39nJ`DzjnPm)u#N!`!QAS1Eu8>I$sa!4TZvO1s?5 zQL|Mu%hV*>vk}9FuEeN*>AAuz zIa1z|G^{A-d0WeIj%#cd@qjzuBgayGE@D%@ZJ9%63(bUDO!3Ml)w6-=&-Gov3Em{= z;}6vdFXo*HT3kBi{SsMGse?Kf66{22q!!7FUsCMWLv}xsZ|vjPHbC7cGyojx_kTk& zix4vaG)VD#N(i3=!o5CIz(4xfa-Tz1J^L-CE!CYO$@_OGJ5s&d*|)SljdXxTD)~#9 zMP(a%@DgGO7^q=5h7%@pIPt}wbiIOpnJU$%>^i%yZ+9(ld?NcuIq{|JNwS8~yA}uq z1VP~p3@BkAWDaIFz+)Fo-6b_LqgHZq$VVm`0?s|UG|k!15#BD`qKqbbU{)(1efZph zU#;}dk5HKGbUf3-eiWPbD8Ek)CQrOY*qqfDX<@6pL85xql*H-FWsecj9$~t8K7*?_ zo@LQ4%Qad-VVdS7T@|J?Z_aK$N92UOCt^3Lm}DQ9(YoD%LgnRE7y=S@G0ic*M6JN7 z?2O|YJZJct@242=z+N68BT)1uF*VmOdog#-xCc#=O;3of2~tv zs9G_Vzt(N{M|D`${p+z@u3tFhl^Jvo6bZoxZJ=dT#dZ<|9CUg!)94Fe;{tVJQp~td z!^&4WF|oS4g8^H$seLlyV$G^trGxpWrdE>^YJZw$%=LwyDMUA&b=ajHSwYt!e+i_5 zz5<4l^e9jfPPkEh5ukJ3aeuVrP!5&p84UgeUE=Y=DaS(Hdd{J9;_;#wYkkE%;t&s){q4vjgdPri5ZdeK zI{(g3;c}E~rp467YjQc#N}z^}5_95{^BKmtPloS>Tg z%m`7LL|Gisr{FV1nX84)Pjj;-kN4(ZI<3z@mW8VwNSr+&Hl$zP2GIrF%djt4$L(L| zl&^vZ3m6|Or)sYODzU`NRCs@YfR~MfkA?BXyTvP$=AG@KYGf0skyq5)_BYM1dYLye z>+b5${SY+;FvPYk$0Lz=9kF%{<4HJpQmGE812(4$YXHJ0Kj5#9c#;F|;Dmu{3# zi>KW?T*&fxk(yO)J+HONmFA_?jIiP@>$_ay{j?{^S?cEbJ8vo>ciu!RtR^4o5`Pjv z{1@_U#vZVM_71qXPkT<|MU7=X`S}GIb5%8p{idgMJP%%62!YLuvM@pYTw|;f_Pi80 zSbrgqUyV>-4bsw}Ku4tHM6O|)w-$dIBhYGX4SRp_#--L+2ziBIplbDei9;8#^o0N7 z=e+No=yAD!7+quTVMrY0C+Yp)~`xOwCht9o|>^AKJq)V%C?84ihp*x zqCxcvy~#+!Kxa2U<uhH3Y&qI0Mc!D%{D{25hgcT! zcRywTJ!|r2_ebi4ml~h6fDEdZoX?m8Xg0Ohy!nLSDbIwnR2+f#11mq` z)a4iA2rir=o6NX<8DC9Q=r{YOB8E+>}|P`?I;Z_vf@nX`dNOG5j*nVL_zC&q~O zuQ{_2vRM-u7BYZ<{Cmw1^Y?n;?kf$*v%l30z(4fgN|*?uYJ$OJsxz3`e&p;_@7rUcU~3`^eWdIj8!b`mtdVJ+vYH z423JmKZ!~Ef8q-v24m)PJC)yuh}UM%>XtOgtF|&es`0G{CY3$G(g)vh z({Jd}%pa>Rn=ZmP6*}eS9n${;z2>hf3)&iX(91FQ_3q0^ymQ@ z%>VSP)^+52H?l*KdV^PJYBzD8^?NwGFs*PIdHyqki8~j6d@5BJ6}UTo&&FnP0gz=> zXB823=)$JLR(xk;J9lKb)RbUEHrdEVePKOc<6RVdOo}$A z@5856i^%}Ya3~*tvl|2HCGK$Y-RM9et4ohnI<(?lC6st!Z^eYb~_Mdu`tA1F?qbKMdGZ(nX>K zLBxN7tLA!B063Y1xZbTk<@;LOo$5S&*Dc$N{F0P0BJ9utDIJAcDuMxy!|!%>?fZqR zm>Izja{~o3T;9fyAaQSRP=1eGkYD&1Ee5*(uj4xd(QJ16iSnh#M7=xfy5gY=Pq_`92f47FrC;3MV20jQ>H+O4(%D!VB*x^Hxys zHl3wX3B$0!3o(tLq>#jbI(GI3jQFTF@Kd0?#LJm8)#A8uuJ7sMU59cu-esNzn2Sbz zWOohOlyQpt-KUktfjNBj%zRS)47HzDRVj9u?q|w=rT4?7<{{o28~GLY&Qc3Hr#ke5 zyMbOfNvMVtRo<1`nkLSd%t= zf_=)yX|FmhloyPTq_ z%#()Wxn#VQf3xj4QICM~^rTC{MDFPnhyV|5>Dp`EO|xV@#`=Et$G3Yf0ebe07=g;B z(l6|f1o=;rh3ob>9y+fXJCBYttG{(DCq9go_ zz%c4`t|xo^A12^uBL9DA(j(wey)B~*ti(R(Cl*FTuySms`|K@9BwvZO6+l+ve- zV)_2f#!cQXL{PjtqtrKS7o5?cGWRGFvu5>P5$>fEtGtEk& zk|z7+Bj#&l0)x+cX7riz(ass@P@2`?-@ zMoZB@*SzJ$K(Ix=n%U#P!$=hB;K=H*a^?a#ym9>U`?;xe0~&q#Og#ogky^ADKVCQl zrG7imgb^Hm3E&qij(k!Vntor!Y~3(=`QCuZ{B@DBOWENM!cx=8HJfO6BWK;%*;%*T z456`JGzS4ViC{mi(^&h<**Q}auBX1ib4M&kiysimi zSEQo+{=m(t*JmNKK=Fs6X*vHtvVbyrUal^Ng0PKr^fi2AG_bvBSfq*;<@-QR_D)Jp zp1rNv^~h&W@foBtT#2X_c4AY=HiDD?s~T?78}`!Rq2tiuZD|GQu5P=sRu27wSS z(KV%Y9!1ORhT&ye-Z|w70|SpCqH$|%%KMmG|MB$xbfN!8Ut-G=hfT9n$0BZeu##O5ZXmYGueZwVB8lR#Fz_&y*JlV<-T(shUTDNFa%`bl5gVz> zW`9&#A=TqosPk-qvOe~F-J|`d?kd2F@jtat|040!s-5Hn2oZX839(0~qY9?%Wo6Id z3-crMnYFvq!(Hgjdp*cYbwt`8fcBp-SJ<)v*ZR%AZ}$bOJ&o0SL;=N#5%gIyicMfI z+#OPU>lIH|czypooRMulb@TfvFngbOEl7u4aN&*lTQc;#xTHIVVa-RW>w)F9esgO+ zjte|zmU-~Wgs-~?j|fSulYQpRS(6RXo!ku@67SM*qso!d&PY2#l`&ci2z=y@XF938 z3^|m{OG~&ER|HYxTM+ojh(vqNoKqFFwA6Q*mL0bCw#>8*#V=?GE}!|TX4C_Det~>C z00Lwa?Lun#%)K(23dhHrvjtOcT+tP>pVpq1qE}rZ3qM+zff)k3r6ReeZLS&O?QggP z494=wc)-{A%r7Xho7R$zf)Rc30k;$CH`En4AQ8x7k!A)Sb3db@J%70-M6zY{yLE~y zqaD4WT-VP%g}pEIIi>|HjZqBe$>#tzfXXuNISN~zSz;ZOelIxTC^=qqsfyfhDr}Si zt2(svED7&9e+!Lr@=28%dPd6x-Ffyb^&T3!Rrn@|&Y$Dh*zX!%>s9snIqhPx-7K8~ zfgkS9j&I*Sz4uO3*5&KXSMrOqv`*ZbuVR+U8D+rVzpy-_k~@zin86vbD_$d5oT>pHlO4e2r-RoY*_bQvDmYjY ztFgXu)x6$S>YNnC9npT*-77OAwWnQNw4uFk5lhF!JA~M+Za?ROQ>}tDBjh*;-fHex zgs6#+u3GS&VQ{D%YRR`1dNUd9wWK59W=rnC97|;#x1nfA$q)fA!bFuA@H&{)1|A?1 zITI$6A_i{p+t_m#orm_m>2Z$ez+pCe7G?UQLIpNh_0fJHqd z-UmyPj3h*e7#?UE8MnlAb1DR%2(}Cks-j+a z)e8>E?ytfn+^WQ7=ko6dH9UIh@#VOS>}RyM$R6;IOwv4gugYifB(ZlbDtmOa>V-;! zML4I-*0t!Y+9{)#Mk`{w7)?SWUPO0Dyo3_(k?nIo7CyH9%<8cX&(L71I`M=`Ugk)M z=3LaheLhZzzblN{@AC9jobdO#cUvzrlW;F}-g{Eo-ftA>O@N#@WEcD~MB9f^6QxE< zPJ}YdkG0v_y_vl*e_A@^4<*h68D3 z)rr$$z8ur+eT*G3Z+XkP-wY0-=t-CTZWQASA|K&zrROxtOg`6f^jG@9ocrx|f=B@a zg_|_l;r<`y59<=TSS!LEMdc@f9dc7hq!~$6a(`EF^FcR_M22}R{ z0RJ!QzZUaj%ttVuXu#`*ZkBWyHwbfAaF-rvx~;C#$@b3a(wRh&xSdG5EPa0t0HPq; z&?OR0XpU7lBYcjx&%HB6xTSksf#qIXpK|1Syyc7bRy9Qij5OK)6zi{MXtP21Sp)9& z1%Jn6Sz3yuf;DZ0pE`Uo*3JGWfFX#e93e8KIV5#*PCeoDW4lWAf{LeCvQBW;=BW4d z35$tS`1VqNwOsBKkF^+?gpUI)g{STmp1)5|CEvE6Mkgw^n|M6m)0A6PQGjk4?{d6a z^fq+!pIKxDda6bVtojcN{@3RVQsz#OO>91;=&yq9qR5T`_Kxd<5p6oZZXuxGDexc9 zk{#Hx;_WtB@$gnz6wAIon;xESDZx66C4xOq__61fhD%Q{kj%l6NAE8ydN%ij%d+9D zL?ZX?$D#E66_wuBejc{gK`K#@fXF_iB5445w%RC-z}A2sEt?s8s7Pdp`_K5BgRx3 z-|*P)v4YNp#UF`Xa&f6wUwr)dW>;{AwN_A1tj*6U|I0%i8lN?VJ;t>msreh+VhI=8LXVe&8hqr8|?Rq#r>jf-ih@kRDc z31CaL<2iUdDWq=3C6W(91??hV#A>mSP4yu^%;1Ux1QzXD6}L&yc9P5 zyq`~FHIy|Z{;8DBxzl4w$LR3utb)S0`igI~WVf1X>L;vfwIj%vLZG`@qBGF!lMv*e z-Y35C%!)nZu9DhaXs`R4PjVm7z_OmFMKV^yom}UAPjttRj<^&yx0k&$Y~qHf>y@u` ztMjxv$EvCBeX2D+$*bijVAb)7=(#Bd)AIhO{q2|E=Vd<5j2laN-bJq)Sk2)25g$W~ zj6g9C1v79vj6s!W+O&QqLr(ZURY-)4pMZ-|r_i;din=Th_+$XCb3mJG;AgLL^`*#`YGXgV!h2Vbc=k+PIb8xOwA zcV4s@$V?F32_tNcD!Pr@#r>zVb0!-uTiV99a^5g~>q6Ff&Sc>mpg?cD|5OX`#L(g) z?8?m|(0v{Ttr3Tb{hrrmvMIP|hOj;}3q41VK=bi$jT|~+B{5)rOEoA^ML6OYjKnzr zbP>{JK{t?`3$RTwj|i;zCkzqiIK%vzorZv4{r4_hO$whHb_-Cv<>toOYs z&Z?ir-Uf(i2;63h>o%6qn)SpZ(M8YtvzBsF62vd|Rgm_8;Nhh@h=RNbJ4FPnxI6BP zJr<|}=T0c_1slfMl_b3C6n%PE$lyDL&&bTs?npCsT#R%U%RWhP(#0J;k+6MPk?0+1 z?G2ezg9=c%s=;{e&HNAYb0$_R9rMaO%Z#b!f`lf!Tfub4LQC z{Va+I?sHS3Bp!u*t|@DAI~v)h(R975a)Qs-1EIG!)nMqM{jvu18A2#>4AGSAvWkE` z9UAxjz)nTB93C&8mzMg<+FC`Es!c#1jT`pCo=_(u383{t)c3Uut`)aeJZ$)Yh5fLV z$y0QxoSu;AGP!d``}!nLXYlq9NPxc{OPrDxzZ8&E%)6q_qs|Io4(rS{W1>2Cc%dkE zwe)~VU8wuYYXK=%i$i(i=Zc=|=1#|^YrYe9!Qy9ZHmAJm7D@xGN>-8NB>7gnfLT@V zw%dcxGaA&XFwxum+JS*%g0yF~(-__c$gV0T#^KGICD%+`%u0r{ojp(Eo4&P+Nhzj= zoJF0YQVtR$udlrIxxgz|w}0=0f`o)fxqh6cg1my@0R*BW5=bVD)vx=U#t@HktPwGG zv}-&XAL|%@VpdXQ@#3wiUg8;lYVOGW`aPLlI{$6_$bul87Hv>i3WyiQC0s*N8*v?) zapaNlWC}e`b_@s(`bbhprv#A}1q=rj$>!hyqio~>(}=$z){`j+yYbrAvcp>BIB{ch z9W9Yk<7=&qXbW*graR)18la9*RTVyoU)Tsh5Pk1E?fUgdJo2$daqKNstC5DyzVr}{ z5(8#dBY|iO1UZV<9!$D}r!OCNxtx5^`84U$70iYE&01f`-$cLX{Z#9RPkr;{alpkP z=^tj@dP<|uxcsE(HE*f&gz*JY%?43n%2X-*9=@7)cp~e`X>jl?{&1kMG-*{<9oWa5svYe9}qcp);wlN(lilDGY1l7F{;c>Spj zQsz_D!?~;7y$Fg0`5zE}I}PucX5?!#RFE>mXF}q5hras$OReuOYt`<)RkhfMU4%t> zVTdwW`Jl~hizb~e1A0zsFbg9KSUn~z5zAC9=$`X*x?`R?s%DVlr->T5Uv_yTx#dp? z>#v{3*X=o@6Y!o*`Z=cU&&tXS+@E#JCkiV|rG`A$`cA>C5fD1kjbee@OAs`aZT;t_ zL>j|$#Dl%B_9!n&OvhBSK)kp?RN-H#G|4D1-o3^Dh3(=X! z8qTqYYkk8cfDU|vj=SFD5)^f_V|LZcR0rS0xIX`D2^_6i4#TPnerCaQ385*AtJrbH z&`(_t!qE>YES_DO!JU5*0ohlaRaF7ot^ya+i*txuY)DiO~EKUZHUXza`=A5F$mHlVek&I?)2P6mfpD%xn@t@2xI0MrC2jSZ2 zlkh!%{c6X&_4y9uM$XbXv=$qwbJ`w*=P6BVasSjY=D{H*v9?q#6Buk^ML)n&Vw+AcVRd(_abd zfoi7(OiOt!Cvx>jKP&D8;FuZ*uS_jkIW-auUC7!m<+qlVV?;rUEF1#Ocl|#(vK8dW z$K<2%C*t7S1I|9VV(;N%*-6o7p~HRslEC%Er=E4o5d@YXYKr^OLr1J%iDrFyB*yD_ zelewR@ypgtvRzk5_76X>ADuS7x+2!tvuT^V`y0aI_|Ij+iTuv{=1X0z(D0+eL>0x= z2ca~f?4~?4i8Sj`jj0LyD&7Wl=-1DQw~7aiEw3Ic84Yy2x7NBG8I_VLxm~|7USk$` zhVc5Ju3ttB8KGs8+I{0&ZArM}7FCCKTlsy49QoPZHWfPmDjxso>r(8E26tXD| zDQ#+)t-9(__aIOy881O_Q29SHj4X-43(+KTEKbP2#d&9b*UaJdYbth1?}f*o?%tEB3F)sb!U@LglVu7mQ*rUMrO>iBi#YGRYcHuWhTW8~Tz|kTR2Zb^wKWv1A#%t4 zVcyl5Sno83AEHMN2g3a_5}PN!7P2l;(&IEnE}D#=r}Mx0#GbyTXk(XyDbN>(1lSU` zwFmd$z|O}b-&Q4vBy6PfxW_;xQC6yBW~a ztja!6eLXub+yOuFg*1Ny@PZAX_}!-2qW) z!F>-ICcRA#dv(?9NqBa4h;f(fY-9Vq?+G>!_9+@ctqf}>jTT36!@*YNHx z!xznkhJT&1S9}%mNv~5&vMX-9M>Ll{!{xlO*7e}}HsuH`pgKlOL-3(_ZfFLt2(ixX zOKp6Qg~AG>tNfZQo80T#TMX15lb3y}fOGk2;*dcirDb@{s&iqFQ-9lzN-XMB+tU(u2P^7) z2hwD#C7|rN0jBI8;}IwR5~SC<$DZ|YrfS9o->TPe+Vwu<>&G6q4{_RH8 z(Lp(Q%Hp{O4dZqj<)rfiBe#akd>a+R_^XeNn|e~oy~Efz_T@2Y%FeSfbY3%>`T|X6 zE*2A-bMqHGly9J%Ux~%l?}LfJbmT2Rap2H)9pCepLtDAn5Pa@K?H}Uzf@ki@#vjV>?fOLwJ9!r#k$=f;)E(^{*QT*@2C|{!69_sI&e< zrzs8xZR|NiI5R#Eqg$P^le@%eYU3&x6&>un%sZ4c&k#97rwq@VdOThh@gNjjodJ{i zj|-ZuqNZmHK+hiRBf!3Vsqws)g;0XqZBB0Txg0!y@7X>zW3oIa^@Gb$!HaH&w-oA@ zA*n4@Um;@kC~$SZgw7!kI6x~`L*HuOlvfP56*HetzkGeZ^?KPn`Pj$ELG$*x)SscVz(%RYHt5HfnR9{JG_+6~$)Fk%vt z88v|+#Wz~rMt&lx67*(KJWq_C_>dBNe7OTmQ>ky>T%Yimsfiw2OlnrHIT(oOT-r z$C?;d3;MJ&*#eJaNxroXs!TiGCVu?U>#kN0n!)4aGY}#klY?^X0j7OUC(kQL&WQXK zps04($Rl>AbTF-NDcRhPiQi(fI&Ok{%{UGuf$N?O1>T57olud zJIoz<2-1bw@rMJz#E4gbo}eaCk|uw`1z7Y_f1-s|jhtrxo&=SbhY}Z6{@xEUgOKl+ z>KR#Ri>hEviEuN>$fxRWq+S@+E8>q%Kv8Fd6Q5OXQ+Ta61jv_Gl;cW^&SfS>Gr8aK znwHX37hU6me&oJC8;N^N{`ph61TzsyaGvvj=*VS`mo^!}S^sz`{xj);fV|MzK>wm- zr_caNKRLYY_tQ}sUa4ZQk0ZmVbQ6F^cL;R+W9Z>Ch{iqE(7?5PF^^*bD z-1-_8a(Ir@_D`P8cNP=>m}9d$5n1?x{gR;#a|@YRZ`_}O?~^y+$fV%=8{#(&4)~Q2 z&oKC6hnvLFFWAvtald9{=27>DSrh@mG&ZA^ogF;fcEv`Z?Ve^9JVI?>tF0sd@MFFE$X%r6>35vkl9wPK7IE?I_CJgw!N@N$_)f=ugdv&-l#*RzyF zp0Q2#ibqOv_0~v$jNg^s5(iLo;w>bn$F;4K5IkTeol!1Pet`dKC~5 zM5K2DDosF8R60QcY0{+{2t|ri0Rfc~5Rgu!OH1fQY0`TjbOA}E1qeCc_MG#c`@VOK zZ;bEHz26W1!IOubz1LoAuDRxl5uf=D@8m z6`hSf%?x6(oY#Lft>33Fm3D(}=JTxw?!}4>>XtOYL6?afEe&}Mz*br=A{qJSB9A4} zE#(F{n)aQGe#ZmzGaa80)2e+lpHt-|dOrT5q<0S#rrfnc@1MTMEJv+l_ovr@RglVE zP?yc&QRI*`ERjoyI8owYrkR<&Av>*^?8ucLS4JM3^y_hpSZe@<4+v_R1~(>Wox#Q@&m0wTGiY>&(+qv(}7CpZ+*h zW2dwR)MQGMd}(P;ta;*Xx`1LC#(waKpBwZ9QP9)OlA&K;66ld``DSohA~?-EPn&YD2f)$E1idtbjGE(U*8)m z+ZNy6?#vsS|9x3n{o~mC;v`lU>%XO#9=rt%%qbmdOi$m#^<;FBTW*&&3<}%?XXmA3 zWCvQ?a@e>e757^J#|Uu#tAgqlk(`e6*9g@0&#t30aDJb^-s%?!`yFvB`0Q_wkD-MeN`PCDbZ~7X-eqXo%DK`9^-1CdU zVt$nP8U(DTj#ujbD$IKM3-V26jJZAWR2~!#^9g@-w+QpC8Y{1OP3$!%~ zgBFJKMRp=DRwJn$8`5WVbMpAQoHLl-b@QZBHEMYarEqf3m9zp)zxSlS>-5j%?gh&Y zaVGck{JBRKLQGU+kM;I<*-&WJOYmz3XW`IsWx3*CFMh9Cnlc{f!czF3-uV#wb-*WM zIllSZS_cYhC(aU6-&qi~XI#s8JA73#x6(fJ`=wxy(K8;u94Mns)4(TP9BDV9BwIBA zP1MpF-GjG0gaEkF0Ae@|EJs-&1zzbOaFldq$)U*zN9Q_>)4l59NS$R8;j5`B$?~w} zsXS$oLQfYvstYFZk@y{Ttuk(O3a28UDt3pFs`C zBGP~S2gIrZvCfCS55Rmo9wx3kvrg!1!~7NV$_F)$W`>yu5o%KVGW+|P4Q5lV&TQ9t z<7^(yX+!7{B9N?a3<-!onj{dItmsIczWXstFa~oXs_j!jq-#OaJeTHNj^lGkrF{E0{3Z7-egbu9FY@(KTzRvfYE1D4QrmPko;lJWDNd-&dC-i_Pvwm1g$4|af3)`QA z?p=c!lJrhL$;|n79oIH4p8B-D+*gWhbl901*Ir4CQH+aHw@40aZEYQp^imc-5hcd+ zTrwFfCx+=;%HFt_d>WqT_<`QN@f}5yMDj>E!+pr{I_*OZ>fj3!KxDvpr3(d&d|zXa zcZA4bExFMe1n5=;?naY(plT$JPNI6sPMl`OOydLP0Eg4%cRFlb9zgaL0_39*M_PqPtKmz5LY4&TM(cwByEe-_pR?0C*W<$YGS?v%vB|wHEnS|f zU;YJZR5j@&5VdOb_D%#a#}sAkp~q9=FUxPP+^iVn(CiEd)}#k6hkyF^|IXGy!EBu| z84UwFNF_kE;6#{M=6x|FAJ`ryb^HHe-^kaw?=#DEZg6Pg8Q%H8XI%N(QVf)l7-nAi z@fE4LNzvcu)-u!4kLTCTF?WF`77RvnnLVRz<4Lt<+?Euk4Sp> z&a#xO*xZ@QelZmSwCKMPx6lwa3c+1LetB8iMgtgB=ru;4V!L0*#1fbix>1Ut}T z2eAh<$e0e-+8N%s6Ac!9$`<;yj9!1N>G%LMa0{;Z$p5D&#(gW>vpsq86^icL>H8*I zp?BuQgu6+XIyC>VJ^bj^%6qM~`z5j{o8IO?4@wBA1}FHz1oZeu$qxI`=7Ls>zHUp< zEw?Mq(_{I42Lx%KuO`eQx}mJ~-ZUXY^lKcWJ;5n}-s!qTu=`rst zAZG$V?1tznqo?x@bZU2zX|P??^1;&MQC!n#4P(@%GxGXnsk*bl&pRpx)kik1RLaQT zRR!25>erRQ=pMgEMZESAK{u(8X&ai+bH|MSV?Y`wijK zjF)0%TzDnYox;hP|9P;=i4CIDRxc}T)|GC?-XbRHp^u@G@9Win5 ztLRbh^Ka^?Igf5sU&@iBQ07cR+){w-|K%S;RRCtqb0ju@1B{LmKrH>R*{_a5k)?tL zjxn)t4_}-c!={gWoSLXbdyvUXZ$r#A4FxKS)@2?GLe~axd{2n(-=lm;;C`1HWW75* z`+b;dP)S|wd^hmGEL8>%xJC@qQce@FIkrI7b=>NjpNFdV*$f$eS@C9Jcfs}S?x|#kfiEi-+ z;v7*|7e80lk#>6nAN6K9m~9aB z27Bji$a}2FHGUO*H%$dA&oWRY>g)50+> z^N${TQmFJ8CBThaX@@kqr{ucOC4CI%Q{!H#_9a{zrE{u z^dw38dvZ^YO(K?Z80aP^g9iQ}wO{&uf>i~M&RDx)q%^2)+{qOuq0}?TK`L%xgjg9o z&+k| zB^eYGF**eLl-xNdSsDXfLD@v<@}$xvOkeSNNF=L@JT!`Cz`g^`4^C^C5dfBYpI=zn&?0fKc77K>SbG!9x^mg1o24KpBjqZtUykv2F> z1Je<~k(^Pg+cWv!S(l$@y;!RLL(`|A0-84;WfD02>P$v|dy8T(!D>2xqy?tkib70t@xnG>&vX4@sm_@ zd*MMhdx46&hRWDWS~AllDm+bW9pjjk}%O zRJFbJ6a9mY%afj-^~|}RUx3Xu0#Z&4tKWOL#mFm1ne>tvx`%?gA6&0XMoiHr-QY-| zlHBKd%XIhP9UH%}4s;#7EaZd28=Uf9MuGm??HRrTviNDUba(7{L_TJ{z55FwGu8?Hz zH#Y!W%CEpO%wX+UPAA21G}3KR-&FA1JHG01PtD-0tv(3l?8N%-M5@#V{gmp--D@HI zaL(9p$k5sE%c{3FS&nJip_iT6E#@`0^V5|jZIsMIkmC)<%V@FnIx-*81;=$oH_o=X zsoFN2?JHX)y%Z6+!@tQAA_!DUCpjOdOEK!4SY#p;4^fjRok6?}gzbZH-{RRh$6XU0 z_n#KpwjI$u6jXoYZHAlf7*P5RE3nJr!o!i8pP&nUpO{ ztZLd*B`M-_QTG?f^l%kI{$R%rUEe&%s0(k!C~woZtlv|M_o8nwrLS63$g@%w2?E>R zdc1M83=g-d^YM!~-@-`)+tkG}K1Wka)3>x&8RegWul)>IRuRR9&EpSt3N+iE-lWYv zxkYXi0tJ`G|E?(3Uz?ws;_nxfsIO-`72VxezPzO6tpQduRi15Rg>rWWmW@2W=e)x} zlEWXw6m?b1Pfg`{@zvg`v#=**(^vkxFUSyuaDuY=#lkH{_xx&NN?vn`UDI`NS&<0G z(t1iLc_f)82z{lj@s~lxkc^yt>E$@-r=k5OXv`JQyh}Y##OR1NW9YoNDX&V>;3@= z1#&*0ASW~jngs4D97CkBV<}80;ML}?3sq9aw}c;>kgS5U31){P1o0`UwqtBW5Z4^S z)l?<6A&yWV!KOJy6=WuSvmiyFXkntx@Lvels@n{hu=t(ew%9MEJ6cd#i{4%Nc1c0g zGhCL`o&oPp$Pbi)$&ja1842`_Zi{oqnj=D^ykB=ee>1n0)c$fO{lU*(o*T=rKu;eq z4Bn)7AogxM0hbMcWX)o$m@y@OF^e)Ot5rUSX}1l;Dt`#we0ZBhsQyrfz(cyGDh2Ky zU2`yw0mqQ^12HGkeqP4-?zVJMI@i%@OMT@uK4z$7?n3$EBp^%_|1+`z+UFnR?Gzm9 zk_oX+6Mq_Cqq~-+Y(N{l2~vq z=A8l1DCsi3wTBfa7p5et-{!x1r^8SBp8f9Jv#^n{ZDbZ^@jL-}dckasd=bzN06k~N zErk7d0nXsfJomUvOtLpZS@LGp?bg7ghBJZqT%r2a#_2^>Y6o^?gz8ly;@-G5^WgzO z+b&9WJAB0#k~mAvcG&Ox>OZmp#!*2Ns@KMqq4?oRN0v&{+MGf$Fe%#3+UMFqIci(D zG2M)AGCrupch%t#5#yUVlf8km$e=pU_DerLba>tnX*LOM51EmsAizihS)gLbJ^=3- z;)#e0)CV$M-5tiz3(9ua=X0-*tQ$6^ibDr}B5z=+vb1Nj7bf};_;9};f_w>|>*^xbW3+eGcz2j}cKj zp@b|pe#6Fp-8K$qBDZfW@i0O=n{D*j`)8RyIcmFnu6y5A{6)qFaFvq`1e<;s5F(>r zZQq$Ek94p{54!OtNlx5qx%4{J-5DStj^Y%=di2d-$7SUqfq`~KS3jLEUGR^fa>|qa z=UrvxQUF&apmPvzHg1b5mUda{>>pp|-J{CNk80sw4TfqdL%qaS!wW2DFO3*DU#+}! z`=aC-zvHpvoa`B0yU&gXUfa7a+t1$~WW;?B=Hw}ZWz!qs&}D%+peMpz@kM`mei}?I zOImZ4($G+2l$Ht5w{|Hjh)Yg(b1@Pp78%LJiS@Kj752jpD1Fo2&l`m(|JQZ`h9$C; z5}dRXYfb2Lggi-UgGDh1MaFZ?j9FUWtQ;t(!Q2MWy^L7GF%7)+EwPw0v6$+W>h!pl zymsfd`pd-8Jyb=>63EE;TZ4ovh#_9(zZAvhkcOxqjK6<5iEo-1m%nQ5^wXrS7l2QFl;K;^E?{Q76y&no}0dU<+7Z zqO#%I*#o_JwZm%}kuM?HeDGPdh~IyO%&x#RgSxHg2)7c-odbTO@+>ySJT|A|@6)nF%)MPn*ML)DH^3RM1vPtEb$rY?PqANcqlPeO+SLeMN@`Jp;k}?v2$&w~ADf|})Ocr4o@uud zTY?^O><^xt1D9GZd$4LGo@HAB7iZ9UY?X#u|D#fzSBh$|&KNLlZb8m>ff371Bp|4+BW}ray>@N*MnjS@=-@e~@&nO~q5z;AWBY0CgIDh^L8%>s1S7__ zSq!7R!dK}tV!b0($SIffMTOcEO$VO;@_q0UWr>bj$ohLHy z&232m_rBO4S!y;nTD##qIpw|wl>mAfDMm~**O8-{mflkwM=my-aRJ{{t40@&PU?I| zrcp;hIgy@_VENww-&|3>az>TePyTYKw61a(01F>A{HwWhUw%^;4Eil2tHnFbQ4waE zAEcWVMf;5&`%{T|CDZC~_pgM$>wAEQQ27cF!ZyACbZ>u$dM~ucO9g*P|g>EvwdWYv)>lQ>0j(RMn^85vgn>LfAqSZ^P0Gvq$3a)-3U(- zzL?muykMK?rI=j8#3#BqT250X-# zVCZ1d6ZAW^nynocGR=A@l3e*)cy_=U^gW0mb{CRlk$NN2`jX?DhD~>segv?6)@~II zF8;221uGBn9po1K>*h!Rx=(+pUoGJ55ceI(Gp{3Kn^fI>G8G~a+a<>RqZEe(A_hVP zzj9IDk&)%k!D)^WH?YyjKMrJSfM}OPcB4Ue;*fnPT+udHv}&e zXn8`%FiY&O7%PoH?3_XZ_5X>QV>mGmTyCva3dlAx=g+O{&Q}d;3<`6<`MxqX4B>J= z^R-u@SA|;Aj{`bg(k<#uY2^?=RokW^B_Q!2z8CpHlug?xcHy1JdVn$i$)hu>io`VD zV2vUk;zcHZeMsOoApv(nlv?~9gHz$x1NNMM_@}R9kH0;6ryzOOTBo&(ViOw==O?q5 z;CN0cn&beRVg%$Mn((<{hj4Uv zDSvwNNN@43t6q}YwMZ(wS1(Ejl?8pCa3u6;;xrtf-S@g-cmIW;|66GGFPcu%Yo&5p z(Dd+&U*qEuW$0%|zP9g+z?onvNyjoVx{5JL7?Rl1y`+zq+g8WL-R%r;MdIifbX-(_ zdx!MD@zfGYRl571Knw>VbMS~;Sp&4am5`_Gql0G=JuBJZY-nPEo+4nlX6$p_aFiMl z7I+cN{I}>rD*!ykX^yt~6fejDEeytE$eZRVjz zwox`uAJ@OV{=_x=HWlU3*r)IKI#LX~)~WwpuwyTb^+Rlk-ByupWacsDyB?IY8U^M} z+gEL=;q<{-DSxCfHL!%(_~$?2iiIZIPeXlpeW;Gwg|KJPABZc1fC^H$oC);qVHwTq zl-tm$lcg*afu^HQm5V@~Vt01%ovYZ|SL1ur-)PIrx4e2c>Rom*q=8PJ-PIkLAtkFJB* zel&&zY6N}xS3>yjoktT&9zt^L!y?W0Q$w;HpZdxrna6xw!#s6pldya8MAImvNFX^r zgOjlQ&JBMYF!8gsmulV4RjT5jsSJ4U!;YxD)VJ zKX6>}NrS3i3R7#VYM&~Y$YJ@}9@ZqV3dMiy!5a{rP*JcwYCM)q3380?5?`De6yH5h zw?OcsM%*Y_6@i>Af$)g(Cn)KRQI{0lwJ+Luq$%iZ5oM<%wBEqG(he3uSGhYJx;eiDfPi@PvuuVDaT*Iha|^s3 zlEhyzJ?j_r8X=4YM za-`Lu8Yh_kCUGnxsAnw9{p>aCM)>R9cl}Mpu2E6~w=5cYwwB&~nM9m;I1hueR*8Pe zrYduF0~*QwawhZQJBcKwU?FUo&z>R8mBqnu7z13cM5AOK%~Dg1esh^~CiA=fx7iNL zo$;EcXA4+bzwgt45Hl^!p5ZfK#UpnuSHODleJI5eaPj31Z2*EofVrtj5(ej9c7ps5 zh%OK*p7=b|PFH;y%#*xpW&Ar&BJw!|%#%Q9Sc@cpAi>0<&K`B4ZhhT@4OMUZ9$|bQRYLrKsVy(>W=yE#eLJueD}+#UqPRA>cl^uS zBsSM2J8XvCrK+#Ah%=g6HE`IB=lwBVt-q83^ykcm2}TYe!$y zK1$SHNlDQekj!J0nmk4tnE>R2ip*?2tD$T~U$x8UzS*F|YRefL+HrZy52f)r_%hQ& zrpx~}4X=KvMw?|)wwUCk$*e zMETkJXYT@{_uSs{{CPk!0=IJv=9F>&2$}Ri9}*M^S0;8ZByj&4-cY>0Ouz~RJy-W< zc|TmZS(W@PKF_;bC0>OYedaV3eDe)NXYxE|FV*biAoxNrK`UYp%H$}PM~u{qd{|*` zs$8VL7^w;L_eX8H-^9NeuenOGipA#!T}2F~JA)SDUnFF+vNr4v9ubks`e;Vy=c{k> zJU;TdV8SP8Ik{J;UIt}B(~SjUxHrL_PfdOvtZ)PBm=K}8^;!R6ZHBA)1BQ{+^7h5S zPB$93de-WUTGM+=^Wc6rFP=1h@x%A_iLi|Q-`I*eQc4kKiwR1O8@z+$R@uelmSt?5P65Fi(gY8E2{|^`f_zmes)SFTIG>edGAtjZ8O zdl-^sz>}*Kx+g+<*X@(*rUTI|c25!NN9RzzDI@yP(3%X#orHtnzP z-~G?Qx4}p7(EHu_ZZ1x@aeL#d9@#dATgRtMQkT(S%?g3znPyq6jVzsWl$xKn)O8}8 z|8R6$y=UTmVQP&g_BeWfnp8IqT^0ku+;SOkzvCc*n+w=@paBavN6oC81#XWuuMA!~ zt>n2O;MzlP(D$1DR1w$6!0N}}mL;kCiXa<9oqzCLPw>w7-mGsT>fJR#a1!YICKWAU zPv}h%>wBo-PmtYLL>_z*mnEFTF@-nw#|@rDr{6M`EceH%P4`r^3_{oRvczv}TdLm4 z4Cbl`iG9U;IreGzf9(YDR_P$o{nu8gz)wV0Gn{w8PBkzisXrEU^^5YkOZwmJd!TbD zluen<^wn9Z1NKM!7jl0?oE+>x`F&q}8oHPne%kr53H|N1f`RP(4BRdLm#3u4rdtbw zy@_Ti(D{T#yZKx6(YtDox%L(RZ@;cCF9TzawVD9v)&{ucO4_jdf3U}5z(Eqq3RlIu z#g@DLZqm)3`6b7^ZWC{OGxlo6_H}ImdjZN3Auj#rX7H;-OGj_IQEvJw#+FS!H-Ao7 z8vbh^Lgm-hshb(2Avo3BPH5_35|;#l^w46%NU^ZiZuZYN1r7-BvUd-EJZWMOi+@l4 z0qS!r1ok7aQGNxhJtW1bg7V?tOpT$x?h9o`fX5JaGaoO~&K!41IF(R79^R1R?oT)1 zznQVT+HicdbRY!24F58aS^@3@=moF;i$nMqN)Y*iDl&06wi_raw(NGabBfR@c)$2i z>u6)XY~mvsXu;l71}ER*IUxJ}9HLRgR!_9wo}4QQ4C;asi?_*~a6>RM39TuJL|M$5 zS@*hlW?3}StGd@KSY+6~QVEC!jN>7m3S&}=m=U=JknoAZlt0me#DW8VZV5E(iUVf4 zp~Loie`nrAw|7keRp9)mj;kLMn1c`Nz)%PfGH&}RX@N++`ZhPz z`P-DQJ=?vTTa=gg1Z;gC1_-j!9HZ@oVy7-06769hxh}2*;_fQbxO#8gj zPect#$>a3Iwm~yyTFy`3?I*UF4?zr9hm{9Uyo2+XH(B#i*Ns}w&- z>V&~aWy;(yML5y+tDfKDS)W@D_st!k;Pg7{(91}9P!gsS?M}DNfd*PPkomWOZTEtq z2P?qnD6d0&7dcW4hU!U+vl3w)S@f#HRioC)8V(-|lAd3^8?q}9d^#-EquSicc7)#S zL{uKtf55!?op-%goRcx!H6Ub_9tynPXh@%69Sq0EfGzj)l5t-|uquIg1qHu``8MjJ z$~AfGqo11N;>Qa=MB+E=--@VEl)K?}-jXS_mUKVBVY#3v*mVEg?WK~T*Q(ufZmT@I zA)eupJlhgw59YtQ;`&c?0{!bONkER7`zsy~IhJb)56|uqn7-op>0LVWgm0pJ z^o`r@I`-BI>)HzogxCZpVjO{|4a-m_R<5k!w|*#AE9!RzUP|+HCM#?lqi_?tbUZ_D zKbsX$WzWeip~FWvhYIR#v6pg?YJ8J(lzQv!z%=@ z2tG+xQ4>*b-P^DjhBFW!$8&+W`p&Db5_y`{%pDEMDO#U?TJ*U;=NI!O3&-Tn<>nHh z9;2_2S6$Q+Ba~dC01pi3!UG@l8Q96G_&*>lR>>L=TcDtB#u6@_lkexN7x|!)e%Sw^ zeX5E{l8#XJ@*c#eEa0!Q$N$|Q265m+AK#4&FAH&b2%O_vzD6%-tmbHTt=#x#pWQmh zv`llLjU-1-g3A6W2$F#?4TvXLcsSRy1?V}D?;51)EQN^)40|VOuB3B?s5BaD9F({n z9mSAOm|B8{roZIok~j&1Ir)}9?EMn&y;T;g8CC9y3!B`2E~&tV!zdfvhnzmt(qhwk z%H1tJk#%n7WwK`6dLIQU0rT9&z!1j18F0HX?L=X3yg|0JGRx@)V>eut z_LluiP6=_xqG_~zr2)BJ*^Sohy-aE@Vm1eYIbzAnSq_!D{W2s(0BXUuEm2Pss zdu_aq%o@~R8^k~eL`Ln4GuD>AH1Dem6@~Gh*V43UuB0j8Bx|iJ3R6_-aN0;}7a?~z zjhI*G&|3ta+-M@=pm9v*A}U6X-Me$P(sXjBO)OhzImVOvIZcs@fmK#$)1&kGby*4v zDY#F9^hx|ow$`5K_;YB0nLyz~i=Fn@GiADqYvKDG8`HSoJ32L4{^o4H4o^O01SC0s z&c4u47G7JDKkt|pZ8iOeCe-e!$bH2W>vJ2eGP`H7TgeTpFRBTHX|A?5S3Ngori_%S zpYc*H6}22+IQ|k1rkTSqRolp5WfQa@dSb@7u`VvP-=^*lv!IiaMyLl(4+> z$2`%lP{+k>mGcA6WJx6W_)IrDpLVqZT6`Ff6F^#xH9Cb>MpvP3s<^b{6^o+X*?Jt zHYP)7-_ekoTWs0u>o3#(QSkjrJU!$X0-^kRm15tm9WhmMNmL~mvKUay*UN*%9a9Z>GcOL;OB zzYs=BFcjVM0UN?PJ4uZ)$sk|MFJU{d*oob(?TPNidY9CbhtcCpKVGu&N%VA{-@C3E z81M?Pv~stfVs{BFrw77W!fr0tL!$g%XvQ66WyfCrM5^rFM-^jiPVJwl(h~ZUVS3}! zFCXgLb8tp+4L|Sn8o7RT>5Jy;PY@Z{)T7>gj*mB$U!|s&U6W9kN&7@V*@>;vCT@mU z8nhWqtr_6mVzLR>v2OwN@uiY#MuQIXu5kHsJ=$lGiqHvnF0`6=lH2{oGQntJ30Njc zru3#C_63_o4;)ALFY?3&06($WubqMXcRiznOpR=0&+YV32dkbYndI{7@Cp7-|3EvU z(O+-<+3&}dV7UuKqXX1f4aZKU>pr zP$yw&)-S+v>08nbo9Csm^morFP;mpXkIy`8Xm%4oJ>F8Z>)c9Zd0Kf?J8{3U?Mf@{ zM~)C~ToaVc# z@aW4I4!5?Y(~ELa0&+|B9w+3S!AP&An!nA<^0xy1B0KyQFJ1@dJNu3@ zX=D3Jh3J9GDERw$#wSrT^_9B93PRZsn~amQS1EQ4;xR8Ql}rixrd3Gg2KwCThrMn7PGZHI#H?%n=vHQwoAHB|V?|)VjDcpllJS55fjRoDGm{!w$#Y%yvMYIYbgRyi$5^1Q; z7Yx3H}M{G?JrF>J&K8O&S$%< zO<{O{*_HJLm96(L#fdd(BVF2FWl7~nU9E{fqwM~9voY-_bh}-4_6+LhrfqV$?^f-6 z=BIcb4fTRdV}DhN_Ta3 zZhNv^^G>*4W;TOYGBTUABXc0WWmNQwC5UmN`lO`yMNLIRl=;2w#7lP1TLUThfl+Pl z-xil`l?Jo&)Fb7+*_Icd(qwLW} zybPW?D8Lf@UA;LeUg|1P+fe5rV5B+u_nro7qBMlY#Rw!p>P}7Vx6yRnrjccF7vH)7 zVV9h-WPaXou2w1rceoarO$V|5c?Pl22sBNf>`X72jWGl((>We&ykXCCk=b%`7qZii zPVKf}{(E~qOL$KN?6o26S@?~iCKWt-v&F?rW~DYSeq!QUt#qYgag{Lna<~P3fLE3d zp$KD}CRrcZ?Zhvz&G5xGWJW-@-xpT#yQt6!c77$E{98BW>u6q0?Jm8Z$2puexK#TS z^p>)jeoIpn5or24bmwPwHxzq&S(T=^{(7G(39d!7kzVZQ$0;oI0xBBgI2$o;m zyu(d;(~q1aK{OklW+O;R*xR{m2I~wcvAZm~EC~ z*g7y!)TD=utCq)%!`jXUxnJqUr|;n|K3P}BxDugc?9jOg25 za2@}0-bk2CdyD~`s${F5=!qL6%x?q4S;6;e^u)GfB!tu zHjE=SI*XSWNS{zZ^mrDr^KkdNbG@NXg<(kb4c0YZm=m6d=&%$0SIz>=v9_H1OJO?( z>n9iC&ArW^43z2c+ z%KrF2-LO%YrBK4h#q{q`(MnEN%G!XX8sBP)U52f6U@$t*LJ%y&iC=b1$Zt7#@m<(e zK2`Hpya~ha;19jO{nnj((bwQ?#5lJl71$Pn0+FP&k*y}`{>-02@(2*a^hgg$jh;jr*80UJW71483uco#! zdeuC=W-_{PG3ro=^-QzYK?NY3Vy_Sj?h|+-U-io9dOWXvbk|i*m*ZxVDh}FkcZvVo zTH)X|{O-Y9Vwl&GzTgk@tO=v9P4&8tyv!Syk0yn!ze}DIQcYRMRu6*McEyjSgWWz7 zF3DSyYOelTW;U@jKTIh*R+N_Zr9hTW&qmlgh781_A^ZV=(S@G;1A>t`gtqR`!-!go z<>8Clh8=D%7oOdITPISbq$-8ws3015WAHyQOaP9O17R)rG7!c&FmBtqs=EY*tj`O6 ztI^d#Es6;|tgw6f)K-2_{<=>0%O#;Qh`01Z{lr?$0ZpG!2Z~YScC(4?8w9@1$Ms14 zrs|P0fp;HVG)g^>bSGWye()G=PQEHfE?~Q{Tu%RhT-wXL(jjH;B&(S5d9t|WUA10! z3<##Fh9d}4PsqXjYO2IdM~;hxBq7h-k9cotUGnm>SU5c1#EAwgoCE z2{eK?H2fl48DbPM(P%beq}4<6{)0AF*YnB4h$}8`+iT63 z8cc0mYwFZCzq-hw<@b7?hoB9w2{#@mF)Sh(v8gh4yRdAXOX zYV#8)K-i^lb*D6982lL?Ff34Vifr;NB&ibvxhqP{yO~aF+D!)t?p2b-NGUh`^6kIY z+18v81|6VsPq~PU!O#R<;{i^f%_B`pUd5&$OSWqFc+oiVvQjEsHC8HArCGvco{=a@ z5IpH<@NOcrHjd;-m8gtOzl@im5lG(SQkuZ-fkvhu!?qm3xg&_8Y4{a_E=p{zF>3aQ zugH1&q5E$xXwoPr?``~*%An&Sn3KbvVu+M1;Jm5vn8RP;(fLc_;l~3ZdFQ5&pK&G% zL{jv5if#`UT@3l{40PBZ^8i0aoiW7DAn5d?mx(I4%>+)8(lA`t*C};E?*7J2kBss? zpF8bCj6h5qoF1Q76uYpkH2hAd71OA9nOlb@HJ;~>4UKdNnA`vIrjcC)0u+gdNf zcS)r)kPaxn`8y%bq6H_hgAQm$bfbwyVb+|S7B9`tjXfD`&>-e~lm9cq-TUMFanyk# z-Fh!R>D=m?J9=tZtd!-v1#B*s2B(b{wc=%9(Jb@)KcyXOoVlLx*~)@*|Fa`8#bc6G zPm@D4%D#KiAPjn?+%=~TT#8B$WBL<~`chR2KLV@dB~*rb%m|`sz7Z7& ziszT#5Tzb@aWKtszE1h-@lJjD^~5@m+x;WA)s3&~`Xt_=L11po?a!NI*akb|0u#nWbbk7py z_!BP>{E<%EqM=0WlF}drNf|v+z3I2sME~yYDl&y_KrRu>|C8vurV4#~pM3<4%VT-=8 z`-4$X{JV9tdktloD@`6DPIAgG!>Dd=XJt=}kG`ikngZ+ecEo`5ZtJHc7VDfWv0}-9 zqyUD1W7rZ)JCxB0NdcEzjJqYj<88oD*4R&bZ`JJ@*X%wvKCI@IKs(KW63seeGmH49noB44hBSS>x4In_Gh2l$Q+ zL*@a8sjG6?jPKT+$pLz)So^CKLD9qqhX;wo9F^G-^IuO>zcyA0T(sx7>11%>;u)a) zmNAh-21lI}ZU;hmY8ER&bOzg6q{SP3FZ`JU7r(#MpM(#W-dG=pWu4>Y#BS5!)4Kx_ zI=PH2f9@&ln{(;qRrET@y=)U$bec=*cbPhygbw4mY%F^i3@>v-xBd zQH4$L=7|b|kjKZeqm@o{zf@*ve>0tW$k8{G@c!MxCHWo!Yfc6|fOLy4A)~R?JP9|I z5oM05JXu)XuOqkeDov)TKIS>hrR3fGF%~sI1F`;VZ!rmyk$LvhJT!kRYO=R@Zu*xG zx5Jfb4hP)sRlJ(FMbR4XF;FsRI;4XzwZmQUUAap<;nO>j)kyK}2_23*m9j%!3Xk;b zx3urjyiBz0_W+nY+|*+M5gE|^woL5Nuhsrb8k}+AvwvD1kdk5NE5tu(6+ZxTpXlH+8O&M5uKDpIoBh>8ALXvjt3_{?1 z|E67+{*v&2!>U+1F^t6Bhhxb43b136Cvaxn!mGyN6v|S^E$__U zC-A#onBrYHwq@V=|7FCJa(Qqj;(UDmqHf~Fqca?DB)66BGT&iQkqUmGQnK>vRBK#1 zQ`D?VR&n5?W04s%t#wn#i#u9(YeLvcd5Fe54AAv^fNXTZ8W0|YfsRD7^(TffcHhj4 zovRiD8_n9Y`9yvdt>)KEw$8`sxJ3p!i(k{RiIa|IFL`S_7Oy^g@CRY+ zD5Z;@NUX}_Pf|>)Qtyjs^9#{r#H(m{D`LXn*GlJW$f!k2mWNH&Oy#%sh+p1?3^CgY zMMFqiL@(MyZYX~FDZW>_;8EQ*{zn}ld`a#q5Fz|u;cDIU1_ilOdXw{h;um0e5{jyiwcF*}DSb2fl%oH;iiXeT>2; zL(lC~Jv}6eCPZTPt`Xh9+{tYOfoaT;Lhr*1HkxAOib3A96#h{|nw2z}?l7SJ zDWSukc)#+tJa1SJ?*U-ss5DLRu#iN)w*7gJiJRn0oz6|_<_iAhyhC@lWuAU?#|{G| z%X6@EeO}i)2Qq1s6uKl$(}aQ*$1<<9Ww8#qT5aCLk0<9F@V)eZS(+RE(JM2rkSg~7 zA@4n-nrydr(I6mA1d-ka6s3q#q!SS73RvhuR60l(2oMNGKzb8UqzNJ*CDJ>gSCI~a z^xjFR0Yct$`>wV2TIU<*jPd>0`}?zhFwkKJk|)nI?>Vn|m4MC)h$E$7PvcFz69(1V zXD7^fI=TNCZG-R5Dx*4!bvAH$by0owLiHNd4o#{d=hFSzUbo{1v3f5HxmY*1g0_4f z5z@i%M#B6|jF59(=jFU2-f!`=%&8Z29MyrAlX(ICnCwXBUAfeBcIC)Hkt(TkX=ePb z1!r=;D?stTZ-~ScD{vwQ!T|f10gz+cfL|W0?0VuBn$(Y%yole;lVtFBaAbybl!~ddD?91bNqo?3}&@>*zovy#g~lsEvu0E4cnH4cG+b zwF(HQ-tjGa@k;g8#xsk#BYutz;F}VTq!M?ck78>}zZ%hY8kOP@uUZo5RJNM-F&WJb zk}BE;)%b?KJXRf2-(t9OQS0ge(bV_|SEX=K7eLm!Q)V91-_@Zn*475g#eAW|6?e#6 z4L^Aye6SL5gGpAi!~wN8@0!csD7)Yfm9yUlbc{)gES#2XFI7jGU76g&9!2*F)+O)A zf=khJrv@3wmQ@@JVB&`S2DdGv-tG)Fb~mVkD`b74;kMzy`NbCM_}1mOXPoUU@4jK~ zP|jVxqk5CLI;+t$;iwQ}S0{k;F?OaP@{b_#K0nQ9)37=Vsu`XfQ=C%c zX6irn9G$+TLFr>bw7o9bIw`{*3g}y2c@=*iQs7o&&rlGH+wjXskSolKsdzr@BE=vE=E5Qxb=Ktl1FW!fn?RO{fcl-=95iM^_li}5DQva z;G?qNkF(n)1L$5&5pqcce1I~@$)6OtI7#27mc0{CUD>TC9(j(2iP5f(fJKNd{xvV_*7eK4{x zOf&N_t_Z^0>a>TNY6R0iHj{B49>%UedlSX4HCWi56!Ue`wlXt`)~M-wCV&29CS}9F zIl-^sE4Liw7#7I6!{-Crky*G^iQl zMltsv8SfeSF*oO{?@Rl14i!rtt}NtGucb%aO#2OLjL; zC#zUp?6+^C8uAc^Q9E^)aia!LD*dn9Yj#e~Mom-oaf^F1lKPvZD0l(Mr#)+FeB zcpY}zU@f;Jh83>dJj3*K;ao=EZ01yn-PpV-!8bGiiT3!3bNxLX1VGFT@VAix=mWKF zB<885BFE~MImrN?zaiexOgzIdMg8OZ0G^Y_x0mHR^g`5})$EQ_mchYO(CR0GaZk#f zw;7w&j#5ESFy7^wU{O3DeB9K;o6F40ZeXBD>gR}u1^PIJ&)kV#V7ca2INiml0_s&^ z&oBA#Sfw3uiBlDr99|FCfQiw;M*U1K!rh*2mkIVilN*nENit({OZuI7D*j9AqQ3?( z-Hdtj;YQ8qR?+rnuiT_J>WfiYVkIvi5FnmGjf9^hHzQdUfG4d^Zc%RQnbYN|`t;}C z@#~e{ON=BIrHCf>xwv2Z9ro+M4z?F*yLO`*$FX1$$a2HEaz|f6=zdY#!_MetC6FAF zawwqoAdZDYnfDUI%UA%d=o^6fwnY4D1~V>X>HZ4oaN?yp?ST%JYi`4Q#GNImufU+@ zIRZG%e7@S6fx8v-7vx=05z~XgazXv_LGe$*x2IF6u z>q_j60@5F%%Hp1`1)V&Jma;J*4B|C{zba*Q!Sn4xAk#-+___^yZD+V`r1xRY148Eu zkvP5j4JXn(*>I42TJ`qe(fz%LPSv%;3rVU?O}(P=&Jg-@wieE-1G@^pvv4rA;oFt( z_qVoqf{)U(UGt>oz>UM>H-Y(qrhmCrmkD55&Mzum8Z<>+9d=PJN zQJCYF_+Pf1A_F{Fjvv|m(cML_GP=zaV^4tLSG4rpcw#Fycg`l!5W!dZE`$Vn^eCgZ z-%J`T-Eb5nW3y!Sg4AuI$~Q|%{`A*I@8(VeX%!7G4Mt_-2Z`y}oAH`_Fj!!G zqP4I6`|tS9{O^6uH@lSdPhX7L`V4d{hPRa{t!h8MQq)nZ zdB;-dgwsKQUtc*l6cj{j8c+ol*^{?r|AO#|wE-Jt(8{bo)H^wW<)-ruiX)kDR-39L zgHGZNs@!b7$oiT~g@y9umuSADNC5z(jTt|u_Rc*&iT0PG`ISx3v?>84bR*~iLgFaO zE2D)bsF4$AEpfOU+}H6r&lL6EvG`67PnRr&-BpqAaL(J{pE{E8n_qQ=z+@t~_eHlY zXGZPCw{I6U_zobt#B0U&gsZc}$bogx*5O%|0ry{jEIEIDvLb6{-QZ~as8HQ#+sa5c zZv4`ha7fi32~_iJMKV6A!GO>JjI5rFM+5&$2Es+0)547!&KehA_ng_2SJGdqh~73i z*H8}xbTYL61`bxt8jD`_kMjHIsztcoN7P$`od6IJTi`Ung(+*EJt6@z%0UWKQ|qP_ zgHUm>oV}QqV>`CV`q6@d{Vq>1q>K{+xs+f8!(_mj2%$RQy3YONA_(|%0jp+>*>L36 z6;IGD8KcDt4zGG7E>N?oMBDpnJwyj*TZbrO``9iN65m>KXJ^H*^-pNE3{Ce7qr==~ z;by8f^H;LH>~yFS#)%^b$U+Id8GEJ(vA-Zruv0AwYXaaa>>0~}g$_&${QRNDBAfw+ ztrbRte}GV8I3FuOvRewL%D`Zyg3sWxS2w)PJ1YR^ae5GMn+r z%*MAPNQA+PHg0^~d z9Yome*`!MYl5|1LITI-t5E9TL2Vax{m}panAkc8!JPYi_bMRuEw8Ks4K?!oLKIiMx zN9en%mX~tb{i{2fFZe^HH>Z*2F(-m@hJijcnQS5gSe~pRm7OujUkvWJBX1uKJ z&0b}9NE4882|+P0ShsE6s=yo5R;AsjR2#h@O+x9c>9-Y2(Ij_D3Bm0ybm9}aSdjnJ zeY%}O@xg`Y&06Hz{l*dO?AH9e69YW!xb4bDRqeCnpH}JQ45qC5d}UEnkqPngZqMo^ z&+fprG{%Jv1sebt`)SCe(zS|22DtOs|c|7P0;1= zp^OTh$ly*|n3#~Qdim`8uS&}vXnzJyZWG(*xe>w znRxqsuo{nas&`+MSBBNa>j)T8PPVL`Kjg-Ttq&K=#6R-$N9bMmW5PARk5vuxR^mWA zAD-m?yXpDU#Qpz$`R1H)!6!Fim_$$+Dp*oXqNua?_&mC8xI6$8Ml)`UG8|X=$FVO0 z*)lY$$ZtD%oe%jNev^1x6KswZM9wRaOJCQaMOu@39d($+MsQ6 z%a5a`5^Ak=!mmE|K=#7}&sd6|D;%fepy6+c#Hj$wMZaF)odTlJzd??W6A1)$&k*Po zVEAfYM?Ee?QQ62?2jiiZnKl?^A))LEI>ssQcn$e(zJwineXu7llvisfTpbqSD;YsN z&Ym!B58z{B!m9CDSXdebu+t)0AyZkUZ(^>(os6q}96;uURgTP!qwt@5(jN`D9j|F{ zTLfgd7>@uS4`nZR2TOb5>FqZCv4MQA8y{PEZfRsqBjYe*o)DG2S=w#AAPQNm{B~4>j@QT5>34_;%q~_x6$zQ z$PBQMH|JxD#tHGgW~)ADw{XTj-78gC&Rhqo8jH88QOo&qw{`eAD@O4`II4M6D1ndj zT7+)hs7RKomQ}UhNpOC6?pY6AtJu|$2Hx>mr5 zf1A}eBjCb#T=m%42wtXAQ}Z!U?UKw}?VZj{N;W?q3t;YgGE)qt|cj|DZX_b?0j6nhU(J$o_(| zf^T1n%GPz-I|ROz+8dQKaSxM_!v!?9i4ahG}d} z7AI#w=?k%~e#iuqiT-8mfWS32~3e zw*3N+&>V5RpSYz=^@`HPMpSU2yh@VYvto&zdt1 z_sRBB8qHCIyH|i;)6Xi*4wR^#&Sc#K`)#uPwN}%{<#CcjxzyF-M*zzf^$bC;0h{4n z;5iX@b_w|dpel~gGZ&bi0U0J?J}DT}uTfhzWSO$Da@aaG-QU{9B+EqXPcQc7P-5o) zr8527E%cv?)BjU{ChG=tC(`z&Modm-y3%W4>orIwOLf8jo{TksGRJ%0t~5@b=QT3H z6FLXhCX+*+2-X$J{s`Mj8@uQ3n(JCeTko*uGLAuIKmU(41!kjFX=F9pWT0QJMWl{Rx?9m_ z@vA-h-<^P{So>PnovD|fp3+woX6ZZ*LM4e6)R*#NEdk@s5d%pje`?axoogy!h5 z8}#x#zh8YLZ1%%}Io z@&yy>F=GBQ`~E~bRFO6m_1n_`%{#- z+Q#$qFMeE`T(k&k>ZG?=wlk{F?i}@RZ)Pl83NXjg&zFjIGA|Y(DuCuy0 zD6r#Y&!l;M9#-sl9TWOacsX4yglcIo+xq&`TL}RJ){TPel-SvcjQN==%vp=UAkM#? zfso^T!`|>0G?v~0Qq4!_jcr=Y(0P@40WFWVrTP>R1JKN?{d`$0LsG3Jvb3sSZuTp6 zFV3Lv+b;p4Cb67nFg%7^%?**Dh zlX)n7t?wnU>@Mt{Q)d4Kxp)tacWsQu`C<|bh6oonW=KbN=c-2CQTIFr>V`(Ab;-NQ zR(M}@vs%_QoFCZ81pb?^~-s> zf^c>zDug%of}-V!@OR%1i#{KwtzHbWoj{E>-3^m_>cp5zT43C2cS9rHv|R&{O`GY^ zncx4~mFS@Tzcd)^sP>qX>y3B8*}$<&c;}5DMGc`Mxc0OV=HM8Wojs>LJSq5(wnVQ_ z?BR&JR^IlT-)s@xOAh($SCXw1{3#*1D&7il7`26@Z^7I3Twb(faq#Zf$XhJ-S_GEO zh1qjT&_4vRIT554IUlLGpB}gBaIW3$yPZG(NSplHUl1dyDiJ!+qmB!nj`MAFwHs;1 zja*Tr{<}i|NANEy6Z$ISM8F$$?!OF128==f;Q`qeIN7f2fSdO_JNi@sn07n?A|0AY zz{&O6F1JLMf7cA++m6D1x8aYsbK@C(Rd0Ui zSk@A}$D%D0C`!IV?H4)T@fU>5j~JmitZ;4ZwDa8?TzLfbJ@-0W)|Fe4IpjPNhEu5t zQ*%VXs2ggbOJqiTW@GwX35e*zS(c(@qm0ErUU^4%GhCG@)TQRx+%e<`<%beH&#osf zz$;fMk+DZMRNs5WamR1NvMj6lojrDU2WikK#|)ctEGZt~B-i!Cvh4^8*&XInNvQ zEw}i>pC$3zEU8{9&Lve}Q`)j$N5^Ck*L$MyY9n$g^9466OynO5MUr4!21K*I4snFI zRe>7h5)J2&JbHsGaL+UOJ-w#;%cPsz=(c4;z}WSj_Q$>K-b5e@?CklBDLTN9v*XyU z+>^Gx>{@J`;JeSQcXnFc=Ft&fEay&;W(&Z1!WX#4N>`!PcmjeG_}y9Q6|E zOUpREsWv(5aHqRMpz_F8c?4MYel|j4%>Qx74HIbW4A#&1abph}YYfU#c0lp@z--VR zR{6QVM^cvk@|KPpOekKXn?L0&VII};$XpFaQ;6f$YYd##8%M0q+U;to!Uylu(fm|P z5>;d2yW~%CG6tKNU6|}yYh(>q(dU-gV8p%L5S?4xAcZ&(I*yqru&?ajq(2CBxc8J# zb4f5!u!{F{hqgk+Dw4Ul_;#Zpyu~DcxjN4MQ(oe-Ya5Sd53qX`@F#l#;SgMlnaG}7 zjYM!PQf1J=vt~G?c`NMU4sW}~dY%WG+-$qa=CkKERNWr|1;c5ZVE;Rrm4f**X7?!Q z)4HUS8z%2EZGIFRcJ%CAeikn(O5QvdpGn;O#{N@;yjjo2keBcdPa3E6E7!tEW?Cio zKpGdAD&cpPBw3pRYuWzN*`4xu7{Av9+E3E_t4oJ{9HxJ36?<Q2BpKg?ebnAJ8|H2jQL~ip zb$i)5NnhxPC#Qy7$~aET&BY$8+~PXf&qv2$HYSRdexjG7XW6y&ELP1~9i@uR*xJSl zK(B5ax22ik(uY4wDc=TcM~!HcM{@l)|;7s54-=Tdcyzq&xMKkjUwsgjOL+NZT&bf z%O}IUKGz$P7Pv3PGkVdANE504I7`@uJsj_tt*lH#G(rat^($fxwQ{c4KHomMR3P-@ zsY10oG-k|SZk50UYZp0X|1~EjF%*TpcH~faV7!mfF2xa7DIP#f;z*rB7>)4#* z$24WjFtS{w@Vy9R6!4LoPnV8~*je72y4QV$vhuRC>Ws=8WW{LkHoATD)B3h$^w{1Q z7mpPz=}{i>i{s`^GZA{%M*=Ez;_ z2Kw%`!%N%Z+i!Y5WQD0}l|5)8+R32X-7zk42(jfE! zu28*O5#muPW(NJO5oP&}@7tr6r~z>oxq$b193FE)o7*t6@#c)ukThRUY_@tsCE#(} zGw<-#CK?FRmDBMg$(R0spz8mDB^O&SSfSnLogka(+@;adw_ghz8z#YOP5i}9#veUb z>*%Fw(3#f>D}3#+R8y1E{nkab#)_8=;s;>82WU3GTXLZDzOxm-u>0D)0nXE zXeF1Uu*m#k8s8IbrX>Tb`vncN_jil~0!!4p ztC#2u7y0+L4QH<4RUBDYPj zg6(FsBjUbSTa!B9V_T+gV@9G z^+25KE2~rG^QrD5iHizzuNbGl!E6Sx0{uFY`ha#T%x!_{SLbgh_3p>xJvtA_2&cUK zS^h*TNGc#VQy61*7vY#<>&*^jb+2=XR#0lK(wA2vj<#QmB2yZyhgwy7-%hsAZGx}~ zFHoeu>1NO9D9`)H^jVGMKnP)UqPSfOY`$L~>}E>p;dy^`D(<+}v(FR2?k!x*3MBB* zrrz5HU!BI@T<;af^6v_dc*dzY9fh*9FVx$ipQRDs(kBX-3mO=2N0;|E;}F+<{(?~G zgcLpW=9tPrS1RpIs_QsCkPlTF{z?i#08H;cc(X7?zjo-}vqRV!5opx2=TI4&Yc#)d z37Y8zU=EN*9w6w6j&lAvJaD>W&)RD<0vlOjn=;ZW4Yy;}cq31IeoE4~Of~{qWbi0f zix4Wi+YJRN;k`%ShIie$f@&i@?v*~T7i$$Ow2#j#N^;6%zyOMW#@S7TDtI;)$Hmcf zW=NZR>AXpTxe#^bBvMc#9YBq|Ki&f?Va)JhpWz>w8B;0!$Q>@-GjBRl^MVFp;pct` zTIW3^AeqBD(mAnffUtl32<~-PI5%;bE3$v#VpF);F6RpHPE*et9n_@{?iU3OL^TMm!y>?1yI$oBf&z0K z{wA9tJr`9}TAwF&gcN0r3%5r124ooH(xqmM=x)XV%L+X`79kx)GVjyY#m3-*PGGDo zFj{`nc53iz^7Ky4y^YGBktPEd1!%xIq)cGaX@~y%L0K1>*j;dXhgZf7w@pUZ@z^mn zaz!nLd(MC5G?Ai-aqm#G!+mS~cAj+Ls|Q?SvGFahB%~~&e{6TAw=g{yLH$rX9UzOD zMY!NTyZOlXz)Lr#G`ALRCSKs?mwjfQxo(zxCWSD={;a^tw6>-Yicc$&^5XN)?)wjv z|EZMzGTY5$ZxK%GWi-10wd&&1aw;>*K7hv?uu_0S%k24M4NcN3=klX(;!20 zhwlptnta-t&eRQ721>j5nPiGY!cG!#8`ERVjlzbQ(>wY}be({!+4N&4or0ye%$>tIgEARdf9M z(ChNw6NOoUBbOM$RUa*Zj1LxQBJ@aVc-In_=91TGFKG08eeX+I@X3`A#REF|%_v#i za4u{e%}{T^`56@L=XS}GcPW_I7UK0STsFvgQvm%&3_HGH8iF==cUzD?yh-K`tqmIp$t3F3S?UbJx*p-b+BI2=RDPk*FMbA9Ma{H_E}XGvJ2ybd+Icdy8ot@ zeN>UOv$NoC_!;D$vmgC2_o&K+-9yb!>jC(kzAB!vnrcrdxRgE}Y_pa!Ftv@h03Npr zxFCpJ@VLNi@RV4yoJE}wZ~caXcZA;}AL)yjn_Q!xddOa_k~kTKWpb5)NBS#*Wl(r` z+p^+gitPi};<3r0vRj?i+yrr=jradVx&9zsUdU~>MPXPo7xmYX14(sMaq*Ku{+9{l z?ypSM9<@5L9HL^tGTt5W1T@?~!OBht=v{Koix18=0lV|{n11PVDlJ?w|K7O5bEW4@ z3Qga;QNek{+o%cX0#}%v!+3ell?+_14<_SwrQo*a6hPW%=!DLm0Kk8nL0JhT|J)e+ z3$nQc7TM4B_OC;H_c2@2k0{*aJ8~mGpT2BcJhLRFw0!RC4)bPOVjTC;eIS7IQemA-UQ$7nd$}@*AcNM-d*PTakwc0FbhSJU!F34jc1JUZ?qbu1FB1@8yMJRO`i({rRDNM0u|m8d<}*H|AY&VfDaR@jsJp3+YnnD zpBKK_G}Ygj)xf+-iF*5lA!Nf;xy#m{Ae{cxfvi&7C2|fU>BmACGIolTGqRnZR&IS_ z(~m&Z*$(Y*U(1na2u;dS0yNZ4N40j}g6co+xYv;_&?CBtlL4}|oW7i#nb`{CR)zk$ zl@2zRgT4nT`bm)-;=ffarAZVsgHH6CRUj@+?=G$+3%=Uy zNw{$)S8ILrq|WZ%&cjAUJ>kR`+3-oRrvBI7xZ=-ex@(Vood*!2fg4u^z1;e}!}uZ+ zLM-=8NTSuJoPE+OOpbuMG5$JMnb2VM7i7o)_0dO{>~{vKpR5h%J**boo^ZtyvwS#W zS@cjeEPgJlOtj?6*CBd3u~_G+ApdTx!BNC8AmMl8CC%r%B|dILn4VP3P9#b4${HLy z)CFFKZ_YG&{cYq|2eKECWAIA2jUZwd}cJEllQwdA1*g_xQ4}Ul!bxpmHuoXy5_GlN4g3 zZyi=qM?<~Fbh?wblcZ1k@$1dHPs9VB61mT5I??~(?In!#4LMGv_Bq49|| z|8o2DMz*0QMHB}R=E%>JIAQnQt##H(`gU{O9zNJPkS*#K>^UJotfIoyI zUd`M-4}L}5-gaqk=5%K5;lQfD2l@NBc%tT?V*cJ-X@6SpC)nikP!Z&T&)7v!pRd_- zo#Mdv`}9P|p`AJr-Xg6ZwuFigbhdw=_0JS#S4Yv^jjONGXX%F!r8mgp03$_$F#!;O zi~9kPE(r3|4xb)Pfyc176@7Hs>73bqWdm}~$VFd&Pk}xr?3yF8G*b4h0&Ex2M}Q?p zf1KT0%|$HK>k`Hx$UbQQc?a}Nm||A@`)=5rpPmP!>piLlyt~pgET&p<{+B`pgK8P1}-< zZ$0PicTM$*RIuWRCgYqyp*FjbmTC6{fy_Z}Fy8oNr3SR+K3+OYfem*4SEA`-MHPPv z?k9H;WUxuOi$3SDCnLLC737W|_-+ZR3Tu{JkRWFM9ci#O?ZP_Vf`Zt~%#R&%Q5!!( z^fxSV2k?4-CnFgkMZcZw$X}odA#jel=cUN2?(}gwG+kKDOn^Wn{n@y-4U8UzOU;Mz zgfZ|cGU%y7h%@OoAScM9KI}d8SB=j$wY~>ODZW!rw4{Q=y&{<*6da+?^^4!kaPEA?k0ZZ7}1 z`(wQdwS1gHXDP}lET87$+;kF;W$Rw%I?G%x+y*NqG{_&UDD1zwmc+@2IVS|?e?8M$ zGiJF%BpBCvDNyt>`C=Z!VkzDy!?ZfSf4)DU0!b&OU|47K!UFFvz_&&+ZjG5IeV9PhyCVSHqa1-H*(>ITf+MnXsBm@wbUt9 z+f=nC40uJn@2_-IWN-M_ip7`cj2@J|r=!R%G7&;dU?cLst}k@SCjZsv@CuYBT3-6{ z0-*HYkP|Wn=fuL$U9j8=HU*wVf#kxAzylpN8R?xzi(n>)V8ltBK1Jm=pFH|tL-L*B3#I>hy~~wDR}1AzGl*Vk`S2; zQX&l0oJqDy-JaVw>8(ybHq049cE8h5#W89i6YnD5SbHLLp1>_%)lBfw!k}H z@;?5ca0_mM=ZDf=n7Wa`E}KHcwfo=`9xccrRv+B2ebVmdZZLcV`cv~NfVZ5D~F^=v0Ym$N=hQO&W)dg^GkKe)h0-F$?S1~b?q%PJRSd=Op2 zkB^c03vwPrMu^gcoN@lyc5&mzvFJxQ%Y37CifpaI4bLlHyG$8JZ^C)h$^1IHuj@cO zq`{0{>Niels*HyU)GQUF;>m`)SE~E}BwDxUo(F<0+@C~iz(79^sVEX_H)AUuJJ=pt zKaITb;Hp}Aq8x1xlGex9uJcF%lx4*_D4q<4DHb{M1E;Upyw2(V-{FIl;BI9uTDYP_ zr^0&hKsVFx!WRH){{goDC%+aIt|KHd=r%1jdnel88Y{*%HjWM-_y&VUm1+H+b|`TK zd(GlSS6gFF&c>0L+fag}*{%x2O`r23rYJcOJx>{<9&m6Yq9=Z$1My?eD8}^Uq{^L) z#QZ{i6@kTbz7v1GiN-O2l^`3+Gs|BJ8iJ&(=f$jUyRI?rYj}DF$M0a<;@>i#PsT#1 z;^HAoi_%SzYd^pK{w|z&dSz+XaxcvgV`C7lS<9*8UIoS5Ihq{O^Hb>B9L(@K_`3 z!OC?MC<^rs+;;=(c?)oiPzt1#iDc*`sR=x@gVtPNZ3;iN>Ir;Nerh8n><{>loUEJz z{g{N>W8%2GrT>B?0Xypqyy$T&^5hg;N?|B&8l3FNPz7e3*Vlf_`7>L|=Z)(-j?8jH ziXdPF35XwXNs&lNfmr3)X-V~e?N3Jn#wc&Q00uDTwhlouMTf+XcU<_*cnwKg!riGRcPW;I z5@Es}glKhj^o$70dY@476(Uzrji#kY$`i4pa!;oynfRpZ`jb;P`Ez%Y>K*-|>I6ZX z^vmN2CQq`cbmli7O0LyflMhHLuKkj7beWazphkdePtLRjnirJ|$ACKrTH?sjn3$m+ zq}6OlPL8WZc(0f6F?x_Wp>ug?$w`54;{;5Ta=^W zi?$H+Dv9|?v_I6UeAI~%#-?M<^JBhfu91{SHd;?3GM-GSdwpaSHj!>*Fqhl*Lw&Kg z^21kVeC}3%*8%E7m^27985tm(LKD|>VVoX~?@Dl8*yJ@Geest1SS-wJf{nYBx*+mE zxVw{uDB81;8*v#U87(n3E}`xuEMlQsCLdYA{2`iHO2E~n!#5a_dnQze>eRU`PzG68 zHzbaXDbsXD(hBbOK(N#zM>aTzmw6^Ll1D{zfrBk+;+-Qm$|yRVU|j? zxbC;`JM@?RFSlfX+!F)6;X-$Zd#eveip=XuYa=_wMT_-D4lfoxR^((5*=oSdFIk(U zIo{Eb)nC6OA7WnIKcCwIB}d3s*VYZprQ82b{N;$^u!y^Q_DzwQWG>@5jgna~01o)0 z6d-Wo_VjBtsyxQRhCZnB^R?k@UuKSMX2!NW94;hsCSaOvnOj_|LduWJNB^CRv9?^^sQjT?H`;MFGZYld*ILq0Pzaaa^C_kphK&@Omd!EEG zI~U?_8bLeGZE?gz_o~6lUn(X9uyAi-HfGl4pbmmMZ`3|{zG+^uGo3bL-m90k8}M=@*=I}}AC-tuQFK$ENNR_d=| zG#3p=655eW68u=s%DLC2poO}ycc1nSIY~nKOSjF0r+z>EF4gyhWvQ!g$@M7o$QD$*KGOd1-iSerXg7ZnQ7-F%_=E?fqZ?JwYs?PntlG*V(> z!f(sTy4&W@Oc^WmhB@%ukE?Rozrz=e_jBK5Rmia{wDm|SPLrxGs|&LXQ6!)GV&Uw? zrL4%f&Z76e;N}bOKrs#8FD~WrcOEDjeMp6nd1KX)wmP9hxGy$-W?y2<35p}e411E+ zY>h|THM=n_{@l3yGg+KT+t}U=?iD#gzJMq;_Qr2p-|)*!8`Z}b8zr$$A%`&(n^iKL zbyi2om8+yC*_VIh)~2@^tp93e;n?i?3t|He86!Y3h*Fcbh@$npp^3h-73Yf&9~`!t z?*m8Ed(00jlC<*f|#|m^J<@?4LZ|_CUsV_IMZ>j95Igm&?GE zmcKjZ2vUsgJWy)d*8cef? z5Td5Ta5cb%`$B`vzx}jwitK7V!oKQ!vm1{yGBPP%w@&>2O<)^wc4k*6(rS85?;Uj^ z_Y0cGOFcHk1L%1-1lYHK9@fVH#tRk<9y(C98}t7b?=5gJ(5lssES(?hyp|Y%S0XW3 zA@%hbV0q8`S^A*%Ch-TXP7H!=-lVG# zue`t}FSGJ&CQh@{1@;``VpNN4&j>kS19)Ww?zuAhLQNK0GBC0mlz1WOu%zu=R9U!{GnXfqZdG_Ykf z;G%U(&{DUYaPW?Qx4atfKew#&9a`QVD8j|4h`{>r?DQwySkqDj}Rt7z% zIY#4{(RjPY=;3>zA;5pnzNd5)MXz=_Ca$ym467I7IF`Vf&bj|2Uv`}M&c)mMQVw;s6mWT%BL zJM@U9r&^=rPQ=YFoTsp=5O;byYH;1E*QFucFLpcnX$56?{k}v%{M~o4rt_s56bV=d zbirC@7%-(8S0`;#E~mYwto?47{q8ck{QgP2Pr~!6|8?JfLI%TOZDAIGi3!Dc&2;*< zL6#1gKLJkBNuWPeS5A6?nyIb!)kezYoQv5OKXe}vTR3tk9Z2K@7pYdEc7$DU^uTNN z=NO9pJ*3i!!RWsvw*Q0QVfP5X%p|aqp$4s|szC+MZJlR@%5;RkM9~-O<`H|R2JEv3 zj5Pv8EI4>^_mmT%0>%_0K!pDWITy@ogb#pNutGx5@yuPHkp$B+F01hE#Y--$d`ew% zOq6X)*jNg(RDAU;EgoR{%AN1f7<<5?W&o$wGlL#kj%{I%1Qg`VS9+_K<(mA4Y@Q2 z1)q5&#P^DNt83yR!$|>Xr)fHYOid0Bj`oXSh{n1ftZoG@I$ec_ceZ(ZpuhEhQ~UbP zvMOaqHU9C6*QfV2(!5K|{SU#3w>sIsRk3JTVV`&4tEE{p9F$Sk>n7HDP5bxzYmlS4 zaUD#=kvmo=6yf#N9%NI$0f`3_q%GG|qj}|b(Wh(d3S^OY6!+g;cU$81v#a~7#3v9qxosoE{Qflj`oI4aUaT?dIEadN}Sc5i)OY?XSWeb_V3~s zbqD5yQziS)Em*; z8msU;P=~k(2Ed@DLy&|yf}Tzq<2Fy-A4Fi^Y-AgAzaGC2)1X@}@=_j=3INyr77!%gCRmVPY=Ld8M~O%Al2@>pD{6F4ervk9>JLCJ z-~dV>K@ee?g5NP%lM))mgDWVs)i>iEdDAN_ z(MGGBJJnFlYxBy)mMx4wKa-{^57oCVxqxgFmVI2+7}{Yx(oYi5<|(uB&%yo#(Po|B^ix{{aya=73)omy~_#s%i8!#~;;ZU_9uYSH0D}gWQgNz>H795|MFQ9jG~rpC zXoR(-)Ay+2~YtjJ&8Qt15uRA zg5_!xY;aQ>dp)c@vLm+{;8v|c1og2YzAP`zaG(|>2eqGvQQ4+*mRi{eabFRv={6rJ zIgQs@Gxlx_gtcKzhfS+#1#p>o8>q{+&E}T& zNxj3zvA}j7*A$z6*5?a4>*84Y`!Rfj%Nm1}d~>7<3$jqv+rh}S@az4ka zW1g%tO6J@8+PYThufPr+=Z!o{AhG1;UAy52z;1B?3dX!JuKnXopa*#d-biD}dAc8q z8D0W2z1^7a!|*dZ@5l9z+}+XKnZcCmBp6pX!^UXxLcz%a>XJQc+IMC7#P`D;?3P+Q zj4YfV>51}H{2a^>TDV|OFZro-iSfs)abGM$b(6$NQs?yyNjd#5Q67p+#4PYf?hpjm zCmC}V*v_BojM7=82%d#zICfdY8Btv=Uv}x#)%JNoT#lPKQ@|y(`^nT}`9meBh#4~( zR>JpGBRl5r&uiv)U32^0;9~3q4!7~f010X}wZOhqiZ3Cap3nNPKg%RJvoM%^uS@EC zZ}7GFrV0J4O$W*c%!k($SV166&pQ7XQhc7M z{&fW&yi$@uT)oQz?a%h;X>CguemTf5E+x>4h#PcelzPS>O!IN4t4d=Mzo+#(7Jh1U zq0?63)0A@sclq3=3Pn0mpNEUzpvuyhRF(cYM?u2*3zdkt3+y`>lTp85 z{1Jyf+4MaRnUYKUFZpBi;ywhD^G~@mbF&`LPM&oTZeq@Ng0pRm8-6A`xlm1Mfx{eB z#>#*O^=mG_bjdQ1t+(-Nf4XNX&^>?o(>)(rx4_7_h;USoFul{~#U?Lyg+4n|)iJ)1 z58$K3T_rGbfu3j0>tBU*a|brD-G^FWM_(zmgg#P2z1qOM^Ji@}t;9(jt`27ief;lQ}>BAMV~W zsL5_^8xDei(iEjvDWcL;dJU*Fxe-JJsSyEb(nNX*p!5z3f=H1rB_d6F?@f9y2}myq zB?J<3f6IIC{p|hJnfJ&0yffdA3`1rDSFUTVbDiZV?x+b*QRXi@`Y)-ANo4viDRX{U zDZ&dAR7A26Z*J%8tMrGGZv2{@)zhJf!wn&+>=wxmfo^J*Y^g}aw*=$4-j5oVN^`fl z+#Vft4@z&Dk^ZN%X7;@fMJ#a;ezB4cz)FfwI}W8-W4>a%;%Dh!g!3#`Q!DX(UP-4g z-}MXn7skkcWRUz{`knOG-mG@p%wQI1`mL5(wOMK_47F zP80++u{Bs+jk&)M+;fxL7RY(W*^b(U2G_hHr@Yei34he5^$C^g1$Lj9Tu!fF&O#u-aH0>f?O;I+G-$%ng4Q$4z_Y2v z|2o%q|3nb)Ec@Lw$&HI5mEZEYoFv>?9uqvL_IM}@k-HUxt6cW<=kmP1*Z$f_VkNBa z>ZP{3zhSbF308oGt{=HRtaLNpMKS^k4{Edw50<$ED+_`OIo&h$l%nmf}8<0qTA zwKJ4HOB^TiFH$E;GW#N9DD2-1Ko&jcv8d?_a=e4X8@CI3`1BNXmXFw@~i z-wk;llNG>yWstJuBBT?O8$Fo6wBw;wuFOBa`q_o-#xt5h%=dzxrPnI{uS{`p(~8>E zku}xE2v0sheyM&{L>ffL`K3t*Je?4@6^z<`)c5(Qgn8~BL^nh@)0%7SE7_kV~>5(~FqwM&;!azb%*);rZDYME$j8 z<0cv#Z<>;+Z>=U;-p?H40{nII^C-8JxNh0{vtY=J3T8JZWF=Or4enAdYx9RZ!^a20 zw1|_6ER#<1TuWP9T9Wo3#lu!24qK%@8n$11O(D(CZ0VBb9nm@h%&vA^@v4jXCvB`A zQy#I6*LWO6w^V=w?J3W$|%uv2niA_p3&7x8fH zA3rM{sF3>eM!lL+jg1MrWbe&WudBC~aj3edSy$2B6P(`S>XbdJ4z3q1#D(nNy#7`@ zFwYa!x!Pr(3Yq%SD3^#uVDzzfaMtRVS9az`5@X@14fmHsy96`BYIha_i#9Gzo8??b zT@co|^Zh2tTSvaPNz5G(pInB0z;lV6vDW=5Z~N88h$(b?t#Z`b#~|q_!>2duY4=5E zCSKAemc77F$k>^D{hG;Qhsq(E>Fb}xzsRHyWDMwd_U^u7r>B+sNtRKS9O_myV4*az zBfe4kinq$%I*EU=O6yku_v^dRctk4nyM)u}dC=7T>sz1Q=kvA}--H~%rzcg$4(Z;W z(tav7QuvGt2emc67a>@?;x6~hWy~PQ;0{K#V}eMI*3U)xe6p@JmeBc@1*t11)6q6h zVp05X0DRv)BFSBlpSdu$+Qb1RE{n%z5@^z8zQrs2V--7^4`aF>e^0Yi~L+?r{>In#wj>cX?eD=NF8~S)_|)f$Dx!1iWHAI?B}+Mz@o+ zzj7{P#)As~5!Y$)s8p@66*C-&NOdH*F0+-m9c|QW7Q$_71a@vZceg2o+=gJ;FRp@) zn)axo9O3C`3F8D8L*K2Nd=lgJPilsWYLlR!EpW>~zrpv_n`M;nJ;9_=OIzVoX?mWh zS=VEwp?O&E50I)I79m3;OX{7LK8v+WZl_yf&eudkf2FmBnpiGbgQ#)Y^r^o;ff`o^ zM=u)Plup=Ft~19L>3BUkI~>B$O*CRa$0XM?R<&~CjpjO{SR~Q+$%AQ%F|K=bWSl_E z&4yGJOmoluew7oHi=M5^3h$N`{_$?=jxk*qPR2TqhRi9AwS|~>%D8bEMT&&!RiH^ylc5%5{p&Gl`wHNK`D*+Y@o92i`-yfK7@Rh?_^`iW_?VyHcJOeWe3 zp@a|}CyLrmWRP>?lF3u#)ZPS|m%FoEVY(UQQ(z^3#QSZC6AbHVV5p%}M)6quCJe~P zD0Y0Qk6ohc=ux~rv#A5y2sbaZ?J&~jPP~%6a+5`d;+G7<ocn@utAWPhx zqGZhq#OvObjnaFGEDyG}Frlb`2g6t<#Ao4|2)$*-1fN0fRaZ^-kYad6~>(GkN6 zNco5&`E4NTZLp?ikuRF??*Kbq$4(%&o~k$Ho5M(a?3n;ikR9-})Y> z)~JaO@DhSJ5Wdb+aujtewv1s8gw$OgO(kslF+Gb=i_p4-F=Z~g8Tg%YQ`TS|+7cUf z2^2X-_I;KU2Dkess&&?G@ITe(r_;@mP4{AtaY>y2Nzn=WP8aG{feUvn(a1AV$)7dU zpib-!hIj~8s(Po>Jv>;9HecTnXU`9~x!nDFGpqpEu**Xre2G6Vp7LUUqOs*Z@JxGmZ{JDD zOKtr13GH=N>M-t0GGKnP`Sk|r{uZ7~A2^I#EuP_k0BIxi<=GWIFo$mCVA7rt<{aF+ z+i#w@cdI_CFMet%Ms_l9QU=4+;l4nLHp@pv)_IA%C{y8K?mj;~|5z1hKZR(Ur&o{+ z55qXPW|}$zFv!pBV6^@_^(vZdg!sPvgt8jU1rQxQch>0C3*#l>cjOJJ2AAWSGqNbE zWFWYMP9g(_Gaj{LpxrcFiuw`rtLGd~bC3qv#A$oS zjq2vMQ^}Xw${+XHzx8G%sPK|q@_uV(2De7MvnE=BHg!ud=*a@*0rIm5=t}Mu7nU4! znAGmWYz#z~oFF*O=Bne{p2q503XXH~L#y&JE7TCvo>#QaVu6X9_8Az_&c%=g_6bLE zzbEZ%PQo$M`cz)mpDKQeS05@Ex=ONf)gaB&dKuORr{B75Gh+}YWj4qDAknyo(si9> zj2f@M7^`>@IxIRi832aOK z3E@fy6z5WR_@LcU2F|+a9sKZCHIZ5bj{D_Kq((c1)HCHZenNg0QIMUU>qZ)W{v7N6 zl0O#ZnMrdh*mo2g%SCkpq7fqD}rd$DqkT?O`H>%V=JqC|v_(i>YlRCi#J=tO={iC`TBN>)l z&DgCgqi@LbJU@*->H=rL8jE6H%`AKxB<(_1I%IJ=VyZ3830UM_lEbFoAInc2ZhFsN^E4CTjzI@dxB?iKoSO0Yn8-#me?i=IcuXTOXX zw7#d@`7{U_n#IlvIkF?{pM8O^RC@jcTkBV{l=7d0TXTvzhqVq#lMbkYjwallLIO_S(zGqwbxii%(J^gk{8-HYhy0oATh3 zIj~bmDTOBPG8*y67@!{b3T4<-#c8l4?Rx3mUa>Ol&1h#mT!KJI0)Wxrd1o#z_)0Xc zoLFys60x@__wib7b;8UlUn+~C!CzCX@Rb%dgBAzJ609PFYQ1bqVM2a`8X@?NgM5uIaGUl9=@J)EUC^ty z0NNpai)WliZ*$p1YCS^@$iZnPE4@*V1)5TCSL~7v&lOIaQ|y!yr+HKHOGUIwM** zhe`4JIOBjLbFup69FOLeV4X)4rFXhZ1&OSJNU1Ywu(+4V018(HB8M#mgJTW^d{Wct z>_$u@Ib!tCfvzK+z4w!YvD#QB)^jyF^aQ~7e{9H9GD1q*Vu3>?Z)!4W4S-x&!XoqUIrT9B z04w<0O>P+cN1cj)9?1X5m}(IaPkaX_>6p=RTWX|nIORdf;wq4Pa54j!JvxEZUpAR^ z2wyh5-um$cc21#TfwE)@zmAjxoh6t3G)^sVQvgd2u6FN{!RRs0cKX&VySbf48LTy2 zB{^k-c7l5s*9ziWtGV^{;m`3!Ns7%5r8>k>80}!m`BWO^63cF%HgZO>S;4gU(Rdgw zIH*8*%aqy&K1o)8Uo>=YsUdKCp#62n^{4gU`M8%GucCcWO^SjZZT`X-u8^c1aRsy0 z9?@$y*6Opr;+ZeK&W~D35 zDV{?o$CT`3VpL-}Glid3h_1>zrZ98MUYe4(w5A~aX}|#*E(hpXVz9?QV=)(of{>yb z3UC7Vfz($n;^WrQx#PAO{=LxIwrW<>@(miwx7#mn^FC{ChR}Z8iTSVwM-Ra$L5J!o zoWMK>O6*?9KHZcD@xCI+Q zxOBlU|LMWf4LQxI5jCA6;9K>u)9bPa56?I-h}I?&-00(2T?}VHvPW3@vy|%zjDk~= zoor>Z647}Ca@^B=*zBUQr{NzHyL{S``(8}WV1IW*?@?ybF`kveS03W*!MgYkF1A2$ z_2C&au6vR0O3N>Q-jX_7a?Abkwv+Uip8JHR4Cz7lKOwZPo%St+#c4sJUnyMSKEZ~x z`YaN@@?jH(W%~pcX^R=&1SpUq&d=g#JJ(w2XOxiz&xNmaPU$`FV-oIuwdTI-ns#9F zuBk$*H7w_i)1}^rWhNz2{j>^{k5nMu;|yImI+Vd%U(e$^p4J9Tu0{ct;V7Fl(m6qh z`Z2@Q-`0THQ#?Qc^<2b=PE3La@&)mh8WTIM$53vP~e=7K-|11G`ur4egj6pja5|=D9E`IuB%m5kjApSyN9Yy~^cvoMB z)Y|MMM|qlSD&+9=?~|ebr#@%DuYu5k$p<{32a)!`!REsP-flGs+$yA^C}L9)zC7Q0Vqe2e1wF8$z)YAd2PY!l!7t&bSoZ70;i2 z6s?G5*qEuRM#=d^YTitO-x`rL@X);&RSmJEo^-|REq(IU92Y;c*B&3$9&^ZK{7&^w zn5~`VqXl~i_1Y#KP}D(}_`f3#Ebc$>75)=q!Uqo45rg30@`AC%kOna#p$GOD$JX{6 z>Xnp&EMmz?UDS?z(IdL>TH~d_C||mq4*AYq_|aW>E&PLGe8?}WBj!xH-rZ-*g{8|& z9)u5j7hIz@AH!v_eim4rF}5c)4!j+b@f+JK&B;1t@dm?TOYvK#6s=9uja#ja65lYi z+8?t`(Pui{rev4#miuztiblGw4H@VIy@9;R*0Mx|b&^L5wF^5yG!=VE6H*YvqF zihofQ0@?obL=3ezD)G<0=1Y7fwTAs_S#3y9N^o!F;v+uPN_1gB58Ps1ivMZvKR7vAV94Vv; zH%JArGs~T3oNGk$60jg>eCJ_azCJif|LdChu&K?y{7p%9^RxY=@MXUAw{g>Fg56ML z;%_y<`})a18PYP7&gLvh8pn}o`6q;9i1=Q^tlVU>E%w3;j;kfrDMVO=uO_C)_|1s3 z>2&0sF7uigirhC9K(g>Ez`6nYT*AMD*`3|Om0rM^E}w@00@zQ;CLBmX{;ST?|0D(Z zU(gERFXG@xo6Ui@6*jMf;5+qzto`{PG=%@*CzmjBE(5c%v<%35q7)l6z8SF0CQe0$ z{Pr{YR!TR~?K8DUAJP}pOozWW{@iSK9abju_L&5IG_bxCfqdrg^{vF%cnQR$TtL_% zHaY@q!~=!{2LB)&7xH=rsMU`bNNSr>m+XSn8fs#Hoh^SYoT5`YS1Pu#XT~eK{Rwe` z_~lvtXn)^*H}D{4`Z$x;8Y>5GRy};11xkGP;h4h{*4z~Mnji~oSepvB9&a*5lLV1f z-wXTN&M2I&vGk&0X*8r1cHU-ot6f)TaU|ooM`rM9K2|; zv~u2b_73R1D+i=j1`*lt7qAmfw_W%uGxX3-4>0t47Qi~pLi_>x?eqqvzNEOji`4mJ zm4ataC1_MJWTf4?WjRVA>s~_YgkXpd|Nn4z|64K7|E14xT_#QdNrhjyLXm{icObF+ zRvk8LT;%YqFY~1|hlawCJju_Cm}e4xx(g_|PkySKH@`SzYX4~Y5ZKMV7DNN+D^Gmo zF&{u}lRtU*`5a#VTT^xP;|jdT51Z+$f)r255CubIoV3cMT|14ez7KxAM?{Jr_se4LOxs-isiB9D;gpsqbib6LmMbqU1>_r9 zVCzbvORp}nOQ)I;s-)U*%pBbv7S@0AqrEsMj?Tn7 z&ERJx680UOwfB+Q6n3t-x7vx15}(uS{o&)MsC>RcxvE&B(ilSR!M9f8hk;%8ABNKSI*E2@!3;+0B?>mHWIqX^2l9;- znjtrIOTKZG#Q(%@C~->U+1Cyx7Z`7{`scejxxGTE=W5VC4ZRQ3@t;iRo!B9;Eful( zJ+u)%vRO$z;<+Y|Re4v6l4%y{;t+cqLoQGC5NCj4(;agt=0WS%I3ClJ(a+oNx6xbI z$=$d}QHaQa+GRj+*=Fd_ce~;Wl3I@|ZmfjtE({xK#C|NhVX1VtnR(EqyIZ=M58@k+ z#V?-VGl;|t3B#eG$MDmE7QV_AV5glk`bq`0_@ zVmhPr=Lh*A*c}8Fzt4D4oPjYqx%IGOa);)_{Lb=%aQUB*;uy&X7un%dzBRLBmZWdv zAPSBkMW-8Fj0Rux2pZvkL#p<3dz0l@sW3MiHVhe~uct%~fzY+Dq$Oy%E?mNJ)?|BT z{4rKBue-Xi$@cCBvS-s-nIedi?~*6?q#q^`85+xt`-GB{nZbsNGmKG@*@o-#VU|%j_B7%Ji%rkA8b$ibPzr$Q0l4;!QpA6H7Ya1z z!9L2OYX_S189;MBlU6rk6BOwG;qC1I^vVB!|GgqvkJh1Z*5En!I8QcH4L)fa7Yja0 zMFjUn=oKY)Q@?yXC+^mk5;_=o=}9mfu|>Nr?_Zug|6FU-_53BtMM`v3x(DU8LJFq9lZ4C7n$ZnK%um(SU8~|CrL2~%_l*&tO2U=p6 z%GIiHZ%v~iVbCitV*XcT$o?EMK;gS|_i7^z-fi~|wgaFUr)I&9 zA_(1QyNuMx96gz9e#RUW-niJ`&rS5-t%FlGtl*@ZeG^Xo(5L9c`dj6*mmw^)NVXd1 zcFt@$H4ed52dDPb-I}lSb~AY;ZmvFWRyC|_DSNP_0TfC=jtqkM+GNQ^ipywo*_x?p z${EU4@#hJlbX>N{}jE`$#AtXco5I(<2C8~dsp`mDjSFDA( z6*Wbk4n&=PXfxXAkDF=bvTF|!e*xjlNzjIM8+^2#St!d}REAp=&Ku7g|46v0`{aZ9 z&&f*vVbIt2b%J(i85oZGN)^~uv8HI4Cu>*v1pJ8BE83y3?L7wV>NMLXDn(v9*5HOH z{|l4mUMrM*HKV^m_=eowb*}m(f^fje`6G2tX;fsF&3AN0g)chbZU>0~>D)xH?#~j- zvSD#NOGJu8Np^T|TF_G^X)LPrCj~#oNw0C(pVc!_Z&ZYJ20PKJ7Gec?|H@g_{vg3r z^Kq&{k}*CxeyUvTVS9pi;(&jgugQ^l57>LI>3Cbd2^*Uc+h`Bchp zxZ|tO0~2jzx~ksNaw$b70-#=kT`*L{ciO;CC=X8LX$(7y^Syv-qT7<4XewQwl+`Mp zXdSt+_rm6-$3n5XDr5)Zrl7GbEe$#8>Ll_^x@9oh)rRDMH8C;Oc#TY}VJ8iMu!9v( z0`wBd3HM8WE4xYXhvd6ls2OwEc}cR*7oovmN-2LynIO#@M`MhsZbgK7J4GC6K1R}* zovF0dnoJpLx==@SaTk4!^Hh+Hc}HrLLD|TJJ6#5_nA{)e(!#u1L)d*Ug~2Rq{(5`% z*MeLr@2z5PLsk+u5+HKWSwYkB$*xrQkkqJ+eC7PyUgK;Bm_)jYh|3sKX)Ry6Q_Gi7 z8+>}(sp5wI%Wli^~mw!SvNitFqv{1StdelXzwBOKlYS z6!|RZ9Gnw|W?s+pO8fJ&=D`orTq$NZNcXscPOGv?eTC7hMPKJ7#*aEO@AfY_zGjX} z^Jr#2E}f|1dY4L@g^Ue3B@g~uZ0~1nM{gBfeg1_;r_Zu_`yrtdQ|!bP<<=h0!ftov*R$ub3#^di zjL^l3{HVQpi(K28qx1|{b06oV{;HoJ=QY3SG$$!s*f0ymtNB__5IMI}ok9r;)~_7% ztC$i^Oht~S`qvA-Qm)<5kU2yo5V?4JmiK5tn|g63sq5lh&kq_1*jeS}uJ)blx>1)a z)W;O1`Lq>P{k}u*C|!jk^BX~x?+TDL&8lF$v$f83i2zIJS~AUOw>RaZsyll2rOIP2 zzSl63?^s7KqqmWIzz>S*zRp#a92ywhCbYidAan8M`?ZZA8BpzL5?Osv2FK!X0+|z0 zq+uEBiYx!>t1?>ov9eE{w{Zk3lRC$saryYU1K(0~Xp;-kC(i-Bkrh8(<8&VaJ0V?hUp&ej*59#BuITpO-IE8dM$h>Ywj(H|7_yRV&g zU$CuJ^9KN#J(32z>rc=Zd~WFcaE|z(hC`ocF|?P~7!E%J$*;-IrWuM3PG-yzfRV|| zSzBCXb-CQbKlW1fLvKiT3o|>aF&8leoe`1Jj#j*}5LH+DG?|x$(`f$&Y56GBj*CmB zz__QtXv2WfZfW4E{&v*X9Qm9x{32dcyM`%=aWW#93?me%F(?IbIi38KS|oN}$MEA4 z>j{gsmfWXMs=agqWv z&#C>6^OpMX=jcdtJ>5(SQ9^7_^{$RSEP8K`kZp}VTY9`4Eq^VhB+T$?{4IJVMBMm|#ZA5O zU%CH~HUnA;VC2p$w#{2>{q$9^dgZt)9J~U2!%f{8-%$P|IU^tVm{wOwb-`JP35|`+d6X@#sMZC&Zitlx@ zO^-mKPOo<99hf5MoIE0E5_iFu2mln8j!9$0{o7{|6;}Icmz~+8^Mj@CUh@CQy|&Q! zJd>{W_Z?wvY7cX>S8yMp+^d*R@WQs)smG|3+6s_4DBvd}?K3LrVy=(x<_;HE%xcNx zK&?Yiqbbw$L4J$$jCQ0yJ!RC2UECdrHRLwmqy^qYbpw-xiMX3R zJ-fr_?Eh$t#GP)&2PLb;rbQbsWY=6vrLWs2AGN!@1?DsuC~AzzhP$7Q8+|zXq4a1x zz*z0hT%343%iSsb6flf`0y}RomchZD3Y~@{Q5Cv^4_?(b>jh=3f2Z}6kZ;{25)NE3sk8oJxxRqDU^EZfi(-!$7x~& zUp-oIDF14%G3OgD&6yt$G?Aoy@H)Qt5_WBcg)2UCTVIz>olpC+7GUh$L)s|Vgrmdk zwJ+Sp990JsG_0+8>@N3Uy-IlReEiI7OS`1eeN7!JjfkIO;D2^WxrL$YpqgaPL;ODA)(A zuE?RwhBGNNbtcLRUzuBjxfz~o48E2r)kG!laG-Vsx@3Wu?v@~xz3|;tP3U1k6oR+& zm*GCpc>yla27-7F-i$~`B=gse#NFr8EH|Sj(+I{Iytu;U^YJXqTLw|2)Xpzo(ES8X zUrL-M$e|gZ&aF~I_^WB#YH!zvjGiUPJKcJi{W6Q7yL69SB0#X7fueTH(3@II0ToORa zd?G-x^Qw+4=x!?zhvBSlix6B$pAvefqLy5dFCQm61439C} z;{qlAeND%y*UBt=-4Z_yG6)d0*rml_`UjTg*2mADpRFR4{=4}2fAxJ@L6k>(6Sc4O zx2jqU*{D3I#Hch?=)?ouo?jrf0g}FJTs^#2jGh$6MCn`<644XMXUVxu*vl5dTPDQ5*s(8qW19Ryk>vs!v`#$)ljG*Fw5I zg7mgkPfJU;bT^C1iFJEMSQ7~kCYZpw7t944%QENa*eS_*&3k((dVpso=G`(zqS+rP z)#)Dnn^GMECId{U*8WYY?kV9eE)vZOXP!2;;k@cZx9OV5(eP_N%_Z-k*|d+>K_GL< z+d6dz6jp7>kU-i`7RzUM+pxp;cLIs+&A*NyyH3-*IN4O5!|hB3>{))i?S zsdME;h~Zci<+$|8_dC@=lro*PP^l9{UOga%UN=V$ou0zZID{c`L`LoHSuoICH~_x8 z01$=QJmv~gyorOOptM6=0k4uZt1CaAoccT{(`1uUwbFgf#?GGp&Yj!eb8;->+akdq zNbuGG$cRh<={k)8GPdo^SSRE0SUqo^$^4w;#!FA~HO2geVw`KUA~kyaWK5ikX3BQb zirW9pW%H`^Y-sSCLmdh*cEb%UEr!P08J-i=;xD%1B|BTL(sh<6F#QFL~PBI!7R4htL=!o(X`yWklPJjK^XSzg|yjw4re znC9icNh6)$c3h#7lTVbAymv;_czK3sihcMYOLuqUBEeJEGzx{UhdecUQVwewyHvIb7io@q*;0(N8|fV zI|Ik^3`dP1h+id{fVVv%$N~r*46uV4!A&x&Swb+_D9WJwSyw}yMP4~7tHr30pnQFY zyX%o5UhKejJ5Cvz;)Y$^2KSNg%Q3M{48C4qL)r*`2dA-oJ>n_&P^gRVM8O{3#(4!j z8JwLCrCqN|HJZCLa+a=8*S(Qw3^*X7D6UZA1q|mGH(}Q{R{tzdoonef$}h4ruV)Pt zS#eYyqYpv>ISLNL?JJeI1+9meG|LjNq?n(dS^%@ui$-5!{ zgg|Uyvx?ln7|DhG_B~QfF>js(u|V*gptts$aCWhw@Ixv<5jxif%#n(iunXYwE!v<* zcu`^`!#)kf|09mHGnFoepukv%#D^r!f6R$jKi5YZQ?PlVW!*bYnT($1joU%jwl;W7 zd&a<6r4GLSv1r{JBG^;!h1qLPr^ZmvoZrKSWO+GKWSnKt%RX3~dokE_dzaiLjq^1{ z@#LC77uSj4f(ZC*+TvhrxJzfpa&J-r3@}Jx``NN1i>dRBO-wmIpd65Y-~m4#OjCsD(oGla!uByw>Rf{ z8C>`Gy%b1hT$UHn-{Jona`i=bz*L!E49tta3BgVky-)UmNYGI%1CA9MGK7*Vs*{Vo zFkt^~Dj=uZ|1o_&m!#_a53)fl)S>!8*CI>UQLOXx@ zQ4xd)+naI?%DFY>9z=SR`OwKlT-{$nqU*W_6Yoj-|1|*Hp1Mun=lV>hBsm%tMFatf zL*5>ptTOiM%_4_QHf)T6D(8}M&WLeVCl#aBYA22i9jH%Pz<2YzaGqP2kcs@vuLviL_!ZWWIGQx8S{%&MPo3!eR^_QSB_FIUlw?rk*n*e?E z1A0gfOh+4VPtYjru6-YZeBS(Z!h3Vt4KvfT@n zN)2vEOPZNa7~sC|;^M#@A)is-VJX0+WWS1-x&L<{6a184Z~*WL5>Z_3FltXKrME^C zr8DZKABBq<11*!*r1>)W&Ou~^)H!5&AScNL31}PQ^gQmL$qjK5x(`Ioz!$THADS@y z&Tmq1yd7X$*GrJsWtmWYy1AdA#!Z5BRQDRt7!)d_ndQl`7HSVc3jCMWA``HifKCCA;x$BSAd=PvW zs4&;F&vHv~_S$1kl9c@ityBygBnqdKPgEeUS)py(koDR$*1Qi$g}Woz`A8Lc=YVc! zRkg+oCBO&9Nf{%NFl~SOv?~8qY=gxQtrp`kZ~e-vBrPCfX8aW~AAJB3)4p7jif{B! zNV)#VWvgFaP-{xbSDli}d3QQ*4O}H@g@fn{4v~A0{)(>I=*bXyDALE!L0%!pXd~_+ z(K6hLS!W>1lK8YNRqQXc33IvtIzLqY3H5DUsA|R72)Quxk68CsZ&b}8?ml{e!_=1{ zm;6bbZ+hrK>23r>eV7-}xF{naVA?~Gnft%2U-u7e^*->1gHu&-g{3;YpUEpb3-x&3 ze&2V>^5WE7RPKBBrROV`V_WNui7feTT!;EX;2ee$6~Ia3u!V20K{uu)-xYpeHYC{0 z_RCsd=>kVL8PRQj!iMzl$wQ#g;;DGGx~+dmG2WGK&)U$1FM*M6wQz;>2RZ47rE`>n zzE_C@oGcvvLWXpBH_)(%ok$Nv*x$sn+Es59&P5im&#K*y+f$Ejx6$Lwba=&vSdEBzq-JeLs#x7y(~ z!SG`ddAM}HTU0&NdE+~Z!(~&yzO5=bKYB!`Z3KoFFMROB%Oy^lOn0eR#H&7+9Q%f- zBuUW4nut;3C#O&WitcFb?Y0~nPJwR|m7-U)i&U(*Q-t+LxA8eoKJdSba;6-lfEjHn z%K2VH@3pgihCXYW>=E5J)${uoSZ$wv;>_WR!t0R762#Pi^I{Q5otK2*1pkIb6L3Ex_k!)wxC(6T z(;V23AGR8vIVRPq?MqVq&*Ga`mf0;1zBiU&{A-GP@SN+_BX$NfZf`c6#5)F0cbF<*jT0O7O;@f9FE@c3!9h>8;tOCRa;74i4_A7lY!qByKZG%+gDkW*LEy>a&DbNx;+@)4$)$&O)ys z&U<$)j=9^}Ws!mtSnq*91Zrfu)&bH^PDnZ0*6Yodp9HU|tW8hc?8nI$D_0JpYsjuF zmUMe9LvJARr#No}nS8Z65cNMM2^1I24oHv=bi(*NjM3(@^?k*-@Z zL0V=nL;La=^d~-ggzA+7E=*VohHa7@wzQbnmy)iuHVh&zM@D`QqKtN z$jhMB#XoA2Zt5W8C$4kHIx9(>e3JjJE!Za>%@?C7R@3vsJHdu@CpM^wlpwaWcU3C< z8s?#A!s*D}s)dJP2~)M-%WoSvv==@X3%EOXC8%-Qr_gpnsK^iQU^jU8i|q%X3@m}J z-p(yBn<3Z@7w?o>Ml77lh3&r3zh?C`NKL1$kwq^y*P1xu9sFypQ}j}Yo2yWd{oRg> zb(X51uI@aZvP{JNq*#cv_m*fcqGoM~`d-ue#e`RkONpdZY+0&xUJ&IUZENf0CGr_= z&a05SV!uj8Q|tiPGuui>gQ z?Qn9rzS{Wn{k3Ve`ni_W_OIhldWSh5z88|)r{rxiyYmZ< z?zsg2c_Ki(2x?9aM4?Rw!fqBf?K0-M@cKiWuTf6OR#clHn8Fhchydk5S2XV!D{P*%d3%=G~169f1+}=9v@?5(7%*x(DR3{*_E{}i%@o!_8rhq?0oC2b{rE7>d{Ur8S!MwH znG#f5nV%#N5lIv{D!D}{)_wZ+=C#0X-b()%!qtYlG5@QcX4q4d=BJcbI`!Wr4P$u| zaXJ(ZDhz67N$TPXs-7k}@xV;2t>Bslc>tSYb+&Q;PZK-r?cPx81QVX`8O~aoR)w~c z{XJF%#-?|vzlB)6rf8(`hnddJnVD+2dB_dP_nT>OeD4Qx1PcuUSo@~>3fus4s>Crj zM6S^6<~ZBA2m64Jw9L&4!EYYxeET+y-P-bjiOI>IJe9I2z3a6Jl|?YteF2tEjAiD0 zF)feje_eRt4H4@>0N9@0z&-5VvVNtli^nazMP zf(55h4SF{F+60DJq8#jjC;ckwxS7aNUzd{i#uP#C3k7Franmk)d(rd5pML{5(p9i(v-vbz=?;ZxW#1vew0%)B+#4CL0 z!#EO;ru+1-H;q)dxgL29Do_kFoXlfRiTileaX4|>6^!Vt8$5XyruUbh1Z4lb z2*GC1`VS~xW}Q!I#0ArqIesTkXkEA(zDD}F?p?9cQS+c!kZkN%FgupLNT?V^AjD^J z5;Tgdn#Oe=^eW<7mGQs#T2B5vEHjhhv5ME)&ILc8JOUd=Vl~5+zofT0;(L3?Gx84% z-_-sHQd?Hy2qo~bQZ8+I!-?kwaN~V)eRAvlRR+%Us`<8_mb4^?-FpN6hKVPW!(3%5 zXPX42i8qzC@1OPLG_=j`o=``MKP-vW;Rx|tC;eG(ix*u#Jq|}4eU@t<-MGEb`_f*& z;-IyTy^Pv~onE75Lt|-g+C%jxLBpbG(@Y>kKWI?-c7?GvuMa{niS`jmOY>Gt<>6Cc z)B)x2zwGIOBCGW;DX?Ee$8IvVZ7@gyTUV;cN5@>r#~b#t#=9qE(L!H`Vs#l*{fgGaemhD4P(JmduW&4~ z*Y69B2xUO^r+#6chs@OT9`J83AA$MY#f3K9Pf36g>CcNCDnH`K=Ud#jTCOqDD zpNGZDq5oe0W@d(!4)o%o9ppee2uOt^g0UX0i#o|q#zd!mxAL`byfWF*N#VBOTjNV3 zGAx*E87UO+oF6Q)BI`NMPCzn*`YTKLHwhof;Ts;Ef?8dtCSh|Q*`KH-=xA**27n&< zN#3O?KD=sH1vB%q(H-U~CV^W4-UbPT7dW~cU=))znLUv=j38t&v{A|*eWa^(PcqG+ zj**sknImMOc~1BHP;0>hpZk+m{qEi1-=B)cltX&`lo?KMDshP_hBm^;&_zjfQHGc;mZ0(&gxf%J`5NpLo*K%RXv88%NreljSF(c?E{ zMUv>s28p#4ZdYqJIy`K6hEIv))g5`*KgH*ELrY0-T$GS@kc75<)`T0n*qm8 z0LiIMn%KQC{w4MLt!~#b)>9~UuBidY>8IMwbql3VXrw|GZ~Eb=)k;C;CW=!R=Q^d? zSj1h;ST<4~%R{$0ilt#{#h{eZ=DYM4`3Qt>rs?vPE>wsm59cnhl?N%EwAX ztU7n%21}=onk7!MQF#1-W(RA@ue1JSsk^2cwT92i+3%dIQlwj4uwA_BD<~1&Uo~{S z$bqWDA$qv}YjOVAC3POqFs`DFscK{4l4ux={Fy+EKFadrSOr1EEaaCS(N&1bcR2zMVwOheSLXYQgadAYj z>bSvqulcdJqp?OOTSQhW)bSb3;>n{*kk#vv96pLfXTxH#uj+AdaV*AODtiZVNA5CV z{JmzS}D4{;eny|p|Y*?k=TDo|3J5A zwfNpW3gskfy~}BrD2KSD;cd!)*^hz~`Q`*r`3wNorcZH(s6`4~s6K`+yjV5$`wZI+ zhtK0VzO^#&_i>QvSY3^c82=K3u5?dzKiLzA#3D-SHj9 z>k))79+3o%TwG+EKCkYB@5VPBvfT`MZ|zINKL+USbrtX~)Z>vuJERyMZ8jTh>Ou)4 zB7ebD%|W;#>04XdH`A7EC1sYO{7P@=jta)LW8ngZF&Q~T1$gA*RrWp#Ywr{4cD*jC zGK>|Fn)59S@aEAf$uk2T2GEnc=Qnf)rXjxvgt_us!0h|SS=uXk3I~b$nZb0%r2=V< zXL2>s8hZyzS6aGWO#+_fPbd?7-4Tit{r#k#9>T-@h3zF5%LkI%*)-W~B>a>p&jlGt zTZ(jB>Uf`x+)MC|N{mrb1mnVB7K}0R-4+6W26|LXFjfCvI4xKjkqWLl82)c%$xrU6 zab(}Ne#`<-3FC8Z3fzG~|Mf{>f9tV^oFx-)cqVqG`qx;aJR)|t9`3JbcBq8s7)lX} zzn;UC7j&7Wy$n!5Mc|~2&axG6nr&7a82MLoez4_x*JjO-MlU2d?ICaV@f0+)LmzG_ zQu%t|-U3X~h&b%Boy#VcHLZ0z+s7vB4x_y6$Dj0SS>4P@~8=G&Yw`gGw#R=i%jd+@E6u(c@67R&;Oh;a~QJn{E&(S_b|Q}3h2>roRwpwpA_4+#2|W5TRqLGMOBgbY2@8@fU^3E%>-1^#FD9}&J#}4uAA%pldFreuO;!GWTr)*98pY4I* zY0(zf^m3PMek_rrtH}Ow$61#B>xuu5w>J-my6@k{$5K@GHM=a4HCxCsq%g!pWoJT? zWJ~sCEFpUcC0kJ_j9uCHHM{KV*!LOB7!23%?Yi#!e(vA%{rma;HOJ8rW9IYPU+3#Q zU3F+C1cbi(Hf37EC?v^PNTWysh}O!^cL;{9fWY-k+4o*4KBREXo9YKsT+SL`G}ItA zX_i3V7^jFDye4lY6&P@lEKlx?@$`$bfGIw-N#~(aq(O<)g_K@XO&+=%3p+d)No~1T zau;b^A$|k*eD_Ws5DdUvcY8mYxtU(WQfW$k*|eEXYf_DooU!qNLPC)D>zz5^R0!t! zRd65F<%RVQ>PL zb{_sf0f*k$f(I*$R%6?nqj_7BF&Y<^x~lFM+uA*idsjr3so)wj-s)aPw3o4&hv{y$ zpr{*hL(v;Y%T3>@7%-_0bq%F3OXMS(+(Tsv0vmsLa@SWkqs8P+ibW}Bg7u2z-@fzq zy|c2%o!`2WFeO+6`05Vt`BsB1qaZdm!q?gs4W%4IzEI=;VhW>h{ElT1dF4Svu2Ai5 z$dAcdSZOisduzg(jCv7g4^BM17~j8x*;DWZ9s^rG&b7r^(9v_%QUv$oxsq#4o>U2cJeV=SH&KFFAU1@g9+? z`G*b??ik5S++4TT5k$&2Ob}Bhjq@MHt-`wsTBI~eRBqqwCW*~0+?f+-%;2(v((TuJ zSPW7n*nE?^-y{9D?argz$Cu%XD=8t4i&1O!Qc!|+icp7p(vrZh@Km8Zv{RZMpMuiKhXK@XK zMAucKse{3KG*YJex#NL2wT4UWySE{;$>i_pAWigu1m7i&a=qltob$ECZ(+&q z9+=arx*N=>tFi`&r>qvBJp2szE4cQV4X4n_y`i2)Cy6=FiS~P^Y-g(0mz0G((-#`+ zsSh&fCQLP4U99OoWW9eE^6?Lm++u*K+=5&i;6=p;5Ge5lsmLad^PuqMJSa@n+c<^z zi($gT-QcwNqz=2cI2{|tvX}zCkaSt`-FOI_#5TNASrYkNJTiw13u4aF?NLBpYglpC zJn_~VhJOH=M+23`C-)U~0D896jbvto> z=Zcv|j^fA{oDT?++gGZ{o2n8aFZoiRF1~K;1B0~xyK+G0n++UlwH5%dVtd-iUA^j^hHFrALgH-%o1#9sL} zmxQQMb1+OHDf(px4D{ zz)my?&p_p&=vc>!shu)0GbrLK>E({GP5nuV%-3h2woI?VLAdk>5X7n{YQAPMo{LBZ zzV)c9t=H3DGBzktjWsr|HImKL&0}=S&Ui8Yk*H4M$-1vLltXt4eqMV`p`wIMflTD@ z(An+$@bq{zm=$N`GYNC}Gxm4ULD*2w^$dG(TRkxE#g}V0ip!H0x5CL^Vd3ld^JV^ z%|1)(wxz8x&|6@cH`DFIfYFRB)7m__Dn)#Gg)=O8rfwp0Plb!b(`in;rMkzvZRSwb zL2`WQqUkT-+FJ5_Ay#l*2nW#2IC%_&WZ}_{(hk)@W_to3Xli}Y-tlWqBdWO$)B_FVkCN!cdW ze6k~u&p@7CikMn4Vi<3Tkwx<%XkH6%ncj5}>T1FXcRGsksgtVDR|Mq41g_@#f?V#a zjig`KfQ|4DzSMV*Vbk2=e$SKFF0u-`ri9d|7znzoj>*>#AMLTQMXx(5V|3s{d1qT# zo;!Zzs{2t4!|k%?REF}69g4BeM7Tu8J&i2|EV7N05~~v%rYP#hnWEfP>1ytEL#p#v z+8xW`fQ!FZSmZe*`l8xT?)aJ;Qp^Zb5p_PyF_}AY;W)>hdV0iw9>>IE^DoFZ&kfzO z%E~|G&hNx@7c1SVKQh8+cr6>`*`rpjLgmlo-1p)2VYIQ4xS>LzUQ}#vjo}&d zVYB!#p8@i4dG`F|@=D6Z5s|6j8{V)%deyEPNrOQWlJVu&wXs2{(*F~oAFpG zn*hV4&Qd_OYn5Cfc*h;5XHZ&A+|0xHW z+=R0{a#vYxwwfUnqfBufi&ViPm%x=S0$|mc{gp9`Ohquo9Q{g%xIF7Xo_J>k$R*__ zK%y~P{tNMTLT1CXLo|C}knh9ivEEWGW7Snns_dA_umV5V98cHg37d|pq1PO@y5@+# znA&SDI8*B-LjI&Z)EYeSWTG_GDI30~K2tocQu4h*;@wqFXSnh{(UBA4k!SV&VS3!H zeoOl+AxSK}>31b=Ek<=;qf(dCXVC@uDGNG(3FCBn*@Wxt%4d-L?cF`CeuioaL<2H+ zL2T8MF&xl}fy4enR36wHD`>wuIxz^e6N>lvq=5cM4u%eZaf)gAmD0tXll zqyyMp&}uzCWn^em6Q z=;3K`3-q!wFIYX&DJmZqCNoqFHVP!V+KY?QyRpZR^QH?| zWD?gaL+ym^BazXZe0Fp3{OM2QdDacRk6y@99I%jOyf`I8k>Fdg2@T3zSkvd;8E>qx zGF$NuDKM+3w^#Rym>;H-E4rv~F-hdWlx8fRC7W9_is-K(eUWdYz0U390t{V zl2OvLXsRYH1uNPLALkaz_n|K?qCSLwg@5)%;%zv~pS91IvBG!MoxTJS_an=0Wh(lx z9=rSaDHsir8XEK?#>~FgCn~BAv{PjNpzsE?oV%d3IE`^|bU35!I%L{yN@mR-Z0ZfU z8(tza^=n-+Us54>%>s;A$qj>%tNH^_j5!z%(i_V7X}~a*VPZ0DnMo!VADw>Y>&;t7 zs}p~Z3C@H(xcHTE?@>m1=$rgWUta(2gdoQUPrVlyX9+;fG1C_|?708K{z|megCM93 z_s8IH>#L-pB*nbJ8!B8Z1k3VYWq8yA`F6ol;H*(qqQN5}V@?Ab%k)-F?n6@66@r3; ziw|mTBI1u?s}Pi?bcAy&sSP2-<(xjeMb>_tm&U}UR0nb9~D*92i&w|{4LD0&4Fo%+A~IC zLhuS_AxiXMr-4OcrHR`7e;ZUXWsnk-IC^T{DR}J)b$o@Xr>h&R6qC~U1Y6|jJYc+& zdpYw7M2 z+j%7GKT$kn((Bj?3{0}&n}Z0F>`l&X74*!uI+1JGIf*uZE4CLF(KPzb_x*P}M^F%K zlL$4)+BHa!&$N@hzMpeU8R!6QA+v+bn%>%Hx8 z#ELc0^Ja3Roaaz|6qj_O#ZGOJxPLJ=cTHIsGhLVUobFKTkLclrB*px=qDF&<_#gn? zd}26)ELtk~?Tw7w;hcF~kz%Te{`l^VyWWi_)OXaEI<|g#kM=_`A)c~lA);Ukd0_yF zmC18*%6UFv7cnw1)6_O@O!&z|n{AzB#i&l^+6poIQs?pq`l6n4Vx5`B*z?61ZbJUD zhO>ZCO8{4GkffA3S2GZMXr7VBDq3(jT;G6I{d$OdmZZwl)$kNW5*z@ZSK*TPAI4SQ z0)496cur-KU}08yJlX4OHhNC1O#0r@UEOihnp+mS5xtGH+EVP0i!hnvz#q@6ZWS*f z?d@*ej!noA-*R6Q`@JAHMI{xOk@?;Ij2e?z-eP!bhG`%(otc`wcmcc()PI^L=D$%w!Rh|MM^iP{= z-o*4=eQi1V&Qm5Dcqt;de)S>nQpCBJs&L)*?>l~$CB8YkfX^!W`h*vcGR_M;FV?A1hT}j`46rXuScf-7+U8q?}m4nhhXwBqST=pgPgs;YSXfu`z1y%RzS=!tK zQ=gu>%|n^$WPG6miq^`JK$1_KZhA~SQRa71`$-OI!d0>CcT)SE(nQ$SujQ>v!?(_Z zfvdt>bSS^bcFP=uj4xrLUOx*h!`W14fob0&!F0-O@XS&_>2FN18|De38#!9==n5?E zliNx;j-pZF{R&}RftEck=viq*GN=OIX_>j}wA2#v>57mPJ5>WQJo92L81@kI8TVy3 z4M814Tki9x^8JKHwjx~aypDPEJ!$n4c?;DE{Hq_p6@zH`U(dnB+amFs-A!p$gEYIG zrmfnn@(?S7m)3lO>~mCpxx`!G@|279Z1Ps6FqY(QoxRtE{1mO01V#mx5j$-`B}tMz z-qX(;h})y-V-UsSmj5H8P;20zcX8a7ALui~o;;~RilME@17wO|wUyk{!Hn7i_ zifD%7qNlHm-b#YbCuD8}*#aOwYfx%Mv18o#`1=$=Y(AcK`-3!^54@>}-mDka+KZaj0BxCs(-LYr_y{e1pD^q7999&-bVPr-4$nWf!#3cpQS z^$ipXrKX)rQxiSicXi!&<+uX@XXj!Z=M7eQ+PMBi8%v-WNRBc-O$4!2S$0)Q&55tr zl>c`^)ioZzTf2hKltP#F5~c!Y>SlgoVnf6n`F{Sww8bj!7JKo>w_UddTCn^;4)%w) zMR?Zp<%ij8Ot)P=aoUcH4vOM?rOGH41ootfGH>fyh+?6*K+H~{ua0<_dZOF+>B-y^ zSa^#)Jq@{qh5#w2lKn?OxjakR=SM2*&r)Aj>2Zbc%)34*=Z6D{0Oxz6eLuz(AZ2nU z^qh;@SKd0`yj}EU{jOic1oivhKlOWlMhRYS*z4o=Q_MJH?Nb9Sl=5E?Ap4O3n90^6 zgcI?03(>r2>X6Uai?Pc*%o1?Y;N_qPM1pX&W|FvJ+Wo@spm#!D!tQMG&evbYe>-f( z9m_1|On*B*=IzYU7+rU|eZMo3r%_ zu0wJwjN{L_$^}AYIGznQfkTJ7A<;rU+S*AiKBLn9L=rw*miEz5A7QEud6kl{PzEW(ip>3S=098oF!NKFg_J6u3^m+G$`ld*Y*5PVXW+)kzwoFrAmt>OFsU|-%CdYBd>Vo7fRu_L7 zF6s!6B&d3eF5UW(^EPUds$u!wHILW5L~=M(E|6!&c${AL7C5_dJ^-|Ts68Y-eSNy1 ztX6rX>7v!Lu#o}3hzd(nfK-4y$iu(;7|$D~#C1bw~!T$xp^_KsHho`trs`5;B zXSapl&}STf3U(Erd0(xp3pWTKa6KM;U0}fwW-#O+LmpSjW3EyX0wNhnuyN#EA35(u z_~qx{yIzqGp*x_5JrKYOF0MWN5O)`oFP<|cQc=<xm8|Tt0 zaff!CwDmpbn}ChDN#_Gp?s{E@wM~x6*KZlN7t%^>U*5vL9eDN1IrGUY7~w_he@;{W zeWLRJ`qek#?;Ane^BX}|;6&g?5b+twR>L@lV0vCr>N)vPhgCsdf+`F5CNRO{(bQZX z8A*_WVFVZeWa5bcu+|2Nak4#cwMk8w2_Kcg90MT$xHQp#+;l%!9U~i@g@0K`T~QXX z(ws0kU+)!n6Ds?$21(0(YrcOjCUNhOAVeJi-8X>l7TEpWu?uYKogaPxlvI&O#<4Q% z$(@={a~=VJWzmA|RHKn~(0;df=I$s!-^I8`tB>pkMnSUVtlQu$s}dXBR=}zAdwg10 zxxxyvevOt#iV9sqSi!t*x8#rY5?M()Q$p+p%)$sFvtfvRB~K*aHPJ(2ACK}1f2q1@ zWBIOOk#^%>6gsXZ8L>6gUuS!A)-A*bUZS`aY=V4N2l^M}xKo1-q{Xj#K15X6>rl6( zSmZfJ)_y6gh8c<9f0vyKC@2}@Q20;HbsW3M%JONb*52NdU&#{BY?LkA+g?dwu~{^< z`6Lp^4G`=zyQK^1+tnc^$ z4CqxD_ngAcs0d_!j-3wv*RB`Krra{RmG=_udOAp}#(qqKfUVySjho*R-||j8DEyey zz^J$pZXh^R5)}{G6ePLxlR7qISDgUh;1-mG*5}Bk9z0l2sEvjJK0zB$VltX~R7}W5 z)F|+meWTRY9G+{_Yu1o{DM`)?z)K)Pkyvb;0LSb!`*ha$5MaGrMhmr1gz${hAKhx* zZ|dF&x0J*E1u-;lZ>kUWXTUpoxgozQO>dnx9U<*%ZET|zzHdE%@DY0gqH`b*6niA( zFA1!1Y3D4~yK$g?Q5?X=5SM_nTs|Q|=QN>$4bP;ihQVLRoD=FM+%J&+M||#$=BBRA zBsU+EVyZJ-smIoncb=^A0#@fuic<_PmU2qpE6#2w&$Nr*hBWFsr2T5I)MPqHT;9S8 zW~0&$!~;$;JqCT+6}IHptJAFX+lxdr6j>VqBH|Nf<17pX8j-m!aUivD``9GJQM9LW z;tlqb-zUO!v*VPl{92SpA>xe5wz(eYltO_{>D79mL$HyXzeQ~}P=2Cb(|=&9q&sfb zHe_in` zyg=DCJypyL+$C&g4uE>@XJ%N(&}+Y2vq9TTa#nirFeL~(Fnxp48W=Fz?L6?{!EN5^ z#UkkZv-6wC&l}AEG${#eE21!1XIMIe#tvyei1T>=p ztmUTEUE+>L98fI?$n>j=RE@{eiZ4G&0O!2plK_rEs`!ta0LqcLX_ zXjA)G)>R1|`uFj&!#ouEROGxKq^ywjjlJZBWI`gaB;=SDVg-@uO%`4wj~0iXkeCl0 zAW9c{PB^->msaF@2VGLRx@tz-!9~wbP>mhhn4s$Fn!8qM$jvg+I-swyy(lRw5VZjC zY80qHSCPXxCs{{^A`Hp#Ogk_8DD+QGmz$Q z2D%oAJRdIe!k1t4SlRjhjfISJlWPMeSFuk`^W1``A|5EjiIf}hAVn8T(`+8Uv?a}x zlLLfitA#$s*!tiy?UCXUARK6S_<}^5*|4~6jx&sO`BN5trLpSI?wB3tjX{|!m8AJh zmBk&KWM+c$nN%?mzemeYshdXzrtK$clM+c$OWBnP`C2_e&R1v?0+2$$2L7l03$bh$ zWS04KZu3M;NJ~oh&hZpV{Y?MR?!8q1KWrzHsxIR57yH}l)wZ~4=7(j&1s)801=o!q zIwX{UFOllVE7y@p>3y_^`vWdd?X!_% zHqib18=0eeuCfpFKJPC+2<^8inZw- z;#nZvx6?m1^1f5Q1SX+b2y_4D*TOAaj@3b0Mf6WF&O= z5y9r)DgeiN2qj+w{9_CN&a@fcPJXLsk%ap{LNO?#q z!##=yYdNw?#b>pyAU*U^`ob#9%%4x0PEF7+nVU%F2f`m81(MpPzb85xYV}(?5d^W5 zZlHdnyzeZ=dl@_<;yi69KA%<>+&VhC65?86U+_astXr!^1HO9M z>xMSkZ$-te6LB#)mtn0Cr-6GwX#2s;^aK!+_VICHEXXH+nq-&Cqn`OR9uA>4S?X(< z4O2tK)SM?1!GDs;cN67;uN&?+a-SbZ7iK+M3`rk(9CwL~^rF7T#lrr7AHMu=y^?ii zR8kC4YBwdJ4R`O$Qdv*AZ+B5Fd;^=1gWUta=9Bw`8>4mCnzLVUawaj?3f#hW1x-}F?rTEilE$<(R}!z=Ap1I&4-2L-^Rq=iCrNAQ>mPZdkv?b>TYBl zg@S5!?lltJF)(lHiDaR#mqQX;;Mr1TmN6ri(kgTc?^uctl*@58z6U2#1Wv3#KzZ#$ z1Xg&mQ5x6Q%cf}tOB9JQE1G*1Ky=k)5)gQD#(O5r#8 zr~=;oNSJ-Tk17*hiJh>XTM1VLW9xJt%hUX9K;B>~rv^1_?QP>ZX`i+Fks z%urm}5WNWrKiYOq^z5UzMW!1Fj~ zY@`R-9wswIAWt#!zv=m)Rk#iDAo^++ZK5KL%UH~x345+mXp5 zT`f;_Tk_<|h7BKLd~ir?MI@FCHmZ0>^rc8y=e=I*kI(Fk;p7lS+u>ty33FmH1^9GX z%E_}6r+F$WjfRV_5k=8KwmJo{sH><#=yPNcr!Y1whB~}Pa^%p?<1$~rw&2^Qj?e0s zpf`Cl`X*}XW^wlTEdCa~ZcA%Y$9WTj+B-D#-8AxY8?6uzpgFVS#DDt9p98R?^&aYt zOD|>fb@_L}drr5#R_bMTdnx@!u}<^1zM`SQ$INfa>{plSF$8Dw`&m}gUM^lwZJ{@~ zSLy-yc@GdcPh%gR1-e7=99-a@t98 zrsKvFH38Z#xNM}TPg>^Usqi%Nxzn48z8GJLdHnvB+^6;Z!Ri%zTq0(?Z-?KFH^nUl z2M^Qts&tSwm*ZA89i&Tq`G3OrJ&ekYlVqOp@oYxTc~iu@tiuaDp#&+9R`~J7e|Zt& zp)Ul(55^=rVc6BSgTqsf=P1*7Un8YZU?k>zx9x!81mfcg|irK6vDV;68Fhkp#Tc%og8~m zi%>PyxzqfxItgn58@aB=nV>vjtX3zol)0}hM>@sDwQ7G0VpNN|0^q&VK<67Sio*{Z8{-CY?ObYNbTCpw z*|;mux30NdXSiv#6vp96vSD!qFi@})8@MpxaYe{i_Xs61x?H0~ee=D*Nja`Wze}qfBjW*ct9>;PbtN$E zxJgOxP4Z=Tz~b)QS8>uyd>Fi{BpeLD{rK$O&t)g1`YNKhVL8t7eeqa-+83<#4F%#2 z-u_t(RWT8~Da>wT$y5<@If!J1-b;W%BORj5fXYW&b}rjPuTGW~TE=M0YIbvc|9zb_ z5TF25m2k1D7NA7d6B;o@x6_GuMnN1Cv*^odifS zNoFCOeIWIcHO!Vv?{#C_TWGt5t5lNmSWD?>b;@4e3GR0Uyzre$8=a~`jql&7g0n|I z1QsL$L}bll?B$JCPF8#S4mZICDK(KV*8>-#t=plVG?^=-yd0PIJ|V_Y=$DlbT9v+r z)QGL4B&rQ)^Asz{1V~aZLr%o!_eTD3KC7_xxSW$&caTuRvfuC^+uLET>2# zsMk_%cRgcbiZi{WIazOYp;y5c0L0iUTaA}0@Yd+@xy1PtCC;ru&9NWCAI{$*+D03rC&MWpdp=%%twoI=+Gv zOtHa#{4tqdc+HPV!?T$qzgzkmjN=l~C$b87fhE^?B|6<%dJ=3+ojT8C09dCK+So_u zc?B_Ca}RCI9b{La^ZwSmp9!x@vyw&R*fSwM=^t1iM-S0_LuQ_bE&6cbt`+^C7t+Gz z=x(xSJh^ah4z^s0tq7E5WJ}TCv5SffU)V~lg2y2!wU>xvNLI5H;ERD zCed0Z9b_Do1j$}U@L$QEWU$Ul2IsLQj`3(oR~p%`MK_v2+t!X*1fXvt!@nQ)qDodr1~u(7hX9lxc2z{K}Z3008(GW1uoWD zdQZE8!jWat2z#%mv{xf2h%d=K0ZA-^hz}wOw?t&mUb>#LFpqq*@-uNMN$2`_+M?C( zmX=?9>EG&9!$9c%vd06nw4{@Q46*_EjF%ic7GZ~%z4sU7jcb#)av(0Y>@Uc(@mBj4 z2jD*;y^=0qve#B9EfS3c0G8!a4MlBp)P0D z1I7c{cBgdx>=btku=W1|IzZfv7RA&y6R+V#Qf{o@htSCLyVf2leO*3 zZPH{edyU;u#q;zIXkSlJVvbymTvP)&iAm3&#@`du^q^G>RnB&Fgy$Wgl7DO?Ca%mY z7cV%G;r)l24<)pbnX#glAC9c%S@d92x(v|)bGSYwsW~o8Pdx(|)UbiMTHpZadLceJ zo0igp1i{)EWAi}{tEb_{;pBCXbOKi;f4i*1b}SAo>`$PcV8AoK5^yp0`;>;v>7OAB z-MCjpr99$EpakIVT!?px>)ihi5#Dn}?+?MTrfuS&xQ%|XMWVoT&EVN%cHCtMNfB|q z<_6Hx$S7@(k>TR9H)#($>&!KJ;h}Hw19rP{Y6WPGVle9}Ep zmH**QCD}b`Db;a7E9v`uw6J?x{l2uC>6tjGV}|sK;-AH{h5j^(yePk9v`Cw35Y|jRKHKh%jM;a1 zXkluu+^ky_vy~m=mYCW$9|Xzyb+ymY|2+{-=A3bu?Ziq)1aXPiXz8`Fo2LFf1Mb^f z@8*u90$kQF_$un!?<*{f20Hn36QG~&4LQ=WjR~0kWP(=<3l)vQUv4WNFKi%$GC*JO z&2w-w_OxE(Oqys7S~NT)%_&uBXV%PSd|TaU%fM33icCfGapz9p8sed0r2p^PE9Dts zw{=UE>!t7f1u=-{cKC9w#)Q?C77k=Rpufh9mYB?GbpU*#(QW6GuCfs98+3J*{!qv0q6K#pq?p z;%WvVwjl+0xo?gN*JotuY(%-!Rf71?LA2we6g-TNTBwjBkcve1MB^6eqErU ziC6m8m^a>SCo-gcTSKt<>U2p#%sBg=MfpF9PQH~VCIk(vQ<&o8ubxiL$N3YF^NLN! z`OB^1ySDhk%PJ35bRq40D~i_ZY6$kTiDTXDI=O*^W?KQ=(-bQc(S8=VyiH;k{n(bX zy7ykx+#E?4bl(14gz57vz;HHHoHNV&v~tn=>M?J2=6H@=nhP@c3w5YNI=Y%&S$Wk<01MmjJ2uPG_kZ=`%|cD zi{6c;4>%4e9XL-8{d-t)k0p}=#kBtL`3&QTTjXnvl%_VH4tnR(*oT|F5Gi`)`EjJIyyG2Jk5Q@ zpy&^8qPe(PhqXk@uJAP_>h9pN;2~~WoRTlciJ%>8;$fST_`|Z&BvVl_`NirqjZSh; zzPp_VA`L{tyWYeaGrXDq5SRXPiZD3)q3i?+(QMi`2FnbrchR3~byaSs7M zz8c_3_oMUyv{^RU z>r%dIbt80xYB{E3Fw*3!N2F05k6owcVVM-j$XRA(?u>BdYo#Sv{^ex5oLL>43$;<- zq%Sc<1!Ne=bqb6YHC(Nu!*^nNnp>(Iey0o7)koVt_OTx~(ahF&Rul819(H3)z?Nza zzs`s_6N=Rm^NM2L<^@n)|oZl90y5SNcnSfC_p8}aApcl)^iLo2Ka*c=Vco(F_X!Ml~tdfpyjd>ji+OuKnNHzzRUZulj2&# zmiS29{7yUWp~9EEp~9)Y(LAL|W;pJfK>JS)9*LwI_|qO;9`tC1@>L60E~=)E*}k&w zI^HjHVREQfi*iRjoY5Ki{(o)%bg(kxrL)l1%Mr zndLf#RM>$gM(YqE5DG}CtTU>(z~((g!_@U^$|0-Yw(}HKA)(3 zD^_mGp_9U5RqJ|v>c*4qJ~Eyf-iJlCg9lfl74$0NM|Q`5yQ&FoJ+&OBwMaaXT)Q2U zg$+}yDlLnDE`7x6^O3=~_<9?p_>d29v;Y4{&rxrc<$aB@n+W>yT3H<|<4m&6w<($Y zo8y{RW;afHg1b3ajtwUdjuuW-!7ok3y0+LpD2tzVa-3NPl(i!FbvfWJ!+xq9_aN7! zu^DKC=&edapmYjRlGEY&*ox4$5E zmu%9}#0H4z&8E;zs=bWNL)5!{2k{3Z<=W-~u(G7x<84*;n|CF9+X;&a4J_B|GEXa{ z)6`9Yyc5>AlRxB3A7#tBWU;~rTZ?Z@z7@ub#P3W_iXgzZ^xy6}ji zHKwWi(I~(1j&kwS{nmtBu#GNUiG1 zH=-<21IcCy6S3z*9*YnI(FyV?4SZVe(*YpMHy&!c|+rLPvqXAA$y zXE``Hw*IZJ<-Cpo7tPG4qu1SUGz*_D8yJ$#PvVO-$O?Rw5KsMEO$@SZ=P6zFs!HS8 zbndK6*$YQq3KG63ym_h3+h09EIqIB^9}TEkdC@sBFa2-4@a9ZFnFDyGrDj%4ml`MG zOR??Bvb@t^%?=#jh!typ5QO!${%&?eN48ri#2%0o=9;g|-^&8L3G2wIty7Apc!?f&7ts`wL);@bB@ zoi0x7`3%li1O>R#%<}(&P^*d-2IiRRe!3ny!WZY7mQO=0cUW*4vV6wBk?hZkp{hk0 zk@x%w`P9RlQl0F~{`Bdyy2}aZzLt+RUhrtQkY`rTmG>byk|I>qnfO{|Reduxyxs!f zH!Ji0a`;dB7_i&-Xkk^oS6`tS&+Ilu(?JPvw1N_%SUGQnV|20okOI2NICO+(O z@}nR&7dM7czM9<0M6FP0b8sr>slv*lUaOzf34@)6yNsr#0;~ZZ|rN!3& zQza^FAn;cK(*eTdY-GY9>$@-SyER0OHL+W*vVP$mePqpW z9WF{NCQAoeTN2>!m=ORA+UGMx&NbHKr)6N0}S|z-f;EglJr973&iVJH9+m#7>vYL z;oyPwOb-46W9wumH`h`^)4NwxxPIS3rR5g{W>meBn5b9YSci@|ok0nffUdd#np#G5 z5dC)(h0R%5gi`z7Pvz=!HsO!O}$O3w^}OyOtno!tW>_iUh39>AWQ>EwWur&q;**i{N44kTruO?{Ex zw$TLYJbVheS4X}q#J&CvJ-5ntgpjkWZ`NRq_a3hFgV}FUUq<@LFiU!E-^X1mK@Ha+ z3#jYg&w7x%cl`yBqhR=PtaY$=4d4=A64*~O9cuj0!>&GH|gJ~ePR zI8vWYgmI~DWq6|kpfeh!8%T0;B9kgz3&{W#(lhW~3_Y^wO;V?C)heWKOo)p zA4#5^?8ZF0AnCqL2#y&)5+3v?_jRenC*)7BR(i`1mXm*$YNA8P@I}$E36PM|$+dgq zO=^d5a$w0{E_)gT9#B6zlUf%aesaHczMgsA39&CDg6K3K%$I|HaHnfh1tUIzA{sNZS#DuK|w1jYC#c@h;LvKXIjZo$W3`rzhuo6 zl_8EO#*m{e$YYvJ>S}*^U+W6|(__E)x3yoSM6Z7&br5(TepRmXC3`Ba)k~Hj^X=ac zA8d$e0Y0i?g%D!_?yiblk*m!J*?zLMu;v{mBm86&D>A%nbur?~AE)V#n;l)P#_)mn zd*&Qo;+rVILJd{@3sRVjIM(C${_~%Po3V{xf?Y~(Mv>{D*{Y4jraHJQD%D~opONMY z;0-Usl&5>#AJ5fSco5z9OKKkOe*Fx*3kzb0bid4U^+@r`Wlw|QseXNuxc80 zX|I=u$|-N(GyUXuGDKZ%E&Rmxny$JW?;0Y#{J>*V=IlpX#*Q&^3rL!#{ObglS_%IL zD>MXlPTa#^i+xV2fBF6}b)IDW@Fcz`IDo9&(+Rjh*0~GR|KkE+VzO0#4s{DfZr~87 z5W^0@CC`jmLAND<>>Xwj7WK6(sa5)YVmUcL2q#(VYys1znGCvZ7&YZTQpw|Wp+3m! zxf+*fcKx-8m8gI)GDhYG=+P}?c*a5R;a?EVkvasB)2X}S>l=*b3eU!D;X+Om-LWj9 zQ4d*?bTX4D+@o{YW7|rU1^fpe;NfkuGn|8jTQU+)X72Cxys|u9J!w>V_)f_lj69## zABmdSp85b5qSaC})PHe$A2WjCa`cH2-$X&aN#z@Z!Nw_(p+bjBtz=eUtxDWIG`dLwN2no-fGmThUbK>OY`1oTMF*@Qrb_vYWxEKc8qC->~2-$Ab$8@3q|Mz5;;8-h#s|Nll z;{kAZAdDX~`T~l$Oejg6*aQJyKFqM@@a+ZDh5JR96k7zxCmM^ORi3ThX}*akR6q^` z(OQ471HO^ZFoa1zkp&kPwJ^*@MYzWtmS4vEeI{R9v@yy0vN~HLWJ@7NcuII)d}85D zTj}uB3mEt+1JR7?iOKZG1$w*?W0>>aWkh_V&m04b%odbJS$Oq~5pQp~;92mjF6^;U zU8AFVGp}aIsLL5`8J&OP(aL{_xE=l`;)V%=SxIOqu-?x+v%q`6=WE-TQoih~Q;PpI zzY&QqI~>~HT%t9bOq%(wtCjeYA_&O*^&TH@;J5+LIV#fy{K=ZK9a6~ z%XM(z1_BN@>Z2gZc194C!{Ll~Bl$Mta^z{EZyZbJ?y_X|TV))Kg!HMQJeC>vF|66* z=J0E9xeJ}pTN{%3|A>39u%^0p3p9ulkfxy02?~f(rAiN|G?5|-L0VLrH0ezsp-2(w z2z(+{K%_>xAP{<$B2{`7keVQ1fFz#D_wT>$i*tAOxlkW@p0!wOt~uX2-Z3i2nWz79 zmtf+&HtBQPQVHkxw7%$XX>73iE)#zssfcucW3Z|6mf*(>7T`;R9#VF_!m$zmY>ll$ zv5$^p@zprK@pI;HEpGVGu{<7-1*L0?6nyUzJ*M^I@K(-HfBwfg$}4R{zR$`+g4;j& z4mcv3jbQj==RXj0mcz&+$-jjIidvETO9JGmEZWR}c)3e|d$}u(u>{Y>4Hzdu8PqKN z4Q|s9#d?XGIyHW;n%IkebO8F@+*K%*`EZ*NJ9zo;sBcUKKb9F-cqBm&wB5| zYDlT#ZEwfl_c(7mpW^gQ_{6WO6n;8!>u}N`jKh)xq{WEz&Nz%J0X;89N1)Y7ZEa1yIwKIU+s*5o zaC=JlMbLKyAa6g(dR>s`P|Q9;ohi6=`Gd^M^`VMb<=NZJ@&|MpW?rQxa|h1}Pw4PE z2OK{YYv{|hs|CNyHFLH5NnfBC@F{A};Kb{N#fI$%$jC_z7M5wSet$A~Baiwxmw`&b z)dCnXx^E}XBDV_Lj-?hPXhP1q%*I0I0G3W z39?~)vtU^P2eJlv}cBkLr42boQgqkLA2ESX_%6PjFAtFkY z^n8BHi6hzbIcuY@u&(L=Pn2sEV_*IF6uj)Mz=GO>Qq`+KAY6Z6pXwdCFOzoJ%hD3{ z<&~^y2P8n~6TuhvrFj;Tyn<%OpfS@*~C= zaDsO(eRzv&{H0m;Zc*o}m#gp>r~F$jPhAs=K3x);#s_uY8-I;{LhixtNm!7*;l$?! zs3}{rkQgH!AUTx7?mfZ0Tcg}NZb^Q@9~&;Ufu_%^TiZDG?zW?$BH`X*Y}}q_pVDH=VbW@9^?|9)O1Q zd5<2t76Nx_@QeHfOMVF~$L*HPi;Xa!3WN@a3SO8c^<_kV9Bb~Yv}@f{4Jv$eX$&qj zYxG|Ghb6eIrNxvG?4}&Z$#RCUcQx)pgkbHB9N+3qv!>TxY^BaTGGlm%>zaA1XTufk z2u>bwjC7-5r~B{2hz#K5-NrEn5#{D}`1i03#;<*H-TBJpX81~`+&yKXl!GYgVfKDJ z3-R@2KIw%v$U@j5OVaRYH0@rW`jTEf-l@)PO4BmQfF@VJHTb6BIW(mSijnYk9?81S zWH-x+r(^jY-?_qi$C=~ewu&bkQ4;y?t2XHS>x{jKuL-S6k9Ttz%x*q5``w8WifcSax>soL+ho% z)Z|K1zS^a~3}2RYQ65gPT3&|x@y7Te`sKqykA+LK-T3fy-##bRh}5Uhv!`WTC_IN4 z3|3bdA4Idr5fDp^$ERk|Yblt%LdY>@-Y6DlGxcyp6vbsxnq9zm_p05**D`gjC)!c5~gl&Pfs~#{(>Rz4X$Ij1c4bM{Ehu zIl3~LZeFE>Ps^0k2!tBMr-mqr00HOWuMsRxx@gp2Jj>34?VCuY`K3M)VW$-PD=~US z%+nbwRe*H|g#V1fcw|#-xW(0RQRyO*jL-8@-JhslxmRpos=syUy>JdwK1Gznt<*Rb zPuNxTxjRn1XL<1UzG;P%Qm6gC$u)@n1qjtiotBce)5wAoi~EoVeSI4Cy-~{{C2irp z2Hv)etE}H)1-U6U@qiqrjXhomUNCw*pFA!HkmWZ$V-HCniWw|gkcW`y370<*rOR>q zU$OM7k=x%^gj`*;*#D4tRn&l&HAf7&625>3!QXF+6mE1?naG1S{GmB!C;`GA7w1)r|mtpgfN=Li6b?AQkuyecz7swfqi6e&3dR>u$}XwS24@>y&zM8c+gg2Va=OghOCdMD z;n9~Pghpf&_>RN~6fd~ie4HF<5BY&1-y{l`)o1iCwT~@6U|`cP;GT~+ zwV)%!&u6k*=vgn6plFZZ`Lre6I=|%dwqu@4m50+RYKd1#4dt{P<0FSLAf;fJ>wIiC zfN9AsAB29(WZy#uB@!}(Vqx~Q`5O{VG?)5#e3@nG7Nr#rPS3|F6-D7zLXX878}jp# zzwozzOFw;MrhbFD(TBd_)RJ;8B9=S^i~$R#ofn&V3BB{xbHxV~kFZ=mEUn5nzwv&n zA}QHahlZ$pDaevnf0Bu(ktbl?@q^UgZ?y^^;CDX{xMYPjGqoa z#AS{Vb;(De=_bof3+~|=(dQ7OjK$kRq2%?%!53r5s79#265gvprncZ`MZ2F1G;J*L zuVi)dA!T%D?M|%8-sCmP?n>CiuZ0hoRA*p>>kIC5 zDv2#bue`Q}x4n)i&eE!$GQ0jnX49&0^GpPQ2@W+#jd#JSe-ooo>_EDP4Zdd$(IJKZ z?elv5{y+jirU~4hb8ASeJS@bC@0)dH6>h5v^X|gh{GBV5{JjBke8($T35RuB2XWv`ciBeP5W<9l&bl)AcM zAP;vZtE_SuR#Iqf?t3CxqXbQEwgj2piCaIhkvEs%oCLjQ*D3g`{f{0yRZN$SjJ*6k zFF#}sitiWi+AlSM$AE6mOp935<5GP)Kp}uo%i*=F1+kUI0J~cTw?-bZiNSY@GRZ*I5k|fT zk`wIyFX(P+qa|SA*9)$5ZtZ0}s{rKgIMU_qzU#OF_+|*fN|=(4vWdq50tN{5>-Bxtf83};{;8s!%J zUq6QZIH?g4$3$UM6HYn3 zXf}@Vk?a6Gap^&gxLFK6QMbMO$~GSgrE;vhuiQ~XNvm<{F<~1&*6Byh`4ykNJ*+G*dze9rT_DS8%TBF^gx+I? zjSLZdJHY-c6n|Gw4deS9u5WvrGmv8Uf_hIJ)a?8Y^4@q0w$eMg9;DxwJWFJh+;+zX zbUlM-IqXvT4JV7;E2GjfJ_O2h2^~DZzqU7OH5}hFF7|I;Z75Et&sNx?XFW~Q@zd+f zdsF}gcqLASH(0zMu|zM~sVqM1N$YovWVT0n z)(7ADC7M1Gzn(iEYl+k*k6B3I8Qv~1Qk&S-y{Q%|=1VlMs{OugX?!=ZHBR!fEO&6A ztRBG{83Eg4$E}i2>wz+f!%t*z2B@Bdy3RlIpmA=T@fPm$^c0Lz=eO23l zKq(6rt;KXP^N6hqO=L&}BYc>GMi#hk!?h>6n!bxBhb^Sd`?)iisy1$NHY z?*^QK1YFiezBTY^ZBI~8@Og&-DvG3PT|Lsi#n$mm^kw=t|{~0IX}xxR&W>%feR-AN=uOL0x&B+bp!2~d5I9NJ-Vkvdv8=! z*fFpArh!PsUlVh__mer}V@7{D6ZuisLES0E2*7}T!ohqf=^Vjwfwi7?K0vYsZ{D!_ zee`vI4~yTIl&8=qwJ}gd%S3Y>-|@{u%sOcMYiF-dj&|2Ix(e%wvx+hqspu87L}l!+ zRf6)X#Whm0|9w$xfJ_L+&cH6A$AU6|@6`=x+708LWhQmiRko&n=y;_*v*$ax>1$b{>RQY5k15h(i=cOfmOw9dr$d>)9KZh| zRl2g~9s6YE6)gw*yuF!-a&^4)%;Z_m5_Tln1{` z`9#U)Q}2WQAPkTL%pFpiqQhbyp(@9&e50mDCiQjY>kohl;MLKZ$OeUuIB)1dIdM zdNN)`<^4+PFOW_!+P~oXnxICN;~-KTd+DZNT@s&d)cklC0#=?Gle}nHJzhz}!*Iu$ zuL5XK1wXu_BD<9s`v7SRh9ne-X9%>#KFsZXi_YiP4<7V7Gix|dW%T#7H`AHbiQbTh z$O&Lr;dkyMPZRy{ov#H5MT@kC`G&W~ajJXLM-Ri@Le>F^?f<;MKWD_CY8gq6m}V52`I1cA{wECYJ(5LCMRK!md;vPf^*il*78C9(f3 z5zS3jK+7rb=A_9mTE^_-28jHPApg&P2g(fp*ydpJ!k3`oi|ZeMSUXyEO-x$p&U<+i zt5#0CbimL4nyXWhVtB)%9VtQXXcWRzl{a16+aat z^zG?-avryoUr<*BYVk9^O@I}|LKg3YZP#N|C1C?x&+JcorJOhGAIj<2q+osO;uo@I z?!0BWB*xr~zp>`^qHiWCG^tYWf>C+pBU^prG6PA%AfR!*PT0zielESk*h%x4&*#V6 z34~(Y2OgSE$Jmb;jG|bM>cS3#T`GNXxi(e016QS+cJ$Ye*{ZTjiinj>WpTU*bEgza zV029LyF{YjJ!vS35z+M<0k$-p{nSoi$h9svWcCh(?a#z_xg19ZRz<5TI&vd9zajt zr=^8VyMbm}&+uTPVGfw8Tjt!QY3A+-gx~czHr{j;DSmJBZ0o)8n;L@)5Xqf(y@yh4 zKk5LjS;2Ry3R4Z%r;90;zQ&2R!24?{8XID;*aon^@VBu9TW?&Hd)sJK) zls~dS;AM)Oltl57%*YYHoqP7)_bE;J5zk zspfBKoNC+bvv-%#we<{=>Gf9J4htB9wxS{E!i!?SMMJ>f818$f#QN*PtzHq0Juk^m zYaLjB;c=ci$P4#Ao}yVDPlfz?a4bkv9UzAJ|LYeWASYJm`?GOl8wR7EB?gl_`t?z! zxkhSHTCA5}3HRid@qU0nya9(Vy2X4E`yU=W2W=S(oZG{3=?2CJqtC^P6GqH=YU8@P zukKs0?ovSWtAM#zL%I4)b!Lc<`ade1H6y&%kDmN7!pFkM1P&*~!$8?nRiNav(LM%a zBFZu&8#5HX5JavV}wUlYujM z>!|n1*4EdG-IIBOZ}|L-@lS$V>qVRyQLb7THQQBdTMpXWE!vU?86MFR7fg?w?@MXz z6*0P#a_?cn(7H8;RT5F^C z*Nii%|N4CT{pF^IOFZ29V~sBjA^Ol2J>q4uFtFB^C)Sz`@LIta81c3wT6QcM0y-6K z<3Ag~(0n{7oq@UTHRDmx4y4i}UyUW)KH&u*HF+OVVf} z1&I7!<&e*TSQkB5_RnWRz5AvDdke#b6Dr7|(V#p4u>!%?Gcf-FZ_nHl29XzmQUJj5 zkstQJvuXVAXX_iOoYBxue>}1&>S$vL76KGZLBiO*TL^?%ehc!D9|d}q zpZHVA5qH6nnN;l0v8eQ*h5Ou>VW|n{6m_l3_1aHX zD=G`QfVZpq6#f*1xlJ9ytK+$Y}x37TR}G-yKV(YuB4AVuBG3YG{dn#z@shhruV zuTAZA$FFs>nm3V#yto-ZVsaf=Ix=`_aF|eOz4PX({B837^pa|96q#HQfqXzN{VSfx zwK!s9glohk@xk`ky}r^oP;Z_B7AElz15P4W;?h#JV`^wf($@zmoEu(=ac7Q$33vkqshqYC5L z9-05Ox^ntlfDc!b@(aSiydH~*zdl|#dg1tRv0=ovKG^gb`$M0+_;g($^TVy7ulB?T z-o!02>aX{~c#7bDUyJYa?K(f(R~IM~v~NHl6qs+jc9JSEVe$kw6P}@{=-G?yw|UdZ zN9~SIRx)c=9sxhX9=ZoB93%SXa1lNja%lR=o&q)QMTfoFNXRej#BbG6MQL2(?IF*1 z&J`9}$r>-??n0B&RXhD}bZuF@*3t%TG(?y3HBtfb0ZW8f_Mnu=6(x!5cU8oelC>f@ zts^yy5>#a=qm~5Ncwk6G7p!U=tefA`l+KG*&AP@<89lNWx1QSFtxM)RmlfZ7c9dXw ztnCwPhVMwsCd8@@k=qNV`QLZvYw)>g2uEFH-H8t}OFQ2Tw$9^yY$dFd3$d^h;Yd`< zvNKA#=>)0$oJx9>GuakuNZsSUo1g{eyLaV%ZntBubTqP#_KKf{e$VC1B+#Qmi2Orh zdyHYT+~4+PU*4=vewL`tpnFxAJU z*OVavQb|CM)+(Q$>3msZcZ+Kze)zUwBMWXN-dgB;N7#0NntAJ&_jM61^7Ch&B)mva zr;?aHld|<_>CKlQjEBf62gCS#-&!wx`|`o}Y=_ZV*TE#&Jt>L<&x^}wR-jnX0))~v z@Z9kqNFR30@A&b3fqi-Ph%%AKs9Qnz2BQVheF26WMV`a1F)m<+knpI;XZXP|`>YfF)tiG;Zsn{?Ow8Q6>JRrUo3~$#hfIT2RB(7}GJYFJ)MvRW zmCTl&_t_(S5r7m7g7zt$o|4=dgSdp6}j0K}ILWCuJn+BkH5DzItdpOVn3z<^G&4YrPFEFk9FLpL}@D&Dx_@Z@rhK9V! zDf>$YE8A?vhRIhWByxm=B+oUcYx2wozpz|^{I&;LyJv-3!R&stNlI&gp3dMNm+08W z{n|kDUL~B(tEa+U7Y*68p7v<*1wuGEiH*RS(i`Gw$$?}JV8856ejy})^7t;n&j>Hp z5^$?ky~?5Jo-Z7r)YL0R&z8|(Q^87(aP<%+xP@O#F@ZEt7EA67?J|YdmwZxZ~_;&B_yX&S9@G5<*~^M+Esl za>YMr{S8%ZtPXD3R;XR6t?`MsR6o7jWOYE<;EAn@4e{4;#S3pFn@G=Wg>F^aaVD2Z zeYP^Y@i;3ERdwLxzmNa5)KM}ocCm8-!}wDM!Q6Vb*Ig#7blp4Zqe@KfV^4u`)+H&5 zU4KAt@A0eiv<2(v*Mu8tR6Qy*0+b0ig!Z%-I#PV!`+%rlIV(s^*yFp%ilH3ED1=w( z@Z#}Gl@VV?@ES5xZQnUyh20`TdZ)6CZWvXNL^qA+C z<-JSebxakrB3Y#A!i#CFt>Qcjn$1dnznY+!`$)D~@@R&vL-%Cot-;Hc65?TUi>Vq9 z2Ppo!n;jz`2OUKC;v^4(#)wW!e<_MADqU&0ijD5eJ#v$G^icO=;F{toOgldq)ZDKE zeFIVgYP>$7s@Ohm&AZ0SB$2QBmr2V|!6|72hCvrMMymQLJ)A()&Q4Z&4X&|hRd<0o zAm{z^QNj&qEd~rOp$6;bLWr)O3mAsA1E!Kd!_V*TYq)Lqa4b`AghFC!`Bp#3EmjUp zR&;2ipL>cxZzs3O{z{m(7Dty70rz^)61ouS|m`;?opRXmKezxp%n~Nke zBY~jMUUnC+wGeG4&s413dU+Y$Z}`P^H3$?$vE*)UMOozc|zN9p!l33NC3fciaj|_kNz2`H5OOmV6RRT zI7Jl&NEzxiR|4{!Xz8a=BNa}6Ab!sjU*Jzb4fF^+KsC*Bf`j|FrSK!wwD;N)U;=Hw zhpb7{#t!^0k~dSYG&yfO^JaYB0=#JptqZV=At4KYd)$*K&H;2HmRc;X3`@^An~m%h zz2>+sqE;yV)kenmW)bIgHXe%c*;;3UJvu_J>s$T5w%~#D#N3i-4qkBe!BAD7#N_z> ziYPO_YSmzGzL1yXTzTMsCS%Z@;m3- zwEusPDkm15v~suSCSYXEz7!J5487*LxxWM}hZE!9i5(FE@@Q~l0>sAk8%x;ORf6}t z(dcHD);{gK(5gzNQ}@k7WL7#K7j|;>pYad9*@=nV;g!WZ79{A7R)tW0x4uTXcQ%rE z1S-hC@RsCo8Z3^9k$h8wD3KaTq zt}YIc7$nj2e4sfN2v>cE8Nd*b@Jr}5SI~f>01FM!mOGo@@Y)81@b~MNUaYHonqN=! z$BTZGa!xPSu<7|E3&>QTm2v>Ww*=UwKalV8$RFl?E_0mN!yc+{fYS;i{24N{D z&0n}lCYb_;&tkOCD`dZf`LHpmR$7*8-sx@W6||Tt8P|Gr&WE?GXb*z;QCjc^a#>;E zXd7Tl&tpak=RE;{c8I>M!K1v_IvXYM5;NyISt)`M~aN*Y4%rXOs(ip|IVD z{s4{rq*si@o%9o44JRgbVK*;A$>IC~4;Gw5;f(b{MIXwt2==oB`q6Am?tBRs=$oe+ zBA}dB?!g+j9By}@PUBLdjVAo{-nV`vy-B2%l#sCfIGxMs=KsD+y{a9AMz?i|u1MU8+qg8$pn^IF!XS3BMJ1(BrFmMfo z?t<7_s2#qLM`rm08R*Xk`&@(QFi3b&f94Qmc@-i2aa~La9rRfZLgF@? z9PQ=kpyqBK?zeoF-ilO-i;NSlo~AP4t>w3Q($8wSU)W;eYCY2`cSgfSm38yksmv`g zupFifu_V9S0AkRMlo($dXirBp8Ih@d>pc(r8W@(dHEX@zmY@CWAT4oD@I`<_6S@Wq z`oot>|3G$LXOiVqn24wUKoV^TDPX(TZ6fDEbp}^;ba|$G z7iG$?o17uj$;yWq$`R?EiDz@uZxxk&K zz!--A1MongXYuXZc;1NjjoY5iB2{0-2TpexXy)qotgD}m^WljRSAiw%??E>W!2;`n zzdmu!9VwqZVkJ@&?x9z7fw?-B4?9-!MA6~&LWwL+uFeb5+*~pW`HV6hwszF4Y&Lvx za!vhIdIf-iF~kC~m>COasFyA^dscH4-8OF2>G)xYQc%d-ylJ27r=mE1V__g-|BDlQ zufgZwm7mpj#`Rg_YOJUA9!v%*(*Dhjm=3rRncyUtl0PZ+`1evH2KT-nP?-{v94;P9 zMf2yLFOpgl4}0<{jM{}_^>k44RA(&O8PzklCp-hH0W5H$+ljoc4L~DtEufE9;p8{k z7MW&1N}wfGBlta@(}geI%$buZ=FkI>;~_lH^AeLIO#*b}tT5$Mn0sDYV= z_!(a8@mD4UEudk>I=&0NOR_-++H|kWHAyrNz8^@B0?Jcr^4Q$&qa6SS>7N6gaLkvY zjz;+n7*$CXz4Fwebb>v5uzJUZAhTX}*v^H=DoR0saZmX?CZ8;i?k+uvU2DWhya}24 z8*}A7v!;I$<=mq93^0n`bdm0(L28G991 zyd|`qI5#+|mlA@kAlH>{K-#@CDXTJr5$vQNHLZP09$Oa1kS8vse91T5Zn4f$OI2mB%JAIRz|I2291fNx^U0|8&y zVe@fD;tB~xx}%q!c%S$Rcug&D9XB^0=Xz7s&+1J+Sa|7HZ}UsvsaX9-#Zv}_SbNAp zq);fq7msLegn#~Ag(DqT^?jxrG*h?qLK;Fp7Gv3nu7Giq%PWBK!H7WW!8FB<%J4qp zspFR+zr$T81C`_(RY*11WqaH>$)_6t7=^y?!6um(N~?-Ze(Kxblo^_VCnuOs{k25* zg6gSH_tLr=lCWCnFH!!@F`E$Y#I?yNv*F-LXF~@S{zCm&f!lSs1cLHM3?IzxT8^Xl zzoUOP+rzj1KqipnP``oW70?iiK@2N%DYK0cVGi7{FQ(6hmzMZ{%)Uc)H6l9i?2#o; z2&iLL?IAa{0Yl{&_7CJSmQ0f?%54(Gb8mlW zbrpDf?bg@+rpMNQC8%&wxkv=)Ks-)B%l;b2Ab9UKt_0>p5q2bDf{Pme=^aI+U4XN9 zI%Wubdpy$EqZOL|mQ8Ym%7B6KgKJZ%x$i2&G&MnZm2L=RR^n!+7_>Fdl{M!n?V@>A;HPMkOZ2(VsI;1Q2V4>?D>4Ok#O0?1@jpBUhrC-71e2T6 zFmgm{qbQ~l?SwwOnhGa{Spu`qjA$SES6qh}L4W!Vy4iTp0R3Q;*>7^_)|f*I199f> z4)}im@L`_{%y^t0-%HkFe2w1Z65B0Z`Rp8n+|~347Ul6zbP@#R1_DHiqX)PYvKCNm z*`xta^PZ`$?uYXKszCo6&F_ync4wR+2p`X(x6U5DKbhuhDS}6`D@A{?rw7vk9#LcC z9ExpckG+eos=Hluac>54r^J0s4ct=fkneZq0V{}x)oz=?h!W<%NsM6DLCl#fhg2c* z;rLTpu~f|q#|rsh+%JEw{p=bmVFVGU1cPP}?vUCnhOZ|NdsqS#!2fe#&cpeT7A(Wx z6LXjn@)h#eZ$60~fXNlW_UUCr)NN{0XZzjD`xf2OgT@HT++nN3Y`FSU4 z!?2&IjLVA+_UFwfIOry5Ic*NcN<2vRvJw7@D%R?>U;8(r40DE5@dvV;ID!6d10!9O z_Um8RFmrClP)zu%<2+`$ztkmsYYX;sc*#*;`|unMa*_2Z_3w2?DT>1eq%OG=yDRJu zb{ED0O=5910I*{|Fy-+JN~Q(&l@nu003~NVg8M?Bo=yU3U#P+Kb$835y*}01kF~N* zUaNcws6YrRU_6K1kKHw?7Xmv}4D5@HhSv{Bi6%O?7IhRlt~=ZG-xJrdvhuKK`sA8f zDpW|&JC<*q&tx_ANxY9|JSa|n{O;NNnj4{R)YB7|88l0g5FF!VELj`#3cEE9JlYeC zAf}ouP9FFKsCF=mPH+oyPhFjwX=O5{L|0c~WN#P;$`G=@ru3_Ag52$1GEMKhq`6$p zbp7p>GQrKahrtQUlu`3>&rV^tB>y|&6CYoLx9h!}M8+sGejwNj{q$HFx_GI@{kSRn z;Z;tn=RHBwoa3*-s$2zAJ%jtsK%WkLy;oXVK4mQG)6GBu z2GNyC3E+-ZcG&$5V6)!$1uR%_;1Kit*w#5_two_3zhjfFE8i!gT7 zsVVQ8X?4L&>flut7JWs}jJ9R$2c0X}0NQGqs z+Bvs-g%C?__J$u_FLHP37o^winR_t&d~YL9#Mzs+`PzeE!#4%ZTg_RFy%wYEndQHU zDM&iN+A?yatK&j)TScL5pr#8=r~lb4Wd_thGMN(Zj}pM* zp$AocxsKKG?G&N+(;u`!7O@ z4!G2;CX#;Sfr<^?{298>PhucTleJt52Ra_ungETC>(Z&8B{k^dH9WT|S!9(7QWSaz zbzrhkXrYGW2cn7hStl|c%uCP~9rz*(m;=13bW2BORkNug4f9vFrlGZUSO2XTHwGY~ z7=v!*M}h-}KRI9^W&ti69s&mM?ND%Fn%-eZ=g;_eIP8fvBWd=W>4_rsxCf+}})6>NW}KckE$#enmL`2(4j zL9ZmOB07Ro;XJhe+680 zY!LDClZ6uHM_0qXX-RbXdOv;iZi+8MK_9C1AFwHCC~5+}s|Bh}pDJNTr<3q!Q1L`# zoJdNW9@Na|#rBG}hNwVIy4>y${aXp}2sc%3aurZqkuW^*#s55c!l+i*F9q~eEO{C+ zYPf=?Y`ie(FFNGBV6Ijra{GQf`^u6{o8Y@@!4JCP4drfbdP5Bo|L3~1aKSYIJ2{@?|eHaP$eC&4ZUl+*x4|D931XF$&1p_bH1s3nnkbj6-f z_wN@tiap=}-MBecV8ImhR;B%Gk@v~1X#DXK>}b`FxRS+K><~uy0e(q%JNAGf2eT6(3qRMW&U^B?%$*pjn43Y# z8n})`CtqQio4aP+o~JlzqENbhvso;bfP#0ON;wq+A(bjZ7xh@yCK<99rkFo+UDX=% zdsh4QWuq}&5mDvBDwb!7tqI!hxu4mBx!@B7fbUH9w1v|F7)`sa&3pl|(@%H>3-zAV zmaW}@LM(x7*aPBSU|t1`K^A3%`9S6%O5&9=H}pnL zzi+y<+0;)WUt*bZ;^PQH#{61rrbav@dOzFEeB4|+8HO)ejo9D1;v_gEbhG;57lVYm z+dU{lr9cb4&smJjgz>_4`I>g*h3*3LZnv??^uAn$f%EO0#UCRTAkfvsk<@0~^Ma4h z8U=^;MFh9C6}p~3uqxv`pBSvvd^(u%G$$h8A4$-{pZ3?p)6c|)?MQ1YsgwQIM1`M9 zWNTU3p4H^M(NC?M-h6t==l1+un;*b<*o_SlO#VRhqWX(}-~3eRs$S@(Eu#b-0VUdd zq-sxdHBSuVS}=G|>4I7Z7nhQ&yI&%_P2xhpBc!WnbV@2$2V3X3kT%r-KGSz9hy2QO;%B1fTp3mTJ;aY3co(<-<9E22)E6cBBaVq} zvG?Lxnz;x;206q$Nj!+%0xs0y&{ga`y z^gj?MF2wzEoJI5#J8!;AtCIN|IF1vncJMXgdm*-q%+w12?1r@2Fw$@?Vw zv+3h?7Nrm-6}ir#M%~Y;4sVw~e@SjR^z$37L<)n zmUmhoYIl6aelj}0J;LLQUoCGrwkBok2Ia0+8SQo65$)L@I7`|uX}~hc>SKE8(m>JoV?|v z+Tz>CS7;WqY9A;$oMtHiznOZ}dr$>BlygizDVlhe@Vx?0Q|t0e2JQBFKcYvMUE0xg zx0iUcIhS~qY5qXYW-cN~)RW`nQvkbHcC2h)7gL4vYkd(@~#OEmChni zJDHQ%z3z*-T`D%;8RiM2_Lmr}S1Mv{oUvm*Y`HpJIKbOD{hJrXAloI|=T!+wI`l3% zxmn?ieO<|NO3Lr&7Iw>yDFTNKO$-nSgV6pZtfD6-c#dhW!DvGdd3J72nCSwGj4Ruf zH(u7y=gFwgfYLfpzBDI~Xoed<2w1Le**#D&4Ge$v)QciM{1*Hp^keD30d}-XC^J$g zcfeQZ`y-RKv4dq|d_}2o(0YJf=IKTzA2R$^-KFf01h1XD6~oUBw(WK-#S0!j_KG!9 zf{+7K*Rl4nFbm1C#>?oI+u3Us>tl#ye1~#J=%kElFsIjmKG4|eJJ&1Btf9!HXwcC- zbmwVtF;H&>4Ed7Ju6VlOk9 zexzSUp0V#S4-WHlE0lPV;6ph*J_K|UlsN$YoItaHrTgj98Bjp*b;|I0!uPiRvx&@k ze0Q%crxoQKXdjn1Vn`|es>S&&VjP%*sIPT*B(xrLZeQA3{z9zX`b-&<`4$5OMN2)r z82M#@1L%RVH>Qx%$sB~tJA@~T@3`zIJf^Vc)bUfzE}A}nAYR>dsq9$l z{zlv!cej|;<8SY4eaoL}B;4P&oSQ`fh52ZWkTTcRtK&Q$A(tY+B>th70gmjqjbmD*o8Z;s-=bQ~eumEdji z9ii^I-|bw#4oR=Geht&ikbcP3Zz}MKa{3g-ff_4d(Ky2jeEEq#7Xn&LC`^GpkQ=Ml zKT2Az(XlI-o>Z;JUl8xJJZw=fC>v#ev0oe zDsZq69j&4_I56!k5H*u3uhyk#;%)-YDX;_`Vp5u)>wh2j^~icTxm>Tnbn31?a66j7 z?R*ANE$_(L8GL^8T*R32^^y7TR+hS#>I*Y}JFhEcT|8uXsJ;JH;C0TBuRLS9nZZ#% zEaK?=q&{`=fS2F<@6OY&9oJ6*Y|0l|o3P$pOZ z1;#&+N0`j>Z&~HVjUL3QHd8fzAQd%o`e56XB|hOVh2-iKPBtW%)O!_LXwE#E;t!yH z&RBLZ5H8RX`%1T2BiJ)2F{-Kp@bV|lC+DM!{jcN3@(N~@W|j{Ed!@pKm&MrxY61Vb zvsq?K^r>UeqhP4yqBIbH*DLfm;ZUdcHj^&JPSj#u&;L1d+kZfLeUTAMbcL3!(MBE7 zr4oYwUnA{Ds5W^W;w6;8m~{}KG|md6VYT!V_{)?#%bD{C&@(s^__g~cFn@xtpTJlE%wXQ}j57}+|M-Zz?(YA?SnspoJjlM9v; zJOAUXSJ91^9|f}bgw82!0Vw23CT<9R5x#ca8uU=bf#nOJork?n4a49h4e_(t`D#~l za%e6mOF0#|t$n<`1QYd+3R-iGnL9f^f;?Ni!*QUAPsl6&Iy~Tlr(nCPMstooK;1A* zBYQ84fk|&tdofW`VqNBbu+~}O93DnpwHLoPC-Cpu$g+5~)o{goBHQma``qcj-Y-^@ zNjh^kC?vgUwOVMYoLZLmv=_#k6#e8aS#Y|)OhJu1<(l`|y9Hw06iO8#hwP9ELf-K$ z!m1AbpeFYjy+i(YyA-zxK}A|21@#|nz2~Q+mOgck{O~~>Bocx%u6!boCk*I+vP-&J zZR#0Omm&LDg1TR0sD5~HPtufVgpYD>!^@48P74(n7=EyMFz}R`=l5y$roktvOJ>9K zHLq4uwf`FBjnK~Cf&GWEs=ogS}{X)SebT#-DJ&nAQ~)1GiC zP79~OmUpzBblIxQo%U=tSI~ty6sdsZjzdv*xPa`>W)!#qXBHluuTZc_BJ^Fb- zD=93~=UD#cey6aymTdRY*!ZRfU1vLr_w#vpg7A5|ce;ZeDk=l_=DC%MR!h9tO7{EI zX(KY&a%$r(+TuV&k9vZX>el6~dX*#8~Ng;it^^dAMVx$z+o^Nh( z;g|9NZEuC_OeJ#^`Qpps05sE>FGMGU^H^;m)U9FL4rbOwO5@mI-{TcX*%mou)tRA-7bUS_PPCKk~y zv20N+nZp*Tin0klKnCq+B8T(rtqQ5A{1mr$j2l8 z|FrkkQB}6<-snV7kW{)s0YOr_LzEIJkuCv|lJ1-c(hUNybcd95%A`A_WzyX+=}FD^ z+^)6vT6?d3zH!bU-}=TGXS{#Rp)jV;czmAgzOMVeu3yC57jK~VAE7MGKtXy?BzwT>M6jES(EY?{;zo4kz5j|usnVXHeP*~-aW*mQJpxWF@$a5% zrmHWEp8Iw^7+fUc_X-RhNu9`Xej4n9+UbOlceU#$$Y)y5L zf`_((tjRSr{4+g)I#R1(dIYq?Q+X8-1B$<2E{&l;Y=Qz7?i)XaJszp6SSi{^0FlT5 zY`XjtjgaVcNUbrtD!a~7qZO_o7tCqn5TosQb~}d$@G4AEPmoM}Te`K}S zWnJn8Z27RQ!-4k9@<`W^oGy~C7`ggpJUd=>RP2|T_N~t$!fJ&B0-y(k(KU8?s;8MC{ zViHryEuD(L7yB}^fHnSr@Yfm(b z^B$B#d~)1?;pb=6Yg(1Q<8-7hKydPSTURR>D5y96g1a?-#YlNeZG-K;c2OR|_lH^i z6J%nL@+jyC;6ri;|U{ye|;yFLkor!xij* zXQQvF{=B~;fUf*hA;{p!oG!p>OX?&*D!5>bu2qefn10O(nlGKwR4t&cpkQ2;jI{9P zzRkLEgRTxyDNY+sgOX;OQ{_H(H}p&I@{1t!3gI4yj6#GRQ3oF_!{np-(3J$vb=z|HJnLY?^%rOR62Dy6F+-ECf{yf= z5B*@8->$$6n=h*jgRDW9b^hj1OxCf8WDouu&x|Caqx|Zjw>TJt^aA44Zeb?A9O_?w zDfzGetgP|>?qX%x!&C5nxTv0}d+Gj)Y7wzK$y-l|$Xmd@`0q{VrsU-oVQAl@BfHyx zJk(pX%RaE-#oSi;7b@BAm;ceE{(GuiGkWMN;V#&^@W|)h@dB_H{1Ny(Rp&8DV6=?M zpl}cHGIfQC!7R!zIV4!YJSPD(xGkmV06pP7>3XRr!pb~@q~oD$S(o{niYDwQvO1jp?9)b$gv?y5TPy=pc?wdn;+@L z7|EE9iMVbe)QL)Y$ar-N{{37_IofxmZ;Q_)uhfxHboTo_Pk=r$-qMf9SK=>TA<`p_JxINq!mY_yV|lQ^ z)>nv8OXlyoemoXA$;?6Tbr@V&dpjO|+QBMQ|NV+{GLTu`;v=^&jDLgB%d-7=tHlXq ze{xWcycjDvZTjH2K_x`W$nLc)MX{tcLMx1HLkPJSt18L0Rv?28gvAg_BZ6aN5;y@w!NDIwRG%UWH~0Idb?n(=A^Ou6MWg6!{_mGcZEV*>~5Evydg7(^4k-I3W13-tx*-rESejfWcCWou?+j@SYikJE> zdYqETe|T>f156eW69%*dnODLPF@!Rnx4iWz%v|M?!;l1oOcDqg6)8G>ZzbgV zGC}aw&XyKpVMoMI>ZI#^q3=ixb})NK9KJno_x&B-eqG;5+PmooSyO%`jh8XIve zVcqxJOoN%x%OS~shZfZR16shoD@9kRMS3L{a;g!vvoNhOc9^Ym=XE%{`lF7gZM_Ux zC3HwlC_ZfKCE@~w1M2yT)n*6ulGzu`z`n~Aph91572c^$Gd9hi=`t(OX*0LR-jBj@ z@3aW`$+CpKc_2Z9ytm@P$&AnpU>|GnVDu4rJP6uTKyauGFfRLk0$=b&0*}RCBBU94 z9>^2lu8%R7e+=_hiux)@YF1_4`I+LXJ1Wd_gDU@O%xH|^TT#j zih3$-i3`IrY1C-D<-sU>`6J;dFE6YdXQA#FcYBwTw=F=wT28~6Wrsrhoj(sA2QgZ2 zMok9p__u7P>NtNf^;TR;%i?HB!j?u(SPK`1Q6rUQttSC_BG zPs{}gG*F#MR_O?5k-j(va<3a2FW-4hNX1FMzpL}Tr`COyM!wv%e3-x=cC2{=UXQW1 ze_eVi45ng6C~CJ zPn9=^p1*!P*SSo=$cUky*x6>f6~Sl8V@tz-&epr6=-hAck)ep`&blPHT!5HFI&~BL ziI-8O3|g8TRdAxb(1@=`x%AWx1wh-M8d>{NjMcDZe!`e`b?CpAwdZL~b%cr_FV4k4 zE2SB_MN?6#IE||{9cPRydx8&ke``uGCeF0Jt|CmBh&~_)C$^2@s7!Aa8s&X= zS&4IayxUXSI>S2No6>D2&Q)l2umhyIZl`qX@geVkYg+(CoL(-!Lc{|(FV(vlNPRj0 zz}bo`Kw*Cb@%HVCa-hfep;55nsOOE`K6q8nl2}Q#ozims zvnBR#{1bECfTUvO_TEW5ih1HjypZ5DO=3Cg16<0U03U4=_gTlUu%1i;X{6$jR;MJM z?WTy8oSU(;n~E9tBLlzIcX9x)?_dZ@5}sK(^k)tzs-L(}BqsnEcnZ7WPXr}VIx-lA z0Yj?&f9NY(88^Df`4yv(+J~0vv(NLktzPum*mgBfW5no0+opPbX#U%TA+M$`{a}Ym z&m!4!(F z5<(K}v`TPaPhZi-X(f&F{<3474Aaf4-qxj}yOmgCCMidER=gz9kWmLy=Kd>qoE!3a zkztk3LPOrk(FY7$pK#3Qy1JPR_ArI^0YfWQ=QO96Q?GO6r`Y+4@Wt9J9pm1)3s$fD0+FKlM6Ku;$QG#2lZ4T^e z`aMq6TT#;e*$cARtu9mvdYFv?@D?OB5OnEzef#Bx-==Z@@?xuGCvQ>9Q}Vl3M6MGCAkKtsa9< z9uP$!F)<|BMN2VJHa4PsM4I5`^u+arofmb~FA5=51+hEUC)2BVLqpSDc4U1n4(tcX zT@$ay0EdUkOatR@evlAN%wSvFk7u_cJ7X()ULj?s7VDw?l{6hOa`eDlp|+rCyHTSK zUZW&E)js!KMPSD)RFF1+U25cBC2Uy`2=C*;=}@>JPFIRI=w@vXHc^D-_`hzmRVu?t zzr1MPTX@glDq;@7ogQ)r9!Xc(B_&y3g5L0831hsOPCyNkX$e9xZX`J-JCg{XrawDw zFd;E_QH%ctfzbN@c;knCx!ajhxf#T7;CP}|Fvai8(AQ6uotb}+QNkd6$)Ls>qt=JXY)R2mCJ_vGF)&gXT~Tncc{8FM zB@2&s6X_(sy3_f7atE1mKG4_@zmiZ|Ji?&Yk^geEQ$Im^@H&JBO^D*GHItw$zB?V? z?X3!_MZ};zvDHIDm4b!RrmxGjb%`4r7EAubT#S{^G-S}3Ld77M!sY!}S2-&!MMVnN zOh^e!t^-5GhSHn{Q#K+V5684JrxV<7sdNX?uOL^vC?e=R=+F5qFkj0Q z>#uSqu21AqAnD$(0a$v#evHr^0{hkex-o~RAZ;YO3qD7+&9xpK4!LAtO5g#JqZT#; zw^v8b5>HwxUw2URcE7~%7mSH;YNIy~LpFMB)PLPfmrK+kuWCx`3SEoyu?8vl!*x?J zG-82D(JS;3(cn@4!?55NRD=aIvQodveYq_H-t>U*dZR!WQjFqQ8js%0u)mlkfGDRdBqS5Bn828bug=_C#m|57$oY))A*?dhA0LYk?dog6%&g@(=@h90 zmQAUNws8cZ^kY9_r!7d)E>0`&h|7`@zW&_c$%sIb4P#ty*VkX2xnS0+B3ugp#{`PC zHm6YAGO^UH$Vl(k>{GP-h6Gb76)Bw%k`FC!Z|gY^NLB9ZlpKeflktc>>XkL)_LJX| z$5^^fhFo#|%Ui57PJFtokQK`mCW9Llfn6CJ1Dwx%wT_J3eEYPt1dT8`B*q=s7c|z6 zdNzb@La^E+PSKgeAm_TER`R%T5xE1G0)ga=5WWGb7<6)5JBG+saj@8BQ&#Not&*h2 zUmxB3>SO=h7_Vl{Q5e0bJak>{Q zVYVZ0UU*9`8_#ah_tjfg&b??{q^E(22K5M2Y+|_V8mUdM=j~5+?N8LXB@ycD53a8# zYq1aa#C4`CNK~VpGq*$iGYI&gsSsvea!1-AB)LQCipNt8z$azcYaRS+W?URA7R9Ml0&R)YLXZG$369^ z%m{zB@3-Zg2WH4NCmZ%-G17B;tQ>PD)4PtYQ8QvkQgmi7R=Kd0SYHr?RqHUt`e#4M zdia6a4}#ZMx^q*45fCGPrL`^BZk-Z75=2Yf)ReAp@MO#;UE#gS;GU^IFdzSksU<@a zzUr}2i!g*a*|2qpiP&`XFn-jC{rvhj;Nc<@qv6Qy^S&)pZNrg7GzOwU;c?s*6Y%hR zphqA5sxHqune+P?XMBnF&J%iE|FLzZne?x4W!`{Gt@V>8`F+L+gI{rjJ&f^l$!8fe zx;q;LkyjWG^c9G;o;t=q7QUlFdi+e2rh8pIU=AJbCxQIksvVIKa7XW;6-7)~uZYWA zqN{)%ZoGWqcUx9+ffeGQ_9(AaS`;|qViPPwZ%KB!t^uM5gvQk3ew&LEZNjqX%XFFA zWUr(+CP$K)m&U+=oGOL(t3=JQ62vzg$-F!Q+*N`K+Ro{=uatA{d!zq~HT&Do;^S`d92XF@mbwU}B6+quh;_eL_LMz~mYTjDvh|!m^`RSTZwy;{_ zU9p6ALLPgOE3KlP^v4KPynQXatSNu{q%!!Sfk7@1twQT~;1J_+ULFZEO(`x!u_;>9 zbcSQVo%7`bXv(Y0ctoh*ukISU9EI)d?5@;>uQk;tuDqN)$kt34co53QS~&796Cw(^ z3-Jxqb}#4-@XpjF z(PkdA7HRe!_c{`<9g7|rVOk_LPe#rY;|e^JF8 zOcz4pA%=~JjYhbl_iw~bV99Xrs*XRxoo;NYFoh}3iZgIqFmFW0z0FL&&A4w65bFPW zRv*;=bY*$Mzs!|eNIhUXRjUuh*3i&=Bv9<+=D^W;Y|wA0qg}%`!BJ+h5W2(zjr8FHXJAPzUvi5K-0GAjB_7Li&hlKX#-@~@ z7*1Et(N=5<7@<$aBDnT%q}cz-j|W)~oqwZIrv*UrC5v-)J}4vI%75ds7l$?iEIfdP zGt&i-oB6PCtmiH!es|BImUIPA8UhrPX(lVqRILxeGG8Jaxy0>4Iq_Bkes;$-FG7F+UC4eTsdtDzkd-XM^&04#!40CwZrf&xm& zXy6Fvjr@H+iAKG(fg<5nY`6GBrCE6gPDgDz)CaDju(pO6G9Bb2Dz z4ew^x;T19TIL8_w=&cWTz+KV(Ki<{V|J%E|6MgQ9Y&>SI3)Fnpd1t?H*4Ss5k@>hh z6NQBuId8#kG4KM6->b69j2#9)qPrC+1UuS?3wqPDa?ihaPk6;t@&)RaOZ8-RKI0YY z41Hu`Yx;1?mg?4Ry_^D@SzQ?LnE_^$-Z@t?*%B=x7*(1d29!cRUF#m!$%3pQ=cad%M6` z&;Td~LvQJ67vR<%tv!Aw+Gl5)#+miGAbD>?K!=*TNT|1$0kZ*}p&eS))46>kSCC(v zBa^Egny}HFczpk*mGE`QdZ-YX2Fc1(PtO5qw-KWny7mlW^k)D1VT)`{**o4hKX z+}Fev@0Qd2V}C^ijF$@`?2zV*;jn*3$MoWaDkcfSyj?3D6X}raZmn7$wsJ7@-Xodb zHOS4Jm3IQ~*9b+oJzLG+j(Y%5wqXaiTLO}yVx}ttl7#NMQ{qaxGdB6PCvjxqmL^o3 ztj{+{}NwDFMYw782bjGp0V8{#a@|I z{+LcMAuSwuTKAHW<%5~1g59%J^j7rLII{VzOalKim6_?4(y|;U8}m%LLcOj>+6UP! zbutNF&$MEmjbxF`-C!cAR`-ckBvMxB2rZ8+Np+2R@unZX9@oX}(qS|^tw@GUr?ViszUcla%a2WpG>;;ywOEx$cz2_H!ikZqr+-JZaSpA5b$va^ z3^Ky&u_lgVZ9ZO{O$Z_K?_>`Qp;1N?v&Zu}k%5@I+bM}~1v~$s9@+$QyE9!j8~d{` zle_Z~spb^jN*1I4Og|*xnb**D5D>bbkI29fgANM@miceJp2V73l1jsPsvb0d+>+nj z4>$w^lh*&SPRk|G-MQ2dbR8+;IGTyfEj;3#17vIuy=lH4$OMc zSOwggfljzn_BQv$_;-mu+5u&j}y2Dhmw7Hz8w&1Ba|+EO#3SiC(g`r5(ihJg6B$t8{c`jy`oVUE#X05sUDvT6gX$F64_;bTGzc>{w4#eEqius(Z-4~f zkx`UfXfu~3X%Wf9ed6yPqwNSD2+!*XboBiZfEC_(1+tsGq%CP)dn@*$Ak9c> z;xvmZ&aRZHj*w)&%9CVRbXA=77Azt0i@lR0L-?^fJz~wxL3Xdmf#kNaB$N6cTIw^* za(X(e)87)L(M~_E3QBNwO6>7!_-^A8*I*E5E5cPaouYlj%i}UfrWQ;EPE!G2; z6wz&=1eJS0&lPE`!*Wg7#)U$n^vAR*UXi}y2<(3(d{AK! zNxGvbbZd$GlXmo=g|P^_0d|VAaePvr15O@(CzPXL#QhY1k_rQ>eM914xTRHl_qzX?EdAx_%)uJPMw*_xA~T#;aIjpI zC^?>V4&q)^E<9%mm1EqKWABLom=qqVKH8ycHE;GKxu}BCGbrv0lGAgwn&w*Bx1k{) z=9_uHt4Dn{7Z;iS2{t+Zb1hZ0U?Zh3S{L!1VfZ_O4^=$yFQ0G|B&iF9jbtsp&k47yxeoWCrXVxBERQ{;?-Xv{YY z1AoQ9Uor4k4E$SS;G5HqW{lAbQ^qFqKg@`u2v)V&c#X#5-&Vu%2SQL589%R7@1>br zCuf{1WzW?#y$)o(BwZfFEw-_bvhjgN&~utDk@`#LSTl~AtRs>=8}1pxryqT#5NtVP zA9E|}Eb^KXx2g^Zp7xo}0%PlBd)SH_?D8q(6!FH;!Sn&kgGx zw-I|8r&lwahnu(?dGwirfrhuX#65AWZSTpCraB!%@(^hSa^?aRCn-{BEnoT7eLd*n zsLWJVzHYxseUtng#KGV5mW`nUelP7wJiru$GFFxyD|^zW-Le|I(|b~t55M&vGj)`A zOMEa$Y?^hmb!YR2QJ3omO!9A!7ngNa6{!)qqXNh^)&7OdY2lSwDjXX^^RWJfr#LKH z@(J>5&O)X8!eXToQ(Onq;g@Jqq%0*=FkGFvxdHnNEvvB#6OswE3}rUDP7#`@r2Lxx z`T6SHKpD$>_a&bLftUSnU*KhF7^TG1-*d|E>a;%2MywaZEzXZ#Da4eW_pOjE*e{gmwEmPv)-$8O zbSsgqz%pjmbQ7mxq?i`Aj8wgQ#g=M!|5ugWwb|zFIqXKs6qzJfJ8t~Rri?SMAzi5= zN2sS^$qPW6MH_R9tZGFpBMe`ep77RT=?Qgv5_qPl)Lk=0kY0!X@$;$a+-j*4ij^B` zvz2ad5cDGLS2s^Jnj4BRk^luIxV?fRa!`b6|M~agKSr-s02rAjU-35Pv1?b-ym+D? zhqmFYQQIcGZ$qxWY(LjgqQ<0Q5j5O-2~hC5)8cf-23TFC%=pTOWb^yYxOC#Pn^H9V^z^BB zc*J*8!1QF|;v7yBk^hwQ-uqaUM;?H$8FbRVZEkTPVl&xp(23)3ba)k&s*0(lYG|mZ zll^028@r;F(;@zCvtSX8x}w4&tyNr{>2(L}nE&f)MT#Q5ClyfrhfJd8K!b6M0rU1k z?tB9zkLTo)hm9(GG{AcofqDylw@r6S5PY0@f8Rw(;=t$${P^j622qVlt93`$^V)M? z0r}Ew?VM|2QLTa(22ZkVFEqd{j53sT=FF6@pBLGO5c{jOE-ofm!U zG8V6?4=qSvJ}%R@8^oLzK2*R@eBT^5DW>3k(h=mm zFh?-#CvaFJ>P7X(qk5eO%J!8@A2ENXuK8+s7Vf$}E!?is{Z19qPc#$!s$iLI%#&gIwnjHQi#zboHx%(a5x0j-kfVLJnpM)uQpcc+t)Zausu^D0lEuRRD2RfUoTWlpuAVA zn07LbyjHYN!qtLR9#0JMB+-!;{FuO4>UUYbgEL#t zUJ75?P~l|i*27p8b1fM`Dx9iZg1p+Rf+7(pbTJi7GPkxAyDRH`EO`ONe!9r_qn~FtM;D;TvnBu4Ikuk$aiT zK-Jm4*x|&X`r>>WRKktOZ~58r8(fbq7+H=;=j5_(3D&jXGHMNfuwGSkIV1!4{OPVn zW!NYUb3>Xboq+q{E$c`q->#mLsB%>xxwIXb|J+IHi4>Ka?nb;7!B?OA3`n&)x?YV^ ziDRnkmyLCd-Iv|61ehe6voFSMdJgs;jTA@roRvQGgP=oPGqDqO6Y|zh+Au!l)(N?6 zqkLEuXA^3wlDC7_Llxh-m87Z5r|#5Vr6=#=F8ig3Q~0P{o)&t)P42606gY%b9Jjpl zMTxHX3G!XXdYF;?l=hby#(SQ#qfpev7fK|z{>qd-bu9uK>D2hx{mZuN#*`j6{$h;6 zh2KT%w{%J+?>48~3`Xo02g3VuNAJuFmZfOcRVVqaIk@_--v^4#?YGCLnu<$gmrtnc zwMmE}O_OK@S%flicex&H3nt;%;f)+p9R$5;kod}9s;?3mH`c$>R$96m^scKfQX0V> zm3{_iz?53e!43PZ)+)I|RkX)XBXBb0V7)=UuIj6nzUISwsd_I$WPT~V&O=yBoApiv7%I=S93 z-e=g9xH-~ICmLfM=h%qzW}n6&BH!{eN8jzs#L-HpA1gU9gm0|n7DbeGi_3ap2crl} z|Fa6@9g(c*e&p8o@n(1aZ$wko`iWqHFd*Qi{Z5`>L z;ghY-hd0~Fd8<0wO3pEt-Ar?;0&`I}lYx<>;$c}@qq1y~Hkc|ZgiHY?8Rp4zjd(Xf znCZTLKQZ-;-AMMo2s%8h)EF@guD>B5VXW#P;8mnHo%}_xsqbV#>4c2^Xd-v~IAHJ# z*$88*6EdvI%h=ju<<~xE_97wt&b<{8qHQV4Izxuh{o$!*9jYrt*@{VL#8<`XA^08v0h6bT`F3{)Es!{}W=6 z1jwX6r!(Ft67to*tZIOJtjiS+IX#1Zbceonwcw!V>`L2kJbbiw56`ZwmNV;zh*^_? zf=xA-zJ!lNMyYveP&)MWavTa0ADDH5*M}Ec|oa%P7$`6g0sZ-c>T)=w16$WleO+4ur^Y33)S@ zncc_YLaR4?io;_oe8gyJWtIfyg>WUBHtVLQx!i)Z90ohu?))SE+6L0NeXqhU2Z-66 z|FeSf@5R1sQ1NPYbCLA}ZsHsa^aRND$=e$x^U{tyzNF-2{ZD*dC?MIE#OGN`twpo2-#vNBvX5^`@}m~E~HQx}xEs~2^SguGjlSC=<|5a*g>aMJK2)#g!SUQ zZ<-@lY;;J_@py*nNEv8NJ)!9{XL+0`%E|Q)(#}oY!*8MA>`tzHSnf=p8->8O#3T=Y zfjQYiBY09C&^fzxl`#2@NVCw%(V!Pkd--O|krzp~-GF5ix&*z1fZdY;)#a3O4aZbP zDeJZs6Q#uok0_j=DS1N-!Hv}~(hDcTY>^F;TKU<~y*d5^E8@%K4rmOGg~9N#pu+hegTF zRKIn> z8;TeBpFjAKHi+6GQ;;nRsI!(-fr2csV=`1(VzcWc>rff9<$T@|I7@fk^9g7xyhZqg zR*kM&VjdPqm=Yb1pJw9quX`KV(M(ljw>^6z7#o1Q%D`rz;pj^re67+sCF0q;^*E&W zAWaLliT4BMZokgAc46#W&;4mCeeF^CAXc(sOZD4IHKEZtrrz4naI%e_SAo z$bohaUxV)^2HdcqEujAii6OhKwc9ozacaQ;KTrSy%+M{H=Z1S}rpKmzY~}!WCRPpA zFLn;vjpgqlB5tNz;=4F)=O+V93l;WFl>0DO$(a$Og4aH*r6q#2%rLT~SEl3iCJfQn zn>S~Yfi7F$!bqgB??Gq^(c=Ry(j^->ri5CI>|&syWt9!0m9@0OK`TayqSZJ}wGMTc zpA2N?j^OYCho|kbd65t&3AFyMLTQPe*W6bCi>+o9%omw9(;v(@7ViP?Y4PW@k`%3%sX!pj`jypBvY0t;_3CsUA673* zzOP_5pZ0_>HnI)%hR5E=@``_d{l{>2EnKZdv%zMV%2rs+ng3B^WBrkGqHXngg&FRh z#QFMn7hMob0Ayom1~;=InLO?c_?l^2>JD5q!rDF4wN4m6p1a_ zT~%iHu6f!y;O^&IuP2*>Ax{p{fplP1!&jTqwY15vrf)jw`&NHy2`{%NiKJcyu7z>s z>vy4vi!Xt`nF+c(pfRB zfWYcYJK@^CAFm~vs%8;G5SjPBGpke$(|_zOKSW1?!ytzNlas_<3%rQk`V<)7*N)r0 z^FJXH4Q0vy(9n3gYjlj+B6)hqjPkb*8CxD-`yN`x=cBx$?51$1LU!#`xNK3&;$E74 z5---{IDv>>Xs~_HS()KXX{oU`$BnLOs3%K^!>|Q9S8pZd)Z*UIOT9~vW9*D)8T#gV zO5bD(m_Co#f5} z(m~tHb`u2@@_*_5nximHM0l@XEX77)eLGK_`mSe9#;g*xx;HH1FbLz0Iz0*GSJ@eYID-T*#*8%Q1 z)m3T@0#XNwD9P=8=pSX#V}8Qo$@v*&mY_Q2%=Oq0B`}~vpl90)kEU0y`S*i=9lUtg zvq606w43bu!wslok0s=*8Z$#B-Nbp--P=!b-i|E~%o{$_(Ac@>2_)u*{a7p(7=p$X z>uRlW8YubUx_rHok#7+TKS%<9^o!?p<|{;GG(tYb2<`40NE^E!e&(mQ$< z&RcP!d}az-#m_J$V>7Y%(gZtBv^&Iya~bMm9e2+Y44u>$EC*axti2fP6N#gSqa%94W1~)#)De<X>`@_c8sjI#- z^|Q5(_Em1yY{Z%=T?T)#48+ussXI2m!!AereyB}zdhm76^a+Xbc*n`yFc<%9fD@P6 zo!c&uZO?pb#sDGnydqmZE>cnU@xl=%cVT<4IJd4Q%wmGivF>|{C39r#1nt`gvOdu@ zMu5(hfqYQ{-8?=-UC}KAm(VH6lS?;Hs0a{c$zhNU$z?o{(lQ2M*IUp)Iz$lISdOeO z-kwUfH+r(5^PDK9E8=*GDqeEDq<`u{@j;b)UxOY|n@%^_80Xa|`qe0UtPb$U?K&G> zI)fF1l|s57B?Za!M#m1xcw|{?eJAIv35)$7NxmbZ)c){x3>*`bH+tE z^HxaainAi;(eI;~&QdU`T;J=A-SSHgzzFl1G&l~{ka!v+1hm^GvAGmSt+fMSZy-GTGv8<0XJ?l{`yNeDQ0kDr8vn|i2{1wQ8!d%QkD2F zIs_LD+%?Y!=fDM_!KiPqGKJ=Rc&yRe&&eKpm1?OA` zs&2CMGcRFD!PCcQ>N<3Yhy`U*s(mZV{V#t)iXjoG_tSt;&$B?jzbyYqDmJsdaE}eA zTT@W*-1SKBrGBX6uRZRmMr;=JufQSq!!9ad8d?te#&S0dWJag7O}9|ww`~wt>WZUw zAH82yiB%d%J(h&|HuNaxH#V|l;zK*mrxVqzqRxA~2FG50UWMjq1#xJp9o_(wqaUu|7l@x5_La;Bo{|bd>J58<;@l0IuR{Hn7 z5OtA_kQ9Ee9x$B02`QdHpFkOHhBf7Am!%ZEu*fA00PTTetMhwKpE7h=?uh#7E@-I5 zsrpG`_ah>!&l}kt^k0rD{0Pl~$N|U5%M1Q-H7V+ahCKZa zm)nY5?>e@@*iW4SFd9@Ess?6p{G-`M%tTn{`Fl)Jm3%8*7WCArFpFkmhsy2z$Q7=~ zP9ev#X!{x41~f8uHg_9h+6cSfn;b~3haOnz{w>1}PAV7bGbNPPbjtYJiCtW2LX*nC zHjVjpSyUKg6;^o$+c(-L*xMHsb-nbs^mMtJJmfL}!!Oyd1~kNIUvtw`MVa>VN}C@! zwmcVidJA7TQLb;Ah(jx!o0BJIO$%h`p=}8`F1=zw&p@mPw$@GHXW6ZX&*((F+zd$l z6Cx5UJ6>SpsMi#3YIisLQeZSbUHtiBn`B0f^MvX>x=xdfj>u(Yn)( zL{yeCRjh-md+a=D`yi80Z)7aj<9!Cq1JusuMx3{E5c4<0|M zH%bE!L8TkKAoSF0qj0i-d|k=L7Ri*ir28ut=p41-W)P?P(9Yeo8^5C522`bICHnJ_n$kkd>wiMfbuV0h8py2Ay{s&x z_v>JIoDUYQxZ2t6tDzETB`L9^pzxP>5T{$V!kX~D+Au<=^-TTSd#nH$iK#kKlTkBZ zS+GsjPkn#gF6KHaW4I^--_Xj*Cr9tcfK6-E9NMR|E|<5em;%Ylbp?LOw%;n-b8txd zPieslJ$nO86Q89jp)FqIxHp?Ie`!gb*1Pbg zx)Vmc3H**T=A%+L3pyT08D@BH)3Hj7O|9L2Y80E}U(ktv3v}W?W*)=D+d0G zfxlwluNe3%2L6hHzhdC882Bp&{{IyN=zq?TDP^GnCl`oJUjQz*xW(IwA?*3UPF=nK E3uV-SL;wH) From 10d3a25916561aab0ab4f1f2bb191ce1c8cdd3ec Mon Sep 17 00:00:00 2001 From: Wei <48665416+hbhflw2000@users.noreply.github.com> Date: Mon, 20 Jul 2026 11:39:59 +0800 Subject: [PATCH 023/240] [Feature] Support per-video audio masks for Qwen Omni (#4656) Signed-off-by: hbhflw2000 <417911774@qq.com> --- .../qwen2_5_omni/test_per_video_audio_mask.py | 708 ++++++++++++++++++ .../qwen2_5_omni/qwen2_5_omni_thinker.py | 562 ++++++++++++-- .../qwen3_omni/qwen3_omni_moe_thinker.py | 193 ++--- vllm_omni/outputs/__init__.py | 12 + 4 files changed, 1338 insertions(+), 137 deletions(-) create mode 100644 tests/model_executor/models/qwen2_5_omni/test_per_video_audio_mask.py diff --git a/tests/model_executor/models/qwen2_5_omni/test_per_video_audio_mask.py b/tests/model_executor/models/qwen2_5_omni/test_per_video_audio_mask.py new file mode 100644 index 000000000..e17cdd6f2 --- /dev/null +++ b/tests/model_executor/models/qwen2_5_omni/test_per_video_audio_mask.py @@ -0,0 +1,708 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + +from types import SimpleNamespace + +import pytest +import torch +from vllm.multimodal.inputs import MultiModalBatchedField, MultiModalSharedField +from vllm.multimodal.processing.processor import PlaceholderFeaturesInfo + +from vllm_omni.model_executor.models.qwen2_5_omni.qwen2_5_omni_thinker import ( + _PER_VIDEO_USE_AUDIO_IN_VIDEO_KEY, + Qwen2_5OmniThinkerMultiModalProcessor, + _coerce_use_audio_in_video_for_hf_processor, + _filter_video_use_audio_in_video_for_uncached_items, + _normalize_use_audio_in_video, +) +from vllm_omni.model_executor.models.qwen3_omni.qwen3_omni_moe_thinker import ( + Qwen3OmniMoeThinkerForConditionalGeneration, + Qwen3OmniMoeThinkerMultiModalProcessor, +) +from vllm_omni.outputs import OmniModelRunnerOutput + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + +AUDIO_TOKEN_ID = 10 +VIDEO_TOKEN_ID = 20 +AUDIO_BOS_TOKEN_ID = 30 +AUDIO_EOS_TOKEN_ID = 31 +IMAGE_TOKEN_ID = 40 + + +class _Feature: + def __init__(self, data: torch.Tensor) -> None: + self.data = data + + +def _fake_processor() -> SimpleNamespace: + tokenizer = SimpleNamespace(get_vocab=lambda: {"<|audio_pad|>": AUDIO_TOKEN_ID}) + processor = SimpleNamespace(audio_token="<|audio_pad|>") + config = SimpleNamespace(audio_token_id=AUDIO_TOKEN_ID) + info = SimpleNamespace( + get_hf_config=lambda: config, + get_hf_processor=lambda: processor, + get_tokenizer=lambda: tokenizer, + ) + return SimpleNamespace(info=info) + + +def _fake_qwen2_prompt_processor() -> SimpleNamespace: + tokenizer = SimpleNamespace( + get_vocab=lambda: { + "<|audio_pad|>": AUDIO_TOKEN_ID, + "<|video_pad|>": VIDEO_TOKEN_ID, + "<|image_pad|>": IMAGE_TOKEN_ID, + } + ) + processor = SimpleNamespace( + audio_token="<|audio_pad|>", + video_token="<|video_pad|>", + image_token="<|image_pad|>", + ) + image_processor = SimpleNamespace(merge_size=1) + config = SimpleNamespace(audio_token_id=AUDIO_TOKEN_ID) + info = SimpleNamespace( + get_hf_config=lambda: config, + get_hf_processor=lambda **_kwargs: processor, + get_image_processor=lambda **_kwargs: image_processor, + get_tokenizer=lambda: tokenizer, + ) + + def omni_get_updates_use_audio_in_video(**_kwargs): + return [ + AUDIO_BOS_TOKEN_ID, + VIDEO_TOKEN_ID, + AUDIO_TOKEN_ID, + AUDIO_EOS_TOKEN_ID, + ] + + return SimpleNamespace( + info=info, + omni_get_updates_use_audio_in_video=omni_get_updates_use_audio_in_video, + ) + + +def _fake_processor_with_spatial_merge_size() -> SimpleNamespace: + hf_config = SimpleNamespace(vision_config=SimpleNamespace(spatial_merge_size=2)) + return SimpleNamespace(info=SimpleNamespace(get_hf_config=lambda: hf_config)) + + +def _fake_qwen3_processor() -> SimpleNamespace: + tokenizer = SimpleNamespace( + get_vocab=lambda: { + "<|audio_pad|>": AUDIO_TOKEN_ID, + "<|video_pad|>": VIDEO_TOKEN_ID, + } + ) + processor = SimpleNamespace( + audio_token="<|audio_pad|>", + video_token="<|video_pad|>", + ) + info = SimpleNamespace( + get_hf_processor=lambda: processor, + get_tokenizer=lambda: tokenizer, + ) + return SimpleNamespace(info=info) + + +def _fake_qwen3_prompt_processor() -> SimpleNamespace: + tokenizer = SimpleNamespace( + get_vocab=lambda: { + "<|audio_pad|>": AUDIO_TOKEN_ID, + "<|video_pad|>": VIDEO_TOKEN_ID, + "<|image_pad|>": IMAGE_TOKEN_ID, + } + ) + processor = SimpleNamespace( + audio_token="<|audio_pad|>", + video_token="<|video_pad|>", + image_token="<|image_pad|>", + ) + image_processor = SimpleNamespace(merge_size=1) + config = SimpleNamespace( + audio_token_id=AUDIO_TOKEN_ID, + video_token_id=VIDEO_TOKEN_ID, + audio_start_token_id=AUDIO_BOS_TOKEN_ID, + audio_end_token_id=AUDIO_EOS_TOKEN_ID, + vision_config=SimpleNamespace(spatial_merge_size=1), + position_id_per_seconds=1, + ) + info = SimpleNamespace( + get_hf_config=lambda: config, + get_hf_processor=lambda **_kwargs: processor, + get_image_processor=lambda **_kwargs: image_processor, + get_tokenizer=lambda: tokenizer, + ) + return SimpleNamespace(info=info) + + +def test_normalize_use_audio_in_video_accepts_bool_and_per_video_mask(): + assert _normalize_use_audio_in_video(True, 2) == [True, True] + assert _normalize_use_audio_in_video(False, 2) == [False, False] + assert _normalize_use_audio_in_video([True, False], 2) == [True, False] + assert _normalize_use_audio_in_video(torch.tensor([1, 0]), 2) == [ + True, + False, + ] + + +def test_normalize_use_audio_in_video_rejects_wrong_length(): + with pytest.raises(ValueError, match="one boolean per video"): + _normalize_use_audio_in_video([True], 2) + + +def test_coerce_use_audio_in_video_for_hf_processor_keeps_global_bool(): + mm_kwargs = {"use_audio_in_video": True} + + result = _coerce_use_audio_in_video_for_hf_processor({"videos": [object(), object()]}, mm_kwargs) + + assert result is mm_kwargs + assert result["use_audio_in_video"] is True + + +def test_coerce_use_audio_in_video_for_hf_processor_hides_per_video_mask_from_hf(): + mm_kwargs = {"use_audio_in_video": [True, False], "other": "kept"} + + result = _coerce_use_audio_in_video_for_hf_processor({"videos": [object(), object()]}, mm_kwargs) + + assert result is not mm_kwargs + assert result["use_audio_in_video"] is False + assert result[_PER_VIDEO_USE_AUDIO_IN_VIDEO_KEY] == [True, False] + assert result["other"] == "kept" + assert mm_kwargs["use_audio_in_video"] == [True, False] + + +def test_coerce_use_audio_in_video_for_hf_processor_does_not_validate_against_empty_mm_data(): + result = _coerce_use_audio_in_video_for_hf_processor( + {}, + {"use_audio_in_video": [True, False]}, + ) + + assert result["use_audio_in_video"] is False + assert result[_PER_VIDEO_USE_AUDIO_IN_VIDEO_KEY] == [True, False] + + +def test_coerce_use_audio_in_video_for_hf_processor_does_not_validate_against_partial_mm_data(): + result = _coerce_use_audio_in_video_for_hf_processor( + {"videos": [object()]}, + {"use_audio_in_video": [True, False]}, + ) + + assert result["use_audio_in_video"] is False + assert result[_PER_VIDEO_USE_AUDIO_IN_VIDEO_KEY] == [True, False] + + +def test_filter_video_use_audio_in_video_for_uncached_items_aligns_partial_cache_mask(): + result = _filter_video_use_audio_in_video_for_uncached_items( + { + "use_audio_in_video": [True, False, True], + "second_per_grid_ts": [0.5, 1.0, 1.5], + }, + [True, False, False], + ) + + assert result["use_audio_in_video"] == [False, True] + assert result[_PER_VIDEO_USE_AUDIO_IN_VIDEO_KEY] == [False, True] + assert result["second_per_grid_ts"] == [1.0, 1.5] + + +def test_filter_video_use_audio_in_video_for_uncached_items_validates_request_mask_length(): + with pytest.raises(ValueError, match="one boolean per video"): + _filter_video_use_audio_in_video_for_uncached_items( + {"use_audio_in_video": [True]}, + [False, False], + ) + + +def test_mm_fields_config_keeps_global_use_audio_in_video_shared(): + fake_self = _fake_processor_with_spatial_merge_size() + hf_inputs = { + "video_grid_thw": torch.tensor([[1, 4, 4], [1, 4, 4]]), + "use_audio_in_video": torch.tensor(False), + } + + result = Qwen2_5OmniThinkerMultiModalProcessor._get_mm_fields_config( + fake_self, + hf_inputs, + {}, + ) + + assert isinstance(result["use_audio_in_video"].field, MultiModalSharedField) + + +def test_mm_fields_config_uses_batched_field_for_per_video_audio_mask(): + fake_self = _fake_processor_with_spatial_merge_size() + hf_inputs = { + "video_grid_thw": torch.tensor([[1, 4, 4], [1, 4, 4]]), + "use_audio_in_video": torch.tensor([True, False]), + } + + result = Qwen2_5OmniThinkerMultiModalProcessor._get_mm_fields_config( + fake_self, + hf_inputs, + {_PER_VIDEO_USE_AUDIO_IN_VIDEO_KEY: [True, False]}, + ) + + assert isinstance(result["use_audio_in_video"].field, MultiModalBatchedField) + + +def test_mm_fields_config_uses_batched_field_for_vector_hf_audio_mask(): + fake_self = _fake_processor_with_spatial_merge_size() + hf_inputs = { + "video_grid_thw": torch.tensor([[1, 4, 4], [1, 4, 4]]), + "use_audio_in_video": torch.tensor([True, False]), + } + + result = Qwen2_5OmniThinkerMultiModalProcessor._get_mm_fields_config( + fake_self, + hf_inputs, + {}, + ) + + assert isinstance(result["use_audio_in_video"].field, MultiModalBatchedField) + + +def test_get_video_use_audio_in_video_preserves_explicit_false(): + fake_self = _fake_processor() + mm_kwargs = { + "video": [ + {"use_audio_in_video": _Feature(torch.tensor(True))}, + {"use_audio_in_video": _Feature(torch.tensor(False))}, + ] + } + + result = Qwen2_5OmniThinkerMultiModalProcessor._get_video_use_audio_in_video( + fake_self, + mm_kwargs, + {}, + ) + + assert result == [True, False] + + +def test_get_video_use_audio_in_video_treats_missing_item_as_false_with_explicit_mask(): + fake_self = _fake_processor() + mm_kwargs = { + "video": [ + {"use_audio_in_video": _Feature(torch.tensor(True))}, + None, + ] + } + + result = Qwen2_5OmniThinkerMultiModalProcessor._get_video_use_audio_in_video( + fake_self, + mm_kwargs, + {}, + ) + + assert result == [True, False] + + +def test_get_video_use_audio_in_video_falls_back_for_partial_cache_hit_with_explicit_mask(): + fake_self = _fake_processor() + update_with_audio = SimpleNamespace(content=SimpleNamespace(full=[VIDEO_TOKEN_ID, AUDIO_TOKEN_ID])) + mm_kwargs = { + "video": [ + {"use_audio_in_video": _Feature(torch.tensor(True))}, + None, + ] + } + + result = Qwen2_5OmniThinkerMultiModalProcessor._get_video_use_audio_in_video( + fake_self, + mm_kwargs, + {"video": [[object()], [update_with_audio]]}, + ) + + assert result == [True, True] + + +def test_qwen2_5_prompt_updates_apply_audio_mask_per_video(): + fake_self = _fake_qwen2_prompt_processor() + out_mm_kwargs = SimpleNamespace( + get_data=lambda **_kwargs: { + "audio_feature_lengths": torch.tensor([100]), + "video_grid_thw": torch.tensor([[1, 1, 2], [1, 1, 2]]), + } + ) + mm_items = SimpleNamespace( + get_all_counts=lambda: {"video": 2, "audio": 1}, + get_items=lambda *_args, **_kwargs: SimpleNamespace(get=lambda _idx: [0] * 100), + ) + + updates = Qwen2_5OmniThinkerMultiModalProcessor._get_prompt_updates( + fake_self, + mm_items, + {"use_audio_in_video": [True, False]}, + out_mm_kwargs, + ) + video_update = next(update for update in updates if update.modality == "video") + + assert video_update.resolve(0).content.full == [ + AUDIO_BOS_TOKEN_ID, + VIDEO_TOKEN_ID, + AUDIO_TOKEN_ID, + AUDIO_EOS_TOKEN_ID, + ] + assert video_update.resolve(1).content.full == [VIDEO_TOKEN_ID, VIDEO_TOKEN_ID] + + +def test_qwen2_5_prompt_updates_prefer_hf_second_per_grid_ts_from_outputs(): + captured = {} + fake_self = _fake_qwen2_prompt_processor() + + def omni_get_updates_use_audio_in_video(**kwargs): + captured["video_second_per_grid_t"] = kwargs["video_second_per_grid_t"] + return [VIDEO_TOKEN_ID] + + fake_self.omni_get_updates_use_audio_in_video = omni_get_updates_use_audio_in_video + out_mm_kwargs = SimpleNamespace( + get_data=lambda **_kwargs: { + "audio_feature_lengths": torch.tensor([100]), + "video_grid_thw": torch.tensor([[1, 1, 1]]), + "second_per_grid_ts": torch.tensor([0.25]), + } + ) + mm_items = SimpleNamespace( + get_all_counts=lambda: {"video": 1, "audio": 1}, + get_items=lambda *_args, **_kwargs: SimpleNamespace(get=lambda _idx: [0] * 100), + ) + + updates = Qwen2_5OmniThinkerMultiModalProcessor._get_prompt_updates( + fake_self, + mm_items, + {"use_audio_in_video": [True], "second_per_grid_ts": torch.tensor([0.5])}, + out_mm_kwargs, + ) + video_update = next(update for update in updates if update.modality == "video") + video_update.resolve(0) + + assert captured["video_second_per_grid_t"] == 0.25 + + +def test_qwen2_5_prompt_updates_accept_tensor_second_per_grid_ts_from_kwargs(): + captured = {} + fake_self = _fake_qwen2_prompt_processor() + + def omni_get_updates_use_audio_in_video(**kwargs): + captured["video_second_per_grid_t"] = kwargs["video_second_per_grid_t"] + return [VIDEO_TOKEN_ID] + + fake_self.omni_get_updates_use_audio_in_video = omni_get_updates_use_audio_in_video + out_mm_kwargs = SimpleNamespace( + get_data=lambda **_kwargs: { + "audio_feature_lengths": torch.tensor([100]), + "video_grid_thw": torch.tensor([[1, 1, 1]]), + } + ) + mm_items = SimpleNamespace( + get_all_counts=lambda: {"video": 1, "audio": 1}, + get_items=lambda *_args, **_kwargs: SimpleNamespace(get=lambda _idx: [0] * 100), + ) + + updates = Qwen2_5OmniThinkerMultiModalProcessor._get_prompt_updates( + fake_self, + mm_items, + {"use_audio_in_video": [True], "second_per_grid_ts": torch.tensor([0.75, 1.0])}, + out_mm_kwargs, + ) + video_update = next(update for update in updates if update.modality == "video") + video_update.resolve(0) + + assert captured["video_second_per_grid_t"] == 0.75 + + +def test_qwen2_5_prompt_updates_validate_mask_against_request_video_count(): + fake_self = _fake_qwen2_prompt_processor() + out_mm_kwargs = SimpleNamespace( + get_data=lambda **_kwargs: { + "video_grid_thw": torch.tensor([[1, 1, 2], [1, 1, 2]]), + } + ) + mm_items = SimpleNamespace(get_all_counts=lambda: {"video": 2}) + + with pytest.raises(ValueError, match="one boolean per video"): + Qwen2_5OmniThinkerMultiModalProcessor._get_prompt_updates( + fake_self, + mm_items, + {"use_audio_in_video": [True]}, + out_mm_kwargs, + ) + + +def test_qwen2_5_mm_only_dummy_counts_subtract_only_videos_using_audio(): + captured_counts = {} + fake_self = SimpleNamespace( + dummy_inputs=SimpleNamespace(get_dummy_text=lambda counts: captured_counts.update(counts) or "dummy"), + _apply_hf_processor_text_mm=lambda **_kwargs: ([], {}, False), + ) + mm_items = SimpleNamespace(get_all_counts=lambda: {"video": 2, "audio": 1}) + + Qwen2_5OmniThinkerMultiModalProcessor._apply_hf_processor_mm_only( + fake_self, + mm_items, + {"use_audio_in_video": [True, False]}, + {}, + ) + + assert captured_counts == {"video": 2, "audio": 0} + + +def test_get_video_use_audio_in_video_falls_back_to_prompt_updates_for_cache_hits(): + fake_self = _fake_processor() + update_with_audio = SimpleNamespace(content=SimpleNamespace(full=[VIDEO_TOKEN_ID, AUDIO_TOKEN_ID])) + update_without_audio = SimpleNamespace(content=SimpleNamespace(full=[VIDEO_TOKEN_ID])) + + result = Qwen2_5OmniThinkerMultiModalProcessor._get_video_use_audio_in_video( + fake_self, + {"video": [None, None]}, + {"video": [[update_with_audio], [update_without_audio]]}, + ) + + assert result == [True, False] + + +def test_derive_audio_from_video_placeholders_only_pairs_true_videos(): + fake_self = _fake_processor() + video_placeholders = [ + PlaceholderFeaturesInfo( + modality="video", + item_idx=0, + start_idx=5, + tokens=[VIDEO_TOKEN_ID, AUDIO_TOKEN_ID, VIDEO_TOKEN_ID], + is_embed=torch.tensor([True, True, True]), + ), + PlaceholderFeaturesInfo( + modality="video", + item_idx=1, + start_idx=12, + tokens=[VIDEO_TOKEN_ID, VIDEO_TOKEN_ID], + is_embed=torch.tensor([True, True]), + ), + ] + + result = Qwen2_5OmniThinkerMultiModalProcessor._derive_audio_from_video_placeholders( + fake_self, + {"video": video_placeholders}, + {"audio": [[object()]]}, + [True, False], + ) + + assert len(result["audio"]) == 1 + assert result["audio"][0].item_idx == 0 + assert result["audio"][0].start_idx == 5 + assert result["audio"][0].is_embed.tolist() == [False, True, False] + + assert len(result["video"]) == 2 + assert result["video"][0].item_idx == 0 + assert result["video"][0].is_embed.tolist() == [True, False, True] + assert result["video"][1].item_idx == 1 + assert result["video"][1].is_embed.tolist() == [True, True] + + +def test_qwen3_processor_inherits_vllm_omni_per_video_helpers(): + assert issubclass( + Qwen3OmniMoeThinkerMultiModalProcessor, + Qwen2_5OmniThinkerMultiModalProcessor, + ) + + +def test_qwen3_prompt_updates_prefer_hf_second_per_grid_ts_from_outputs(): + captured = {} + fake_self = _fake_qwen3_prompt_processor() + + def get_updates_use_audio_in_video(**kwargs): + captured["video_second_per_grid_t"] = kwargs["video_second_per_grid_t"] + return [VIDEO_TOKEN_ID] + + fake_self.get_updates_use_audio_in_video = get_updates_use_audio_in_video + out_mm_kwargs = SimpleNamespace( + get_data=lambda **_kwargs: { + "audio_feature_lengths": torch.tensor([100]), + "video_grid_thw": torch.tensor([[1, 1, 1]]), + "second_per_grid_ts": torch.tensor([0.25]), + } + ) + mm_items = SimpleNamespace( + get_items=lambda *_args, **_kwargs: SimpleNamespace(get=lambda _idx: [0] * 100), + ) + + updates = Qwen3OmniMoeThinkerMultiModalProcessor._get_prompt_updates( + fake_self, + mm_items, + {"use_audio_in_video": [True], "second_per_grid_ts": torch.tensor([0.5])}, + out_mm_kwargs, + ) + video_update = next(update for update in updates if update.modality == "video") + video_update.resolve(0) + + assert captured["video_second_per_grid_t"] == 0.25 + + +def test_qwen3_prompt_updates_accept_tensor_second_per_grid_ts_from_kwargs(): + captured = {} + fake_self = _fake_qwen3_prompt_processor() + + def get_updates_use_audio_in_video(**kwargs): + captured["video_second_per_grid_t"] = kwargs["video_second_per_grid_t"] + return [VIDEO_TOKEN_ID] + + fake_self.get_updates_use_audio_in_video = get_updates_use_audio_in_video + out_mm_kwargs = SimpleNamespace( + get_data=lambda **_kwargs: { + "audio_feature_lengths": torch.tensor([100]), + "video_grid_thw": torch.tensor([[1, 1, 1]]), + } + ) + mm_items = SimpleNamespace( + get_items=lambda *_args, **_kwargs: SimpleNamespace(get=lambda _idx: [0] * 100), + ) + + updates = Qwen3OmniMoeThinkerMultiModalProcessor._get_prompt_updates( + fake_self, + mm_items, + {"use_audio_in_video": [True], "second_per_grid_ts": torch.tensor([0.75, 1.0])}, + out_mm_kwargs, + ) + video_update = next(update for update in updates if update.modality == "video") + video_update.resolve(0) + + assert captured["video_second_per_grid_t"] == 0.75 + + +def test_qwen3_derive_audio_from_video_placeholders_excludes_audio_wrappers_from_embeds(): + fake_self = _fake_qwen3_processor() + video_placeholders = [ + PlaceholderFeaturesInfo( + modality="video", + item_idx=0, + start_idx=5, + tokens=[ + AUDIO_BOS_TOKEN_ID, + VIDEO_TOKEN_ID, + AUDIO_TOKEN_ID, + AUDIO_EOS_TOKEN_ID, + VIDEO_TOKEN_ID, + ], + is_embed=torch.ones(5, dtype=torch.bool), + ) + ] + + result = Qwen3OmniMoeThinkerMultiModalProcessor._derive_audio_from_video_placeholders( + fake_self, + {"video": video_placeholders}, + {"audio": [[object()]]}, + [True], + ) + + assert result["audio"][0].is_embed.tolist() == [ + False, + False, + True, + False, + False, + ] + assert result["video"][0].is_embed.tolist() == [ + False, + True, + False, + False, + True, + ] + + +def test_qwen3_mrope_positions_match_plain_video_placeholder_length(): + fake_self = SimpleNamespace( + iter_mm_features=lambda _: iter( + [ + ( + 2, + "video", + { + "grid_t": 1, + "grid_h": 2, + "grid_w": 2, + "t_factor": 1.0, + "use_audio_in_video": False, + "placeholder_len": 4, + }, + ) + ] + ) + ) + + positions, _ = Qwen3OmniMoeThinkerForConditionalGeneration.get_mrope_input_positions( + fake_self, + list(range(8)), + [], + ) + + assert positions.shape == (3, 8) + + +def test_qwen3_mrope_positions_count_single_interleaved_audio_wrappers(): + fake_self = SimpleNamespace( + iter_mm_features=lambda _: iter( + [ + ( + 2, + "video", + { + "grid_t": 1, + "grid_h": 2, + "grid_w": 2, + "t_factor": 1.0, + "use_audio_in_video": True, + "placeholder_len": 8, + }, + ) + ] + ), + _compute_interleaved_positions=lambda _start_idx, _data: ( + torch.zeros((3, 6), dtype=torch.long).numpy(), + 6, + ), + ) + + positions, _ = Qwen3OmniMoeThinkerForConditionalGeneration.get_mrope_input_positions( + fake_self, + list(range(10)), + [], + ) + + assert positions.shape == (3, 10) + + +def test_qwen3_interleaved_mrope_prefers_placeholder_length_for_audio_count(): + fake_self = SimpleNamespace( + _compute_audio_token_count=lambda _audio_feature_length: 100, + ) + data = { + "grid_t": 1, + "grid_h": 2, + "grid_w": 2, + "t_factor": 1.0, + "audio_feature_length": 999, + "placeholder_len": 8, + } + + positions, token_count = Qwen3OmniMoeThinkerForConditionalGeneration._compute_interleaved_positions( + fake_self, + 0, + data, + ) + + assert token_count == 6 + assert positions.shape == (3, 6) + + +def test_omni_model_runner_output_can_carry_kv_connector_output_only(): + marker = object() + + output = OmniModelRunnerOutput.with_kv_conn_output_only(marker) + + assert output.req_ids == [] + assert output.req_id_to_index == {} + assert output.sampled_token_ids == [] + assert output.kv_connector_output is marker diff --git a/vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py b/vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py index d0df1c2ea..03edcfa10 100644 --- a/vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py +++ b/vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py @@ -1,6 +1,7 @@ """Thin Omni wrapper: reuse upstream Qwen2.5-Omni thinker with minimal overrides.""" -from collections.abc import Iterable, Iterator, Mapping +from collections.abc import Iterable, Iterator, Mapping, Sequence +from functools import partial from typing import Any import numpy as np @@ -28,6 +29,7 @@ from vllm.model_executor.models.qwen2_5_omni_thinker import ( Qwen2_5OmniAudioFeatureInputs, Qwen2_5OmniThinkerDummyInputsBuilder, check_interleaved_audio_video, + create_qwen2_5_omni_thinker_field_factory, merge_interleaved_embeddings, ) from vllm.model_executor.models.qwen2_5_omni_thinker import ( @@ -51,6 +53,9 @@ from vllm.model_executor.models.qwen2_5_vl import ( Qwen2_5_VLVideoInputs, Qwen2_5_VLVideoPixelInputs, ) +from vllm.model_executor.models.qwen2_audio import ( + _get_feat_extract_output_lengths, +) from vllm.model_executor.models.utils import ( AutoWeightsLoader, WeightsMapper, @@ -60,12 +65,19 @@ from vllm.model_executor.models.utils import ( from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.multimodal.inputs import ( MultiModalFeatureSpec, + MultiModalFieldConfig, MultiModalKwargsItems, ) -from vllm.multimodal.parse import MultiModalDataItems, VideoProcessorItems +from vllm.multimodal.parse import AudioProcessorItems, MultiModalDataItems, VideoProcessorItems +from vllm.multimodal.processing.context import TimingContext +from vllm.multimodal.processing.inputs import ProcessorInputs from vllm.multimodal.processing.processor import ( + MultiModalProcessingInfo, MultiModalPromptUpdates, PlaceholderFeaturesInfo, + PromptReplacement, + PromptUpdate, + PromptUpdateDetails, ) from vllm.sequence import IntermediateTensors from vllm.utils.collection_utils import is_list_of @@ -80,6 +92,158 @@ except (ImportError, ModuleNotFoundError): flash_attn = None logger = init_logger(__name__) +_PER_VIDEO_USE_AUDIO_IN_VIDEO_KEY = "_vllm_omni_per_video_use_audio_in_video" + + +def _normalize_use_audio_in_video( + value: object, + num_videos: int, +) -> list[bool]: + if isinstance(value, torch.Tensor): + if value.numel() == 1: + return [bool(value.item())] * num_videos + values = value.flatten().tolist() + elif isinstance(value, np.ndarray): + if value.size == 1: + return [bool(value.item())] * num_videos + values = value.flatten().tolist() + elif isinstance(value, bool): + return [value] * num_videos + elif isinstance(value, Sequence) and not isinstance(value, (str, bytes)): + values = list(value) + else: + return [bool(value)] * num_videos + + if len(values) != num_videos: + raise ValueError( + "use_audio_in_video must be a boolean or contain one boolean per " + f"video, but found {len(values)} values for {num_videos} videos." + ) + return [bool(v) for v in values] + + +def _normalize_per_video_use_audio_in_video(value: object) -> list[bool]: + if isinstance(value, torch.Tensor): + values = value.flatten().tolist() + elif isinstance(value, np.ndarray): + values = value.flatten().tolist() + elif isinstance(value, Sequence) and not isinstance(value, (str, bytes)): + values = list(value) + else: + values = [value] + return [bool(v) for v in values] + + +def _is_per_video_use_audio_in_video(value: object) -> bool: + if isinstance(value, bool): + return False + if isinstance(value, torch.Tensor): + return value.numel() != 1 + if isinstance(value, np.ndarray): + return value.size != 1 + return isinstance(value, Sequence) and not isinstance(value, (str, bytes)) + + +def _get_request_video_use_audio_in_video( + hf_processor_mm_kwargs: Mapping[str, object], + num_videos: int, +) -> list[bool]: + value = hf_processor_mm_kwargs.get( + _PER_VIDEO_USE_AUDIO_IN_VIDEO_KEY, + hf_processor_mm_kwargs.get("use_audio_in_video", False), + ) + return _normalize_use_audio_in_video(value, num_videos) + + +def _prompt_update_has_audio_token( + mm_prompt_updates: MultiModalPromptUpdates, + item_idx: int, + audio_token_id: int, +) -> bool: + updates = mm_prompt_updates.get("video", []) + if item_idx >= len(updates): + return False + return any( + audio_token_id in update.content.full for update in updates[item_idx] if isinstance(update.content.full, list) + ) + + +def _get_video_second_per_grid_t( + out_mm_data: Mapping[str, object], + hf_processor_mm_kwargs: Mapping[str, object], + item_idx: int, + default: float, +) -> float: + second_per_grid_ts = out_mm_data.get("second_per_grid_ts") + if second_per_grid_ts is None: + second_per_grid_ts = hf_processor_mm_kwargs.get("second_per_grid_ts", None) + if second_per_grid_ts is None: + return default + + if isinstance(second_per_grid_ts, torch.Tensor): + if second_per_grid_ts.numel() == 1: + return float(second_per_grid_ts.item()) + return float(second_per_grid_ts.flatten()[item_idx].item()) + + if isinstance(second_per_grid_ts, np.ndarray): + if second_per_grid_ts.size == 1: + return float(second_per_grid_ts.item()) + return float(second_per_grid_ts.flatten()[item_idx].item()) + + if isinstance(second_per_grid_ts, Sequence) and not isinstance(second_per_grid_ts, (str, bytes)): + return float(second_per_grid_ts[item_idx]) + + return float(second_per_grid_ts) + + +def _filter_video_use_audio_in_video_for_uncached_items( + hf_processor_mm_kwargs: Mapping[str, object], + video_is_cached: Sequence[bool] | None, +) -> Mapping[str, object]: + use_audio_in_video = hf_processor_mm_kwargs.get( + _PER_VIDEO_USE_AUDIO_IN_VIDEO_KEY, + hf_processor_mm_kwargs.get("use_audio_in_video"), + ) + if video_is_cached is None or not _is_per_video_use_audio_in_video(use_audio_in_video): + return hf_processor_mm_kwargs + + mask = _normalize_use_audio_in_video(use_audio_in_video, len(video_is_cached)) + missing_mask = [use_audio for use_audio, is_cached in zip(mask, video_is_cached) if not is_cached] + + filtered_kwargs = dict(hf_processor_mm_kwargs) + filtered_kwargs["use_audio_in_video"] = missing_mask + filtered_kwargs[_PER_VIDEO_USE_AUDIO_IN_VIDEO_KEY] = missing_mask + second_per_grid_ts = hf_processor_mm_kwargs.get("second_per_grid_ts") + if ( + isinstance(second_per_grid_ts, Sequence) + and not isinstance(second_per_grid_ts, (str, bytes)) + and len(second_per_grid_ts) == len(video_is_cached) + ): + filtered_kwargs["second_per_grid_ts"] = [ + second_per_grid_t + for second_per_grid_t, is_cached in zip(second_per_grid_ts, video_is_cached) + if not is_cached + ] + return filtered_kwargs + + +def _coerce_use_audio_in_video_for_hf_processor( + mm_data: Mapping[str, object], + mm_kwargs: Mapping[str, object], +) -> Mapping[str, object]: + use_audio_in_video = mm_kwargs.get("use_audio_in_video") + if not _is_per_video_use_audio_in_video(use_audio_in_video): + return mm_kwargs + + video_use_audio_in_video = _normalize_per_video_use_audio_in_video(use_audio_in_video) + + hf_mm_kwargs = dict(mm_kwargs) + # HF processors only support a global bool. For per-video masks, vLLM + # consumes the list while building prompt updates and placeholders. + hf_mm_kwargs["use_audio_in_video"] = False + hf_mm_kwargs[_PER_VIDEO_USE_AUDIO_IN_VIDEO_KEY] = video_use_audio_in_video + return hf_mm_kwargs + def _presampled_videos_hf_kwargs( mm_data: Mapping[str, object], @@ -87,12 +251,9 @@ def _presampled_videos_hf_kwargs( ) -> Mapping[str, object]: """Adjust HF video kwargs for videos pre-sampled by vLLM's video loader. - When ``video_metadata`` is present (emitted by - ``Qwen2_5OmniVideoProcessorItems``), the frames were already sampled - according to ``media_io_kwargs``, so the HF processor must not re-sample - them, and it needs the sampled fps (instead of its default) to compute - ``video_second_per_grid`` correctly. Otherwise the audio/video temporal - alignment breaks under ``use_audio_in_video=True``. + When ``video_metadata`` is present, the frames were already sampled + according to ``media_io_kwargs``. The HF processor should not resample + them and should use the sampled fps when computing temporal metadata. """ video_metadata = mm_data.get("video_metadata") if not video_metadata: @@ -109,18 +270,17 @@ def _presampled_videos_hf_kwargs( return None return len(indices) / float(duration) - # An explicit user-provided fps takes precedence. if "fps" not in videos_kwargs: fps_values = [_compute_sampled_video_fps(m) for m in video_metadata] - # HF accepts a single fps per call and uses it for every video's - # video_second_per_grid. known_fps = [fps for fps in fps_values if fps is not None] unique_fps = set(known_fps) if len(unique_fps) == 1: videos_kwargs["fps"] = known_fps[0] elif len(unique_fps) > 1: logger.warning( - f"Mixed sampled FPS {sorted(unique_fps)} in one request; HF accepts a single fps, using {known_fps[0]}." + "Mixed sampled FPS %s in one request; HF accepts a single fps, using %s.", + sorted(unique_fps), + known_fps[0], ) videos_kwargs["fps"] = known_fps[0] @@ -129,12 +289,7 @@ def _presampled_videos_hf_kwargs( class Qwen2_5OmniVideoProcessorItems(VideoProcessorItems): - """Video items that carry the loader's ``(frames, metadata)`` tuples. - - The tuples are kept in ``data`` so that mm hashing and cache-miss - re-parsing retain the metadata; ``get_processor_data`` unpacks them into - the ``videos`` + ``video_metadata`` arguments of the HF processor. - """ + """Video items that carry the loader's ``(frames, metadata)`` tuples.""" def get_processor_data(self) -> Mapping[str, object]: from transformers.video_utils import VideoMetadata @@ -151,7 +306,6 @@ class Qwen2_5OmniThinkerMultiModalDataParser(_Qwen2_5OmniThinkerMultiModalDataPa if data is None or isinstance(data, dict) or self.is_embeddings(data): return super()._parse_video_data(data) - # Normalize to a list of per-video items (same as the base parser). if (is_list_of(data, PILImage.Image) and len(data) > 0) or ( isinstance(data, (np.ndarray, torch.Tensor)) and data.ndim == 4 ): @@ -164,11 +318,6 @@ class Qwen2_5OmniThinkerMultiModalDataParser(_Qwen2_5OmniThinkerMultiModalDataPa data_items = data videos_with_metadata = [self._get_video_with_metadata(item) for item in data_items] - - # Metadata is optional: videos fetched by vLLM's media connector - # arrive as (frames, metadata) tuples, while plain arrays (e.g. - # offline `multi_modal_data` or dummy profiling inputs) carry no - # metadata and parse as before. if any(metadata is None for _, metadata in videos_with_metadata): return super()._parse_video_data(data) @@ -195,14 +344,6 @@ class Qwen2_5OmniThinkerMultiModalProcessor( ): """Override to fix use_audio_in_video detection when mm cache returns None.""" - def _cached_apply_hf_processor(self, inputs, timing_ctx): - # If use_audio_in_video, process video and audio together; otherwise, - # skip cache to avoid errors. - # Prevents partial cache hits from causing processing failures. - if inputs.hf_processor_mm_kwargs.get("use_audio_in_video"): - return self._apply_hf_processor(inputs, timing_ctx) - return super()._cached_apply_hf_processor(inputs, timing_ctx) - def _call_hf_processor( self, prompt: str, @@ -211,12 +352,341 @@ class Qwen2_5OmniThinkerMultiModalProcessor( tok_kwargs: Mapping[str, object], ): mm_kwargs = _presampled_videos_hf_kwargs(mm_data, mm_kwargs) - return super()._call_hf_processor( + mm_kwargs = _coerce_use_audio_in_video_for_hf_processor(mm_data, mm_kwargs) + hf_inputs = super()._call_hf_processor( prompt=prompt, mm_data=mm_data, mm_kwargs=mm_kwargs, tok_kwargs=tok_kwargs, ) + per_video_mask = mm_kwargs.get(_PER_VIDEO_USE_AUDIO_IN_VIDEO_KEY) + if per_video_mask is not None: + hf_inputs["use_audio_in_video"] = torch.tensor(per_video_mask) + return hf_inputs + + def _get_mm_fields_config( + self, + hf_inputs, + hf_processor_mm_kwargs: Mapping[str, object], + ) -> Mapping[str, MultiModalFieldConfig]: + fields_config = dict( + create_qwen2_5_omni_thinker_field_factory(self.info.get_hf_config().vision_config.spatial_merge_size)( + hf_inputs + ) + ) + use_audio_in_video = hf_inputs.get("use_audio_in_video") + is_batched_mask = isinstance(use_audio_in_video, torch.Tensor) and use_audio_in_video.numel() > 1 + if _PER_VIDEO_USE_AUDIO_IN_VIDEO_KEY in hf_processor_mm_kwargs or is_batched_mask: + fields_config["use_audio_in_video"] = MultiModalFieldConfig.batched("video") + return fields_config + + def _get_video_use_audio_in_video( + self, + mm_kwargs: MultiModalKwargsItems, + mm_prompt_updates: MultiModalPromptUpdates, + ) -> list[bool]: + video_kwargs = mm_kwargs.get("video", []) + if not video_kwargs: + return [] + + audio_token_id = self.info.get_hf_config().audio_token_id + video_use_audio_in_video = [] + has_use_audio_in_video = any(item is not None and "use_audio_in_video" in item for item in video_kwargs) + for item_idx, item in enumerate(video_kwargs): + if has_use_audio_in_video: + if item is None or "use_audio_in_video" not in item: + video_use_audio_in_video.append( + _prompt_update_has_audio_token( + mm_prompt_updates, + item_idx, + audio_token_id, + ) + ) + continue + use_audio_in_video_tensor = item["use_audio_in_video"].data + if use_audio_in_video_tensor.numel() > 0: + video_use_audio_in_video.append(bool(use_audio_in_video_tensor.item())) + continue + video_use_audio_in_video.append(False) + continue + + video_use_audio_in_video.append( + _prompt_update_has_audio_token( + mm_prompt_updates, + item_idx, + audio_token_id, + ) + ) + + return video_use_audio_in_video + + def _get_prompt_updates( + self, + mm_items: MultiModalDataItems, + hf_processor_mm_kwargs: Mapping[str, Any], + out_mm_kwargs: MultiModalKwargsItems, + ) -> Sequence[PromptUpdate]: + processor = self.info.get_hf_processor(**hf_processor_mm_kwargs) + tokenizer = self.info.get_tokenizer() + image_processor = self.info.get_image_processor(**hf_processor_mm_kwargs) + vocab = tokenizer.get_vocab() + + audio_token = processor.audio_token + image_token = processor.image_token + video_token = processor.video_token + audio_token_id = vocab[audio_token] + image_token_id = vocab[image_token] + video_token_id = vocab[video_token] + + out_mm_data = out_mm_kwargs.get_data() + audio_feature_lengths = out_mm_data.get("audio_feature_lengths") + feature_attention_mask = out_mm_data.get("feature_attention_mask") + if audio_feature_lengths is None and feature_attention_mask is None: + audio_output_lengths = [] + elif audio_feature_lengths is not None: + _, audio_output_lens = _get_feat_extract_output_lengths(audio_feature_lengths) + audio_output_lengths = audio_output_lens.tolist() + elif feature_attention_mask is not None: + assert isinstance(feature_attention_mask, torch.Tensor) + _, audio_output_lens = _get_feat_extract_output_lengths(feature_attention_mask.sum(-1)) + audio_output_lengths = audio_output_lens.tolist() + + audio_in_video_item_idx = 0 + + def get_replacement_qwen2_audio(item_idx: int): + item_idx += audio_in_video_item_idx + + num_features = audio_output_lengths[item_idx] + if num_features == 0: + audios = mm_items.get_items("audio", AudioProcessorItems) + audio = audios.get(item_idx) + raise ValueError( + f"The audio {audio} (len={len(audio)}) is too short to be represented inside the model" + ) + + return [audio_token_id] * num_features + + def get_replacement_qwen2_vision(item_idx: int, modality: str): + grid_thw = out_mm_data[f"{modality}_grid_thw"][item_idx] + assert isinstance(grid_thw, torch.Tensor) + merge_length = image_processor.merge_size**2 + + token_id = image_token_id if modality == "image" else video_token_id + return [token_id] * (int(grid_thw.prod()) // merge_length) + + num_videos = mm_items.get_all_counts().get("video", 0) + video_use_audio_in_video = _get_request_video_use_audio_in_video( + hf_processor_mm_kwargs, + num_videos, + ) + thinker_config = self.info.get_hf_config() + + def get_replacement_qwen2_video(item_idx: int): + nonlocal audio_in_video_item_idx + + if not video_use_audio_in_video[item_idx]: + return get_replacement_qwen2_vision(item_idx, modality="video") + + audio_num_features = audio_output_lengths[audio_in_video_item_idx] + video_grid_thw = out_mm_data["video_grid_thw"][item_idx] + + audio_in_video_item_idx += 1 + + video_second_per_grid_t = _get_video_second_per_grid_t( + out_mm_data, + hf_processor_mm_kwargs, + item_idx, + default=1.0, + ) + + updates = self.omni_get_updates_use_audio_in_video( + thinker_config=thinker_config, + audio_len=audio_num_features, + video_grid_thw=video_grid_thw, + video_second_per_grid_t=video_second_per_grid_t, + ) + + return PromptUpdateDetails.select_token_id( + seq=updates, + embed_token_id=video_token_id, + ) + + return [ + PromptReplacement( + modality="audio", + target=audio_token, + replacement=get_replacement_qwen2_audio, + ), + PromptReplacement( + modality="image", + target=image_token, + replacement=partial(get_replacement_qwen2_vision, modality="image"), + ), + PromptReplacement( + modality="video", + target=video_token, + replacement=get_replacement_qwen2_video, + ), + ] + + def _apply_hf_processor_mm_only( + self, + mm_items: MultiModalDataItems, + hf_processor_mm_kwargs: Mapping[str, object], + tokenization_kwargs: Mapping[str, object], + ): + mm_counts = mm_items.get_all_counts() + + if "video" in mm_counts: + video_use_audio_in_video = _get_request_video_use_audio_in_video( + hf_processor_mm_kwargs, + mm_counts["video"], + ) + if any(video_use_audio_in_video): + assert "audio" in mm_counts + mm_counts["audio"] -= sum(video_use_audio_in_video) + + _, mm_processed_data, _ = self._apply_hf_processor_text_mm( + prompt_text=self.dummy_inputs.get_dummy_text(mm_counts), + mm_items=mm_items, + hf_processor_mm_kwargs=hf_processor_mm_kwargs, + tokenization_kwargs=tokenization_kwargs, + ) + + return mm_processed_data + + def _cached_apply_hf_processor( + self, + inputs: ProcessorInputs, + timing_ctx: TimingContext, + ) -> tuple[list[int], MultiModalProcessingInfo, bool]: + cache = self.cache + + _, passthrough_data = self._get_hf_mm_data(inputs.mm_data_items) + if cache is None or passthrough_data: + return self._apply_hf_processor(inputs, timing_ctx) + + with timing_ctx.record("get_mm_hashes"): + mm_hashes = inputs.get_mm_hashes(self.info.model_id) + + with timing_ctx.record("get_cache_missing_items"): + mm_is_cached, mm_missing_data_items = self._get_cache_missing_items( + cache=cache, + mm_data_items=inputs.mm_data_items, + mm_hashes=mm_hashes, + ) + + hf_processor_mm_kwargs = _filter_video_use_audio_in_video_for_uncached_items( + inputs.hf_processor_mm_kwargs, + mm_is_cached.get("video"), + ) + + with timing_ctx.record("apply_hf_processor"): + ( + prompt_ids, + mm_missing_processed_data, + is_update_applied, + ) = self._apply_hf_processor_main( + prompt=inputs.prompt, + mm_items=mm_missing_data_items, + hf_processor_mm_kwargs=hf_processor_mm_kwargs, + tokenization_kwargs=inputs.tokenization_kwargs, + enable_hf_prompt_update=False, + ) + + mm_missing_kwargs = MultiModalKwargsItems.from_hf_inputs( + mm_missing_processed_data, + self._get_mm_fields_config( + mm_missing_processed_data, + hf_processor_mm_kwargs, + ), + ) + + mm_missing_prompt_updates = self._get_mm_prompt_updates( + mm_missing_data_items, + hf_processor_mm_kwargs, + mm_missing_kwargs, + ) + + with timing_ctx.record("merge_mm_kwargs"): + mm_kwargs, mm_prompt_updates = self._merge_mm_kwargs( + cache, + mm_hashes=mm_hashes, + mm_is_cached=mm_is_cached, + mm_missing_kwargs=mm_missing_kwargs, + mm_missing_prompt_updates=mm_missing_prompt_updates, + ) + + mm_info = MultiModalProcessingInfo( + kwargs=mm_kwargs, + hashes=mm_hashes, + prompt_updates=mm_prompt_updates, + ) + + return prompt_ids, mm_info, is_update_applied + + def _derive_audio_from_video_placeholders( + self, + placeholders: Mapping[str, list[PlaceholderFeaturesInfo]], + mm_prompt_updates: MultiModalPromptUpdates, + video_use_audio_in_video: Sequence[bool] | None = None, + ) -> Mapping[str, list[PlaceholderFeaturesInfo]]: + if "video" not in placeholders: + return placeholders + + num_videos = len(placeholders["video"]) + if video_use_audio_in_video is None: + video_use_audio_in_video = [True] * num_videos + elif len(video_use_audio_in_video) != num_videos: + raise ValueError( + "use_audio_in_video must contain one boolean per video, " + f"but found {len(video_use_audio_in_video)} values for " + f"{num_videos} videos." + ) + + num_audio_in_video = sum(video_use_audio_in_video) + num_audios = len(mm_prompt_updates.get("audio", [])) + if num_audios != num_audio_in_video: + raise ValueError( + "use_audio_in_video requires equal number of audio and video " + f"items using audio, got {num_audios=}, {num_audio_in_video=}" + ) + + tokenizer = self.info.get_tokenizer() + processor = self.info.get_hf_processor() + audio_token_id = tokenizer.get_vocab()[processor.audio_token] + + result_placeholders = dict(placeholders) + audio_placeholders = [] + video_placeholders = [] + + audio_idx = 0 + for video_idx, video_placeholder in enumerate(placeholders["video"]): + audio_is_embed = torch.tensor(video_placeholder.tokens) == audio_token_id + + if video_use_audio_in_video[video_idx]: + audio_placeholder = PlaceholderFeaturesInfo( + modality="audio", + item_idx=audio_idx, + start_idx=video_placeholder.start_idx, + tokens=video_placeholder.tokens, + is_embed=audio_is_embed, + ) + audio_placeholders.append(audio_placeholder) + audio_idx += 1 + + video_placeholder_with_mask = PlaceholderFeaturesInfo( + modality="video", + item_idx=video_idx, + start_idx=video_placeholder.start_idx, + tokens=video_placeholder.tokens, + is_embed=~audio_is_embed, + ) + video_placeholders.append(video_placeholder_with_mask) + + result_placeholders["audio"] = audio_placeholders + result_placeholders["video"] = video_placeholders + return result_placeholders def _maybe_apply_prompt_updates( self, @@ -230,24 +700,8 @@ class Qwen2_5OmniThinkerMultiModalProcessor( self._validate_mm_kwargs(mm_kwargs, mm_item_counts) self._validate_mm_updates(mm_prompt_updates, mm_item_counts) - # Detect use_audio_in_video from mm_kwargs - use_audio_in_video = False - if "video" in mm_kwargs: - for item in mm_kwargs["video"]: - if item and item.get("use_audio_in_video"): - use_audio_in_video_tensor = item["use_audio_in_video"].data - if use_audio_in_video_tensor.numel() > 0: - use_audio_in_video = bool(use_audio_in_video_tensor.item()) - break - # for mutilmodality cache - if any(item is None for item in mm_kwargs["video"]): - video_token_id = self.info.get_hf_config().video_token_id - audio_token_id = self.info.get_hf_config().audio_token_id - video_audio_item_num = sum(id in (video_token_id, audio_token_id) for id in prompt_ids) - audio_updates_num = len(mm_prompt_updates.get("audio", [])) - video_updates_num = len(mm_prompt_updates.get("video", [])) - if video_audio_item_num != video_updates_num + audio_updates_num: - use_audio_in_video = True + video_use_audio_in_video = self._get_video_use_audio_in_video(mm_kwargs, mm_prompt_updates) + use_audio_in_video = any(video_use_audio_in_video) if is_update_applied: mm_placeholders = self._find_mm_placeholders( @@ -265,7 +719,11 @@ class Qwen2_5OmniThinkerMultiModalProcessor( prompt_ids, filtered_updates, ) - mm_placeholders = self._derive_audio_from_video_placeholders(mm_placeholders, mm_prompt_updates) + mm_placeholders = self._derive_audio_from_video_placeholders( + mm_placeholders, + mm_prompt_updates, + video_use_audio_in_video, + ) else: prompt_ids, mm_placeholders = self._apply_prompt_updates( prompt_ids, diff --git a/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py b/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py index 011496f86..fde493deb 100644 --- a/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py +++ b/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py @@ -121,6 +121,8 @@ from vllm_omni.model_executor.models.qwen2_5_omni.qwen2_5_omni_thinker import ( Qwen2_5OmniConditionalGenerationMixin, Qwen2_5OmniThinkerMultiModalDataParser, Qwen2_5OmniThinkerMultiModalProcessor, + _get_request_video_use_audio_in_video, + _get_video_second_per_grid_t, _presampled_videos_hf_kwargs, ) from vllm_omni.quantization.component_config import ( @@ -626,9 +628,6 @@ class Qwen3OmniMoeThinkerProcessingInfo(Qwen2AudioProcessingInfo, Qwen2_5_VLProc def get_data_parser(self): feature_extractor = self.get_feature_extractor() - # Install the Omni parser, which keeps video metadata from vLLM's - # video loader (see Qwen2_5OmniVideoProcessorItems); the parser - # inherited from Qwen2AudioProcessingInfo would drop it. return Qwen2_5OmniThinkerMultiModalDataParser( spatial_merge_size=self.get_hf_config().vision_config.spatial_merge_size, target_sr=feature_extractor.sampling_rate, @@ -686,10 +685,8 @@ class Qwen3OmniMoeThinkerMultiModalProcessor( tok_kwargs: Mapping[str, object], ) -> BatchFeature: mm_data = dict(mm_data) - mm_kwargs = dict(mm_kwargs) - audios = mm_data.pop("audios", []) - mm_kwargs = dict(_presampled_videos_hf_kwargs(mm_data, mm_kwargs)) + audios = mm_data.pop("audios", []) def pad_to_hop_length(x: np.ndarray, hop_length: int) -> np.ndarray: length = x.shape[-1] @@ -787,22 +784,8 @@ class Qwen3OmniMoeThinkerMultiModalProcessor( mm_item_counts = mm_items.get_all_counts() self._validate_mm_kwargs(mm_kwargs, mm_item_counts) - use_audio_in_video = False - if "video" in mm_kwargs: - for item in mm_kwargs["video"]: - if item and item["use_audio_in_video"].data: - use_audio_in_video = True - else: - use_audio_in_video = False - # for mutilmodality cache - if any(item is None for item in mm_kwargs["video"]): - video_token_id = self.info.get_hf_config().video_token_id - audio_token_id = self.info.get_hf_config().audio_token_id - video_audio_item_num = sum(id in (video_token_id, audio_token_id) for id in prompt_ids) - audio_updates_num = len(mm_prompt_updates.get("audio", [])) - video_updates_num = len(mm_prompt_updates.get("video", [])) - if video_audio_item_num != video_updates_num + audio_updates_num: - use_audio_in_video = True + video_use_audio_in_video = self._get_video_use_audio_in_video(mm_kwargs, mm_prompt_updates) + use_audio_in_video = any(video_use_audio_in_video) # normal case with `use_audio_in_video=False` if is_update_applied: @@ -821,7 +804,11 @@ class Qwen3OmniMoeThinkerMultiModalProcessor( prompt_ids, filtered_updates, ) - mm_placeholders = self._derive_audio_from_video_placeholders(mm_placeholders, mm_prompt_updates) + mm_placeholders = self._derive_audio_from_video_placeholders( + mm_placeholders, + mm_prompt_updates, + video_use_audio_in_video, + ) else: prompt_ids, mm_placeholders = self._apply_prompt_updates( prompt_ids, @@ -935,25 +922,30 @@ class Qwen3OmniMoeThinkerMultiModalProcessor( token_id = image_token_id if modality == "image" else video_token_id return [token_id] * (int(grid_thw.prod()) // merge_length) - use_audio_in_video = hf_processor_mm_kwargs.get("use_audio_in_video", False) + num_videos = len(out_mm_data.get("video_grid_thw", [])) + video_use_audio_in_video = _get_request_video_use_audio_in_video( + hf_processor_mm_kwargs, + num_videos, + ) thinker_config = self.info.get_hf_config() - def get_replacement_qwen2_use_audio_in_video(item_idx: int): + def get_replacement_qwen2_video(item_idx: int): nonlocal audio_in_video_item_idx + + if not video_use_audio_in_video[item_idx]: + return get_replacement_qwen2_vision(item_idx, modality="video") + audio_num_features = audio_output_lengths[audio_in_video_item_idx] video_grid_thw = out_mm_data["video_grid_thw"][item_idx] audio_in_video_item_idx += 1 - # Prefer the HF-computed value (temporal_patch_size / sampled fps) - # over request kwargs; - second_per_grid_ts = out_mm_data.get("second_per_grid_ts") - if second_per_grid_ts is None: - second_per_grid_ts = hf_processor_mm_kwargs.get("second_per_grid_ts", None) - if second_per_grid_ts is not None: - video_second_per_grid_t = float(second_per_grid_ts[item_idx]) - else: - video_second_per_grid_t = 2.0 + video_second_per_grid_t = _get_video_second_per_grid_t( + out_mm_data, + hf_processor_mm_kwargs, + item_idx, + default=2.0, + ) placeholder = self.get_updates_use_audio_in_video( thinker_config=thinker_config, @@ -963,12 +955,6 @@ class Qwen3OmniMoeThinkerMultiModalProcessor( ) return PromptUpdateDetails.select_token_id(placeholder, embed_token_id=video_token_id) - video_replacement_fn = ( - get_replacement_qwen2_use_audio_in_video - if use_audio_in_video - else partial(get_replacement_qwen2_vision, modality="video") - ) - return [ PromptReplacement( modality="audio", @@ -983,7 +969,7 @@ class Qwen3OmniMoeThinkerMultiModalProcessor( PromptReplacement( modality="video", target=video_token, - replacement=video_replacement_fn, + replacement=get_replacement_qwen2_video, ), ] @@ -991,6 +977,7 @@ class Qwen3OmniMoeThinkerMultiModalProcessor( self, placeholders: Mapping[str, list[PlaceholderFeaturesInfo]], mm_prompt_updates: MultiModalPromptUpdates, + video_use_audio_in_video: Sequence[bool] | None = None, ) -> Mapping[str, list[PlaceholderFeaturesInfo]]: """ Helper to derive audio placeholders from video placeholders when @@ -1000,34 +987,62 @@ class Qwen3OmniMoeThinkerMultiModalProcessor( return placeholders num_videos = len(placeholders["video"]) - num_audios = len(mm_prompt_updates.get("audio", [])) - if num_audios != num_videos: + if video_use_audio_in_video is None: + video_use_audio_in_video = [True] * num_videos + elif len(video_use_audio_in_video) != num_videos: raise ValueError( - f"use_audio_in_video requires equal number of audio and video items, got {num_audios=}, {num_videos=}" + "use_audio_in_video must contain one boolean per video, " + f"but found {len(video_use_audio_in_video)} values for " + f"{num_videos} videos." + ) + + num_audio_in_video = sum(video_use_audio_in_video) + num_audios = len(mm_prompt_updates.get("audio", [])) + if num_audios != num_audio_in_video: + raise ValueError( + "use_audio_in_video requires equal number of audio and video " + f"items using audio, got {num_audios=}, {num_audio_in_video=}" ) tokenizer = self.info.get_tokenizer() processor = self.info.get_hf_processor() audio_token_id = tokenizer.get_vocab()[processor.audio_token] + video_token_id = tokenizer.get_vocab()[processor.video_token] result_placeholders = dict(placeholders) audio_placeholders = [] + video_placeholders = [] - # Each video is paired with one audio + # Each video using audio is paired with one audio + audio_idx = 0 for video_idx, video_placeholder in enumerate(placeholders["video"]): # Create is_embed mask selecting only audio tokens - audio_is_embed = torch.tensor(video_placeholder.tokens) == audio_token_id + placeholder_tokens = torch.tensor(video_placeholder.tokens) + audio_is_embed = placeholder_tokens == audio_token_id + video_is_embed = placeholder_tokens == video_token_id - audio_placeholder = PlaceholderFeaturesInfo( - modality="audio", + if video_use_audio_in_video[video_idx]: + audio_placeholder = PlaceholderFeaturesInfo( + modality="audio", + item_idx=audio_idx, + start_idx=video_placeholder.start_idx, + tokens=video_placeholder.tokens, + is_embed=audio_is_embed, + ) + audio_placeholders.append(audio_placeholder) + audio_idx += 1 + + video_placeholder_with_mask = PlaceholderFeaturesInfo( + modality="video", item_idx=video_idx, start_idx=video_placeholder.start_idx, tokens=video_placeholder.tokens, - is_embed=audio_is_embed, + is_embed=video_is_embed, ) - audio_placeholders.append(audio_placeholder) + video_placeholders.append(video_placeholder_with_mask) result_placeholders["audio"] = audio_placeholders + result_placeholders["video"] = video_placeholders return result_placeholders def _get_raw_input_ids( @@ -1529,10 +1544,12 @@ class Qwen3OmniMoeThinkerForConditionalGeneration( Iterate over multimodal features sorted by position offset. Yields: (offset, modality, feature_data) where feature_data contains: - - image: {"grid_t", "grid_h", "grid_w", "t_factor"} + - image: {"grid_t", "grid_h", "grid_w", "t_factor", + "placeholder_len"} - video: {"grid_t", "grid_h", "grid_w", "t_factor", - "use_audio_in_video", "audio_feature_length"} - - audio: {"audio_feature_length"} + "use_audio_in_video", "audio_feature_length", + "placeholder_len"} + - audio: {"audio_feature_length", "placeholder_len"} """ config = self.config spatial_merge_size = config.vision_config.spatial_merge_size @@ -1555,6 +1572,7 @@ class Qwen3OmniMoeThinkerForConditionalGeneration( "grid_h": h // spatial_merge_size, "grid_w": w // spatial_merge_size, "t_factor": position_id_per_seconds, + "placeholder_len": mm_feature.mm_position.length, }, ) elif modality == "video": @@ -1576,12 +1594,20 @@ class Qwen3OmniMoeThinkerForConditionalGeneration( "t_factor": second_per_grid_ts * position_id_per_seconds, "use_audio_in_video": use_audio_in_video, "audio_feature_length": audio_for_video.get(offset), + "placeholder_len": mm_feature.mm_position.length, }, ) elif modality == "audio": if offset not in paired_audio_offsets: audio_len = mm_feature.data["audio_feature_lengths"].data.item() - yield offset, "audio", {"audio_feature_length": audio_len} + yield ( + offset, + "audio", + { + "audio_feature_length": audio_len, + "placeholder_len": mm_feature.mm_position.length, + }, + ) def _compute_interleaved_positions(self, start_idx: int, data: dict[str, Any]) -> tuple[np.ndarray, int]: """ @@ -1594,9 +1620,13 @@ class Qwen3OmniMoeThinkerForConditionalGeneration( grid_h = data["grid_h"] grid_w = data["grid_w"] t_factor = data["t_factor"] - audio_feature_length = data["audio_feature_length"] - - audio_len = self._compute_audio_token_count(audio_feature_length) + num_video = grid_t * grid_h * grid_w + placeholder_len = data.get("placeholder_len") + if placeholder_len is None: + audio_feature_length = data["audio_feature_length"] + audio_len = self._compute_audio_token_count(audio_feature_length) + else: + audio_len = max(0, int(placeholder_len) - num_video - 2) h_index = np.tile(np.arange(grid_h).reshape(1, -1, 1), (grid_t, 1, grid_w)).flatten() w_index = np.tile(np.arange(grid_w).reshape(1, 1, -1), (grid_t, grid_h, 1)).flatten() @@ -1612,8 +1642,6 @@ class Qwen3OmniMoeThinkerForConditionalGeneration( pos_ids_list: list[np.ndarray] = [] video_idx, audio_idx = 0, 0 - num_video = grid_t * grid_h * grid_w - while video_idx < num_video and audio_idx < audio_len: if video_t_values[video_idx] <= audio_t_values[audio_idx]: pos_ids_list.append(video_pos[:, video_idx : video_idx + 1]) @@ -1703,9 +1731,18 @@ class Qwen3OmniMoeThinkerForConditionalGeneration( seq_len = len(input_tokens) llm_pos_ids_list: list[np.ndarray] = [] + feature_summaries: list[dict[str, Any]] = [] st = 0 for offset, modality, data in self.iter_mm_features(mm_features): + feature_summaries.append( + { + "offset": offset, + "modality": modality, + "placeholder_len": data.get("placeholder_len"), + "use_audio_in_video": data.get("use_audio_in_video"), + } + ) text_len = offset - st st_idx = int(llm_pos_ids_list[-1].max()) + 1 if llm_pos_ids_list else 0 @@ -1713,19 +1750,11 @@ class Qwen3OmniMoeThinkerForConditionalGeneration( llm_pos_ids_list.append(np.broadcast_to(np.arange(text_len), (3, text_len)) + st_idx) st_idx += text_len - bos_pos = np.broadcast_to(np.array([st_idx]), (3, 1)) - llm_pos_ids_list.append(bos_pos) - st_idx += 1 - if modality == "audio": audio_tokens = self._compute_audio_token_count(data["audio_feature_length"]) audio_pos = np.broadcast_to(np.arange(audio_tokens), (3, audio_tokens)) + st_idx llm_pos_ids_list.append(audio_pos) - st_idx = int(audio_pos.max()) + 1 - - eos_pos = np.broadcast_to(np.array([st_idx]), (3, 1)) - llm_pos_ids_list.append(eos_pos) - st = offset + 1 + audio_tokens + 1 + st = offset + int(data.get("placeholder_len", audio_tokens)) elif modality == "image": grid_t = data["grid_t"] @@ -1739,11 +1768,7 @@ class Qwen3OmniMoeThinkerForConditionalGeneration( llm_pos_ids_list.append(grid_indices.reshape(3, -1) + st_idx) image_len = grid_t * grid_h * grid_w - st_idx = int(llm_pos_ids_list[-1].max()) + 1 - - eos_pos = np.broadcast_to(np.array([st_idx]), (3, 1)) - llm_pos_ids_list.append(eos_pos) - st = offset + 1 + image_len + 1 + st = offset + int(data.get("placeholder_len", image_len)) elif modality == "video": grid_t = data["grid_t"] @@ -1758,26 +1783,20 @@ class Qwen3OmniMoeThinkerForConditionalGeneration( llm_pos_ids_list.append(grid_indices.reshape(3, -1) + st_idx) video_len = grid_t * grid_h * grid_w - st_idx = int(llm_pos_ids_list[-1].max()) + 1 - - eos_pos = np.broadcast_to(np.array([st_idx]), (3, 1)) - llm_pos_ids_list.append(eos_pos) - st = offset + 1 + video_len + 1 + st = offset + int(data.get("placeholder_len", video_len)) else: - audio_bos_pos = np.broadcast_to(np.array([st_idx - 1]), (3, 1)) + audio_bos_pos = np.broadcast_to(np.array([st_idx]), (3, 1)) llm_pos_ids_list.append(audio_bos_pos) + st_idx += 1 - pos_ids, _ = self._compute_interleaved_positions(st_idx, data) + pos_ids, token_count = self._compute_interleaved_positions(st_idx, data) llm_pos_ids_list.append(pos_ids) st_idx = int(pos_ids.max()) + 1 eos_pos = np.broadcast_to(np.array([st_idx]), (3, 1)) llm_pos_ids_list.append(eos_pos) - llm_pos_ids_list.append(eos_pos) - video_len = grid_t * grid_h * grid_w - audio_len = self._compute_audio_token_count(data["audio_feature_length"]) - st = offset + 2 + video_len + audio_len + 2 + st = offset + int(data.get("placeholder_len", token_count + 2)) if st < seq_len: st_idx = int(llm_pos_ids_list[-1].max()) + 1 if llm_pos_ids_list else 0 @@ -1786,7 +1805,11 @@ class Qwen3OmniMoeThinkerForConditionalGeneration( llm_positions = np.concatenate(llm_pos_ids_list, axis=1).reshape(3, -1) if llm_positions.shape[1] != seq_len: - raise RuntimeError("Position ids length mismatch with input ids length") + raise RuntimeError( + "Position ids length mismatch with input ids length: " + f"positions={llm_positions.shape[1]}, input_ids={seq_len}, " + f"features={feature_summaries}" + ) mrope_position_delta = int(llm_positions.max()) + 1 - seq_len return torch.from_numpy(llm_positions), mrope_position_delta diff --git a/vllm_omni/outputs/__init__.py b/vllm_omni/outputs/__init__.py index 2f2a6f2ee..34731eeca 100644 --- a/vllm_omni/outputs/__init__.py +++ b/vllm_omni/outputs/__init__.py @@ -58,6 +58,18 @@ class OmniModelRunnerOutput(ModelRunnerOutput): kv_extracted_req_ids: list[str] | None = None omni_connector_output: OmniConnectorOutput | None = None + @classmethod + def with_kv_conn_output_only(cls, kv_connector_output: Any) -> "OmniModelRunnerOutput": + return cls( + req_ids=[], + req_id_to_index={}, + sampled_token_ids=[], + logprobs=None, + prompt_logprobs_dict={}, + pooler_output=[], + kv_connector_output=kv_connector_output, + ) + @dataclass class OmniRequestOutput: From 32dddfcef41fcd6bf0cb2935e566fd2278a4a15d Mon Sep 17 00:00:00 2001 From: boatman <109857087+sphinxkkkbc@users.noreply.github.com> Date: Mon, 20 Jul 2026 14:03:32 +0800 Subject: [PATCH 024/240] [Cleanup]Migrate model validation to adapters and clean up serving_speech.py (#4833) Signed-off-by: boatman <109857087+sphinxkkkbc@users.noreply.github.com> Co-authored-by: Yueqian Lin <70319226+linyueqian@users.noreply.github.com> --- .../test_higgs_audio_v2_expansion.py | 2 +- .../openai_api/test_serving_speech.py | 63 ++- .../entrypoints/openai/serving_speech.py | 518 +----------------- .../entrypoints/openai/tts_adapters/base.py | 4 + .../openai/tts_adapters/cosyvoice3.py | 26 +- .../openai/tts_adapters/covo_audio.py | 5 + .../openai/tts_adapters/fish_speech.py | 22 +- .../openai/tts_adapters/glm_tts.py | 22 +- .../openai/tts_adapters/higgs_audio_v2.py | 70 ++- .../openai/tts_adapters/higgs_audio_v3.py | 14 +- .../openai/tts_adapters/ming_tts.py | 91 ++- .../openai/tts_adapters/moss_tts.py | 63 ++- .../openai/tts_adapters/qwen3_tts.py | 145 ++++- .../openai/tts_adapters/step_audio2.py | 5 +- .../openai/tts_adapters/voxcpm2.py | 20 +- .../openai/tts_adapters/voxtral.py | 27 +- 16 files changed, 547 insertions(+), 550 deletions(-) diff --git a/tests/e2e/online_serving/test_higgs_audio_v2_expansion.py b/tests/e2e/online_serving/test_higgs_audio_v2_expansion.py index e7371fe17..e564baf9d 100644 --- a/tests/e2e/online_serving/test_higgs_audio_v2_expansion.py +++ b/tests/e2e/online_serving/test_higgs_audio_v2_expansion.py @@ -6,7 +6,7 @@ E2E Online expansion tests for higgs-audio v2 against /v1/audio/speech. v1 scope is plain text -> 24 kHz speech plus shallow voice clone via ref_audio + ref_text (inline) or voice= (after POST /v1/audio/voices). The model-aware request validator -(vllm_omni/entrypoints/openai/serving_speech.py::_validate_higgs_audio_v2_request) +(vllm_omni/entrypoints/openai/tts_adapters/higgs_audio_v2::validate) rejects multi-speaker tags, language overrides, task_type, and bare voice= for names that do not match an uploaded speaker — this suite exercises both the happy path (plain text in, audio bytes out) and the diff --git a/tests/entrypoints/openai_api/test_serving_speech.py b/tests/entrypoints/openai_api/test_serving_speech.py index 904e5735c..26a66450c 100644 --- a/tests/entrypoints/openai_api/test_serving_speech.py +++ b/tests/entrypoints/openai_api/test_serving_speech.py @@ -1549,7 +1549,7 @@ class TestTTSMethods: req = OpenAICreateSpeechRequest.model_validate({"input": "Hello", "speaker": "UTESF", "task_type": "Base"}) assert req.voice == "UTESF" mocker.patch("pathlib.Path.exists", return_value=True) - result = speech_server._validate_qwen_tts_request(req) + result = speech_server._validate_tts_request(req) assert result is None # ── uploaded voice with embedding ── @@ -1684,7 +1684,7 @@ class TestTTSMethods: # Validation should pass (file exists) mocker.patch("pathlib.Path.exists", return_value=True) - err = speech_server._validate_qwen_tts_request(req) + err = speech_server._validate_tts_request(req) assert err is None, f"Validation failed: {err}" # Build params should auto-set ref_audio from stored file @@ -1722,7 +1722,7 @@ class TestTTSMethods: # Validation should pass mocker.patch("pathlib.Path.exists", return_value=True) - err = speech_server._validate_qwen_tts_request(req) + err = speech_server._validate_tts_request(req) assert err is None, f"Validation failed: {err}" # Build params should use embedding, NOT audio @@ -1866,6 +1866,35 @@ class TestTTSMethods: assert error is not None assert "max 10 characters" in error + def test_batch_voice_not_treated_as_inline_ref_audio( + self, + speech_server, + mocker: MockerFixture, + ): + speech_server._check_model = mocker.AsyncMock(return_value=None) + speech_server._batch_max_items = 32 + + def validate_and_resolve_uploaded_voice(request): + assert request.voice == "alice" + assert request.ref_audio is None + + request.ref_audio = "data:audio/wav;base64,dGVzdA==" + request.ref_text = "reference text" + return None + + speech_server._validate_tts_request = mocker.Mock(side_effect=validate_and_resolve_uploaded_voice) + speech_server._generate_audio_bytes = mocker.AsyncMock(return_value=("YWJj", "audio/wav")) + + batch = BatchSpeechRequest(voice="alice", items=[SpeechBatchItem(input="hello")]) + + response = asyncio.run(speech_server.create_speech_batch(batch)) + call = speech_server._generate_audio_bytes.await_args + request = call.args[0] + + assert response.results[0].status == "success" + assert request.ref_audio is not None + assert call.kwargs["has_inline_ref_audio"] is False + class TestFileValidationFunctions: """Unit tests for file validation helper functions.""" @@ -3221,25 +3250,25 @@ class TestCosyVoice3Serving: def test_validate_cosyvoice3_empty_input(self, cosyvoice3_server): request = OpenAICreateSpeechRequest(input="", ref_audio="data:audio/wav;base64,abc", ref_text="hello") - error = cosyvoice3_server._validate_cosyvoice3_request(request) + error = cosyvoice3_server._validate_tts_request(request) assert error is not None assert "empty" in error.lower() def test_validate_cosyvoice3_missing_ref_audio(self, cosyvoice3_server): request = OpenAICreateSpeechRequest(input="Hello", ref_text="hello") - error = cosyvoice3_server._validate_cosyvoice3_request(request) + error = cosyvoice3_server._validate_tts_request(request) assert error is not None assert "ref_audio" in error.lower() def test_validate_cosyvoice3_missing_ref_text(self, cosyvoice3_server): request = OpenAICreateSpeechRequest(input="Hello", ref_audio="data:audio/wav;base64,abc") - error = cosyvoice3_server._validate_cosyvoice3_request(request) + error = cosyvoice3_server._validate_tts_request(request) assert error is not None assert "ref_text" in error.lower() def test_validate_cosyvoice3_invalid_ref_audio_format(self, cosyvoice3_server): request = OpenAICreateSpeechRequest(input="Hello", ref_audio="/local/path.wav", ref_text="hello") - error = cosyvoice3_server._validate_cosyvoice3_request(request) + error = cosyvoice3_server._validate_tts_request(request) assert error is not None assert "url" in error.lower() or "format" in error.lower() @@ -3249,7 +3278,7 @@ class TestCosyVoice3Serving: ref_audio="data:audio/wav;base64,abc123", ref_text="Reference transcript", ) - error = cosyvoice3_server._validate_cosyvoice3_request(request) + error = cosyvoice3_server._validate_tts_request(request) assert error is None def test_validate_cosyvoice3_max_new_tokens_range(self, cosyvoice3_server): @@ -3259,7 +3288,7 @@ class TestCosyVoice3Serving: ref_text="hello", max_new_tokens=0, ) - error = cosyvoice3_server._validate_cosyvoice3_request(request) + error = cosyvoice3_server._validate_tts_request(request) assert error is not None assert "max_new_tokens" in error @@ -3325,13 +3354,13 @@ def glm_tts_server(mocker: MockerFixture): class TestGLMTTSServing: def test_validate_glm_tts_requires_ref_audio(self, glm_tts_server): request = OpenAICreateSpeechRequest(input="Hello", ref_text="Reference transcript") - error = glm_tts_server._validate_glm_tts_request(request) + error = glm_tts_server._validate_tts_request(request) assert error is not None assert "ref_audio" in error def test_validate_glm_tts_requires_ref_text(self, glm_tts_server): request = OpenAICreateSpeechRequest(input="Hello", ref_audio="data:audio/wav;base64,abc") - error = glm_tts_server._validate_glm_tts_request(request) + error = glm_tts_server._validate_tts_request(request) assert error is not None assert "ref_text" in error @@ -3438,7 +3467,7 @@ class TestTTSAsyncOffloading: def test_prepare_speech_generation_awaits_qwen3_tts_async(self, qwen3_tts_server, mocker: MockerFixture): """Qwen3 TTS path should call _estimate_prompt_len_async.""" - qwen3_tts_server._validate_tts_request = mocker.MagicMock(return_value=None) + qwen3_tts_server._adapter.validate = mocker.MagicMock(return_value=None) qwen3_tts_server._build_tts_params = mocker.MagicMock( return_value={"text": ["hello"], "task_type": ["CustomVoice"], "speaker": ["Vivian"]} ) @@ -3455,7 +3484,7 @@ class TestTTSAsyncOffloading: qwen3_tts_server.engine_client.default_sampling_params_list = [ SimpleNamespace(max_tokens=2048, seed=42, extra_args=None) ] - qwen3_tts_server._validate_tts_request = mocker.MagicMock(return_value=None) + qwen3_tts_server._adapter.validate = mocker.MagicMock(return_value=None) qwen3_tts_server._build_tts_params = mocker.MagicMock( return_value={"text": ["hello"], "task_type": ["CustomVoice"], "speaker": ["Vivian"]} ) @@ -3505,7 +3534,7 @@ class TestTTSAsyncOffloading: def test_prepare_speech_generation_treats_sse_as_streaming(self, qwen3_tts_server, mocker: MockerFixture): """stream_format=sse should request delta-style multimodal outputs.""" - qwen3_tts_server._validate_tts_request = mocker.MagicMock(return_value=None) + qwen3_tts_server._adapter.validate = mocker.MagicMock(return_value=None) qwen3_tts_server._build_tts_params = mocker.MagicMock( return_value={"text": ["hello"], "task_type": ["CustomVoice"], "speaker": ["Vivian"]} ) @@ -3522,7 +3551,7 @@ class TestTTSAsyncOffloading: def test_prepare_speech_generation_treats_stream_true_as_streaming(self, qwen3_tts_server, mocker: MockerFixture): """stream=True should request delta-style multimodal outputs for SSE streaming.""" - qwen3_tts_server._validate_tts_request = mocker.MagicMock(return_value=None) + qwen3_tts_server._adapter.validate = mocker.MagicMock(return_value=None) qwen3_tts_server._build_tts_params = mocker.MagicMock( return_value={"text": ["hello"], "task_type": ["CustomVoice"], "speaker": ["Vivian"]} ) @@ -3539,7 +3568,7 @@ class TestTTSAsyncOffloading: def test_prepare_speech_generation_treats_audio_as_streaming(self, qwen3_tts_server, mocker: MockerFixture): """stream_format=audio should request delta-style multimodal outputs.""" - qwen3_tts_server._validate_tts_request = mocker.MagicMock(return_value=None) + qwen3_tts_server._adapter.validate = mocker.MagicMock(return_value=None) qwen3_tts_server._build_tts_params = mocker.MagicMock( return_value={"text": ["hello"], "task_type": ["CustomVoice"], "speaker": ["Vivian"]} ) @@ -3559,7 +3588,7 @@ class TestTTSAsyncOffloading: ): """Full-payload TTS streaming should not request delta multimodal outputs.""" qwen3_tts_server.engine_client.model_config.async_chunk = False - qwen3_tts_server._validate_tts_request = mocker.MagicMock(return_value=None) + qwen3_tts_server._adapter.validate = mocker.MagicMock(return_value=None) qwen3_tts_server._build_tts_params = mocker.MagicMock( return_value={"text": ["hello"], "task_type": ["CustomVoice"], "speaker": ["Vivian"]} ) diff --git a/vllm_omni/entrypoints/openai/serving_speech.py b/vllm_omni/entrypoints/openai/serving_speech.py index 3db6b63a6..a935157e1 100644 --- a/vllm_omni/entrypoints/openai/serving_speech.py +++ b/vllm_omni/entrypoints/openai/serving_speech.py @@ -150,7 +150,6 @@ _HIGGS_V3_REF_CODE_CACHE_MAX_ENTRIES = 256 _HIGGS_V3_REF_CODE_CACHE_MAX_BYTES = 64 * 1024 * 1024 _QWEN3_TTS_REF_AUDIO_CACHE_KEY = "_qwen3_tts_ref_audio_cache_key" _TTS_MAX_INSTRUCTIONS_LENGTH = 500 -_TTS_MAX_NEW_TOKENS_MIN = 1 _TTS_MAX_NEW_TOKENS_MAX = 4096 _MING_DEFAULT_PROMPT = MING_DEFAULT_PROMPT @@ -1490,48 +1489,22 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): def _validate_tts_request(self, request: OpenAICreateSpeechRequest) -> str | None: """Validate TTS request parameters. Returns error message or None.""" - if self._tts_model_type == "step_audio2": - # StepAudio2 only requires non-empty input text - if not request.input or not request.input.strip(): - return "Input text cannot be empty" - return None - if self._tts_model_type == "voxtral_tts": - return self._validate_voxtral_tts_request(request) - if self._tts_model_type == "fish_tts": - return self._validate_fish_tts_request(request) - if self._tts_model_type == "cosyvoice3": - return self._validate_cosyvoice3_request(request) - if self._tts_model_type == "voxcpm2": - return self._validate_voxcpm2_request(request) if self._tts_model_type == "ming_flash_omni_tts": return self._validate_ming_flash_omni_tts_request(request) - if self._tts_model_type == "ming_tts": - return self._validate_ming_tts_request(request) - if self._tts_model_type in ("moss_tts_nano", "moss_tts"): - return self._validate_moss_tts_request(request) - if self._tts_model_type == "glm_tts": - return self._validate_glm_tts_request(request) - return self._validate_qwen_tts_request(request) - def _validate_voxcpm2_request(self, request: OpenAICreateSpeechRequest) -> str | None: - """Validate VoxCPM2 request parameters. Returns error message or None.""" - if not request.input or not request.input.strip(): - return "Input text cannot be empty" + adapter = self._get_tts_adapter() + if adapter is not None: + return adapter.validate(request) - if request.voice is not None: - request.voice = request.voice.lower() - available_voices = set(self.uploaded_speakers) | set(self.precomputed_speakers) | {"default"} - if request.voice not in available_voices: - supported = ", ".join(sorted(available_voices)) or "none" - return f"Invalid voice '{request.voice}'. Supported: {supported}" + adapter_cls = resolve_adapter("qwen3_tts") + if adapter_cls is None: + raise ValueError("Qwen3-TTS adapter is not registered") - if request.max_new_tokens is not None: - if request.max_new_tokens < _TTS_MAX_NEW_TOKENS_MIN: - return f"max_new_tokens must be at least {_TTS_MAX_NEW_TOKENS_MIN}" - if request.max_new_tokens > _TTS_MAX_NEW_TOKENS_MAX: - return f"max_new_tokens cannot exceed {_TTS_MAX_NEW_TOKENS_MAX}" - - return None + ctx = SpeechServingContext( + server=self, + engine_client=self.engine_client, + ) + return adapter_cls(ctx).validate(request) def _voxcpm2_encode(self, text: str) -> list[int]: """Tokenize text for VoxCPM2, splitting multichar Chinese tokens.""" @@ -1599,158 +1572,6 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): return "ref_audio must be a URL (http/https), base64 data URL (data:...), or file URI (file://...)" return None - def _validate_voxtral_tts_request(self, request: OpenAICreateSpeechRequest) -> str | None: - """Validate Voxtral TTS request parameters. Returns error message or None.""" - if not request.input or not request.input.strip(): - return "Input text cannot be empty" - - # Voxtral TTS requires either a preset voice or ref_audio for voice cloning. - if request.voice is None and request.ref_audio is None: - return "Either 'voice' (preset speaker) or 'ref_audio' (voice cloning) must be provided" - - if request.ref_audio is not None: - fmt_err = self._validate_ref_audio_format(request.ref_audio) - if fmt_err: - return fmt_err - - if request.voice is not None: - request.voice = request.voice.lower() - if self.supported_speakers and request.voice not in self.supported_speakers: - return f"Invalid speaker '{request.voice}'. Supported: {', '.join(sorted(self.supported_speakers))}" - - if request.max_new_tokens is not None: - if request.max_new_tokens < _TTS_MAX_NEW_TOKENS_MIN: - return f"max_new_tokens must be at least {_TTS_MAX_NEW_TOKENS_MIN}" - if request.max_new_tokens > _TTS_MAX_NEW_TOKENS_MAX: - return f"max_new_tokens cannot exceed {_TTS_MAX_NEW_TOKENS_MAX}" - - return None - - def _validate_qwen_tts_request(self, request: OpenAICreateSpeechRequest) -> str | None: - """Validate Qwen TTS request parameters. Returns error message or None.""" - # Infer Base task when ref_audio or ref_text is provided without explicit task_type. - if request.task_type is None and (request.ref_audio is not None or request.ref_text is not None): - request.task_type = "Base" - - # Normalize voice to lowercase for case-insensitive matching - if request.voice is not None: - request.voice = request.voice.lower() - if request.task_type is None and request.voice in self.precomputed_speakers: - request.task_type = "Base" - task_type = request.task_type or "CustomVoice" - - # Validate input is not empty - if not request.input or not request.input.strip(): - return "Input text cannot be empty" - - # Validate language (case-insensitive; normalized to the title-cased config form) - if request.language is not None: - request.language = request.language.title() - if request.language not in self.supported_languages: - return ( - f"Invalid language '{request.language}'. Supported: {', '.join(sorted(self.supported_languages))}" - ) - - # Validate speaker for CustomVoice task - if task_type == "CustomVoice": - if not self.supported_speakers: - return ( - "This model does not support CustomVoice task (no speakers configured). " - "Use task_type='Base' with ref_audio/ref_text for voice cloning, " - "or use a CustomVoice model." - ) - if request.voice is not None and request.voice not in self.supported_speakers: - return f"Invalid voice '{request.voice}'. Supported: {', '.join(sorted(self.supported_speakers))}" - - # Validate speaker_embedding constraints - if request.speaker_embedding is not None: - if task_type != "Base": - return "'speaker_embedding' is only valid for Base task" - if not request.speaker_embedding: - return "'speaker_embedding' must be a non-empty list of floats" - # speaker_embedding implies x_vector_only_mode — set it before - # Base task validation so callers don't need to pass it explicitly. - request.x_vector_only_mode = True - emb_len = len(request.speaker_embedding) - dim_err = self._validate_qwen_tts_speaker_embedding_dim(emb_len) - if dim_err is not None: - return dim_err - # Validate Base task requirements - if task_type == "Base": - if request.voice is None: - # 1. Ensure a voice source is provided - if request.ref_audio is None and getattr(request, "speaker_embedding", None) is None: - return "Base task requires 'ref_audio' or 'speaker_embedding' for voice cloning" - # 2. Validate ref_audio format if it exists (using the helper from main) - if request.ref_audio is not None: - fmt_err = self._validate_ref_audio_format(request.ref_audio) - if fmt_err: - return fmt_err - # 3. Validate text requirements based on the mode - if not getattr(request, "x_vector_only_mode", False): - if not request.ref_text or not request.ref_text.strip(): - return ( - "Base task requires non-empty 'ref_text' (transcript of " - "the reference audio) unless 'x_vector_only_mode' is enabled" - ) - else: - voice_lower = request.voice.lower() - if voice_lower in self.uploaded_speakers: - # Check if data file exists for uploaded speaker - speaker_info = self.uploaded_speakers[voice_lower] - file_path = Path(speaker_info["file_path"]) - if not file_path.exists(): - return f"Data file for uploaded speaker '{request.voice}' not found on disk" - elif voice_lower in self.precomputed_speakers: - profile = self.precomputed_speakers[voice_lower] - mode = str(profile.get("mode") or "xvec").lower() - ref_text = request.ref_text or profile.get("ref_text") - if mode == "icl" and (not isinstance(ref_text, str) or not ref_text.strip()): - return ( - f"Precomputed voice '{request.voice}' uses ICL mode but has no ref_text in " - "the request or custom voice manifest" - ) - else: - # need ref_audio for built-in speaker - if request.ref_audio is None: - return ( - f"Base task with built-in speaker '{request.voice}' requires 'ref_audio' for voice cloning" - ) - fmt_err = self._validate_ref_audio_format(request.ref_audio) - if fmt_err: - return fmt_err - if not getattr(request, "x_vector_only_mode", False) and ( - not request.ref_text or not request.ref_text.strip() - ): - return ( - "Base task requires non-empty 'ref_text' (transcript of " - "the reference audio) unless 'x_vector_only_mode' is enabled" - ) - - # Validate cross-parameter dependencies - if task_type != "Base": - if request.ref_text is not None: - return "'ref_text' is only valid for Base task" - if request.x_vector_only_mode is not None: - return "'x_vector_only_mode' is only valid for Base task" - - # Validate VoiceDesign task requirements - if task_type == "VoiceDesign" and not request.instructions: - return "VoiceDesign task requires 'instructions' to describe the voice" - - # Validate instructions length (using cached value from initialization) - if request.instructions and len(request.instructions) > self._max_instructions_length: - return f"Instructions too long (max {self._max_instructions_length} characters)" - - # Validate max_new_tokens range - if request.max_new_tokens is not None: - if request.max_new_tokens < _TTS_MAX_NEW_TOKENS_MIN: - return f"max_new_tokens must be at least {_TTS_MAX_NEW_TOKENS_MIN}" - if request.max_new_tokens > _TTS_MAX_NEW_TOKENS_MAX: - return f"max_new_tokens cannot exceed {_TTS_MAX_NEW_TOKENS_MAX}" - - return None - def _detect_moss_variant(self) -> str: """Sub-classify a ``moss_tts``-stage server into the actual MOSS-TTS variant family (tts, ttsd, sound_effect, voice_generator, realtime). @@ -1776,66 +1597,6 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): return "voice_generator" return "tts" - def _validate_moss_tts_request(self, request: OpenAICreateSpeechRequest) -> str | None: - """Validate any MOSS-TTS-family request (nano + 5 full variants). - - Dispatches by ``self._moss_variant``: - - ``tts``/``realtime``: require ``ref_audio`` (voice cloning). - - ``ttsd``: require ``ref_audio`` (speaker 1); ``ref_audio_2`` - optional (defaults to the same ref for both speakers). - - ``sound_effect``: require ``ambient_sound`` (no ref_audio). - - ``voice_generator``: require ``instructions`` (no ref_audio). - - For the legacy moss_tts_nano model_type the variant is None and - we fall through to the original nano contract (ref_audio only). - """ - if not request.input or not request.input.strip(): - # SoundEffect can legitimately have empty input (just ambient_sound). - if self._moss_variant != "sound_effect": - return "Input text cannot be empty" - - v = self._moss_variant - if v in (None, "tts", "realtime", "local"): - if request.ref_audio is None: - label = ( - "MOSS-TTS-Nano" - if v is None - else ( - "MOSS-TTS-Realtime" - if v == "realtime" - else ("MOSS-TTS-Local-Transformer" if v == "local" else "MOSS-TTS") - ) - ) - return f"{label} requires 'ref_audio' (reference audio for voice cloning)." - return self._validate_ref_audio_format(request.ref_audio) - - if v == "ttsd": - if request.ref_audio is None: - return "MOSS-TTSD requires 'ref_audio' (speaker 1 reference)." - fmt_err = self._validate_ref_audio_format(request.ref_audio) - if fmt_err: - return fmt_err - if request.ref_audio_2 is not None: - return self._validate_ref_audio_format(request.ref_audio_2) - return None - - if v == "sound_effect": - if not request.ambient_sound or not request.ambient_sound.strip(): - return ( - "MOSS-SoundEffect requires 'ambient_sound' (natural language " - "description of the sound effect to synthesise)." - ) - return None - - if v == "voice_generator": - if not request.instructions or not request.instructions.strip(): - return ( - "MOSS-VoiceGenerator requires 'instructions' (natural language " - "voice description, e.g. 'a warm female voice with an American accent')." - ) - return None - - return None # unreachable - def _get_moss_processor(self): """Lazily load the upstream MOSS-TTS processor once per server. @@ -1994,93 +1755,6 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): params["max_new_frames"] = [request.max_new_tokens] return params - def _validate_higgs_audio_v2_request(self, request: OpenAICreateSpeechRequest) -> str | None: - """Validate higgs_audio_v2 request parameters. Returns error message or None. - - Accepted: plain text -> speech, or shallow voice clone via ``ref_audio`` - + ``ref_text`` (both required together). Still out of scope: preset - ``voice``/``speaker`` selection, ``x_vector_only_mode`` / - ``speaker_embedding`` helpers, ``task_type``/``language``/ - ``instructions``/``speed`` overrides, and multi-speaker ``[SPEAKERn]`` - tags inside the input body. - """ - from vllm_omni.model_executor.models.higgs_audio_v2.higgs_audio_v2_tokenizer import ( - MULTI_SPEAKER_TAG_PATTERN, - ) - - if not request.input or not request.input.strip(): - return "higgs_audio_v2: input text cannot be empty" - - # Voice clone: ref_audio and ref_text must come together. - if request.ref_audio is not None and not request.ref_text: - return ( - "higgs_audio_v2 voice clone requires both 'ref_audio' and " - "'ref_text'; received ref_audio without ref_text" - ) - if request.ref_text and request.ref_audio is None: - return ( - "higgs_audio_v2 voice clone requires both 'ref_audio' and " - "'ref_text'; received ref_text without ref_audio" - ) - - if request.x_vector_only_mode is not None: - return "higgs_audio_v2 v1 does not support 'x_vector_only_mode' (voice-cloning helper field)" - if request.speaker_embedding is not None: - return "higgs_audio_v2 v1 does not support 'speaker_embedding' (voice-cloning helper field)" - if request.voice and request.ref_audio is None: - # _apply_uploaded_speaker runs before this validator; if voice was - # an uploaded speaker, ref_audio is now populated and ref_text is - # backfilled from the speaker entry. A bare voice= with no - # ref_audio means the name didn't resolve to an uploaded speaker - # (and higgs has no built-in preset voices). - return ( - "higgs_audio_v2 v1 does not support 'voice'/'speaker' selection for built-in voices; " - f"upload a voice first via POST /v1/audio/voices, or use ref_audio + ref_text. " - f"Got voice={request.voice!r}" - ) - if request.instructions: - return ( - "higgs_audio_v2 v1 does not support 'instructions' (voice " - "style/emotion control); supply plain text instead" - ) - if request.task_type is not None: - return "higgs_audio_v2 v1 does not support 'task_type'; the model is single-mode plain text -> speech" - if request.language is not None: - return ( - "higgs_audio_v2 v1 does not accept 'language' overrides; the model infers language from the input text" - ) - if request.speed is not None and request.speed != 1.0: - return ( - "higgs_audio_v2 v1 does not support 'speed' adjustments; the audio is rendered at native rate (24 kHz)" - ) - - if MULTI_SPEAKER_TAG_PATTERN.search(request.input): - return "higgs_audio_v2 v1 does not support multi-speaker [SPEAKERn] tags; remove the tag from the input" - - if request.max_new_tokens is not None: - if request.max_new_tokens < _TTS_MAX_NEW_TOKENS_MIN: - return f"max_new_tokens must be at least {_TTS_MAX_NEW_TOKENS_MIN}" - - def _validate_glm_tts_request(self, request: OpenAICreateSpeechRequest) -> str | None: - """Validate GLM-TTS request — requires ref_audio for voice cloning.""" - if not request.input or not request.input.strip(): - return "Input text cannot be empty" - - if request.ref_audio is None: - return "GLM-TTS requires 'ref_audio' for zero-shot voice cloning" - fmt_err = self._validate_ref_audio_format(request.ref_audio) - if fmt_err: - return fmt_err - if not request.ref_text or not request.ref_text.strip(): - return "GLM-TTS voice cloning requires 'ref_text' (transcript of the reference audio)" - - if request.max_new_tokens is not None: - if request.max_new_tokens < _TTS_MAX_NEW_TOKENS_MIN: - return f"max_new_tokens must be >= {_TTS_MAX_NEW_TOKENS_MIN}" - if request.max_new_tokens > _TTS_MAX_NEW_TOKENS_MAX: - return f"max_new_tokens cannot exceed {_TTS_MAX_NEW_TOKENS_MAX}" - return None - async def _build_higgs_audio_v2_params(self, request: OpenAICreateSpeechRequest): """Build prompt_token_ids for higgs_audio_v2 via the upstream processor. @@ -2157,18 +1831,6 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): # ---- higgs-audio v3 ---- - def _validate_higgs_audio_v3_request(self, request: OpenAICreateSpeechRequest) -> str | None: - """Validate higgs_audio_v3 request parameters.""" - if not request.input or not request.input.strip(): - return "higgs_audio_v3: input text cannot be empty" - if request.ref_audio is not None and not request.ref_text: - # Voice clone ref_text is optional for v3 (improves fidelity but not required) - pass - if request.max_new_tokens is not None: - if request.max_new_tokens < _TTS_MAX_NEW_TOKENS_MIN: - return f"max_new_tokens must be at least {_TTS_MAX_NEW_TOKENS_MIN}" - return None - async def _build_higgs_audio_v3_params(self, request: OpenAICreateSpeechRequest): """Build prompt_token_ids for higgs_audio_v3. @@ -2302,143 +1964,6 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): self._higgs_audio_v3_adapter = adapter return adapter - def _validate_fish_tts_request(self, request: OpenAICreateSpeechRequest) -> str | None: - """Validate Fish Speech request parameters. Returns error message or None.""" - if not request.input or not request.input.strip(): - return "Input text cannot be empty" - - if request.ref_audio is not None: - fmt_err = self._validate_ref_audio_format(request.ref_audio) - if fmt_err: - return fmt_err - if not request.ref_text or not request.ref_text.strip(): - return "Voice cloning requires 'ref_text' (transcript of the reference audio)" - - if request.max_new_tokens is not None: - if request.max_new_tokens < _TTS_MAX_NEW_TOKENS_MIN: - return f"max_new_tokens must be at least {_TTS_MAX_NEW_TOKENS_MIN}" - if request.max_new_tokens > _TTS_MAX_NEW_TOKENS_MAX: - return f"max_new_tokens cannot exceed {_TTS_MAX_NEW_TOKENS_MAX}" - - return None - - def _validate_cosyvoice3_request(self, request: OpenAICreateSpeechRequest) -> str | None: - """Validate CosyVoice3 request parameters. Returns error message or None.""" - if not request.input or not request.input.strip(): - return "Input text cannot be empty" - - # CosyVoice3 requires reference audio for voice cloning - if request.ref_audio is None: - return "CosyVoice3 requires 'ref_audio' (reference audio for voice cloning)" - - fmt_err = self._validate_ref_audio_format(request.ref_audio) - if fmt_err: - return fmt_err - - if not request.ref_text or not request.ref_text.strip(): - return "CosyVoice3 requires 'ref_text' (transcript of the reference audio)" - - if request.max_new_tokens is not None: - if request.max_new_tokens < _TTS_MAX_NEW_TOKENS_MIN: - return f"max_new_tokens must be at least {_TTS_MAX_NEW_TOKENS_MIN}" - if request.max_new_tokens > _TTS_MAX_NEW_TOKENS_MAX: - return f"max_new_tokens cannot exceed {_TTS_MAX_NEW_TOKENS_MAX}" - - return None - - def _validate_ming_tts_request(self, request: OpenAICreateSpeechRequest) -> str | None: - """Validate Ming TTS request parameters. Returns error message or None.""" - if not request.input or not request.input.strip(): - return "Input text cannot be empty" - - if isinstance(request.ref_audio, list): - return self._validate_ming_tts_podcast_request(request) - return self._validate_ming_tts_single_speaker_request(request) - - def _validate_ming_tts_single_speaker_request(self, request: OpenAICreateSpeechRequest) -> str | None: - if request.ref_audio is not None: - fmt_err = self._validate_ref_audio_format(request.ref_audio) - if fmt_err: - return fmt_err - - if request.speaker_embedding is not None: - if not request.speaker_embedding: - return "'speaker_embedding' must be a non-empty list of floats" - emb_len = len(request.speaker_embedding) - if emb_len != 192: - logger.warning( - "speaker_embedding has %d dimensions; Ming dense expects 192. " - "Wrong dimensions will likely fail or degrade output.", - emb_len, - ) - - voice_lower = request.voice.lower() if isinstance(request.voice, str) else None - uploaded_voice = bool(voice_lower and voice_lower in self.uploaded_speakers) - clone_source_present = request.ref_audio is not None or request.speaker_embedding is not None or uploaded_voice - - if request.task_type == "Base" and not clone_source_present: - return "Base task requires 'ref_audio', 'speaker_embedding', or an uploaded voice sample" - - if request.ref_audio is not None and request.ref_text is not None and not request.ref_text.strip(): - return "'ref_text' must be non-empty when provided with 'ref_audio'" - - # Ming offline ref-audio cases use prompt_waveform without prompt_text; - # keep the transcript requirement for other TTS models. - if request.ref_audio is not None and request.speaker_embedding is None and not self._is_ming_tts_model(): - uploaded_ref_text = self.uploaded_speakers[voice_lower].get("ref_text") if uploaded_voice else None - if not (request.ref_text and request.ref_text.strip()) and not uploaded_ref_text: - return "Reference-audio cloning requires non-empty 'ref_text'" - - if request.ref_text is not None and request.ref_audio is None and not uploaded_voice: - return "'ref_text' requires 'ref_audio' or an uploaded voice sample" - - if request.instructions and len(request.instructions) > self._max_instructions_length: - return f"Instructions too long (max {self._max_instructions_length} characters)" - - if request.max_new_tokens is not None: - if request.max_new_tokens < _TTS_MAX_NEW_TOKENS_MIN: - return f"max_new_tokens must be at least {_TTS_MAX_NEW_TOKENS_MIN}" - if request.max_new_tokens > _TTS_MAX_NEW_TOKENS_MAX: - return f"max_new_tokens cannot exceed {_TTS_MAX_NEW_TOKENS_MAX}" - - return None - - def _validate_ming_tts_podcast_request(self, request: OpenAICreateSpeechRequest) -> str | None: - if len(request.ref_audio) < 2: - return "Podcast-style Ming requests require at least two 'ref_audio' clips" - - for ref_audio in request.ref_audio: - fmt_err = self._validate_ref_audio_format(ref_audio) - if fmt_err: - return fmt_err - - if not request.ref_text or not request.ref_text.strip(): - return "Podcast-style Ming requests require non-empty 'ref_text'" - - if request.speaker_embedding is not None: - embeddings = request.speaker_embedding - embedding_count = len(embeddings) if embeddings and isinstance(embeddings[0], list) else 1 - if embedding_count != len(request.ref_audio): - return ( - "Podcast-style Ming requests require one speaker embedding per ref_audio clip; " - f"got {embedding_count} embeddings for {len(request.ref_audio)} clips" - ) - if embeddings and isinstance(embeddings[0], list): - for item in embeddings: - if len(item) != 192: - return "Podcast-style Ming speaker embeddings must each have 192 dimensions" - - if request.instructions and len(request.instructions) > self._max_instructions_length: - return f"Instructions too long (max {self._max_instructions_length} characters)" - - if request.max_new_tokens is not None: - if request.max_new_tokens < _TTS_MAX_NEW_TOKENS_MIN: - return f"max_new_tokens must be at least {_TTS_MAX_NEW_TOKENS_MIN}" - if request.max_new_tokens > _TTS_MAX_NEW_TOKENS_MAX: - return f"max_new_tokens cannot exceed {_TTS_MAX_NEW_TOKENS_MAX}" - - return None - async def _resolve_ref_audio(self, ref_audio_str: str) -> tuple[list[float], int]: """Resolve ref_audio to (wav_samples, sample_rate). @@ -2573,9 +2098,6 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): # ---- Ming TTS helpers ---- - def _is_ming_tts_model(self) -> bool: - return self._tts_model_type == "ming_tts" - def _coerce_ming_prompt_waveform(self, wav_samples, sample_rate): from torchaudio.functional import resample as resample_audio @@ -3488,6 +3010,7 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): self, request: OpenAICreateSpeechRequest, request_id: str | None = None, + has_inline_ref_audio: bool | None = None, ) -> tuple[str, Any, dict[str, Any]]: if self.engine_client.errored: raise self.engine_client.dead_error @@ -3520,7 +3043,7 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): # in place. The builders need to know whether the caller supplied audio # inline vs. via an uploaded voice. model_type: str | None = None - has_inline_ref_audio = request.ref_audio is not None + has_inline_ref_audio = (request.ref_audio is not None) if has_inline_ref_audio is None else has_inline_ref_audio if self._tts_model_type == "ming_flash_omni_tts": # ming_flash_omni is intentionally NOT migrated onto the adapter # framework in this PR (it has no registered adapter); keep it on the @@ -3743,12 +3266,15 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): base64_encode: bool = False, request_id: str | None = None, usage_out: list[SpeechTokenUsage] | None = None, + has_inline_ref_audio: bool | None = None, ) -> tuple[bytes | str, str]: # ``usage_out`` is an opt-in output channel: when a list is passed, the # computed SpeechTokenUsage is appended to it. The return stays a # 2-tuple so existing callers (and their test mocks) are unaffected; # only the batch path, which surfaces per-item usage, opts in. - request_id, generator, bytes_tts_params = await self._prepare_speech_generation(request, request_id=request_id) + request_id, generator, bytes_tts_params = await self._prepare_speech_generation( + request, request_id=request_id, has_inline_ref_audio=has_inline_ref_audio + ) artifact_ready = False try: @@ -4205,17 +3731,15 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): merged_requests = [self._merge_batch_item(batch_request, item) for item in batch_request.items] async def _run_item(idx: int, req: OpenAICreateSpeechRequest) -> SpeechBatchItemResult: - # Batch validation still goes through _validate_tts_request directly - # (not the adapter). The single-request path validates via the - # adapter; both ultimately call the same per-model validators, so the - # surfaces stay in sync. Routing batch through the adapter (and its - # uploaded-speaker handling) is a follow-up (RFC #4327). + has_inline_ref_audio = req.ref_audio is not None validation_error = self._validate_tts_request(req) if validation_error is not None: return SpeechBatchItemResult(index=idx, status="error", error=validation_error) usage_box: list[SpeechTokenUsage] = [] try: - audio_data, media_type = await self._generate_audio_bytes(req, base64_encode=True, usage_out=usage_box) + audio_data, media_type = await self._generate_audio_bytes( + req, base64_encode=True, usage_out=usage_box, has_inline_ref_audio=has_inline_ref_audio + ) except Exception as e: logger.exception("Batch item %d failed: %s", idx, e) return SpeechBatchItemResult(index=idx, status="error", error=str(e)) diff --git a/vllm_omni/entrypoints/openai/tts_adapters/base.py b/vllm_omni/entrypoints/openai/tts_adapters/base.py index ad753a91e..58c135044 100644 --- a/vllm_omni/entrypoints/openai/tts_adapters/base.py +++ b/vllm_omni/entrypoints/openai/tts_adapters/base.py @@ -104,6 +104,10 @@ class TTSModelAdapter(ABC): #: Serving backend: ``"ar"`` (engine_client) or ``"diffusion"``. backend: ClassVar[str] = "ar" + max_new_tokens_min = 1 + + max_new_tokens_max = 4096 + def __init__(self, ctx: SpeechServingContext) -> None: self.ctx = ctx diff --git a/vllm_omni/entrypoints/openai/tts_adapters/cosyvoice3.py b/vllm_omni/entrypoints/openai/tts_adapters/cosyvoice3.py index 6a7a89702..85648442f 100644 --- a/vllm_omni/entrypoints/openai/tts_adapters/cosyvoice3.py +++ b/vllm_omni/entrypoints/openai/tts_adapters/cosyvoice3.py @@ -16,10 +16,32 @@ class CosyVoice3Adapter(ARTTSAdapter): name = "cosyvoice3" def validate(self, request: "OpenAICreateSpeechRequest") -> str | None: - err = self.ctx.server._apply_uploaded_speaker(request) + """Validate CosyVoice3 request parameters. Returns error message or None.""" + server = self.ctx.server + err = server._apply_uploaded_speaker(request) if err: return err - return self.ctx.server._validate_cosyvoice3_request(request) + if not request.input or not request.input.strip(): + return "Input text cannot be empty" + + # CosyVoice3 requires reference audio for voice cloning + if request.ref_audio is None: + return "CosyVoice3 requires 'ref_audio' (reference audio for voice cloning)" + + fmt_err = server._validate_ref_audio_format(request.ref_audio) + if fmt_err: + return fmt_err + + if not request.ref_text or not request.ref_text.strip(): + return "CosyVoice3 requires 'ref_text' (transcript of the reference audio)" + + if request.max_new_tokens is not None: + if request.max_new_tokens < self.max_new_tokens_min: + return f"max_new_tokens must be at least {self.max_new_tokens_min}" + if request.max_new_tokens > self.max_new_tokens_max: + return f"max_new_tokens cannot exceed {self.max_new_tokens_max}" + + return None async def build( self, request: "OpenAICreateSpeechRequest", sampling_params_list: list, has_inline_ref_audio: bool diff --git a/vllm_omni/entrypoints/openai/tts_adapters/covo_audio.py b/vllm_omni/entrypoints/openai/tts_adapters/covo_audio.py index f98f89619..70673ac54 100644 --- a/vllm_omni/entrypoints/openai/tts_adapters/covo_audio.py +++ b/vllm_omni/entrypoints/openai/tts_adapters/covo_audio.py @@ -15,6 +15,11 @@ class CovoAudioAdapter(ARTTSAdapter): stage_keys = frozenset({"fused_thinker_talker"}) name = "covo_audio" + def validate(self, request: "OpenAICreateSpeechRequest") -> str | None: + if not request.input or not request.input.strip(): + return "Input text cannot be empty" + return None + async def build( self, request: "OpenAICreateSpeechRequest", sampling_params_list: list, has_inline_ref_audio: bool ) -> PreparedRequest: diff --git a/vllm_omni/entrypoints/openai/tts_adapters/fish_speech.py b/vllm_omni/entrypoints/openai/tts_adapters/fish_speech.py index d216117b7..e8080a427 100644 --- a/vllm_omni/entrypoints/openai/tts_adapters/fish_speech.py +++ b/vllm_omni/entrypoints/openai/tts_adapters/fish_speech.py @@ -16,10 +16,28 @@ class FishSpeechAdapter(ARTTSAdapter): name = "fish_tts" def validate(self, request: "OpenAICreateSpeechRequest") -> str | None: - err = self.ctx.server._apply_uploaded_speaker(request) + """Validate Fish Speech request parameters. Returns error message or None.""" + server = self.ctx.server + err = server._apply_uploaded_speaker(request) if err: return err - return self.ctx.server._validate_fish_tts_request(request) + if not request.input or not request.input.strip(): + return "Input text cannot be empty" + + if request.ref_audio is not None: + fmt_err = server._validate_ref_audio_format(request.ref_audio) + if fmt_err: + return fmt_err + if not request.ref_text or not request.ref_text.strip(): + return "Voice cloning requires 'ref_text' (transcript of the reference audio)" + + if request.max_new_tokens is not None: + if request.max_new_tokens < self.max_new_tokens_min: + return f"max_new_tokens must be at least {self.max_new_tokens_min}" + if request.max_new_tokens > self.max_new_tokens_max: + return f"max_new_tokens cannot exceed {self.max_new_tokens_max}" + + return None async def build( self, request: "OpenAICreateSpeechRequest", sampling_params_list: list, has_inline_ref_audio: bool diff --git a/vllm_omni/entrypoints/openai/tts_adapters/glm_tts.py b/vllm_omni/entrypoints/openai/tts_adapters/glm_tts.py index e9e8700e6..ddfd194e2 100644 --- a/vllm_omni/entrypoints/openai/tts_adapters/glm_tts.py +++ b/vllm_omni/entrypoints/openai/tts_adapters/glm_tts.py @@ -16,10 +16,28 @@ class GlmTTSAdapter(ARTTSAdapter): name = "glm_tts" def validate(self, request: "OpenAICreateSpeechRequest") -> str | None: - err = self.ctx.server._apply_uploaded_speaker(request) + """Validate GLM-TTS request — requires ref_audio for voice cloning.""" + server = self.ctx.server + err = server._apply_uploaded_speaker(request) if err: return err - return self.ctx.server._validate_glm_tts_request(request) + if not request.input or not request.input.strip(): + return "Input text cannot be empty" + + if request.ref_audio is None: + return "GLM-TTS requires 'ref_audio' for zero-shot voice cloning" + fmt_err = server._validate_ref_audio_format(request.ref_audio) + if fmt_err: + return fmt_err + if not request.ref_text or not request.ref_text.strip(): + return "GLM-TTS voice cloning requires 'ref_text' (transcript of the reference audio)" + + if request.max_new_tokens is not None: + if request.max_new_tokens < self.max_new_tokens_min: + return f"max_new_tokens must be >= {self.max_new_tokens_min}" + if request.max_new_tokens > self.max_new_tokens_max: + return f"max_new_tokens cannot exceed {self.max_new_tokens_max}" + return None async def build( self, request: "OpenAICreateSpeechRequest", sampling_params_list: list, has_inline_ref_audio: bool diff --git a/vllm_omni/entrypoints/openai/tts_adapters/higgs_audio_v2.py b/vllm_omni/entrypoints/openai/tts_adapters/higgs_audio_v2.py index 606b4bca1..99949d8b3 100644 --- a/vllm_omni/entrypoints/openai/tts_adapters/higgs_audio_v2.py +++ b/vllm_omni/entrypoints/openai/tts_adapters/higgs_audio_v2.py @@ -16,10 +16,76 @@ class HiggsAudioV2Adapter(ARTTSAdapter): name = "higgs_audio_v2" def validate(self, request: "OpenAICreateSpeechRequest") -> str | None: - err = self.ctx.server._apply_uploaded_speaker(request) + """Validate higgs_audio_v2 request parameters. Returns error message or None. + + Accepted: plain text -> speech, or shallow voice clone via ``ref_audio`` + + ``ref_text`` (both required together). Still out of scope: preset + ``voice``/``speaker`` selection, ``x_vector_only_mode`` / + ``speaker_embedding`` helpers, ``task_type``/``language``/ + ``instructions``/``speed`` overrides, and multi-speaker ``[SPEAKERn]`` + tags inside the input body. + """ + from vllm_omni.model_executor.models.higgs_audio_v2.higgs_audio_v2_tokenizer import ( + MULTI_SPEAKER_TAG_PATTERN, + ) + + server = self.ctx.server + err = server._apply_uploaded_speaker(request) if err: return err - return self.ctx.server._validate_higgs_audio_v2_request(request) + + if not request.input or not request.input.strip(): + return "higgs_audio_v2: input text cannot be empty" + + # Voice clone: ref_audio and ref_text must come together. + if request.ref_audio is not None and not request.ref_text: + return ( + "higgs_audio_v2 voice clone requires both 'ref_audio' and " + "'ref_text'; received ref_audio without ref_text" + ) + if request.ref_text and request.ref_audio is None: + return ( + "higgs_audio_v2 voice clone requires both 'ref_audio' and " + "'ref_text'; received ref_text without ref_audio" + ) + + if request.x_vector_only_mode is not None: + return "higgs_audio_v2 v1 does not support 'x_vector_only_mode' (voice-cloning helper field)" + if request.speaker_embedding is not None: + return "higgs_audio_v2 v1 does not support 'speaker_embedding' (voice-cloning helper field)" + if request.voice and request.ref_audio is None: + # _apply_uploaded_speaker runs before this validator; if voice was + # an uploaded speaker, ref_audio is now populated and ref_text is + # backfilled from the speaker entry. A bare voice= with no + # ref_audio means the name didn't resolve to an uploaded speaker + # (and higgs has no built-in preset voices). + return ( + "higgs_audio_v2 v1 does not support 'voice'/'speaker' selection for built-in voices; " + f"upload a voice first via POST /v1/audio/voices, or use ref_audio + ref_text. " + f"Got voice={request.voice!r}" + ) + if request.instructions: + return ( + "higgs_audio_v2 v1 does not support 'instructions' (voice " + "style/emotion control); supply plain text instead" + ) + if request.task_type is not None: + return "higgs_audio_v2 v1 does not support 'task_type'; the model is single-mode plain text -> speech" + if request.language is not None: + return ( + "higgs_audio_v2 v1 does not accept 'language' overrides; the model infers language from the input text" + ) + if request.speed is not None and request.speed != 1.0: + return ( + "higgs_audio_v2 v1 does not support 'speed' adjustments; the audio is rendered at native rate (24 kHz)" + ) + + if MULTI_SPEAKER_TAG_PATTERN.search(request.input): + return "higgs_audio_v2 v1 does not support multi-speaker [SPEAKERn] tags; remove the tag from the input" + + if request.max_new_tokens is not None: + if request.max_new_tokens < self.max_new_tokens_min: + return f"max_new_tokens must be at least {self.max_new_tokens_min}" async def build( self, request: "OpenAICreateSpeechRequest", sampling_params_list: list, has_inline_ref_audio: bool diff --git a/vllm_omni/entrypoints/openai/tts_adapters/higgs_audio_v3.py b/vllm_omni/entrypoints/openai/tts_adapters/higgs_audio_v3.py index dde5e0815..9cae9aa2d 100644 --- a/vllm_omni/entrypoints/openai/tts_adapters/higgs_audio_v3.py +++ b/vllm_omni/entrypoints/openai/tts_adapters/higgs_audio_v3.py @@ -16,10 +16,20 @@ class HiggsAudioV3Adapter(ARTTSAdapter): name = "higgs_audio_v3" def validate(self, request: "OpenAICreateSpeechRequest") -> str | None: - err = self.ctx.server._apply_uploaded_speaker(request) + """Validate higgs_audio_v3 request parameters.""" + server = self.ctx.server + err = server._apply_uploaded_speaker(request) if err: return err - return self.ctx.server._validate_higgs_audio_v3_request(request) + if not request.input or not request.input.strip(): + return "higgs_audio_v3: input text cannot be empty" + if request.ref_audio is not None and not request.ref_text: + # Voice clone ref_text is optional for v3 (improves fidelity but not required) + pass + if request.max_new_tokens is not None: + if request.max_new_tokens < self.max_new_tokens_min: + return f"max_new_tokens must be at least {self.max_new_tokens_min}" + return None async def build( self, request: "OpenAICreateSpeechRequest", sampling_params_list: list, has_inline_ref_audio: bool diff --git a/vllm_omni/entrypoints/openai/tts_adapters/ming_tts.py b/vllm_omni/entrypoints/openai/tts_adapters/ming_tts.py index 7e1a8e66b..c736afb51 100644 --- a/vllm_omni/entrypoints/openai/tts_adapters/ming_tts.py +++ b/vllm_omni/entrypoints/openai/tts_adapters/ming_tts.py @@ -9,6 +9,10 @@ from vllm_omni.entrypoints.openai.tts_adapters.base import ARTTSAdapter, Prepare if TYPE_CHECKING: from vllm_omni.entrypoints.openai.protocol.audio import OpenAICreateSpeechRequest +from vllm.logger import init_logger + +logger = init_logger(__name__) + @register_tts_adapter class MingTTSAdapter(ARTTSAdapter): @@ -16,7 +20,13 @@ class MingTTSAdapter(ARTTSAdapter): name = "ming_tts" def validate(self, request: "OpenAICreateSpeechRequest") -> str | None: - return self.ctx.server._validate_ming_tts_request(request) + """Validate Ming TTS request parameters. Returns error message or None.""" + if not request.input or not request.input.strip(): + return "Input text cannot be empty" + + if isinstance(request.ref_audio, list): + return self._validate_ming_tts_podcast_request(request) + return self._validate_ming_tts_single_speaker_request(request) async def build( self, request: "OpenAICreateSpeechRequest", sampling_params_list: list, has_inline_ref_audio: bool @@ -51,3 +61,82 @@ class MingTTSAdapter(ARTTSAdapter): tts_params = prompt.get("additional_information", {}) # Ming stop-token / max_tokens sampling stays in the orchestrator tail. return PreparedRequest(prompt=prompt, tts_params=tts_params, model_type="ming_tts") + + def _validate_ming_tts_single_speaker_request(self, request: "OpenAICreateSpeechRequest") -> str | None: + server = self.ctx.server + if request.ref_audio is not None: + fmt_err = server._validate_ref_audio_format(request.ref_audio) + if fmt_err: + return fmt_err + + if request.speaker_embedding is not None: + if not request.speaker_embedding: + return "'speaker_embedding' must be a non-empty list of floats" + emb_len = len(request.speaker_embedding) + if emb_len != 192: + logger.warning( + "speaker_embedding has %d dimensions; Ming dense expects 192. " + "Wrong dimensions will likely fail or degrade output.", + emb_len, + ) + + voice_lower = request.voice.lower() if isinstance(request.voice, str) else None + uploaded_voice = bool(voice_lower and voice_lower in server.uploaded_speakers) + clone_source_present = request.ref_audio is not None or request.speaker_embedding is not None or uploaded_voice + + if request.task_type == "Base" and not clone_source_present: + return "Base task requires 'ref_audio', 'speaker_embedding', or an uploaded voice sample" + + if request.ref_audio is not None and request.ref_text is not None and not request.ref_text.strip(): + return "'ref_text' must be non-empty when provided with 'ref_audio'" + + if request.ref_text is not None and request.ref_audio is None and not uploaded_voice: + return "'ref_text' requires 'ref_audio' or an uploaded voice sample" + + if request.instructions and len(request.instructions) > server._max_instructions_length: + return f"Instructions too long (max {server._max_instructions_length} characters)" + + if request.max_new_tokens is not None: + if request.max_new_tokens < self.max_new_tokens_min: + return f"max_new_tokens must be at least {self.max_new_tokens_min}" + if request.max_new_tokens > self.max_new_tokens_max: + return f"max_new_tokens cannot exceed {self.max_new_tokens_max}" + + return None + + def _validate_ming_tts_podcast_request(self, request: "OpenAICreateSpeechRequest") -> str | None: + server = self.ctx.server + if len(request.ref_audio) < 2: + return "Podcast-style Ming requests require at least two 'ref_audio' clips" + + for ref_audio in request.ref_audio: + fmt_err = server._validate_ref_audio_format(ref_audio) + if fmt_err: + return fmt_err + + if not request.ref_text or not request.ref_text.strip(): + return "Podcast-style Ming requests require non-empty 'ref_text'" + + if request.speaker_embedding is not None: + embeddings = request.speaker_embedding + embedding_count = len(embeddings) if embeddings and isinstance(embeddings[0], list) else 1 + if embedding_count != len(request.ref_audio): + return ( + "Podcast-style Ming requests require one speaker embedding per ref_audio clip; " + f"got {embedding_count} embeddings for {len(request.ref_audio)} clips" + ) + if embeddings and isinstance(embeddings[0], list): + for item in embeddings: + if len(item) != 192: + return "Podcast-style Ming speaker embeddings must each have 192 dimensions" + + if request.instructions and len(request.instructions) > server._max_instructions_length: + return f"Instructions too long (max {server._max_instructions_length} characters)" + + if request.max_new_tokens is not None: + if request.max_new_tokens < self.max_new_tokens_min: + return f"max_new_tokens must be at least {self.max_new_tokens_min}" + if request.max_new_tokens > self.max_new_tokens_max: + return f"max_new_tokens cannot exceed {self.max_new_tokens_max}" + + return None diff --git a/vllm_omni/entrypoints/openai/tts_adapters/moss_tts.py b/vllm_omni/entrypoints/openai/tts_adapters/moss_tts.py index 36d98c2d9..ad43a1c01 100644 --- a/vllm_omni/entrypoints/openai/tts_adapters/moss_tts.py +++ b/vllm_omni/entrypoints/openai/tts_adapters/moss_tts.py @@ -18,10 +18,69 @@ if TYPE_CHECKING: class _MossTTSAdapterBase(ARTTSAdapter): def validate(self, request: "OpenAICreateSpeechRequest") -> str | None: - err = self.ctx.server._apply_uploaded_speaker(request) + """Validate any MOSS-TTS-family request (nano + 5 full variants). + + Dispatches by ``self._moss_variant``: + - ``tts``/``realtime``: require ``ref_audio`` (voice cloning). + - ``ttsd``: require ``ref_audio`` (speaker 1); ``ref_audio_2`` + optional (defaults to the same ref for both speakers). + - ``sound_effect``: require ``ambient_sound`` (no ref_audio). + - ``voice_generator``: require ``instructions`` (no ref_audio). + - For the legacy moss_tts_nano model_type the variant is None and + we fall through to the original nano contract (ref_audio only). + """ + server = self.ctx.server + err = server._apply_uploaded_speaker(request) if err: return err - return self.ctx.server._validate_moss_tts_request(request) + + if not request.input or not request.input.strip(): + # SoundEffect can legitimately have empty input (just ambient_sound). + if server._moss_variant != "sound_effect": + return "Input text cannot be empty" + + v = server._moss_variant + if v in (None, "tts", "realtime", "local"): + if request.ref_audio is None: + label = ( + "MOSS-TTS-Nano" + if v is None + else ( + "MOSS-TTS-Realtime" + if v == "realtime" + else ("MOSS-TTS-Local-Transformer" if v == "local" else "MOSS-TTS") + ) + ) + return f"{label} requires 'ref_audio' (reference audio for voice cloning)." + return server._validate_ref_audio_format(request.ref_audio) + + if v == "ttsd": + if request.ref_audio is None: + return "MOSS-TTSD requires 'ref_audio' (speaker 1 reference)." + fmt_err = server._validate_ref_audio_format(request.ref_audio) + if fmt_err: + return fmt_err + if request.ref_audio_2 is not None: + return server._validate_ref_audio_format(request.ref_audio_2) + return None + + if v == "sound_effect": + if not request.ambient_sound or not request.ambient_sound.strip(): + return ( + "MOSS-SoundEffect requires 'ambient_sound' (natural language " + "description of the sound effect to synthesise)." + ) + return None + + if v == "voice_generator": + if not request.instructions or not request.instructions.strip(): + return ( + "MOSS-VoiceGenerator requires 'instructions' (natural language " + "voice description, e.g. 'a warm female voice with an American accent')." + ) + return None + + return None # unreachable async def build( self, request: "OpenAICreateSpeechRequest", sampling_params_list: list, has_inline_ref_audio: bool diff --git a/vllm_omni/entrypoints/openai/tts_adapters/qwen3_tts.py b/vllm_omni/entrypoints/openai/tts_adapters/qwen3_tts.py index 0f62a169c..c0be8c685 100644 --- a/vllm_omni/entrypoints/openai/tts_adapters/qwen3_tts.py +++ b/vllm_omni/entrypoints/openai/tts_adapters/qwen3_tts.py @@ -1,18 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 -"""Qwen3-TTS serving adapter. - -First model migrated onto the adapter framework (RFC #4327). It is the dispatch -surface for Qwen3-TTS: the serving orchestrator routes prompt/param building -through ``build()`` instead of the inline ``else`` branch in -``_prepare_speech_generation``. - -To stay behaviour-identical and avoid divergence from the batch path (which -shares ``_validate_qwen_tts_request`` and ``_build_tts_params``), the adapter -reuses the single-source helper implementations on the serving instance through -``ctx.server`` rather than copying them. Subsequent PRs relocate those bodies as -more models migrate and the batch path also routes through adapters. -""" +"""Qwen3-TTS serving adapter.""" +from pathlib import Path from typing import TYPE_CHECKING from vllm.logger import init_logger @@ -36,13 +25,133 @@ class Qwen3TTSAdapter(ARTTSAdapter): def normalize(self, request: "OpenAICreateSpeechRequest") -> None: """Qwen3-TTS normalization (Base-task inference, voice lowercasing) is performed inside ``validate`` today; kept fused for a strict behaviour - match. See ``_validate_qwen_tts_request``.""" + match.""" def validate(self, request: "OpenAICreateSpeechRequest") -> str | None: - # Go through the shared dispatcher (which routes to - # _validate_qwen_tts_request for this model-type) rather than the leaf - # validator directly, matching the pre-adapter qwen3 path. - return self.ctx.server._validate_tts_request(request) + """Validate Qwen TTS request parameters. Returns error message or None.""" + # Infer Base task when ref_audio or ref_text is provided without explicit task_type. + server = self.ctx.server + if request.task_type is None and (request.ref_audio is not None or request.ref_text is not None): + request.task_type = "Base" + + # Normalize voice to lowercase for case-insensitive matching + if request.voice is not None: + request.voice = request.voice.lower() + if request.task_type is None and request.voice in server.precomputed_speakers: + request.task_type = "Base" + task_type = request.task_type or "CustomVoice" + + # Validate input is not empty + if not request.input or not request.input.strip(): + return "Input text cannot be empty" + + # Validate language (case-insensitive; normalized to the title-cased config form) + if request.language is not None: + request.language = request.language.title() + if request.language not in server.supported_languages: + return ( + f"Invalid language '{request.language}'. Supported: {', '.join(sorted(server.supported_languages))}" + ) + + # Validate speaker for CustomVoice task + if task_type == "CustomVoice": + if not server.supported_speakers: + return ( + "This model does not support CustomVoice task (no speakers configured). " + "Use task_type='Base' with ref_audio/ref_text for voice cloning, " + "or use a CustomVoice model." + ) + if request.voice is not None and request.voice not in server.supported_speakers: + return f"Invalid voice '{request.voice}'. Supported: {', '.join(sorted(server.supported_speakers))}" + + # Validate speaker_embedding constraints + if request.speaker_embedding is not None: + if task_type != "Base": + return "'speaker_embedding' is only valid for Base task" + if not request.speaker_embedding: + return "'speaker_embedding' must be a non-empty list of floats" + # speaker_embedding implies x_vector_only_mode — set it before + # Base task validation so callers don't need to pass it explicitly. + request.x_vector_only_mode = True + emb_len = len(request.speaker_embedding) + dim_err = server._validate_qwen_tts_speaker_embedding_dim(emb_len) + if dim_err is not None: + return dim_err + # Validate Base task requirements + if task_type == "Base": + if request.voice is None: + # 1. Ensure a voice source is provided + if request.ref_audio is None and getattr(request, "speaker_embedding", None) is None: + return "Base task requires 'ref_audio' or 'speaker_embedding' for voice cloning" + # 2. Validate ref_audio format if it exists (using the helper from main) + if request.ref_audio is not None: + fmt_err = server._validate_ref_audio_format(request.ref_audio) + if fmt_err: + return fmt_err + # 3. Validate text requirements based on the mode + if not getattr(request, "x_vector_only_mode", False): + if not request.ref_text or not request.ref_text.strip(): + return ( + "Base task requires non-empty 'ref_text' (transcript of " + "the reference audio) unless 'x_vector_only_mode' is enabled" + ) + else: + voice_lower = request.voice.lower() + if voice_lower in server.uploaded_speakers: + # Check if data file exists for uploaded speaker + speaker_info = server.uploaded_speakers[voice_lower] + file_path = Path(speaker_info["file_path"]) + if not file_path.exists(): + return f"Data file for uploaded speaker '{request.voice}' not found on disk" + elif voice_lower in server.precomputed_speakers: + profile = server.precomputed_speakers[voice_lower] + mode = str(profile.get("mode") or "xvec").lower() + ref_text = request.ref_text or profile.get("ref_text") + if mode == "icl" and (not isinstance(ref_text, str) or not ref_text.strip()): + return ( + f"Precomputed voice '{request.voice}' uses ICL mode but has no ref_text in " + "the request or custom voice manifest" + ) + else: + # need ref_audio for built-in speaker + if request.ref_audio is None: + return ( + f"Base task with built-in speaker '{request.voice}' requires 'ref_audio' for voice cloning" + ) + fmt_err = server._validate_ref_audio_format(request.ref_audio) + if fmt_err: + return fmt_err + if not getattr(request, "x_vector_only_mode", False) and ( + not request.ref_text or not request.ref_text.strip() + ): + return ( + "Base task requires non-empty 'ref_text' (transcript of " + "the reference audio) unless 'x_vector_only_mode' is enabled" + ) + + # Validate cross-parameter dependencies + if task_type != "Base": + if request.ref_text is not None: + return "'ref_text' is only valid for Base task" + if request.x_vector_only_mode is not None: + return "'x_vector_only_mode' is only valid for Base task" + + # Validate VoiceDesign task requirements + if task_type == "VoiceDesign" and not request.instructions: + return "VoiceDesign task requires 'instructions' to describe the voice" + + # Validate instructions length (using cached value from initialization) + if request.instructions and len(request.instructions) > server._max_instructions_length: + return f"Instructions too long (max {server._max_instructions_length} characters)" + + # Validate max_new_tokens range + if request.max_new_tokens is not None: + if request.max_new_tokens < self.max_new_tokens_min: + return f"max_new_tokens must be at least {self.max_new_tokens_min}" + if request.max_new_tokens > self.max_new_tokens_max: + return f"max_new_tokens cannot exceed {self.max_new_tokens_max}" + + return None async def build( self, request: "OpenAICreateSpeechRequest", sampling_params_list: list, has_inline_ref_audio: bool diff --git a/vllm_omni/entrypoints/openai/tts_adapters/step_audio2.py b/vllm_omni/entrypoints/openai/tts_adapters/step_audio2.py index dd1365204..2c720d8e1 100644 --- a/vllm_omni/entrypoints/openai/tts_adapters/step_audio2.py +++ b/vllm_omni/entrypoints/openai/tts_adapters/step_audio2.py @@ -23,8 +23,9 @@ class StepAudio2Adapter(ARTTSAdapter): name = "step_audio2" def validate(self, request: "OpenAICreateSpeechRequest") -> str | None: - # Dispatcher routes to the step_audio2 case (non-empty input check). - return self.ctx.server._validate_tts_request(request) + if not request.input or not request.input.strip(): + return "Input text cannot be empty" + return None async def build( self, request: "OpenAICreateSpeechRequest", sampling_params_list: list, has_inline_ref_audio: bool diff --git a/vllm_omni/entrypoints/openai/tts_adapters/voxcpm2.py b/vllm_omni/entrypoints/openai/tts_adapters/voxcpm2.py index 221e2f64b..361aa2522 100644 --- a/vllm_omni/entrypoints/openai/tts_adapters/voxcpm2.py +++ b/vllm_omni/entrypoints/openai/tts_adapters/voxcpm2.py @@ -19,7 +19,25 @@ class VoxCPM2Adapter(ARTTSAdapter): name = "voxcpm2" def validate(self, request: "OpenAICreateSpeechRequest") -> str | None: - return self.ctx.server._validate_voxcpm2_request(request) + """Validate VoxCPM2 request parameters. Returns error message or None.""" + server = self.ctx.server + if not request.input or not request.input.strip(): + return "Input text cannot be empty" + + if request.voice is not None: + request.voice = request.voice.lower() + available_voices = set(server.uploaded_speakers) | set(server.precomputed_speakers) | {"default"} + if request.voice not in available_voices: + supported = ", ".join(sorted(available_voices)) or "none" + return f"Invalid voice '{request.voice}'. Supported: {supported}" + + if request.max_new_tokens is not None: + if request.max_new_tokens < self.max_new_tokens_min: + return f"max_new_tokens must be at least {self.max_new_tokens_min}" + if request.max_new_tokens > self.max_new_tokens_max: + return f"max_new_tokens cannot exceed {self.max_new_tokens_max}" + + return None async def build( self, request: "OpenAICreateSpeechRequest", sampling_params_list: list, has_inline_ref_audio: bool diff --git a/vllm_omni/entrypoints/openai/tts_adapters/voxtral.py b/vllm_omni/entrypoints/openai/tts_adapters/voxtral.py index 72e18dc84..0b2d400b1 100644 --- a/vllm_omni/entrypoints/openai/tts_adapters/voxtral.py +++ b/vllm_omni/entrypoints/openai/tts_adapters/voxtral.py @@ -16,7 +16,32 @@ class VoxtralTTSAdapter(ARTTSAdapter): name = "voxtral_tts" def validate(self, request: "OpenAICreateSpeechRequest") -> str | None: - return self.ctx.server._validate_voxtral_tts_request(request) + """Validate Voxtral TTS request parameters. Returns error message or None.""" + server = self.ctx.server + if not request.input or not request.input.strip(): + return "Input text cannot be empty" + + # Voxtral TTS requires either a preset voice or ref_audio for voice cloning. + if request.voice is None and request.ref_audio is None: + return "Either 'voice' (preset speaker) or 'ref_audio' (voice cloning) must be provided" + + if request.ref_audio is not None: + fmt_err = server._validate_ref_audio_format(request.ref_audio) + if fmt_err: + return fmt_err + + if request.voice is not None: + request.voice = request.voice.lower() + if server.supported_speakers and request.voice not in server.supported_speakers: + return f"Invalid speaker '{request.voice}'. Supported: {', '.join(sorted(server.supported_speakers))}" + + if request.max_new_tokens is not None: + if request.max_new_tokens < self.max_new_tokens_min: + return f"max_new_tokens must be at least {self.max_new_tokens_min}" + if request.max_new_tokens > self.max_new_tokens_max: + return f"max_new_tokens cannot exceed {self.max_new_tokens_max}" + + return None async def build( self, request: "OpenAICreateSpeechRequest", sampling_params_list: list, has_inline_ref_audio: bool From 7b2edd66507823c20d30ea097e1df116e9f80984 Mon Sep 17 00:00:00 2001 From: Shenglei Fu <117230642+ShengleiFu@users.noreply.github.com> Date: Mon, 20 Jul 2026 00:38:07 -0700 Subject: [PATCH 025/240] [Bugfix][Qwen3-TTS] Mode-aware ref_audio artifact readiness (fixes #5049) (#5157) --- ...t_qwen3_tts_ref_audio_artifact_mode_e2e.py | 88 +++++++++++++++++++ .../openai_api/test_serving_speech.py | 44 ++++++++-- tests/helpers/runtime.py | 1 + .../entrypoints/openai/serving_speech.py | 37 ++++++-- 4 files changed, 152 insertions(+), 18 deletions(-) create mode 100644 tests/e2e/online_serving/test_qwen3_tts_ref_audio_artifact_mode_e2e.py diff --git a/tests/e2e/online_serving/test_qwen3_tts_ref_audio_artifact_mode_e2e.py b/tests/e2e/online_serving/test_qwen3_tts_ref_audio_artifact_mode_e2e.py new file mode 100644 index 000000000..e6c396932 --- /dev/null +++ b/tests/e2e/online_serving/test_qwen3_tts_ref_audio_artifact_mode_e2e.py @@ -0,0 +1,88 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""E2E regression for Qwen3-TTS ref_audio artifact mode collision (#5049). + +Reproduces the issue's exact sequence on a live server: an +``x_vector_only_mode=true`` request followed by an ICL +(``x_vector_only_mode=false`` + ``ref_text``) request using the **same** +``ref_audio``. + +Before the fix, the ref_audio artifact readiness was mode-agnostic: the +x-vector-only request (which caches a speaker embedding but no ``ref_code``) +marked the artifact "ready", so the ICL request took the artifact-only serving +path (``ref_audio`` stripped before dispatch), found no ``ref_code`` and no +audio to recompute from, and raised in ``build_prompt_embeds`` -> +``EngineDeadError`` — taking the whole server down rather than failing the one +request. + +After the fix (readiness keyed by ``(artifact_key, x_vector_only)``), the ICL +request no longer matches the x-vector artifact, so it sends ``ref_audio`` and +the worker computes ``ref_code``. Both requests succeed and the server stays up. +""" + +import os + +os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" + +import pytest + +from tests.helpers.mark import hardware_test +from tests.helpers.media import load_test_audio_data_url +from tests.helpers.runtime import OmniServerParams +from tests.helpers.stage_config import get_deploy_config_path + +MODEL = "Qwen/Qwen3-TTS-12Hz-0.6B-Base" +DEFAULT_AUDIO_SPEECH_TIMEOUT_S = 180.0 + +# Vendored under tests/assets/qwen3_tts/clone_2.wav (see test_qwen3_tts_base.py). +REF_AUDIO_URL = load_test_audio_data_url("qwen3_tts/clone_2.wav") +REF_TEXT = "Okay. Yeah. I resent you. I love you. I respect you. But you know what? You blew it! And thanks to you." +INPUT_TEXT = "The weather is nice today, perfect for a walk in the park." + +tts_server_params = [ + pytest.param( + OmniServerParams( + model=MODEL, + stage_config_path=get_deploy_config_path("qwen3_tts.yaml"), + server_args=["--trust-remote-code"], + ), + id="async_chunk", + ) +] + + +@pytest.mark.advanced_model +@pytest.mark.core_model +@pytest.mark.tts +@hardware_test(res={"cuda": "L4"}, num_cards=1) +@pytest.mark.parametrize("omni_server", tts_server_params, indirect=True) +def test_xvector_then_icl_same_ref_audio_keeps_engine_alive(omni_server, openai_client) -> None: + """Regression for #5049: x-vector-only then ICL on the same ref_audio. + + Both requests must return audio and the engine must stay alive (pre-fix the + second request killed EngineCore). + """ + base_request = { + "model": omni_server.model, + "input": INPUT_TEXT, + "stream": False, + "timeout": DEFAULT_AUDIO_SPEECH_TIMEOUT_S, + "response_format": "wav", + "task_type": "Base", + "ref_audio": REF_AUDIO_URL, + # Assert non-empty audio came back (not just HTTP 200). + "min_audio_bytes": 1, + } + + # 1) x-vector-only request: caches a speaker embedding (no ref_code) and, on + # completion, marks the ref_audio artifact "ready". + openai_client.send_audio_speech_request({**base_request, "x_vector_only_mode": True}) + + # 2) ICL request with the SAME ref_audio. Pre-fix this reused the x-vector + # artifact via the artifact-only path, hit the missing ref_code, and + # killed EngineCore (#5049). It must now succeed with audio. + openai_client.send_audio_speech_request({**base_request, "x_vector_only_mode": False, "ref_text": REF_TEXT}) + + # 3) The server must still be serving after the ICL request (pre-fix this + # would fail with EngineDeadError / connection refused). + openai_client.send_audio_speech_request({**base_request, "x_vector_only_mode": True}) diff --git a/tests/entrypoints/openai_api/test_serving_speech.py b/tests/entrypoints/openai_api/test_serving_speech.py index 26a66450c..8faca1b78 100644 --- a/tests/entrypoints/openai_api/test_serving_speech.py +++ b/tests/entrypoints/openai_api/test_serving_speech.py @@ -3682,7 +3682,7 @@ class TestTTSAsyncOffloading: 24000, artifact_key, ) - qwen3_tts_server._ref_audio_model_artifact_ready.add(artifact_key) + qwen3_tts_server._ref_audio_model_artifact_ready.add((artifact_key, False)) qwen3_tts_server._codec_frame_rate = 25.0 qwen3_tts_server._tts_tokenizer = lambda _text, padding=False: {"input_ids": list(range(10))} qwen3_tts_server.engine_client.model_config.hf_config.talker_config = SimpleNamespace( @@ -3712,10 +3712,10 @@ class TestTTSAsyncOffloading: qwen3_tts_server._ref_audio_resolve_cache_max_bytes = 1_000_000 qwen3_tts_server._put_resolved_ref_audio("ref-a", [0.0] * 8, 24000, "artifact-a") - qwen3_tts_server._ref_audio_model_artifact_ready.add("artifact-a") + qwen3_tts_server._ref_audio_model_artifact_ready.add(("artifact-a", False)) qwen3_tts_server._put_resolved_ref_audio("ref-b", [0.0] * 8, 24000, "artifact-b") - assert "artifact-a" not in qwen3_tts_server._ref_audio_model_artifact_ready + assert ("artifact-a", False) not in qwen3_tts_server._ref_audio_model_artifact_ready assert "artifact-b" in {entry[3] for entry in qwen3_tts_server._ref_audio_resolve_cache.values()} @pytest.mark.asyncio @@ -3724,13 +3724,13 @@ class TestTTSAsyncOffloading: raise ValueError("boom") yield # pragma: no cover - qwen3_tts_server._request_ref_audio_artifact_keys["req-fail"] = "artifact-fail" + qwen3_tts_server._request_ref_audio_artifact_keys["req-fail"] = ("artifact-fail", False) with pytest.raises(ValueError, match="boom"): await anext(qwen3_tts_server._generate_audio_chunks(failing_generator(), "req-fail")) assert "req-fail" not in qwen3_tts_server._request_ref_audio_artifact_keys - assert "artifact-fail" not in qwen3_tts_server._ref_audio_model_artifact_ready + assert ("artifact-fail", False) not in qwen3_tts_server._ref_audio_model_artifact_ready @pytest.mark.asyncio async def test_generate_audio_chunks_discards_ref_audio_artifact_warmup_on_close(self, qwen3_tts_server): @@ -3743,22 +3743,48 @@ class TestTTSAsyncOffloading: ) await asyncio.sleep(0) - qwen3_tts_server._request_ref_audio_artifact_keys["req-close"] = "artifact-close" + qwen3_tts_server._request_ref_audio_artifact_keys["req-close"] = ("artifact-close", False) stream = qwen3_tts_server._generate_audio_chunks(pcm_generator(), "req-close") assert await anext(stream) await stream.aclose() assert "req-close" not in qwen3_tts_server._request_ref_audio_artifact_keys - assert "artifact-close" not in qwen3_tts_server._ref_audio_model_artifact_ready + assert ("artifact-close", False) not in qwen3_tts_server._ref_audio_model_artifact_ready def test_qwen3_ref_audio_artifact_ready_requires_live_resolve_cache_entry(self, qwen3_tts_server): - qwen3_tts_server._request_ref_audio_artifact_keys["req-evicted"] = "artifact-evicted" + qwen3_tts_server._request_ref_audio_artifact_keys["req-evicted"] = ("artifact-evicted", False) qwen3_tts_server._mark_ref_audio_artifact_ready_for_request("req-evicted") assert "req-evicted" not in qwen3_tts_server._request_ref_audio_artifact_keys - assert "artifact-evicted" not in qwen3_tts_server._ref_audio_model_artifact_ready + assert ("artifact-evicted", False) not in qwen3_tts_server._ref_audio_model_artifact_ready + + def test_qwen3_xvector_ready_artifact_does_not_enable_icl_artifact_only(self, qwen3_tts_server): + # An x-vector-only artifact (speaker embedding, no ref_code) must not enable + # the artifact-only path for a later ICL request with the same ref_audio (#5049). + qwen3_tts_server._put_resolved_ref_audio("ref-a", [0.0] * 8, 24000, "artifact-a") + qwen3_tts_server._track_ref_audio_artifact_warmup("req-xvec", "artifact-a", x_vector_only=True) + qwen3_tts_server._mark_ref_audio_artifact_ready_for_request("req-xvec") + + icl_params = {"task_type": ["Base"], "x_vector_only_mode": [False]} + assert qwen3_tts_server._qwen3_tts_can_use_ref_audio_artifact_only(icl_params, "artifact-a") is False + + def test_qwen3_xvector_ready_artifact_still_reusable_by_xvector_request(self, qwen3_tts_server): + qwen3_tts_server._put_resolved_ref_audio("ref-a", [0.0] * 8, 24000, "artifact-a") + qwen3_tts_server._track_ref_audio_artifact_warmup("req-xvec", "artifact-a", x_vector_only=True) + qwen3_tts_server._mark_ref_audio_artifact_ready_for_request("req-xvec") + + xvec_params = {"task_type": ["Base"], "x_vector_only_mode": [True]} + assert qwen3_tts_server._qwen3_tts_can_use_ref_audio_artifact_only(xvec_params, "artifact-a") is True + + def test_qwen3_icl_ready_artifact_enables_icl_artifact_only(self, qwen3_tts_server): + qwen3_tts_server._put_resolved_ref_audio("ref-a", [0.0] * 8, 24000, "artifact-a") + qwen3_tts_server._track_ref_audio_artifact_warmup("req-icl", "artifact-a", x_vector_only=False) + qwen3_tts_server._mark_ref_audio_artifact_ready_for_request("req-icl") + + icl_params = {"task_type": ["Base"], "x_vector_only_mode": [False]} + assert qwen3_tts_server._qwen3_tts_can_use_ref_audio_artifact_only(icl_params, "artifact-a") is True def test_shutdown_is_idempotent(self, mocker: MockerFixture): """Calling shutdown() twice should not raise.""" diff --git a/tests/helpers/runtime.py b/tests/helpers/runtime.py index 885a4654f..263f34da2 100644 --- a/tests/helpers/runtime.py +++ b/tests/helpers/runtime.py @@ -1795,6 +1795,7 @@ class OpenAIClientHandler: "instructions", "speed", "stream_format", + "x_vector_only_mode", ): if key in request_config: extra_body[key] = request_config[key] diff --git a/vllm_omni/entrypoints/openai/serving_speech.py b/vllm_omni/entrypoints/openai/serving_speech.py index a935157e1..f07ee7e10 100644 --- a/vllm_omni/entrypoints/openai/serving_speech.py +++ b/vllm_omni/entrypoints/openai/serving_speech.py @@ -322,8 +322,11 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): self._ref_audio_resolve_cache_bytes = 0 self._ref_audio_resolve_cache_max_entries = _REF_AUDIO_RESOLVE_CACHE_MAX_ENTRIES self._ref_audio_resolve_cache_max_bytes = _REF_AUDIO_RESOLVE_CACHE_MAX_BYTES - self._ref_audio_model_artifact_ready: set[str] = set() - self._request_ref_audio_artifact_keys: dict[str, str] = {} + # Readiness is keyed by (artifact_key, x_vector_only). An x-vector-only + # request caches a speaker embedding but no ref_code, so its artifact + # must not satisfy a later ICL request that needs ref_code (#5049). + self._ref_audio_model_artifact_ready: set[tuple[str, bool]] = set() + self._request_ref_audio_artifact_keys: dict[str, tuple[str, bool]] = {} self._higgs_audio_v3_ref_code_cache: OrderedDict[str, tuple[torch.Tensor, int]] = OrderedDict() self._higgs_audio_v3_ref_code_cache_bytes = 0 self._higgs_audio_v3_ref_code_inflight: dict[str, asyncio.Task[torch.Tensor]] = {} @@ -2069,23 +2072,35 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): def _discard_ref_audio_artifact_ready_if_unreferenced(self, artifact_key: str) -> None: if artifact_key and all(entry[3] != artifact_key for entry in self._ref_audio_resolve_cache.values()): - self._ref_audio_model_artifact_ready.discard(artifact_key) + self._ref_audio_model_artifact_ready = { + (key, mode) for (key, mode) in self._ref_audio_model_artifact_ready if key != artifact_key + } + + @staticmethod + def _tts_x_vector_only(tts_params: dict[str, Any]) -> bool: + return bool((tts_params.get("x_vector_only_mode") or [False])[0]) def _qwen3_tts_can_use_ref_audio_artifact_only(self, tts_params: dict[str, Any], artifact_key: str | None) -> bool: if self._tts_model_type != "qwen3_tts": return False - if not artifact_key or artifact_key not in self._ref_audio_model_artifact_ready: + x_vector_only = self._tts_x_vector_only(tts_params) + if not artifact_key or (artifact_key, x_vector_only) not in self._ref_audio_model_artifact_ready: return False return (tts_params.get("task_type") or ["CustomVoice"])[0] == "Base" - def _track_ref_audio_artifact_warmup(self, request_id: str, artifact_key: str | None) -> None: + def _track_ref_audio_artifact_warmup( + self, request_id: str, artifact_key: str | None, x_vector_only: bool = False + ) -> None: if artifact_key: - self._request_ref_audio_artifact_keys[request_id] = artifact_key + self._request_ref_audio_artifact_keys[request_id] = (artifact_key, bool(x_vector_only)) def _mark_ref_audio_artifact_ready_for_request(self, request_id: str) -> None: - artifact_key = self._request_ref_audio_artifact_keys.pop(request_id, None) + tracked = self._request_ref_audio_artifact_keys.pop(request_id, None) + if tracked is None: + return + artifact_key, x_vector_only = tracked if artifact_key and any(entry[3] == artifact_key for entry in self._ref_audio_resolve_cache.values()): - self._ref_audio_model_artifact_ready.add(artifact_key) + self._ref_audio_model_artifact_ready.add((artifact_key, x_vector_only)) def _discard_ref_audio_artifact_warmup(self, request_id: str) -> None: self._request_ref_audio_artifact_keys.pop(request_id, None) @@ -3234,7 +3249,11 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): sampling_params_list=sampling_params_list, output_modalities=["audio"], ) - self._track_ref_audio_artifact_warmup(request_id, qwen3_ref_audio_warmup_artifact_key) + self._track_ref_audio_artifact_warmup( + request_id, + qwen3_ref_audio_warmup_artifact_key, + x_vector_only=self._tts_x_vector_only(tts_params), + ) return request_id, generator, tts_params async def _generate_pcm_chunks(self, generator, request_id: str, *, include_sample_rate: bool = False): From 6cd1ff7e645030e6e76adbe2ee2b15cf8bda968a Mon Sep 17 00:00:00 2001 From: Diya Peng <50388704+dpeng123@users.noreply.github.com> Date: Mon, 20 Jul 2026 01:16:31 -0700 Subject: [PATCH 026/240] [Quantization] Add FLUX.2-dev Mistral TE online FP8 support (#5136) Signed-off-by: diyapeng <50388704+diyapeng@users.noreply.github.com> Co-authored-by: diyapeng <50388704+diyapeng@users.noreply.github.com> --- docs/user_guide/quantization/fp8.md | 1 + .../flux2/test_pipeline_flux2_quantization.py | 50 +++++++++++++++++ .../test_mistral_encoder_tp.py | 55 +++++++++++++++++++ .../quantization/test_quantization_quality.py | 25 ++++++++- .../diffusion/models/flux2/pipeline_flux2.py | 37 ++++++++++++- .../models/mistral_encoder/mistral_encoder.py | 25 ++++++++- 6 files changed, 185 insertions(+), 8 deletions(-) create mode 100644 tests/diffusion/models/flux2/test_pipeline_flux2_quantization.py diff --git a/docs/user_guide/quantization/fp8.md b/docs/user_guide/quantization/fp8.md index 6114812db..c76cfd32e 100644 --- a/docs/user_guide/quantization/fp8.md +++ b/docs/user_guide/quantization/fp8.md @@ -88,6 +88,7 @@ warmup_quack_fp8([(14040, 2048, 6144), (14040, 2048, 2048)]) | Wan2.2 | Wan2.2 diffusion pipelines | Not validated | Not validated | Validate against BF16 before documenting as supported | TBD | | | Z-Image | `Tongyi-MAI/Z-Image-Turbo` | Yes | Yes | All layers | None | ✅︎ | | FLUX.1 | `black-forest-labs/FLUX.1-dev`, `black-forest-labs/FLUX.1-schnell` | Yes | Yes | All layers | None | | +| FLUX.2-dev | `black-forest-labs/FLUX.2-dev` | Yes | Not validated | All layers | None | ✅︎ | | FLUX.2-klein | `black-forest-labs/FLUX.2-klein-4B` | Yes | Yes | All layers | None | | | HunyuanImage-3.0 | `tencent/HunyuanImage-3.0`, `tencent/HunyuanImage-3.0-Instruct` | Yes | Yes | All layers; use the Hunyuan stage config for multi-stage runs | None | | | HunyuanVideo-1.5 | `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v`, `720p_t2v`, `480p_i2v` | Yes | Yes | All layers | None | | diff --git a/tests/diffusion/models/flux2/test_pipeline_flux2_quantization.py b/tests/diffusion/models/flux2/test_pipeline_flux2_quantization.py new file mode 100644 index 000000000..3900c0c94 --- /dev/null +++ b/tests/diffusion/models/flux2/test_pipeline_flux2_quantization.py @@ -0,0 +1,50 @@ +from unittest.mock import Mock, patch + +import pytest +import torch + +from vllm_omni.diffusion.models.flux2.pipeline_flux2 import ( + Flux2Pipeline, + _resolve_component_quant_config, +) + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +def test_resolve_component_quant_config_routes_components(): + text_encoder_config = object() + component_config = Mock() + component_config.resolve.side_effect = { + "text_encoder": text_encoder_config, + "transformer": None, + }.get + + assert _resolve_component_quant_config(component_config, "text_encoder") is text_encoder_config + assert _resolve_component_quant_config(component_config, "transformer") is None + + +def test_resolve_component_quant_config_preserves_global_config(): + global_config = object() + assert _resolve_component_quant_config(global_config, "transformer") is global_config + assert _resolve_component_quant_config(None, "text_encoder") is None + + +def test_offload_transformer_before_text_encoder_weights(): + pipeline = object.__new__(Flux2Pipeline) + pipeline.transformer = Mock() + weights = [ + ("transformer.layer.weight", torch.empty(1)), + ("text_encoder.layer.weight", torch.empty(1)), + ("text_encoder.layer.bias", torch.empty(1)), + ] + + iterator = iter(pipeline._offload_transformer_before_text_encoder(weights)) + assert next(iterator)[0] == "transformer.layer.weight" + pipeline.transformer.to.assert_not_called() + + with patch("vllm_omni.diffusion.models.flux2.pipeline_flux2.current_omni_platform.empty_cache") as empty_cache: + assert next(iterator)[0] == "text_encoder.layer.weight" + assert next(iterator)[0] == "text_encoder.layer.bias" + + pipeline.transformer.to.assert_called_once_with("cpu") + empty_cache.assert_called_once_with() diff --git a/tests/diffusion/models/mistral_encoder/test_mistral_encoder_tp.py b/tests/diffusion/models/mistral_encoder/test_mistral_encoder_tp.py index 27d1e757e..20024abe7 100644 --- a/tests/diffusion/models/mistral_encoder/test_mistral_encoder_tp.py +++ b/tests/diffusion/models/mistral_encoder/test_mistral_encoder_tp.py @@ -5,6 +5,7 @@ from types import SimpleNamespace import pytest import torch from vllm.config import DeviceConfig, VllmConfig, set_current_vllm_config +from vllm.model_executor.layers.linear import UnquantizedLinearMethod from vllm_omni.diffusion.models.mistral_encoder.mistral_encoder import ( MistralEncoderModel, @@ -491,3 +492,57 @@ class TestComputeLogits: assert logits.dim() == 3 assert logits.shape[0] == 1 assert logits.shape[1] == 4 + + +class TestQuantConfig: + """Verify quant_config is threaded into weight linears only, with correct prefixes.""" + + def test_quant_config_on_weight_linears(self, mocker): + quant_config = mocker.MagicMock(name="QuantizationConfig") + # Keep construction on the CPU-safe unquantized path while still + # verifying LinearBase consults the supplied quant_config. + quant_config.get_quant_method.return_value = UnquantizedLinearMethod() + + config = _make_config(num_hidden_layers=1) + model = MistralEncoderModel(config, prefix="text_encoder", quant_config=quant_config) + + layer = model.language_model.model.layers[0] + for linear in ( + layer.self_attn.qkv_proj, + layer.self_attn.o_proj, + layer.mlp.gate_up_proj, + layer.mlp.down_proj, + ): + assert linear.quant_config is quant_config + + # get_quant_method should be consulted for the four weight linears. + assert quant_config.get_quant_method.call_count == 4 + seen_prefixes = set() + for call in quant_config.get_quant_method.call_args_list: + if "prefix" in call.kwargs: + seen_prefixes.add(call.kwargs["prefix"]) + elif len(call.args) > 1: + seen_prefixes.add(call.args[1]) + assert any(p.endswith("self_attn.qkv_proj") for p in seen_prefixes) + assert any(p.endswith("self_attn.o_proj") for p in seen_prefixes) + assert any(p.endswith("mlp.gate_up_proj") for p in seen_prefixes) + assert any(p.endswith("mlp.down_proj") for p in seen_prefixes) + + # Embeddings / lm_head stay unquantized (quant_config never passed). + embed = model.language_model.model.embed_tokens + lm_head = model.language_model.lm_head + assert getattr(embed, "quant_config", None) is None + assert getattr(lm_head, "quant_config", None) is None + + def test_layer_prefixes_include_text_encoder(self): + config = _make_config(num_hidden_layers=2) + model = MistralEncoderModel(config, prefix="text_encoder") + + layer0 = model.language_model.model.layers[0] + assert layer0.self_attn.qkv_proj.prefix.startswith("text_encoder.language_model.model.layers.0") + assert layer0.self_attn.o_proj.prefix.startswith("text_encoder.language_model.model.layers.0") + assert layer0.mlp.gate_up_proj.prefix.startswith("text_encoder.language_model.model.layers.0") + assert layer0.mlp.down_proj.prefix.startswith("text_encoder.language_model.model.layers.0") + + layer1 = model.language_model.model.layers[1] + assert "layers.1" in layer1.self_attn.qkv_proj.prefix diff --git a/tests/diffusion/quantization/test_quantization_quality.py b/tests/diffusion/quantization/test_quantization_quality.py index 5f0dc4d62..9b53d8941 100644 --- a/tests/diffusion/quantization/test_quantization_quality.py +++ b/tests/diffusion/quantization/test_quantization_quality.py @@ -82,6 +82,7 @@ class QualityTestConfig: gpu: str = "H100" # minimum GPU requirement negative_prompt: str = "" guidance_scale: float | None = None + enable_cpu_offload: bool = False def baseline_ref(self) -> str: return self.baseline_model or self.model or "" @@ -181,6 +182,18 @@ QUALITY_CONFIGS = [ max_lpips=0.20, num_inference_steps=10, ), + QualityTestConfig( + id="fp8_flux2_dev_text_encoder", + model="black-forest-labs/FLUX.2-dev", + quantization={"text_encoder": "fp8", "transformer": None, "vae": None}, + task="t2i", + prompt="a cup of coffee on a wooden table, morning light", + max_lpips=0.15, + num_inference_steps=10, + enable_cpu_offload=True, + height=1024, + width=1024, + ), QualityTestConfig( id="fp8_qwen_image", model="Qwen/Qwen-Image", @@ -459,7 +472,10 @@ def test_quantization_quality(config: QualityTestConfig): generate_fn = _generate_video if config.task == "t2v" else _generate_image # --- BF16 baseline --- - omni_bl = Omni(model=config.baseline_ref()) + bl_kwargs: dict = {"model": config.baseline_ref()} + if config.enable_cpu_offload: + bl_kwargs["enable_cpu_offload"] = True + omni_bl = Omni(**bl_kwargs) baseline_out, bl_mem = generate_fn(omni_bl, config) omni_bl.shutdown() del omni_bl @@ -468,10 +484,13 @@ def test_quantization_quality(config: QualityTestConfig): # --- Quantized --- quantization = config.quantization_ref() + qt_kwargs: dict = {"model": config.quantized_ref()} + if config.enable_cpu_offload: + qt_kwargs["enable_cpu_offload"] = True if quantization is None: - omni_qt = Omni(model=config.quantized_ref()) + omni_qt = Omni(**qt_kwargs) else: - omni_qt = Omni(model=config.quantized_ref(), quantization_config=quantization) + omni_qt = Omni(**qt_kwargs, quantization_config=quantization) quant_out, qt_mem = generate_fn(omni_qt, config) omni_qt.shutdown() del omni_qt diff --git a/vllm_omni/diffusion/models/flux2/pipeline_flux2.py b/vllm_omni/diffusion/models/flux2/pipeline_flux2.py index f88e582e3..351d94f7e 100644 --- a/vllm_omni/diffusion/models/flux2/pipeline_flux2.py +++ b/vllm_omni/diffusion/models/flux2/pipeline_flux2.py @@ -37,10 +37,17 @@ from vllm_omni.diffusion.profiler.diffusion_pipeline_profiler import DiffusionPi from vllm_omni.diffusion.utils.tf_utils import get_transformer_config_kwargs from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch from vllm_omni.model_executor.model_loader.weight_utils import download_weights_from_hf_specific +from vllm_omni.platforms import current_omni_platform logger = logging.getLogger(__name__) +def _resolve_component_quant_config(quant_config, component: str): + if hasattr(quant_config, "resolve"): + return quant_config.resolve(component) + return quant_config + + class Flux2ImageProcessor(VaeImageProcessor): """Image processor to preprocess the reference image for Flux2.""" @@ -393,10 +400,14 @@ class Flux2Pipeline( text_encoder_config = AutoConfig.from_pretrained( model, subfolder="text_encoder", local_files_only=local_files_only ) + te_quant_config = _resolve_component_quant_config(od_config.quantization_config, "text_encoder") self.text_encoder = MistralEncoderModel( text_encoder_config, prefix="text_encoder", - ).to(self._execution_device) + quant_config=te_quant_config, + ) + if not any(param.is_meta for param in self.text_encoder.parameters()): + self.text_encoder.to(self._execution_device) self.tokenizer = PixtralProcessor.from_pretrained( model, subfolder="tokenizer", local_files_only=local_files_only ) @@ -409,8 +420,9 @@ class Flux2Pipeline( self._execution_device ) transformer_kwargs = get_transformer_config_kwargs(od_config.tf_model_config, Flux2Transformer2DModel) + transformer_quant_config = _resolve_component_quant_config(od_config.quantization_config, "transformer") self.transformer = Flux2Transformer2DModel( - quant_config=od_config.quantization_config, od_config=od_config, **transformer_kwargs + quant_config=transformer_quant_config, od_config=od_config, **transformer_kwargs ) self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) if getattr(self, "vae", None) else 8 @@ -1148,5 +1160,26 @@ class Flux2Pipeline( return DiffusionOutput(output=image) def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: + text_encoder_quant_config = _resolve_component_quant_config(self.od_config.quantization_config, "text_encoder") + transformer_quant_config = _resolve_component_quant_config(self.od_config.quantization_config, "transformer") + if ( + self.od_config.enable_cpu_offload + and text_encoder_quant_config is not None + and transformer_quant_config is None + ): + weights = self._offload_transformer_before_text_encoder(weights) + loader = AutoWeightsLoader(self) return loader.load_weights(weights) + + def _offload_transformer_before_text_encoder( + self, weights: Iterable[tuple[str, torch.Tensor]] + ) -> Iterable[tuple[str, torch.Tensor]]: + transformer_offloaded = False + for name, weight in weights: + if not transformer_offloaded and name.startswith("text_encoder."): + self.transformer.to("cpu") + current_omni_platform.empty_cache() + transformer_offloaded = True + logger.info("Offloaded the unquantized transformer before text encoder quantization") + yield name, weight diff --git a/vllm_omni/diffusion/models/mistral_encoder/mistral_encoder.py b/vllm_omni/diffusion/models/mistral_encoder/mistral_encoder.py index abeacff2a..518453a17 100644 --- a/vllm_omni/diffusion/models/mistral_encoder/mistral_encoder.py +++ b/vllm_omni/diffusion/models/mistral_encoder/mistral_encoder.py @@ -17,7 +17,7 @@ from __future__ import annotations import logging from collections.abc import Iterable -from typing import Any +from typing import TYPE_CHECKING, Any import torch import torch.nn as nn @@ -37,6 +37,9 @@ from vllm.model_executor.layers.linear import ( from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead, VocabParallelEmbedding from vllm.model_executor.model_loader.weight_utils import default_weight_loader +if TYPE_CHECKING: + from vllm.model_executor.layers.quantization.base_config import QuantizationConfig + logger = logging.getLogger(__name__) @@ -118,6 +121,7 @@ class MistralEncoderAttention(nn.Module): num_kv_heads: int, head_dim: int, prefix: str = "", + quant_config: QuantizationConfig | None = None, ): super().__init__() self.hidden_size = hidden_size @@ -140,6 +144,7 @@ class MistralEncoderAttention(nn.Module): total_num_heads=num_heads, total_num_kv_heads=num_kv_heads, bias=False, + quant_config=quant_config, prefix=f"{prefix}.qkv_proj", ) @@ -147,6 +152,7 @@ class MistralEncoderAttention(nn.Module): input_size=num_heads * head_dim, output_size=hidden_size, bias=False, + quant_config=quant_config, prefix=f"{prefix}.o_proj", ) @@ -194,18 +200,21 @@ class MistralEncoderMLP(nn.Module): hidden_size: int, intermediate_size: int, prefix: str = "", + quant_config: QuantizationConfig | None = None, ): super().__init__() self.gate_up_proj = MergedColumnParallelLinear( input_size=hidden_size, output_sizes=[intermediate_size, intermediate_size], bias=False, + quant_config=quant_config, prefix=f"{prefix}.gate_up_proj", ) self.down_proj = RowParallelLinear( input_size=intermediate_size, output_size=hidden_size, bias=False, + quant_config=quant_config, prefix=f"{prefix}.down_proj", ) self.act_fn = SiluAndMul() @@ -227,6 +236,7 @@ class MistralEncoderLayer(nn.Module): intermediate_size: int, rms_norm_eps: float, prefix: str = "", + quant_config: QuantizationConfig | None = None, ): super().__init__() self.self_attn = MistralEncoderAttention( @@ -235,11 +245,13 @@ class MistralEncoderLayer(nn.Module): num_kv_heads=num_kv_heads, head_dim=head_dim, prefix=f"{prefix}.self_attn", + quant_config=quant_config, ) self.mlp = MistralEncoderMLP( hidden_size=hidden_size, intermediate_size=intermediate_size, prefix=f"{prefix}.mlp", + quant_config=quant_config, ) self.input_layernorm = RMSNorm(hidden_size, eps=rms_norm_eps) self.post_attention_layernorm = RMSNorm(hidden_size, eps=rms_norm_eps) @@ -295,7 +307,12 @@ class MistralEncoderModel(nn.Module): parallel layers for TP but simple SDPA for attention (no PagedAttention). """ - def __init__(self, config: PretrainedConfig, prefix: str = ""): + def __init__( + self, + config: PretrainedConfig, + prefix: str = "", + quant_config: QuantizationConfig | None = None, + ): super().__init__() self._processor = None self._system_message_t2i: str | None = None @@ -339,6 +356,7 @@ class MistralEncoderModel(nn.Module): m = self.language_model.model m.embed_tokens = VocabParallelEmbedding(self.vocab_size, self.hidden_size) + layer_prefix_root = f"{prefix}.language_model.model.layers" if prefix else "language_model.model.layers" m.layers = nn.ModuleList( [ @@ -349,7 +367,8 @@ class MistralEncoderModel(nn.Module): head_dim=self.head_dim, intermediate_size=self.intermediate_size, rms_norm_eps=self.rms_norm_eps, - prefix=f"language_model.model.layers.{i}", + prefix=f"{layer_prefix_root}.{i}", + quant_config=quant_config, ) for i in range(self.num_layers) ] From a06664a243d87fe4094bfbb5383e5e7bc5436262 Mon Sep 17 00:00:00 2001 From: Mu GuanLin <1203789601@qq.com> Date: Mon, 20 Jul 2026 20:02:30 +0800 Subject: [PATCH 027/240] [Refactor] Unify the LTX-2 and LTX-2.3 pipeline runtime (#5147) Signed-off-by: mglyn <1203789601@qq.com> --- docs/models/supported_models.md | 4 +- docs/user_guide/diffusion_features.md | 4 +- .../offline_inference/image_to_video.md | 4 +- .../offline_inference/text_to_video.md | 6 +- .../examples/online_serving/text_to_video.md | 12 + .../image_to_video/README.md | 4 +- .../image_to_video/image_to_video.py | 2 +- .../text_to_video/text_to_video.md | 6 +- recipes/LTX/LTX-2.3.md | 18 +- .../dmd2/test_dmd2_request_sanitization.py | 31 +- .../models/dmd2/test_dmd2_scheduler.py | 8 +- .../ltx2/test_ltx2_cfg_parallel_adaptation.py | 58 - ..._pipeline.py => test_ltx2_conditioning.py} | 65 +- tests/diffusion/models/ltx2/test_ltx2_hsdp.py | 25 - .../models/ltx2/test_ltx2_latents.py | 121 ++ ...ward_parallel.py => test_ltx2_parallel.py} | 62 +- ...x2_3_pipeline.py => test_ltx2_pipeline.py} | 533 +++-- .../models/ltx2/test_ltx2_transformer.py | 32 +- ..._ltx2_3_vae_tiling.py => test_ltx2_vae.py} | 58 +- .../accuracy/test_ltx2_3_video_similarity.py | 7 +- vllm_omni/diffusion/models/dmd2/mixin.py | 63 +- vllm_omni/diffusion/models/ltx2/__init__.py | 20 +- .../diffusion/models/ltx2/ltx2_components.py | 297 +++ .../models/ltx2/ltx2_conditioning.py | 578 ++++++ .../diffusion/models/ltx2/ltx2_denoise.py | 407 ++++ .../diffusion/models/ltx2/ltx2_guidance.py | 379 ++++ .../diffusion/models/ltx2/ltx2_latents.py | 293 +++ .../models/ltx2/ltx2_pipeline_runtime.py | 619 ++++++ .../diffusion/models/ltx2/ltx2_recipes.py | 20 + .../diffusion/models/ltx2/ltx2_request.py | 257 +++ .../diffusion/models/ltx2/pipeline_ltx2.py | 1314 ++---------- .../diffusion/models/ltx2/pipeline_ltx2_3.py | 1832 ----------------- .../ltx2/pipeline_ltx2_3_image2video.py | 454 ---- .../models/ltx2/pipeline_ltx2_image2video.py | 909 -------- .../models/ltx2/pipeline_ltx2_two_stage.py | 207 ++ vllm_omni/diffusion/registry.py | 12 +- 36 files changed, 3886 insertions(+), 4835 deletions(-) delete mode 100644 tests/diffusion/models/ltx2/test_ltx2_cfg_parallel_adaptation.py rename tests/diffusion/models/ltx2/{test_ltx2_3_image2video_pipeline.py => test_ltx2_conditioning.py} (71%) delete mode 100644 tests/diffusion/models/ltx2/test_ltx2_hsdp.py create mode 100644 tests/diffusion/models/ltx2/test_ltx2_latents.py rename tests/diffusion/models/ltx2/{test_ltx2_3_forward_parallel.py => test_ltx2_parallel.py} (88%) rename tests/diffusion/models/ltx2/{test_ltx2_3_pipeline.py => test_ltx2_pipeline.py} (57%) rename tests/diffusion/models/ltx2/{test_ltx2_3_vae_tiling.py => test_ltx2_vae.py} (85%) create mode 100644 vllm_omni/diffusion/models/ltx2/ltx2_components.py create mode 100644 vllm_omni/diffusion/models/ltx2/ltx2_conditioning.py create mode 100644 vllm_omni/diffusion/models/ltx2/ltx2_denoise.py create mode 100644 vllm_omni/diffusion/models/ltx2/ltx2_guidance.py create mode 100644 vllm_omni/diffusion/models/ltx2/ltx2_latents.py create mode 100644 vllm_omni/diffusion/models/ltx2/ltx2_pipeline_runtime.py create mode 100644 vllm_omni/diffusion/models/ltx2/ltx2_recipes.py create mode 100644 vllm_omni/diffusion/models/ltx2/ltx2_request.py delete mode 100644 vllm_omni/diffusion/models/ltx2/pipeline_ltx2_3.py delete mode 100644 vllm_omni/diffusion/models/ltx2/pipeline_ltx2_3_image2video.py delete mode 100644 vllm_omni/diffusion/models/ltx2/pipeline_ltx2_image2video.py create mode 100644 vllm_omni/diffusion/models/ltx2/pipeline_ltx2_two_stage.py diff --git a/docs/models/supported_models.md b/docs/models/supported_models.md index 1ab76588f..c1dac6ca7 100644 --- a/docs/models/supported_models.md +++ b/docs/models/supported_models.md @@ -45,8 +45,8 @@ th { | `LTX2ImageToVideoPipeline` | LTX-2-I2V | `Lightricks/LTX-2` | ✅︎ | ✅︎ | | | | `LTX2TwoStagesPipeline` | LTX-2-T2V | `rootonchair/LTX-2-19b-distilled` | ✅︎ | ✅︎ | | | | `LTX2ImageToVideoTwoStagesPipeline` | LTX-2-I2V | `rootonchair/LTX-2-19b-distilled` | ✅︎ | ✅︎ | | | -| `LTX23Pipeline` | LTX-2.3-T2V | `dg845/LTX-2.3-Diffusers` | ✅︎ | ✅︎ | | | -| `LTX23ImageToVideoPipeline` | LTX-2.3-I2V | `dg845/LTX-2.3-Diffusers` | ✅︎ | ✅︎ | | | +| `LTX23Pipeline` | LTX-2.3-T2V | `diffusers/LTX-2.3-Diffusers` | ✅︎ | ✅︎ | | | +| `LTX23ImageToVideoPipeline` | LTX-2.3-I2V | `diffusers/LTX-2.3-Diffusers` | ✅︎ | ✅︎ | | | | `DreamZeroPipeline` | DreamZero-DROID | `GEAR-Dreams/DreamZero-DROID` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | | `HeliosPipeline`, `HeliosPyramidPipeline` | Helios | `BestWishYsh/Helios-Base`, `BestWishYsh/Helios-Mid`, `BestWishYsh/Helios-Distilled` | ✅︎ | ✅︎ | ✅︎ | | | `MagiHumanPipeline` | MagiHuman | `SII-GAIR/daVinci-MagiHuman-Base-1080p` | ✅︎ | ✅︎ | | | diff --git a/docs/user_guide/diffusion_features.md b/docs/user_guide/diffusion_features.md index d3ed3bd67..c34c1338c 100644 --- a/docs/user_guide/diffusion_features.md +++ b/docs/user_guide/diffusion_features.md @@ -154,8 +154,8 @@ The following tables show which models support each feature: | **Wan2.2** | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ (encode/decode) | ❌ | ❌ | | **Wan2.2-S2V** | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ (encode/decode) | ❌ | ❌ | | **Wan2.1-VACE** | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ (decode) | ❌ | ❌ | -| **LTX-2** | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | -| **LTX-2.3** | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ✅ (decode) | ❌ | ❌ | +| **LTX-2** | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ (decode) | ❌ | ❌ | +| **LTX-2.3** | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ (decode) | ❌ | ❌ | | **Helios** | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | | **HunyuanVideo-1.5 T2V I2V** | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ (encode/decode) | ✅ | ❌ | | **DreamID-Omni** | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | diff --git a/docs/user_guide/examples/offline_inference/image_to_video.md b/docs/user_guide/examples/offline_inference/image_to_video.md index dc1d3d6b2..b3d76ac46 100644 --- a/docs/user_guide/examples/offline_inference/image_to_video.md +++ b/docs/user_guide/examples/offline_inference/image_to_video.md @@ -57,7 +57,7 @@ python image_to_video.py \ ```bash python image_to_video.py \ - --model dg845/LTX-2.3-Diffusers \ + --model diffusers/LTX-2.3-Diffusers \ --model-class-name LTX23ImageToVideoPipeline \ --image cherry_blossom.jpg \ --prompt "Cherry blossoms swaying gently in the breeze with synchronized ambient sound" \ @@ -72,7 +72,7 @@ python image_to_video.py \ --output ltx23_i2v_output.mp4 ``` -Use a Diffusers-format checkpoint such as `dg845/LTX-2.3-Diffusers`; the +Use the Diffusers-format checkpoint `diffusers/LTX-2.3-Diffusers`; the upstream `Lightricks/LTX-2.3` raw safetensors repo is not directly loadable by this pipeline. Pass `--model-class-name LTX23ImageToVideoPipeline` to select the LTX-2.3 image-to-video pipeline. diff --git a/docs/user_guide/examples/offline_inference/text_to_video.md b/docs/user_guide/examples/offline_inference/text_to_video.md index 5d89eac34..61c498245 100644 --- a/docs/user_guide/examples/offline_inference/text_to_video.md +++ b/docs/user_guide/examples/offline_inference/text_to_video.md @@ -12,7 +12,7 @@ For backend selection and SageAttention usage, see the [Diffusion Attention Back | Model | Default Resolution | Default Frames | Default Steps | Guidance | VRAM (BF16) | |---|---|---|---|---|---| | `Wan-AI/Wan2.2-T2V-A14B-Diffusers` | 720x1280 | 81 | 40 | 4.0 | ~60 GiB | -| `dg845/LTX-2.3-Diffusers` | 384x512 | 25 | 20 | 4.0 | 96GB-class GPU | +| `diffusers/LTX-2.3-Diffusers` | 384x512 | 25 | 20 | 4.0 | 96GB-class GPU | | `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v` | 480x832 | 121 | 50 | 6.0 | 1×A100 80GB | | `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-720p_t2v` | 720x1280 | 121 | 50 | 6.0 | FP8 + VAE tiling required | @@ -55,7 +55,7 @@ python text_to_video.py \ ```bash python text_to_video.py \ - --model dg845/LTX-2.3-Diffusers \ + --model diffusers/LTX-2.3-Diffusers \ --model-class-name LTX23Pipeline \ --prompt "Cherry blossoms swaying gently in the breeze with synchronized ambient sound" \ --negative-prompt "worst quality, inconsistent motion, blurry, jittery, distorted" \ @@ -70,7 +70,7 @@ python text_to_video.py \ --output ltx23_t2v_output.mp4 ``` -Use a Diffusers-format checkpoint such as `dg845/LTX-2.3-Diffusers`; the +Use the Diffusers-format checkpoint `diffusers/LTX-2.3-Diffusers`; the upstream `Lightricks/LTX-2.3` raw safetensors repo is not directly loadable by this pipeline. Pass `--model-class-name LTX23Pipeline` to select the LTX-2.3 text-to-video pipeline explicitly. diff --git a/docs/user_guide/examples/online_serving/text_to_video.md b/docs/user_guide/examples/online_serving/text_to_video.md index d4d520d5e..0f1f33a25 100644 --- a/docs/user_guide/examples/online_serving/text_to_video.md +++ b/docs/user_guide/examples/online_serving/text_to_video.md @@ -13,6 +13,7 @@ This example demonstrates how to deploy text-to-video models for online video ge | Wan2.1 T2V (14B) | `Wan-AI/Wan2.1-T2V-14B-Diffusers` | | Wan2.2 T2V | `Wan-AI/Wan2.2-T2V-A14B-Diffusers` | | LTX-2 | `Lightricks/LTX-2` | +| LTX-2.3 | `diffusers/LTX-2.3-Diffusers` | ## Wan2.2 T2V @@ -374,6 +375,17 @@ curl -sS -X POST http://localhost:8098/v1/videos \ -F "seed=42" ``` +## LTX-2.3 + +LTX-2.3 uses a Diffusers-format checkpoint and a dedicated pipeline class: + +```bash +vllm serve diffusers/LTX-2.3-Diffusers --omni --port 8098 \ + --model-class-name LTX23Pipeline +``` + +Send requests through the same `/v1/videos` API shown in the LTX-2 example. + ## Example materials ??? abstract "response.json" diff --git a/examples/offline_inference/image_to_video/README.md b/examples/offline_inference/image_to_video/README.md index c161022a5..4604a64bd 100644 --- a/examples/offline_inference/image_to_video/README.md +++ b/examples/offline_inference/image_to_video/README.md @@ -237,7 +237,7 @@ python image_to_video.py \ ```bash python image_to_video.py \ - --model dg845/LTX-2.3-Diffusers \ + --model diffusers/LTX-2.3-Diffusers \ --model-class-name LTX23ImageToVideoPipeline \ --image cherry_blossom.jpg \ --prompt "Cherry blossoms swaying gently in the breeze with synchronized ambient sound" \ @@ -252,7 +252,7 @@ python image_to_video.py \ --output ltx23_i2v_output.mp4 ``` -Use a Diffusers-format checkpoint such as `dg845/LTX-2.3-Diffusers`; the +Use the Diffusers-format checkpoint `diffusers/LTX-2.3-Diffusers`; the upstream `Lightricks/LTX-2.3` raw safetensors repo is not directly loadable by this pipeline. Pass `--model-class-name LTX23ImageToVideoPipeline` to select the LTX-2.3 image-to-video pipeline. diff --git a/examples/offline_inference/image_to_video/image_to_video.py b/examples/offline_inference/image_to_video/image_to_video.py index d64367bc2..164290d43 100644 --- a/examples/offline_inference/image_to_video/image_to_video.py +++ b/examples/offline_inference/image_to_video/image_to_video.py @@ -29,7 +29,7 @@ Usage: --frame-rate 24 --fps 24 --output ltx2_i2v.mp4 # LTX-2.3 image-to-video - python image_to_video.py --model dg845/LTX-2.3-Diffusers \ + python image_to_video.py --model diffusers/LTX-2.3-Diffusers \ --model-class-name LTX23ImageToVideoPipeline \ --image input.jpg --prompt "A cinematic dolly shot of a boat" \ --height 384 --width 512 --num-frames 25 --num-inference-steps 20 \ diff --git a/examples/offline_inference/text_to_video/text_to_video.md b/examples/offline_inference/text_to_video/text_to_video.md index edc75bb26..876b8429b 100644 --- a/examples/offline_inference/text_to_video/text_to_video.md +++ b/examples/offline_inference/text_to_video/text_to_video.md @@ -8,7 +8,7 @@ A unified script for text-to-video generation. Supports multiple models with mod |---|---|---|---|---|---| | `Wan-AI/Wan2.1-VACE-1.3B-diffusers` | 480x832 | 81 | 30 | 5.0 | ~20 GiB (RTX 5090, VAE tiling) | | `Wan-AI/Wan2.2-T2V-A14B-Diffusers` | 720x1280 | 81 | 40 | 4.0 | ~60 GiB | -| `dg845/LTX-2.3-Diffusers` | 384x512 | 25 | 20 | 4.0 | 96GB-class GPU | +| `diffusers/LTX-2.3-Diffusers` | 384x512 | 25 | 20 | 4.0 | 96GB-class GPU | | `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v` | 480x832 | 121 | 50 | 6.0 | 1×A100 80GB | | `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-720p_t2v` | 720x1280 | 121 | 50 | 6.0 | FP8 + VAE tiling required | | `nvidia/Cosmos3-Nano` | 720x1280 | 189 | 35 | 6.0 | ~46 GiB (peak, 720p) | @@ -75,7 +75,7 @@ python text_to_video.py \ ```bash python text_to_video.py \ - --model dg845/LTX-2.3-Diffusers \ + --model diffusers/LTX-2.3-Diffusers \ --model-class-name LTX23Pipeline \ --prompt "Cherry blossoms swaying gently in the breeze with synchronized ambient sound" \ --negative-prompt "worst quality, inconsistent motion, blurry, jittery, distorted" \ @@ -90,7 +90,7 @@ python text_to_video.py \ --output ltx23_t2v_output.mp4 ``` -Use a Diffusers-format checkpoint such as `dg845/LTX-2.3-Diffusers`; the +Use the Diffusers-format checkpoint `diffusers/LTX-2.3-Diffusers`; the upstream `Lightricks/LTX-2.3` raw safetensors repo is not directly loadable by this pipeline. Pass `--model-class-name LTX23Pipeline` to select the LTX-2.3 text-to-video pipeline explicitly. diff --git a/recipes/LTX/LTX-2.3.md b/recipes/LTX/LTX-2.3.md index 7a4f1507c..471ee7ae0 100644 --- a/recipes/LTX/LTX-2.3.md +++ b/recipes/LTX/LTX-2.3.md @@ -5,7 +5,7 @@ ## Summary - Vendor: Lightricks -- Model: `dg845/LTX-2.3-Diffusers` +- Model: `diffusers/LTX-2.3-Diffusers` - Task: Text-to-video and image-to-video with synchronized audio generation - Mode: Online serving (pure diffusion) - Maintainer: @oglok @@ -22,7 +22,7 @@ encoder, VAE, and vocoder components. ## References - Upstream raw checkpoints: -- Diffusers-format checkpoint: +- Diffusers-format checkpoint: - Requires `diffusers >= 0.38.0` (install from git: `pip install git+https://github.com/huggingface/diffusers.git`) ## Serving @@ -30,7 +30,7 @@ encoder, VAE, and vocoder components. ### Command ```bash -vllm serve dg845/LTX-2.3-Diffusers \ +vllm serve diffusers/LTX-2.3-Diffusers \ --omni \ --model-class-name LTX23Pipeline \ --stage-init-timeout 600 @@ -39,7 +39,7 @@ vllm serve dg845/LTX-2.3-Diffusers \ For image-to-video: ```bash -vllm serve dg845/LTX-2.3-Diffusers \ +vllm serve diffusers/LTX-2.3-Diffusers \ --omni \ --model-class-name LTX23ImageToVideoPipeline \ --stage-init-timeout 600 @@ -55,7 +55,7 @@ curl http://localhost:8000/health curl -X POST http://localhost:8000/v1/videos \ -F "prompt=A majestic bald eagle soaring over a misty mountain valley at dawn, golden sunlight breaking through clouds" \ -F "negative_prompt=blurry, low quality, distorted, watermark" \ - -F "model=dg845/LTX-2.3-Diffusers" \ + -F "model=diffusers/LTX-2.3-Diffusers" \ -F "num_frames=81" \ -F "fps=24" \ -F "size=768x512" \ @@ -66,7 +66,7 @@ curl -X POST http://localhost:8000/v1/videos \ # Generate a 10-second video (241 frames) curl -X POST http://localhost:8000/v1/videos \ -F "prompt=A cozy Japanese ramen shop at night in the rain, steam rising from bowls, neon signs reflecting on wet cobblestone streets" \ - -F "model=dg845/LTX-2.3-Diffusers" \ + -F "model=diffusers/LTX-2.3-Diffusers" \ -F "num_frames=241" \ -F "fps=24" \ -F "size=768x512" \ @@ -76,7 +76,7 @@ curl -X POST http://localhost:8000/v1/videos \ # Generate a 20-second video (481 frames) curl -X POST http://localhost:8000/v1/videos \ -F "prompt=An underwater coral reef teeming with tropical fish, sea turtles gliding gracefully, National Geographic documentary style" \ - -F "model=dg845/LTX-2.3-Diffusers" \ + -F "model=diffusers/LTX-2.3-Diffusers" \ -F "num_frames=481" \ -F "fps=24" \ -F "size=768x512" \ @@ -96,7 +96,7 @@ curl http://localhost:8000/health curl -X POST http://localhost:8000/v1/videos/sync \ -F "prompt=A plush toy astronaut gently waving while the camera slowly pushes in." \ -F "input_reference=@/absolute/path/to/reference.png" \ - -F "model=dg845/LTX-2.3-Diffusers" \ + -F "model=diffusers/LTX-2.3-Diffusers" \ -F "num_frames=81" \ -F "fps=24" \ -F "size=768x512" \ @@ -111,7 +111,7 @@ curl -X POST http://localhost:8000/v1/videos/sync \ - Memory usage: run a latest-head serving smoke or DFX benchmark on the target GPU before publishing load/peak VRAM numbers. - Checkpoint format: use a Diffusers-format checkpoint such as - `dg845/LTX-2.3-Diffusers`; the upstream `Lightricks/LTX-2.3` repository ships + `diffusers/LTX-2.3-Diffusers`; the upstream `Lightricks/LTX-2.3` repository ships raw safetensors and does not contain the subfolder configs required by this pipeline loader. - Key flags: diff --git a/tests/diffusion/models/dmd2/test_dmd2_request_sanitization.py b/tests/diffusion/models/dmd2/test_dmd2_request_sanitization.py index 21679b5b7..5f8de588e 100644 --- a/tests/diffusion/models/dmd2/test_dmd2_request_sanitization.py +++ b/tests/diffusion/models/dmd2/test_dmd2_request_sanitization.py @@ -6,12 +6,17 @@ import pytest import torch from vllm_omni.diffusion.models.flux.pipeline_flux import FluxDMD2Pipeline, FluxPipeline -from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import LTX2Pipeline, LTX2T2VDMD2Pipeline -from vllm_omni.diffusion.models.ltx2.pipeline_ltx2_image2video import LTX2I2VDMD2Pipeline, LTX2ImageToVideoPipeline +from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import ( + LTX2I2VDMD2Pipeline, + LTX2ImageToVideoPipeline, + LTX2Pipeline, + LTX2T2VDMD2Pipeline, +) from vllm_omni.diffusion.models.qwen_image.pipeline_qwen_image import QwenImageDMD2Pipeline, QwenImagePipeline from vllm_omni.diffusion.models.wan2_2.pipeline_wan2_2 import Wan22Pipeline, WanT2VDMD2Pipeline from vllm_omni.diffusion.models.wan2_2.pipeline_wan2_2_i2v import Wan22I2VPipeline, WanI2VDMD2Pipeline from vllm_omni.diffusion.request import OmniDiffusionRequest, OmniDiffusionSamplingParams +from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch pytestmark = [pytest.mark.core_model, pytest.mark.cpu] @@ -152,6 +157,28 @@ def test_string_prompt_not_mutated(pipeline): assert req.prompt == "a cat dancing" +def test_request_batch_sanitizes_each_request(pipeline): + requests = [ + _make_request( + prompt={"prompt": f"prompt-{index}", "negative_prompt": "blurry"}, + num_inference_steps=40 + index, + guidance_scale=5.0 + index, + guidance_scale_provided=True, + ) + for index in range(2) + ] + + pipeline._sanitize_dmd2_request(DiffusionRequestBatch(requests)) + + for request in requests: + assert request.sampling_params.num_inference_steps == pipeline.dmd2_config.num_inference_steps + assert request.sampling_params.guidance_scale == pipeline.dmd2_config.guidance_scale + assert request.sampling_params.guidance_scale_provided is False + assert request.sampling_params.do_classifier_free_guidance is False + assert request.sampling_params.is_cfg_negative is False + assert "negative_prompt" not in request.prompt + + # --------------------------------------------------------------------------- # Clean request — nothing changes # --------------------------------------------------------------------------- diff --git a/tests/diffusion/models/dmd2/test_dmd2_scheduler.py b/tests/diffusion/models/dmd2/test_dmd2_scheduler.py index 0b6ca3a9e..a395af959 100644 --- a/tests/diffusion/models/dmd2/test_dmd2_scheduler.py +++ b/tests/diffusion/models/dmd2/test_dmd2_scheduler.py @@ -6,8 +6,12 @@ import pytest import torch from vllm_omni.diffusion.models.flux.pipeline_flux import FluxDMD2Pipeline, FluxPipeline -from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import LTX2Pipeline, LTX2T2VDMD2Pipeline -from vllm_omni.diffusion.models.ltx2.pipeline_ltx2_image2video import LTX2I2VDMD2Pipeline, LTX2ImageToVideoPipeline +from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import ( + LTX2I2VDMD2Pipeline, + LTX2ImageToVideoPipeline, + LTX2Pipeline, + LTX2T2VDMD2Pipeline, +) from vllm_omni.diffusion.models.qwen_image.pipeline_qwen_image import QwenImageDMD2Pipeline, QwenImagePipeline from vllm_omni.diffusion.models.wan2_2.pipeline_wan2_2 import Wan22Pipeline, WanT2VDMD2Pipeline from vllm_omni.diffusion.models.wan2_2.pipeline_wan2_2_i2v import Wan22I2VPipeline, WanI2VDMD2Pipeline diff --git a/tests/diffusion/models/ltx2/test_ltx2_cfg_parallel_adaptation.py b/tests/diffusion/models/ltx2/test_ltx2_cfg_parallel_adaptation.py deleted file mode 100644 index bbfe63dfa..000000000 --- a/tests/diffusion/models/ltx2/test_ltx2_cfg_parallel_adaptation.py +++ /dev/null @@ -1,58 +0,0 @@ -from types import SimpleNamespace - -import pytest -import torch - -from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import LTX2Pipeline - -pytestmark = [pytest.mark.core_model, pytest.mark.cpu] - - -def _make_pipeline(sequence_parallel_size: int = 1) -> LTX2Pipeline: - pipeline = object.__new__(LTX2Pipeline) - torch.nn.Module.__init__(pipeline) - pipeline.audio_vae_temporal_compression_ratio = 4 - pipeline.audio_vae_mel_compression_ratio = 4 - pipeline.od_config = SimpleNamespace(parallel_config=SimpleNamespace(sequence_parallel_size=sequence_parallel_size)) - # Mock audio_vae with identity normalization (mean=0, std=1) so - # _normalize_audio_latents is a no-op and test values are preserved. - pipeline.audio_vae = SimpleNamespace( - latents_mean=torch.tensor(0.0), - latents_std=torch.tensor(1.0), - ) - return pipeline - - -def test_prepare_audio_latents_pads_packed_sequence_dim_for_provided_latents(): - pipeline = _make_pipeline(sequence_parallel_size=4) - latents = torch.arange(40, dtype=torch.float32).view(1, 10, 4) - - padded, original_num_frames, padded_num_frames = pipeline.prepare_audio_latents( - batch_size=1, - num_channels_latents=2, - num_mel_bins=8, - audio_latent_length=10, - dtype=torch.float32, - device=torch.device("cpu"), - latents=latents, - ) - - assert original_num_frames == 10 - assert padded_num_frames == 12 - assert padded.shape == (1, 12, 4) - torch.testing.assert_close(padded[:, :10], latents) - torch.testing.assert_close(padded[:, 10:], torch.zeros(1, 2, 4)) - - -def test_unpad_audio_latents_restores_original_frames_before_unpack(): - pipeline = _make_pipeline() - original = torch.arange(40, dtype=torch.float32).view(1, 10, 4) - padded = torch.cat([original, torch.full((1, 2, 4), 999.0)], dim=1) - - unpadded = pipeline._unpad_audio_latents(padded, 10) - unpacked = pipeline._unpack_audio_latents(unpadded, latent_length=10, num_mel_bins=2) - expected = pipeline._unpack_audio_latents(original, latent_length=10, num_mel_bins=2) - - assert unpacked.shape == (1, 2, 10, 2) - assert not (unpacked == 999.0).any() - torch.testing.assert_close(unpacked, expected) diff --git a/tests/diffusion/models/ltx2/test_ltx2_3_image2video_pipeline.py b/tests/diffusion/models/ltx2/test_ltx2_conditioning.py similarity index 71% rename from tests/diffusion/models/ltx2/test_ltx2_3_image2video_pipeline.py rename to tests/diffusion/models/ltx2/test_ltx2_conditioning.py index 9b883ff34..43737f8c1 100644 --- a/tests/diffusion/models/ltx2/test_ltx2_3_image2video_pipeline.py +++ b/tests/diffusion/models/ltx2/test_ltx2_conditioning.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Unit tests for the LTX-2.3 image-to-video pipeline.""" +"""Unit tests for LTX image-to-video input and conditioning behavior.""" from types import SimpleNamespace @@ -19,9 +19,9 @@ def _make_ltx23_request_pipe(cls): return pipe -class TestLTX23ImageToVideoForwardStages: +class TestLTXImageToVideoForwardStages: def test_forward_resolves_request_image_and_delegates_to_shared_forward_impl(self): - from vllm_omni.diffusion.models.ltx2.pipeline_ltx2_3_image2video import LTX23ImageToVideoPipeline + from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import LTX23ImageToVideoPipeline from vllm_omni.diffusion.request import OmniDiffusionRequest from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch from vllm_omni.inputs.data import OmniDiffusionSamplingParams @@ -66,7 +66,7 @@ class TestLTX23ImageToVideoForwardStages: assert seen["kwargs"]["noise_scale"] == 0.5 def test_denoise_timestep_kwargs_masks_video_only(self): - from vllm_omni.diffusion.models.ltx2.pipeline_ltx2_3_image2video import LTX23ImageToVideoPipeline + from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import LTX23ImageToVideoPipeline pipe = object.__new__(LTX23ImageToVideoPipeline) ts = torch.tensor([2.0, 4.0]) @@ -86,16 +86,14 @@ class TestLTX23ImageToVideoForwardStages: torch.testing.assert_close(kwargs["sigma"], ts) -class TestLTX23ImageToVideoPipeline: - def test_ltx23_i2v_pipeline_reuses_ltx23_semantics(self): - from vllm_omni.diffusion.models.ltx2.pipeline_ltx2_3 import LTX23Pipeline - from vllm_omni.diffusion.models.ltx2.pipeline_ltx2_3_image2video import LTX23ImageToVideoPipeline +class TestLTXImageToVideoConditioning: + def test_ltx23_i2v_supports_image_input(self): + from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import LTX23ImageToVideoPipeline - assert issubclass(LTX23ImageToVideoPipeline, LTX23Pipeline) assert LTX23ImageToVideoPipeline.support_image_input is True def test_ltx23_i2v_rejects_multi_image_prompt_list(self): - from vllm_omni.diffusion.models.ltx2.pipeline_ltx2_3_image2video import LTX23ImageToVideoPipeline + from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import LTX23ImageToVideoPipeline image = object() @@ -104,7 +102,7 @@ class TestLTX23ImageToVideoPipeline: LTX23ImageToVideoPipeline._resolve_single_prompt_image([object(), object()]) def test_ltx23_i2v_additional_image_resolution_is_tensor_safe(self): - from vllm_omni.diffusion.models.ltx2.pipeline_ltx2_3_image2video import LTX23ImageToVideoPipeline + from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import LTX23ImageToVideoPipeline image = torch.zeros(1, 3, 4, 4) additional = { @@ -116,8 +114,9 @@ class TestLTX23ImageToVideoPipeline: assert LTX23ImageToVideoPipeline._resolve_additional_image(additional) is image def test_ltx23_i2v_packed_latents_are_not_noised(self, monkeypatch): - import vllm_omni.diffusion.models.ltx2.pipeline_ltx2_3_image2video as ltx23_i2v - from vllm_omni.diffusion.models.ltx2.pipeline_ltx2_3_image2video import LTX23ImageToVideoPipeline + import vllm_omni.diffusion.models.ltx2.ltx2_conditioning as ltx2_conditioning + import vllm_omni.diffusion.models.ltx2.ltx2_latents as ltx2_latents + from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import LTX23ImageToVideoPipeline pipe = object.__new__(LTX23ImageToVideoPipeline) torch.nn.Module.__init__(pipe) @@ -129,7 +128,8 @@ class TestLTX23ImageToVideoPipeline: def fake_randn_tensor(shape, generator=None, device=None, dtype=None): raise AssertionError("packed I2V latents should not be noised") - monkeypatch.setattr(ltx23_i2v, "randn_tensor", fake_randn_tensor) + monkeypatch.setattr(ltx2_conditioning, "randn_tensor", fake_randn_tensor) + monkeypatch.setattr(ltx2_latents, "randn_tensor", fake_randn_tensor) latents = torch.tensor([[[10.0, 11.0], [20.0, 21.0], [30.0, 31.0]]]) @@ -150,9 +150,9 @@ class TestLTX23ImageToVideoPipeline: torch.testing.assert_close(out, latents) def test_ltx23_i2v_5d_latents_noise_preserves_conditioning_frame(self, monkeypatch): - import vllm_omni.diffusion.models.ltx2.pipeline_ltx2 as ltx2 - import vllm_omni.diffusion.models.ltx2.pipeline_ltx2_3_image2video as ltx23_i2v - from vllm_omni.diffusion.models.ltx2.pipeline_ltx2_3_image2video import LTX23ImageToVideoPipeline + import vllm_omni.diffusion.models.ltx2.ltx2_conditioning as ltx2_conditioning + import vllm_omni.diffusion.models.ltx2.ltx2_latents as ltx2_latents + from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import LTX23ImageToVideoPipeline pipe = object.__new__(LTX23ImageToVideoPipeline) torch.nn.Module.__init__(pipe) @@ -169,8 +169,8 @@ class TestLTX23ImageToVideoPipeline: def fake_randn_tensor(shape, generator=None, device=None, dtype=None): return torch.ones(shape, device=device, dtype=dtype) - monkeypatch.setattr(ltx23_i2v, "randn_tensor", fake_randn_tensor) - monkeypatch.setattr(ltx2, "randn_tensor", fake_randn_tensor) + monkeypatch.setattr(ltx2_conditioning, "randn_tensor", fake_randn_tensor) + monkeypatch.setattr(ltx2_latents, "randn_tensor", fake_randn_tensor) latents = torch.tensor([[[[[10.0]], [[20.0]], [[30.0]]], [[[11.0]], [[21.0]], [[31.0]]]]]) @@ -191,7 +191,7 @@ class TestLTX23ImageToVideoPipeline: torch.testing.assert_close(out, torch.tensor([[[10.0, 11.0], [1.0, 1.0], [1.0, 1.0]]])) def test_ltx23_i2v_video_step_preserves_conditioning_frame(self): - from vllm_omni.diffusion.models.ltx2.pipeline_ltx2_3_image2video import LTX23ImageToVideoPipeline + from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import LTX23ImageToVideoPipeline pipe = object.__new__(LTX23ImageToVideoPipeline) torch.nn.Module.__init__(pipe) @@ -217,28 +217,3 @@ class TestLTX23ImageToVideoPipeline: torch.testing.assert_close(out[:, :1], latents[:, :1]) torch.testing.assert_close(out[:, 1:], latents[:, 1:] + noise_pred[:, 1:] + 0.5) - - -class TestLTX23ImageToVideoModule: - """Test that pipeline_ltx2_3_image2video.py exposes I2V entry points.""" - - def test_i2v_classes_importable(self): - """I2V classes must be importable from the implementation module.""" - from vllm_omni.diffusion.models.ltx2.pipeline_ltx2_3_image2video import LTX23ImageToVideoPipeline - - assert LTX23ImageToVideoPipeline is not None - - def test_post_process_func_importable(self): - """get_ltx2_post_process_func must be importable for registry lookup.""" - from vllm_omni.diffusion.models.ltx2.pipeline_ltx2_3_image2video import get_ltx2_post_process_func - - assert callable(get_ltx2_post_process_func) - - def test_i2v_class_matches_package_export(self): - """Package export must point at the I2V implementation module.""" - from vllm_omni.diffusion.models.ltx2 import LTX23ImageToVideoPipeline as PackageExported - from vllm_omni.diffusion.models.ltx2.pipeline_ltx2_3_image2video import ( - LTX23ImageToVideoPipeline as ModuleExported, - ) - - assert PackageExported is ModuleExported diff --git a/tests/diffusion/models/ltx2/test_ltx2_hsdp.py b/tests/diffusion/models/ltx2/test_ltx2_hsdp.py deleted file mode 100644 index 4dd07e1bf..000000000 --- a/tests/diffusion/models/ltx2/test_ltx2_hsdp.py +++ /dev/null @@ -1,25 +0,0 @@ -import pytest -import torch.nn as nn - -from vllm_omni.diffusion.models.ltx2.ltx2_transformer import LTX2VideoTransformer3DModel - -pytestmark = [pytest.mark.core_model, pytest.mark.cpu] - - -def test_ltx2_exposes_hsdp_shard_conditions_for_transformer_blocks(): - model = object.__new__(LTX2VideoTransformer3DModel) - nn.Module.__init__(model) - model.transformer_blocks = nn.ModuleList([nn.Linear(4, 4) for _ in range(2)]) - model.norm_out = nn.LayerNorm(4) - - conditions = getattr(model, "_hsdp_shard_conditions", None) - - assert conditions is not None - assert len(conditions) == 1 - - matched = [] - for name, module in model.named_modules(): - if any(cond(name, module) for cond in conditions): - matched.append(name) - - assert matched == ["transformer_blocks.0", "transformer_blocks.1"] diff --git a/tests/diffusion/models/ltx2/test_ltx2_latents.py b/tests/diffusion/models/ltx2/test_ltx2_latents.py new file mode 100644 index 000000000..74cad09d1 --- /dev/null +++ b/tests/diffusion/models/ltx2/test_ltx2_latents.py @@ -0,0 +1,121 @@ +from types import SimpleNamespace + +import pytest +import torch + +from vllm_omni.diffusion.models.ltx2 import ltx2_latents as latent_ops +from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import LTX2Pipeline, LTX23Pipeline + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +def _make_pipeline(pipeline_cls, sequence_parallel_size: int = 1): + pipeline = object.__new__(pipeline_cls) + torch.nn.Module.__init__(pipeline) + pipeline.audio_vae_temporal_compression_ratio = 4 + pipeline.audio_vae_mel_compression_ratio = 4 + pipeline.od_config = SimpleNamespace(parallel_config=SimpleNamespace(sequence_parallel_size=sequence_parallel_size)) + # Mock audio_vae with identity normalization (mean=0, std=1). + pipeline.audio_vae = SimpleNamespace( + latents_mean=torch.tensor(0.0), + latents_std=torch.tensor(1.0), + ) + return pipeline + + +def test_prepare_audio_latents_pads_generated_dummy_length_for_sp(): + pipeline = _make_pipeline(LTX23Pipeline, sequence_parallel_size=2) + + latents, original_num_frames, padded_num_frames = pipeline.prepare_audio_latents( + batch_size=1, + num_channels_latents=8, + num_mel_bins=64, + audio_latent_length=1, + dtype=torch.float32, + device=torch.device("cpu"), + ) + + assert original_num_frames == 1 + assert padded_num_frames == 2 + assert latents.shape == (1, 2, 128) + + +def test_prepare_audio_latents_pads_packed_sequence_dim_for_provided_latents(): + pipeline = _make_pipeline(LTX2Pipeline, sequence_parallel_size=4) + latents = torch.arange(40, dtype=torch.float32).view(1, 10, 4) + + padded, original_num_frames, padded_num_frames = pipeline.prepare_audio_latents( + batch_size=1, + num_channels_latents=2, + num_mel_bins=8, + audio_latent_length=10, + dtype=torch.float32, + device=torch.device("cpu"), + latents=latents, + ) + + assert original_num_frames == 10 + assert padded_num_frames == 12 + assert padded.shape == (1, 12, 4) + torch.testing.assert_close(padded[:, :10], latents) + torch.testing.assert_close(padded[:, 10:], torch.zeros(1, 2, 4)) + + +def test_unpad_audio_latents_restores_original_frames_before_unpack(): + original = torch.arange(40, dtype=torch.float32).view(1, 10, 4) + padded = torch.cat([original, torch.full((1, 2, 4), 999.0)], dim=1) + + unpadded = latent_ops.unpad_audio_latents(padded, 10) + unpacked = latent_ops.unpack_audio_latents(unpadded, latent_length=10, num_mel_bins=2) + expected = latent_ops.unpack_audio_latents(original, latent_length=10, num_mel_bins=2) + + assert unpacked.shape == (1, 2, 10, 2) + assert not (unpacked == 999.0).any() + torch.testing.assert_close(unpacked, expected) + + +def test_prepare_audio_latents_accepts_already_padded_4d_latents_for_sp(): + pipeline = _make_pipeline(LTX23Pipeline, sequence_parallel_size=4) + latents = torch.arange(96, dtype=torch.float32).view(1, 2, 12, 4) + + audio_latent_length = pipeline._resolve_audio_latent_length(10, latents) + padded, original_num_frames, padded_num_frames = pipeline.prepare_audio_latents( + batch_size=1, + num_channels_latents=2, + num_mel_bins=16, + audio_latent_length=audio_latent_length, + dtype=torch.float32, + device=torch.device("cpu"), + latents=latents, + ) + + assert audio_latent_length == 10 + assert original_num_frames == 10 + assert padded_num_frames == 12 + assert padded.shape == (1, 12, 8) + torch.testing.assert_close(padded, latent_ops.pack_audio_latents(latents)) + + +def test_resolve_audio_latent_length_preserves_legacy_4d_shape_inference(): + pipeline = _make_pipeline(LTX23Pipeline, sequence_parallel_size=4) + latents = torch.zeros(1, 2, 13, 4) + + audio_latent_length = pipeline._resolve_audio_latent_length(10, latents) + + assert audio_latent_length == 13 + + +def test_prepare_audio_latents_rejects_incompatible_provided_length(): + pipeline = _make_pipeline(LTX23Pipeline, sequence_parallel_size=4) + latents = torch.zeros(1, 11, 4) + + with pytest.raises(ValueError, match="incompatible audio frame count"): + pipeline.prepare_audio_latents( + batch_size=1, + num_channels_latents=2, + num_mel_bins=8, + audio_latent_length=10, + dtype=torch.float32, + device=torch.device("cpu"), + latents=latents, + ) diff --git a/tests/diffusion/models/ltx2/test_ltx2_3_forward_parallel.py b/tests/diffusion/models/ltx2/test_ltx2_parallel.py similarity index 88% rename from tests/diffusion/models/ltx2/test_ltx2_3_forward_parallel.py rename to tests/diffusion/models/ltx2/test_ltx2_parallel.py index 3e592f14b..0c6b12fbe 100644 --- a/tests/diffusion/models/ltx2/test_ltx2_3_forward_parallel.py +++ b/tests/diffusion/models/ltx2/test_ltx2_parallel.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Unit tests for LTX-2.3 forward-parallel behavior.""" +"""Unit tests for LTX guidance and forward-parallel behavior.""" from types import SimpleNamespace @@ -11,29 +11,11 @@ import torch pytestmark = [pytest.mark.core_model, pytest.mark.cpu] -class TestCFGParallelIntegration: - def test_ltx23_pipeline_has_cfg_parallel_mixin(self): - """LTX23Pipeline must use the shared CFG parallel implementation.""" - from vllm_omni.diffusion.distributed.cfg_parallel import CFGParallelMixin - from vllm_omni.diffusion.models.ltx2.pipeline_ltx2_3 import LTX23Pipeline - - assert issubclass(LTX23Pipeline, CFGParallelMixin) - - def test_ltx2_transformer_has_dit_cache_config(self): - """Ensure LTX2 has a Cache DiT adapter config and that it uses separate CFG.""" - from vllm_omni.diffusion.cache.cache_dit_backend import CacheDiTAdapterConfig - from vllm_omni.diffusion.models.ltx2.ltx2_transformer import LTX2VideoTransformer3DModel - - adapter_config = getattr(LTX2VideoTransformer3DModel, "_cache_dit_adapter_config") - assert isinstance(adapter_config, CacheDiTAdapterConfig) - assert adapter_config.has_separate_cfg - - class TestCFGParallelHelpers: """Test LTX-2.3 CFG helper math without loading model weights.""" def test_combine_cfg_noise_matches_x0_space_formula(self): - from vllm_omni.diffusion.models.ltx2.pipeline_ltx2_3 import LTX23Pipeline + from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import LTX23Pipeline pipe = object.__new__(LTX23Pipeline) video_sample = torch.tensor([[[1.0, -2.0]]]) @@ -69,7 +51,8 @@ class TestCFGParallelHelpers: assert torch.allclose(audio_combined, expected_audio) def test_two_rank_cfg_parallel_smoke_uses_rank_local_branch_and_x0_formula(self, monkeypatch): - from vllm_omni.diffusion.models.ltx2 import pipeline_ltx2_3 as ltx23 + from vllm_omni.diffusion.models.ltx2 import ltx2_guidance + from vllm_omni.diffusion.models.ltx2 import pipeline_ltx2 as ltx23 pipe = object.__new__(ltx23.LTX23Pipeline) video_sample = torch.tensor([[[1.0, -2.0]]]) @@ -89,17 +72,17 @@ class TestCFGParallelHelpers: return [video_pos, video_neg] return [audio_pos, audio_neg] - monkeypatch.setattr(ltx23, "get_classifier_free_guidance_world_size", lambda: 2) - monkeypatch.setattr(ltx23, "get_cfg_group", lambda: FakeCfgGroup()) + monkeypatch.setattr(ltx2_guidance, "get_classifier_free_guidance_world_size", lambda: 2) + monkeypatch.setattr(ltx2_guidance, "get_cfg_group", lambda: FakeCfgGroup()) - expected_video = ltx23.LTX23Pipeline._combine_x0_space_cfg( + expected_video = ltx2_guidance.combine_velocity_via_x0( video_sample, video_pos, video_neg, video_sigma, scale, ) - expected_audio = ltx23.LTX23Pipeline._combine_x0_space_cfg( + expected_audio = ltx2_guidance.combine_velocity_via_x0( audio_sample, audio_pos, audio_neg, @@ -109,7 +92,7 @@ class TestCFGParallelHelpers: for rank, expected_branch in ((0, "positive"), (1, "negative")): calls = [] - monkeypatch.setattr(ltx23, "get_classifier_free_guidance_rank", lambda rank=rank: rank) + monkeypatch.setattr(ltx2_guidance, "get_classifier_free_guidance_rank", lambda rank=rank: rank) def fake_predict_noise(**kwargs): calls.append(kwargs["branch"]) @@ -140,17 +123,20 @@ class TestCFGParallelHelpers: class TestCFGParallelForwardPath: """Test the LTX-2.3 CFG-parallel denoising path without loading model weights.""" - def test_forward_collates_request_prompt_embeds_and_mask_aliases(self, monkeypatch): - from vllm_omni.diffusion.models.ltx2 import pipeline_ltx2_3 as ltx23 + @pytest.mark.parametrize("pipeline_cls_name", ["LTX2Pipeline", "LTX23Pipeline"]) + def test_forward_collates_request_prompt_embeds_and_mask_aliases(self, monkeypatch, pipeline_cls_name): + from vllm_omni.diffusion.models.ltx2 import ltx2_pipeline_runtime + from vllm_omni.diffusion.models.ltx2 import pipeline_ltx2 as ltx23 from vllm_omni.diffusion.request import OmniDiffusionRequest from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch from vllm_omni.inputs.data import OmniDiffusionSamplingParams - pipe = object.__new__(ltx23.LTX23Pipeline) + pipeline_cls = getattr(ltx23, pipeline_cls_name) + pipe = object.__new__(pipeline_cls) torch.nn.Module.__init__(pipe) pipe.device = torch.device("cpu") pipe.tokenizer_max_length = 4 - monkeypatch.setattr(ltx23, "get_classifier_free_guidance_world_size", lambda: 1) + monkeypatch.setattr(ltx2_pipeline_runtime, "get_classifier_free_guidance_world_size", lambda: 1) class StopAtEncodePromptError(Exception): pass @@ -247,7 +233,8 @@ class TestCFGParallelForwardPath: audio_sampling_rate, expected_frame_rate, ): - from vllm_omni.diffusion.models.ltx2 import pipeline_ltx2_3 as ltx23 + from vllm_omni.diffusion.models.ltx2 import ltx2_denoise, ltx2_guidance, ltx2_pipeline_runtime + from vllm_omni.diffusion.models.ltx2 import pipeline_ltx2 as ltx23 from vllm_omni.diffusion.request import OmniDiffusionRequest from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch from vllm_omni.inputs.data import OmniDiffusionSamplingParams @@ -291,15 +278,16 @@ class TestCFGParallelForwardPath: return [audio_pos, audio_neg] raise AssertionError(f"Unexpected gathered tensor: {tensor}") - monkeypatch.setattr(ltx23, "get_classifier_free_guidance_world_size", lambda: 2) - monkeypatch.setattr(ltx23, "get_classifier_free_guidance_rank", lambda: cfg_rank) - monkeypatch.setattr(ltx23, "get_cfg_group", lambda: FakeCfgGroup()) + monkeypatch.setattr(ltx2_pipeline_runtime, "get_classifier_free_guidance_world_size", lambda: 2) + monkeypatch.setattr(ltx2_guidance, "get_classifier_free_guidance_world_size", lambda: 2) + monkeypatch.setattr(ltx2_guidance, "get_classifier_free_guidance_rank", lambda: cfg_rank) + monkeypatch.setattr(ltx2_guidance, "get_cfg_group", lambda: FakeCfgGroup()) def fake_retrieve_timesteps(scheduler, num_inference_steps, device, timesteps, sigmas=None, mu=None): scheduler.sigmas = torch.tensor([0.25, 0.25], device=device) return torch.tensor([1.0, 0.5], device=device), 2 - monkeypatch.setattr(ltx23, "retrieve_timesteps", fake_retrieve_timesteps) + monkeypatch.setattr(ltx2_denoise, "retrieve_timesteps", fake_retrieve_timesteps) class FakeScheduler: def __init__(self, name="video", calls=None): @@ -402,14 +390,14 @@ class TestCFGParallelForwardPath: output = pipe.forward(DiffusionRequestBatch(requests=[req]))[0] - expected_video_noise = ltx23.LTX23Pipeline._combine_x0_space_cfg( + expected_video_noise = ltx2_guidance.combine_velocity_via_x0( video_latents, video_pos, video_neg, pipe.scheduler.sigmas[0], 4.0, ) - expected_audio_noise = ltx23.LTX23Pipeline._combine_x0_space_cfg( + expected_audio_noise = ltx2_guidance.combine_velocity_via_x0( audio_latents, audio_pos, audio_neg, diff --git a/tests/diffusion/models/ltx2/test_ltx2_3_pipeline.py b/tests/diffusion/models/ltx2/test_ltx2_pipeline.py similarity index 57% rename from tests/diffusion/models/ltx2/test_ltx2_3_pipeline.py rename to tests/diffusion/models/ltx2/test_ltx2_pipeline.py index 25aae4510..2f9155fe0 100644 --- a/tests/diffusion/models/ltx2/test_ltx2_3_pipeline.py +++ b/tests/diffusion/models/ltx2/test_ltx2_pipeline.py @@ -1,16 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Unit tests for LTX-2.3 pipeline integration. - -These tests verify: -- Pipeline is properly registered in the diffusion registry -- Post-process function is registered -- Cache-DiT enablers are registered -- Pipeline does NOT inherit from LTX2Pipeline -- Vocoder sample rate detection logic -- Re-export module works correctly -""" +"""Unit tests for the shared LTX pipeline runtime and public contracts.""" import json import os @@ -20,25 +11,37 @@ from types import SimpleNamespace import pytest import torch +from vllm_omni.diffusion.data import DiffusionOutput +from vllm_omni.diffusion.models.ltx2.ltx2_components import ( + LTX2_COMPONENT_PROFILE, + LTX23_COMPONENT_PROFILE, +) +from vllm_omni.diffusion.models.ltx2.ltx2_conditioning import LTXI2VConditioningMixin +from vllm_omni.diffusion.models.ltx2.ltx2_denoise import LTXDenoiseExecutor, LTXPhaseResult +from vllm_omni.diffusion.models.ltx2.ltx2_guidance import ( + LTX_LEGACY_VELOCITY_GUIDANCE, + LTX_OFFICIAL_X0_GUIDANCE, +) +from vllm_omni.diffusion.models.ltx2.ltx2_latents import LTXAVState +from vllm_omni.diffusion.models.ltx2.ltx2_pipeline_runtime import LTXPipelineRuntime +from vllm_omni.diffusion.models.ltx2.ltx2_recipes import LTX2_ONE_STAGE_RECIPE, LTX23_ONE_STAGE_RECIPE +from vllm_omni.diffusion.models.ltx2.ltx2_request import LTXRequestInputs +from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import ( + LTX2ImageToVideoPipeline, + LTX2Pipeline, + LTX23ImageToVideoPipeline, + LTX23Pipeline, + LTXOneStagePipeline, +) +from vllm_omni.diffusion.models.ltx2.pipeline_ltx2_two_stage import ( + LTX2ImageToVideoTwoStagesPipeline, + LTX2TwoStagesPipeline, +) + pytestmark = [pytest.mark.core_model, pytest.mark.cpu] -def _make_ltx23_pipeline(sequence_parallel_size: int = 1): - from vllm_omni.diffusion.models.ltx2.pipeline_ltx2_3 import LTX23Pipeline - - pipeline = object.__new__(LTX23Pipeline) - torch.nn.Module.__init__(pipeline) - pipeline.audio_vae_temporal_compression_ratio = 4 - pipeline.audio_vae_mel_compression_ratio = 4 - pipeline.od_config = SimpleNamespace(parallel_config=SimpleNamespace(sequence_parallel_size=sequence_parallel_size)) - pipeline.audio_vae = SimpleNamespace( - latents_mean=torch.tensor(0.0), - latents_std=torch.tensor(1.0), - ) - return pipeline - - -def _make_ltx23_request_pipe(cls): +def _make_ltx_request_pipe(cls): pipe = object.__new__(cls) torch.nn.Module.__init__(pipe) pipe.device = torch.device("cpu") @@ -73,10 +76,263 @@ def _resolve_request_inputs_for_test(pipe, req): ) -class TestLTX23RequestParsing: - def test_t2v_and_i2v_share_request_input_resolution(self): - from vllm_omni.diffusion.models.ltx2.pipeline_ltx2_3 import LTX23Pipeline - from vllm_omni.diffusion.models.ltx2.pipeline_ltx2_3_image2video import LTX23ImageToVideoPipeline +def test_ltx_versions_share_runtime_without_cross_version_inheritance(): + from vllm_omni.diffusion.distributed.autoencoders.autoencoder_kl_ltx2 import ( + DistributedAutoencoderKLLTX2Video, + ) + + assert issubclass(LTX2Pipeline, LTXPipelineRuntime) + assert issubclass(LTX23Pipeline, LTXPipelineRuntime) + assert issubclass(LTX2TwoStagesPipeline, LTXPipelineRuntime) + assert not issubclass(LTX23Pipeline, LTX2Pipeline) + assert LTX2Pipeline.component_profile is LTX2_COMPONENT_PROFILE + assert LTX23Pipeline.component_profile is LTX23_COMPONENT_PROFILE + assert LTX2Pipeline.one_stage_recipe is LTX2_ONE_STAGE_RECIPE + assert LTX23Pipeline.one_stage_recipe is LTX23_ONE_STAGE_RECIPE + assert LTX2_COMPONENT_PROFILE.video_vae_cls is DistributedAutoencoderKLLTX2Video + assert LTX23_COMPONENT_PROFILE.video_vae_cls is DistributedAutoencoderKLLTX2Video + + +def test_ltx_one_stage_variants_expose_shared_request_batch_and_distributed_decode_contracts(): + one_stage_variants = ( + LTX2Pipeline, + LTX2ImageToVideoPipeline, + LTX23Pipeline, + LTX23ImageToVideoPipeline, + ) + assert all(pipeline_cls.supports_request_batch for pipeline_cls in one_stage_variants) + assert all(pipeline_cls.distributed_video_decode for pipeline_cls in one_stage_variants) + assert not LTX2TwoStagesPipeline.supports_request_batch + assert not LTX2ImageToVideoTwoStagesPipeline.supports_request_batch + + +@pytest.mark.parametrize("pipeline_cls", [LTX2Pipeline, LTX23Pipeline]) +def test_ltx_request_batch_decode_splits_video_and_audio_per_request(pipeline_cls): + from vllm_omni.diffusion.request import OmniDiffusionRequest + from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch + from vllm_omni.inputs.data import OmniDiffusionSamplingParams + + requests = [ + OmniDiffusionRequest( + prompt=f"prompt-{index}", + sampling_params=OmniDiffusionSamplingParams(), + request_id=f"request-{index}", + ) + for index in range(2) + ] + request_batch = DiffusionRequestBatch(requests) + video = torch.tensor([[1.0], [2.0]]) + audio = torch.tensor([[3.0], [4.0]]) + pipe = object.__new__(pipeline_cls) + torch.nn.Module.__init__(pipe) + object.__setattr__(pipe, "_decode_output", lambda **_kwargs: DiffusionOutput(output=(video, audio))) + forward_ctx = SimpleNamespace( + req=request_batch, + request_inputs=SimpleNamespace( + output_type="np", + generator=None, + decode_timestep=0.0, + decode_noise_scale=None, + ), + prompt_context=SimpleNamespace(connector_prompt_embeds=torch.empty(2, 1)), + device=torch.device("cpu"), + batch_size=2, + num_videos_per_prompt=1, + ) + + outputs = pipe.decode_phase( + LTXPhaseResult( + forward_context=forward_ctx, + video=video, + audio=audio, + ) + ) + + assert len(outputs) == 2 + torch.testing.assert_close(outputs[0].output[0], video[:1]) + torch.testing.assert_close(outputs[0].output[1], audio[:1]) + torch.testing.assert_close(outputs[1].output[0], video[1:]) + torch.testing.assert_close(outputs[1].output[1], audio[1:]) + + +def test_ltx_one_stage_variants_share_forward_template(): + assert LTX2Pipeline._forward_impl is LTXOneStagePipeline._forward_impl + assert LTX2ImageToVideoPipeline._forward_impl is LTXOneStagePipeline._forward_impl + assert LTX23Pipeline._forward_impl is LTXOneStagePipeline._forward_impl + assert LTX23ImageToVideoPipeline._forward_impl is LTXOneStagePipeline._forward_impl + assert LTX2ImageToVideoPipeline.forward is LTX23ImageToVideoPipeline.forward + + +def test_ltx_versions_share_request_prompt_and_step_templates(): + shared_methods = ( + "_get_gemma_prompt_embeds", + "encode_prompt", + "check_inputs", + "_resolve_request_inputs", + "_prepare_prompt_context", + "_denoise_step", + ) + for method_name in shared_methods: + base_method = getattr(LTXPipelineRuntime, method_name) + assert getattr(LTX2Pipeline, method_name) is base_method + assert getattr(LTX23Pipeline, method_name) is base_method + + +def test_ltx_versions_select_guidance_without_overriding_control_flow(): + assert LTX2Pipeline.guidance_strategy is LTX_LEGACY_VELOCITY_GUIDANCE + assert LTX23Pipeline.guidance_strategy is LTX_OFFICIAL_X0_GUIDANCE + assert LTX2Pipeline._predict_noise_for_step is LTXPipelineRuntime._predict_noise_for_step + assert LTX23Pipeline._predict_noise_for_step is LTXPipelineRuntime._predict_noise_for_step + assert LTX2Pipeline.combine_cfg_noise is LTXPipelineRuntime.combine_cfg_noise + assert LTX23Pipeline.combine_cfg_noise is LTXPipelineRuntime.combine_cfg_noise + + +def test_ltx2_two_stage_variants_share_stage_orchestration(): + assert issubclass(LTX2ImageToVideoTwoStagesPipeline, LTX2TwoStagesPipeline) + assert issubclass(LTX2ImageToVideoTwoStagesPipeline, LTXI2VConditioningMixin) + assert LTX2ImageToVideoTwoStagesPipeline._run_two_stage is LTX2TwoStagesPipeline._run_two_stage + + +def test_ltx_variants_share_denoise_loop_and_i2v_conditioning(): + assert "_denoise_loop" not in LTX2Pipeline.__dict__ + assert "_denoise_loop" not in LTX23Pipeline.__dict__ + assert issubclass(LTX2ImageToVideoPipeline, LTXI2VConditioningMixin) + assert issubclass(LTX23ImageToVideoPipeline, LTXI2VConditioningMixin) + assert LTX2ImageToVideoPipeline.prepare_latents is LTXI2VConditioningMixin.prepare_latents + assert LTX23ImageToVideoPipeline.prepare_latents is LTXI2VConditioningMixin.prepare_latents + assert LTX2Pipeline.prepare_latents is LTXPipelineRuntime.prepare_latents + assert LTX23Pipeline.prepare_latents is LTXPipelineRuntime.prepare_latents + assert LTX2Pipeline.prepare_audio_latents is LTXPipelineRuntime.prepare_audio_latents + assert LTX23Pipeline.prepare_audio_latents is LTXPipelineRuntime.prepare_audio_latents + assert LTX2Pipeline.decode_phase is LTXPipelineRuntime.decode_phase + assert LTX23Pipeline.decode_phase is LTXPipelineRuntime.decode_phase + + +def test_denoise_executor_owns_progress_and_interrupt(): + updates = [] + + class Progress: + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, traceback): + return False + + def update(self): + updates.append(True) + + pipeline = SimpleNamespace( + interrupt=False, + progress_bar=lambda total: Progress(), + ) + seen: list[tuple[int, float]] = [] + timesteps = torch.tensor([3.0, 2.0, 1.0]) + initial_state = LTXAVState(video=torch.tensor(0.0), audio=torch.tensor(10.0)) + + def step(index, timestep, state): + seen.append((index, timestep.item())) + pipeline.interrupt = True + return LTXAVState(video=state.video + 1, audio=state.audio + 1) + + state = LTXDenoiseExecutor.run(pipeline, initial_state, timesteps, step) + + assert seen == [(0, 3.0)] + assert updates == [True] + torch.testing.assert_close(state.video, torch.tensor(1.0)) + torch.testing.assert_close(state.audio, torch.tensor(11.0)) + + +def test_ltx2_two_stage_reuses_prompt_context_between_phases(): + request_inputs = LTXRequestInputs( + prompt="prompt", + negative_prompt="negative", + height=32, + width=32, + num_frames=1, + frame_rate=24.0, + num_inference_steps=4, + guidance_scale=4.0, + guidance_rescale=0.0, + num_videos_per_prompt=1, + generator=None, + latents=None, + audio_latents=None, + prompt_embeds=None, + negative_prompt_embeds=None, + prompt_attention_mask=None, + negative_prompt_attention_mask=None, + decode_timestep=0.25, + decode_noise_scale=0.5, + output_type="np", + max_sequence_length=16, + ) + prompt_context = object() + phase_calls = [] + + def resolve_request_inputs(req, **kwargs): + return request_inputs + + def run_phase(req, inputs, *, prompt_context=None, **kwargs): + phase_calls.append((inputs, prompt_context)) + if len(phase_calls) == 1: + assert prompt_context is None + assert inputs.decode_timestep == 0.25 + assert inputs.decode_noise_scale == 0.5 + context = prompt_context_sentinel + video = torch.tensor([1.0]) + audio = torch.tensor([2.0]) + else: + assert prompt_context is prompt_context_sentinel + torch.testing.assert_close(inputs.latents, torch.tensor([11.0])) + torch.testing.assert_close(inputs.audio_latents, torch.tensor([2.0])) + assert inputs.guidance_scale == 1.0 + assert inputs.num_inference_steps == 3 + assert inputs.decode_timestep == 0.0 + assert inputs.decode_noise_scale is None + context = prompt_context + video = torch.tensor([3.0]) + audio = torch.tensor([4.0]) + return LTXPhaseResult( + forward_context=SimpleNamespace(prompt_context=context), + video=video, + audio=audio, + ) + + def decode_phase(phase): + return DiffusionOutput(output=(phase.video, phase.audio)) + + class FakeUpsampler(torch.nn.Module): + def forward(self, *, latents, output_type, return_dict): + assert output_type == "latent" + assert not return_dict + return (latents + 10,) + + prompt_context_sentinel = prompt_context + pipeline = object.__new__(LTX2TwoStagesPipeline) + torch.nn.Module.__init__(pipeline) + pipeline.distilled = True + pipeline.upsample_pipe = FakeUpsampler() + object.__setattr__(pipeline, "_resolve_request_inputs", resolve_request_inputs) + object.__setattr__(pipeline, "run_phase", run_phase) + object.__setattr__(pipeline, "decode_phase", decode_phase) + + output = pipeline.forward(SimpleNamespace()) + + assert len(phase_calls) == 2 + assert phase_calls[1][1] is prompt_context_sentinel + torch.testing.assert_close(output.output[0], torch.tensor([3.0])) + torch.testing.assert_close(output.output[1], torch.tensor([4.0])) + + +class TestLTXRequestParsing: + @pytest.mark.parametrize( + ("t2v_cls", "i2v_cls"), + [ + (LTX2Pipeline, LTX2ImageToVideoPipeline), + (LTX23Pipeline, LTX23ImageToVideoPipeline), + ], + ) + def test_t2v_and_i2v_share_request_input_resolution(self, t2v_cls, i2v_cls): from vllm_omni.diffusion.request import OmniDiffusionRequest from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch from vllm_omni.inputs.data import OmniDiffusionSamplingParams @@ -124,11 +380,11 @@ class TestLTX23RequestParsing: ) resolved_t2v = _resolve_request_inputs_for_test( - _make_ltx23_request_pipe(LTX23Pipeline), + _make_ltx_request_pipe(t2v_cls), req, ) resolved_i2v = _resolve_request_inputs_for_test( - _make_ltx23_request_pipe(LTX23ImageToVideoPipeline), + _make_ltx_request_pipe(i2v_cls), req, ) @@ -169,12 +425,12 @@ class TestLTX23RequestParsing: ) def test_request_input_resolution_rejects_mixed_precomputed_prompt_fields(self): - from vllm_omni.diffusion.models.ltx2.pipeline_ltx2_3 import LTX23Pipeline + from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import LTX23Pipeline from vllm_omni.diffusion.request import OmniDiffusionRequest from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch from vllm_omni.inputs.data import OmniDiffusionSamplingParams - pipe = _make_ltx23_request_pipe(LTX23Pipeline) + pipe = _make_ltx_request_pipe(LTX23Pipeline) req = DiffusionRequestBatch( [ OmniDiffusionRequest( @@ -209,9 +465,9 @@ class TestLTX23RequestParsing: class TestLTX23ForwardStages: def test_encode_prompt_repeats_precomputed_embeds_for_num_outputs(self): - from vllm_omni.diffusion.models.ltx2.pipeline_ltx2_3 import LTX23Pipeline + from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import LTX23Pipeline - pipe = _make_ltx23_request_pipe(LTX23Pipeline) + pipe = _make_ltx_request_pipe(LTX23Pipeline) prompt_embeds = torch.tensor([[[1.0], [2.0]], [[3.0], [4.0]]]) negative_prompt_embeds = torch.tensor([[[-1.0], [-2.0]], [[-3.0], [-4.0]]]) prompt_attention_mask = torch.tensor([[True, False], [False, True]]) @@ -251,12 +507,12 @@ class TestLTX23ForwardStages: ) def test_t2v_forward_delegates_to_shared_forward_impl(self): - from vllm_omni.diffusion.models.ltx2.pipeline_ltx2_3 import LTX23Pipeline + from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import LTX23Pipeline from vllm_omni.diffusion.request import OmniDiffusionRequest from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch from vllm_omni.inputs.data import OmniDiffusionSamplingParams - pipe = _make_ltx23_request_pipe(LTX23Pipeline) + pipe = _make_ltx_request_pipe(LTX23Pipeline) req = DiffusionRequestBatch( [ OmniDiffusionRequest( @@ -311,36 +567,10 @@ class TestLTX23ForwardStages: } -class TestPipelineIndependence: - """Verify LTX23Pipeline is fully independent from LTX2Pipeline.""" - - def test_ltx23_pipeline_does_not_inherit_from_ltx2(self): - """LTX23Pipeline must NOT inherit from LTX2Pipeline.""" - from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import LTX2Pipeline - from vllm_omni.diffusion.models.ltx2.pipeline_ltx2_3 import LTX23Pipeline - - assert not issubclass(LTX23Pipeline, LTX2Pipeline), ( - "LTX23Pipeline should be fully independent and not inherit from LTX2Pipeline" - ) - - def test_ltx23_pipeline_is_nn_module(self): - """LTX23Pipeline must be an nn.Module.""" - import torch.nn as nn - - from vllm_omni.diffusion.models.ltx2.pipeline_ltx2_3 import LTX23Pipeline - - assert issubclass(LTX23Pipeline, nn.Module) - - def test_ltx23_pipeline_has_progress_bar(self): - """LTX23Pipeline must mix in ProgressBarMixin.""" - from vllm_omni.diffusion.models.ltx2.pipeline_ltx2_3 import LTX23Pipeline - from vllm_omni.diffusion.models.progress_bar import ProgressBarMixin - - assert issubclass(LTX23Pipeline, ProgressBarMixin) - +class TestPipelineComponents: def test_ltx23_pipeline_declares_offload_components(self): """LTX23Pipeline must expose LTX-2.3-specific modules to offload discovery.""" - from vllm_omni.diffusion.models.ltx2.pipeline_ltx2_3 import LTX23Pipeline + from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import LTX23Pipeline from vllm_omni.diffusion.offloader.module_collector import ModuleDiscovery pipe = object.__new__(LTX23Pipeline) @@ -363,17 +593,10 @@ class TestPipelineIndependence: assert modules.resident_names == ["vocoder"] assert len(modules.vaes) == 2 - def test_ltx23_pipeline_has_diffusion_pipeline_profiler_mixin(self): - """LTX23Pipeline must support lightweight stage timing.""" - from vllm_omni.diffusion.models.ltx2.pipeline_ltx2_3 import LTX23Pipeline - from vllm_omni.diffusion.profiler.diffusion_pipeline_profiler import DiffusionPipelineProfilerMixin - - assert issubclass(LTX23Pipeline, DiffusionPipelineProfilerMixin) - class TestLTX23DecodeConditioning: def test_decode_conditioning_expands_per_prompt_values_to_effective_batch(self): - from vllm_omni.diffusion.models.ltx2.pipeline_ltx2_3 import _expand_per_prompt_decode_value + from vllm_omni.diffusion.models.ltx2.ltx2_pipeline_runtime import _expand_per_prompt_decode_value assert _expand_per_prompt_decode_value( [0.1, 0.2], @@ -395,7 +618,7 @@ class TestLTX23DecodeConditioning: ) == [0.1, 0.2, 0.3, 0.4] def test_decode_conditioning_rejects_ambiguous_lengths(self): - from vllm_omni.diffusion.models.ltx2.pipeline_ltx2_3 import _expand_per_prompt_decode_value + from vllm_omni.diffusion.models.ltx2.ltx2_pipeline_runtime import _expand_per_prompt_decode_value with pytest.raises(ValueError, match="decode_timestep"): _expand_per_prompt_decode_value( @@ -409,28 +632,15 @@ class TestLTX23DecodeConditioning: class TestRegistryIntegration: """Verify all LTX-2.3 pipeline variants are registered.""" - def test_pipeline_models_registered(self): - """LTX-2.3 pipeline variants must be in _DIFFUSION_MODELS.""" - from vllm_omni.diffusion.registry import _DIFFUSION_MODELS - - expected = [ - "LTX23Pipeline", - "LTX23ImageToVideoPipeline", - ] - for name in expected: - assert name in _DIFFUSION_MODELS, f"{name} not found in _DIFFUSION_MODELS" - def test_pipeline_module_paths(self): """Registry entries must point to the correct modules.""" from vllm_omni.diffusion.registry import _DIFFUSION_MODELS - # T2V -> pipeline_ltx2_3 - assert _DIFFUSION_MODELS["LTX23Pipeline"] == ("ltx2", "pipeline_ltx2_3", "LTX23Pipeline") + assert _DIFFUSION_MODELS["LTX23Pipeline"] == ("ltx2", "pipeline_ltx2", "LTX23Pipeline") - # I2V -> pipeline_ltx2_3_image2video assert _DIFFUSION_MODELS["LTX23ImageToVideoPipeline"] == ( "ltx2", - "pipeline_ltx2_3_image2video", + "pipeline_ltx2", "LTX23ImageToVideoPipeline", ) @@ -439,6 +649,12 @@ class TestRegistryIntegration: from vllm_omni.diffusion.registry import _DIFFUSION_POST_PROCESS_FUNCS expected = [ + "LTX2Pipeline", + "LTX2ImageToVideoPipeline", + "LTX2TwoStagesPipeline", + "LTX2ImageToVideoTwoStagesPipeline", + "LTX2T2VDMD2Pipeline", + "LTX2I2VDMD2Pipeline", "LTX23Pipeline", "LTX23ImageToVideoPipeline", ] @@ -446,6 +662,35 @@ class TestRegistryIntegration: assert name in _DIFFUSION_POST_PROCESS_FUNCS, f"{name} not in _DIFFUSION_POST_PROCESS_FUNCS" assert _DIFFUSION_POST_PROCESS_FUNCS[name] == "get_ltx2_post_process_func" + @pytest.mark.parametrize( + "model_class_name", + [ + "LTX2Pipeline", + "LTX2ImageToVideoPipeline", + "LTX2TwoStagesPipeline", + "LTX2ImageToVideoTwoStagesPipeline", + "LTX2T2VDMD2Pipeline", + "LTX2I2VDMD2Pipeline", + "LTX23Pipeline", + "LTX23ImageToVideoPipeline", + ], + ) + def test_post_process_func_resolves_from_every_entry_module(self, model_class_name): + from vllm_omni.diffusion.registry import get_diffusion_post_process_func + + with tempfile.TemporaryDirectory() as tmpdir: + vocoder_dir = os.path.join(tmpdir, "vocoder") + os.makedirs(vocoder_dir) + with open(os.path.join(vocoder_dir, "config.json"), "w") as config_file: + json.dump({"output_sampling_rate": 48000}, config_file) + + post_process = get_diffusion_post_process_func( + SimpleNamespace(model_class_name=model_class_name, model=tmpdir) + ) + + result = post_process((torch.zeros(1), torch.zeros(1))) + assert result["audio_sample_rate"] == 48000 + def test_cache_dit_for_ltx2_does_not_have_custom_enablers_registered(self): """Pipeline variants are *not* registered in CUSTOM_DIT_ENABLERS.""" from vllm_omni.diffusion.cache.cache_dit_backend import CUSTOM_DIT_ENABLERS @@ -465,7 +710,7 @@ class TestVocoderSampleRateDetection: def test_detects_48khz_from_config(self): """Should detect output_sampling_rate=48000 from vocoder/config.json.""" - from vllm_omni.diffusion.models.ltx2.pipeline_ltx2_3 import _detect_vocoder_output_sample_rate + from vllm_omni.diffusion.models.ltx2.ltx2_components import _detect_vocoder_output_sample_rate with tempfile.TemporaryDirectory() as tmpdir: vocoder_dir = os.path.join(tmpdir, "vocoder") @@ -478,7 +723,7 @@ class TestVocoderSampleRateDetection: def test_returns_none_for_no_output_sr(self): """Should return None if vocoder config has no output_sampling_rate.""" - from vllm_omni.diffusion.models.ltx2.pipeline_ltx2_3 import _detect_vocoder_output_sample_rate + from vllm_omni.diffusion.models.ltx2.ltx2_components import _detect_vocoder_output_sample_rate with tempfile.TemporaryDirectory() as tmpdir: vocoder_dir = os.path.join(tmpdir, "vocoder") @@ -491,7 +736,7 @@ class TestVocoderSampleRateDetection: def test_returns_none_for_missing_directory(self): """Should return None if vocoder directory doesn't exist.""" - from vllm_omni.diffusion.models.ltx2.pipeline_ltx2_3 import _detect_vocoder_output_sample_rate + from vllm_omni.diffusion.models.ltx2.ltx2_components import _detect_vocoder_output_sample_rate result = _detect_vocoder_output_sample_rate("/nonexistent/path") assert result is None @@ -504,7 +749,7 @@ class TestPostProcessFunction: """Post-process func should include audio_sample_rate when detected.""" import torch - from vllm_omni.diffusion.models.ltx2.pipeline_ltx2_3 import get_ltx2_post_process_func + from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import get_ltx2_post_process_func with tempfile.TemporaryDirectory() as tmpdir: vocoder_dir = os.path.join(tmpdir, "vocoder") @@ -531,7 +776,7 @@ class TestPostProcessFunction: """Post-process func should work without vocoder config (no audio_sample_rate key).""" import torch - from vllm_omni.diffusion.models.ltx2.pipeline_ltx2_3 import get_ltx2_post_process_func + from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import get_ltx2_post_process_func class MockConfig: model = "/nonexistent/path" @@ -546,101 +791,3 @@ class TestPostProcessFunction: assert "video" in result assert "audio" in result assert "audio_sample_rate" not in result - - -class TestInitExports: - """Test that __init__.py exports all LTX-2.3 classes.""" - - def test_all_ltx23_classes_exported(self): - """All LTX-2.3 pipeline classes must be in the ltx2 package __all__.""" - from vllm_omni.diffusion.models import ltx2 - - expected_classes = [ - "LTX23Pipeline", - "LTX23ImageToVideoPipeline", - ] - for name in expected_classes: - assert hasattr(ltx2, name), f"{name} not exported from ltx2 package" - assert name in ltx2.__all__, f"{name} not in ltx2.__all__" - - -class TestAudioLatentSPPadding: - def test_prepare_audio_latents_pads_generated_dummy_length_for_sp(self): - pipeline = _make_ltx23_pipeline(sequence_parallel_size=2) - - latents, original_num_frames, padded_num_frames = pipeline.prepare_audio_latents( - batch_size=1, - num_channels_latents=8, - num_mel_bins=64, - audio_latent_length=1, - dtype=torch.float32, - device=torch.device("cpu"), - ) - - assert original_num_frames == 1 - assert padded_num_frames == 2 - assert latents.shape == (1, 2, 128) - - def test_prepare_audio_latents_pads_provided_packed_sequence_dim_for_sp(self): - pipeline = _make_ltx23_pipeline(sequence_parallel_size=4) - latents = torch.arange(40, dtype=torch.float32).view(1, 10, 4) - - padded, original_num_frames, padded_num_frames = pipeline.prepare_audio_latents( - batch_size=1, - num_channels_latents=2, - num_mel_bins=8, - audio_latent_length=10, - dtype=torch.float32, - device=torch.device("cpu"), - latents=latents, - ) - - assert original_num_frames == 10 - assert padded_num_frames == 12 - assert padded.shape == (1, 12, 4) - torch.testing.assert_close(padded[:, :10], latents) - torch.testing.assert_close(padded[:, 10:], torch.zeros(1, 2, 4)) - - def test_prepare_audio_latents_accepts_already_padded_4d_latents_for_sp(self): - pipeline = _make_ltx23_pipeline(sequence_parallel_size=4) - latents = torch.arange(96, dtype=torch.float32).view(1, 2, 12, 4) - - audio_latent_length = pipeline._resolve_audio_latent_length(10, latents) - padded, original_num_frames, padded_num_frames = pipeline.prepare_audio_latents( - batch_size=1, - num_channels_latents=2, - num_mel_bins=16, - audio_latent_length=audio_latent_length, - dtype=torch.float32, - device=torch.device("cpu"), - latents=latents, - ) - - assert audio_latent_length == 10 - assert original_num_frames == 10 - assert padded_num_frames == 12 - assert padded.shape == (1, 12, 8) - torch.testing.assert_close(padded, pipeline._pack_audio_latents(latents)) - - def test_resolve_audio_latent_length_preserves_legacy_4d_shape_inference(self): - pipeline = _make_ltx23_pipeline(sequence_parallel_size=4) - latents = torch.zeros(1, 2, 13, 4) - - audio_latent_length = pipeline._resolve_audio_latent_length(10, latents) - - assert audio_latent_length == 13 - - def test_prepare_audio_latents_rejects_incompatible_provided_length(self): - pipeline = _make_ltx23_pipeline(sequence_parallel_size=4) - latents = torch.zeros(1, 11, 4) - - with pytest.raises(ValueError, match="incompatible audio frame count"): - pipeline.prepare_audio_latents( - batch_size=1, - num_channels_latents=2, - num_mel_bins=8, - audio_latent_length=10, - dtype=torch.float32, - device=torch.device("cpu"), - latents=latents, - ) diff --git a/tests/diffusion/models/ltx2/test_ltx2_transformer.py b/tests/diffusion/models/ltx2/test_ltx2_transformer.py index e71253297..01a1fa3cf 100644 --- a/tests/diffusion/models/ltx2/test_ltx2_transformer.py +++ b/tests/diffusion/models/ltx2/test_ltx2_transformer.py @@ -1,9 +1,13 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import pytest from torch import nn -from vllm_omni.diffusion.models.ltx2.ltx2_transformer import _make_rms_norm +from vllm_omni.diffusion.cache.cache_dit_backend import CacheDiTAdapterConfig +from vllm_omni.diffusion.models.ltx2.ltx2_transformer import LTX2VideoTransformer3DModel, _make_rms_norm + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] def test_ltx_rms_norm_no_affine_identity_weight_is_non_persistent_buffer(): @@ -20,3 +24,29 @@ def test_ltx_rms_norm_affine_weight_remains_parameter(): assert isinstance(dict(norm.named_parameters())["weight"], nn.Parameter) assert "weight" not in dict(norm.named_buffers()) assert "weight" in norm.state_dict() + + +def test_ltx_transformer_has_separate_cfg_cache_dit_config(): + adapter_config = getattr(LTX2VideoTransformer3DModel, "_cache_dit_adapter_config") + + assert isinstance(adapter_config, CacheDiTAdapterConfig) + assert adapter_config.has_separate_cfg + + +def test_ltx_transformer_exposes_hsdp_shard_conditions_for_blocks(): + model = object.__new__(LTX2VideoTransformer3DModel) + nn.Module.__init__(model) + model.transformer_blocks = nn.ModuleList([nn.Linear(4, 4) for _ in range(2)]) + model.norm_out = nn.LayerNorm(4) + + conditions = getattr(model, "_hsdp_shard_conditions", None) + + assert conditions is not None + assert len(conditions) == 1 + + matched = [] + for name, module in model.named_modules(): + if any(condition(name, module) for condition in conditions): + matched.append(name) + + assert matched == ["transformer_blocks.0", "transformer_blocks.1"] diff --git a/tests/diffusion/models/ltx2/test_ltx2_3_vae_tiling.py b/tests/diffusion/models/ltx2/test_ltx2_vae.py similarity index 85% rename from tests/diffusion/models/ltx2/test_ltx2_3_vae_tiling.py rename to tests/diffusion/models/ltx2/test_ltx2_vae.py index 9b53bb4f8..4ae7a0721 100644 --- a/tests/diffusion/models/ltx2/test_ltx2_3_vae_tiling.py +++ b/tests/diffusion/models/ltx2/test_ltx2_vae.py @@ -11,24 +11,56 @@ import torch pytestmark = [pytest.mark.core_model, pytest.mark.cpu] -class TestLTX23OutputRank: - def test_single_process_rank_is_output_rank(self, monkeypatch): - import vllm_omni.diffusion.models.ltx2.pipeline_ltx2_3 as ltx23 +class TestLTXOutputRank: + @pytest.mark.parametrize( + ("distributed_vae_state", "expected_decode_calls"), + [(False, 0), (True, 1), (RuntimeError("unavailable"), 0)], + ) + def test_non_output_rank_only_enters_collective_vae_decode( + self, + monkeypatch, + distributed_vae_state, + expected_decode_calls, + ): + import vllm_omni.diffusion.models.ltx2.pipeline_ltx2 as ltx - monkeypatch.setattr(ltx23.torch.distributed, "is_initialized", lambda: False) + class FakeVae: + dtype = torch.float32 + config = SimpleNamespace(timestep_conditioning=False) - assert ltx23._is_output_rank() is True - assert ltx23._should_decode_video_on_rank(SimpleNamespace(is_distributed_enabled=lambda: False)) is True + def __init__(self): + self.decode_calls = 0 - def test_non_output_rank_skips_decode_unless_vae_decode_is_distributed(self, monkeypatch): - import vllm_omni.diffusion.models.ltx2.pipeline_ltx2_3 as ltx23 + def is_distributed_enabled(self): + if isinstance(distributed_vae_state, Exception): + raise distributed_vae_state + return distributed_vae_state - monkeypatch.setattr(ltx23.torch.distributed, "is_initialized", lambda: True) - monkeypatch.setattr(ltx23.torch.distributed, "get_rank", lambda: 1) + def decode(self, *_args, **_kwargs): + self.decode_calls += 1 + return (torch.ones(1, 1),) - assert ltx23._is_output_rank() is False - assert ltx23._should_decode_video_on_rank(SimpleNamespace(is_distributed_enabled=lambda: False)) is False - assert ltx23._should_decode_video_on_rank(SimpleNamespace(is_distributed_enabled=lambda: True)) is True + monkeypatch.setattr(ltx.torch.distributed, "is_initialized", lambda: True) + monkeypatch.setattr(ltx.torch.distributed, "get_rank", lambda: 1) + pipe = object.__new__(ltx.LTX2Pipeline) + torch.nn.Module.__init__(pipe) + pipe.vae = FakeVae() + + output = pipe._decode_output( + latents=torch.ones(1, 1), + audio_latents=torch.ones(1, 1), + output_type="np", + connector_prompt_embeds=torch.ones(1, 1), + generator=None, + device=torch.device("cpu"), + decode_timestep=0.0, + decode_noise_scale=None, + prompt_batch_size=1, + ) + + assert pipe.vae.decode_calls == expected_decode_calls + assert output.output[0].numel() == 0 + assert output.output[1].numel() == 0 class TestLTX23VaeDistributedDecode: diff --git a/tests/e2e/accuracy/test_ltx2_3_video_similarity.py b/tests/e2e/accuracy/test_ltx2_3_video_similarity.py index 7fb2297a2..a169817a2 100644 --- a/tests/e2e/accuracy/test_ltx2_3_video_similarity.py +++ b/tests/e2e/accuracy/test_ltx2_3_video_similarity.py @@ -40,7 +40,7 @@ from tests.helpers.runtime import OmniServer _DIFFUSERS_VERSION = tuple(int(x) for x in diffusers.__version__.split(".")[:2] if x.isdigit()) _DIFFUSERS_038 = _DIFFUSERS_VERSION >= (0, 38) -MODEL_ID = "dg845/LTX-2.3-Diffusers" +MODEL_ID = "diffusers/LTX-2.3-Diffusers" MODEL_ENV_VAR = "VLLM_TEST_LTX23_MODEL" PROMPT = "A lighthouse on a rocky cliff at sunset, waves crashing below, golden hour lighting" NEGATIVE_PROMPT = "blurry, low quality, distorted, watermark" @@ -216,7 +216,10 @@ def _run_with_custom_transformer(model: str, output_dir: Path) -> list[Image.Ima from vllm.config import VllmConfig, set_current_vllm_config from vllm.distributed.parallel_state import init_distributed_environment, initialize_model_parallel - from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import create_transformer_from_config, load_transformer_config + from vllm_omni.diffusion.models.ltx2 import ( + create_transformer_from_config, + load_transformer_config, + ) vllm_config = VllmConfig() ctx = set_current_vllm_config(vllm_config) diff --git a/vllm_omni/diffusion/models/dmd2/mixin.py b/vllm_omni/diffusion/models/dmd2/mixin.py index 873120fc1..9c98e96b4 100644 --- a/vllm_omni/diffusion/models/dmd2/mixin.py +++ b/vllm_omni/diffusion/models/dmd2/mixin.py @@ -37,43 +37,42 @@ class DMD2PipelineMixin: def _sanitize_dmd2_request(self, req) -> None: """Sanitize CFG-related fields in-place. Works with both OmniDiffusionRequest and DiffusionRequestBatch.""" - sp = req.sampling_params + sampling_params = req.sampling_params_list if isinstance(req, DiffusionRequestBatch) else [req.sampling_params] + for sp in sampling_params: + if sp.num_inference_steps and sp.num_inference_steps != self.dmd2_config.num_inference_steps: + logger.warning( + "DMD2: ignoring num_inference_steps=%d, forcing %d.", + sp.num_inference_steps, + self.dmd2_config.num_inference_steps, + ) + sp.num_inference_steps = self.dmd2_config.num_inference_steps - if sp.num_inference_steps and sp.num_inference_steps != self.dmd2_config.num_inference_steps: - logger.warning( - "DMD2: ignoring num_inference_steps=%d, forcing %d.", - sp.num_inference_steps, - self.dmd2_config.num_inference_steps, - ) - sp.num_inference_steps = self.dmd2_config.num_inference_steps + if sp.guidance_scale_provided and sp.guidance_scale != self.dmd2_config.guidance_scale: + logger.warning( + "DMD2: ignoring guidance_scale=%.2f, forcing %.2f.", + sp.guidance_scale, + self.dmd2_config.guidance_scale, + ) + sp.guidance_scale = self.dmd2_config.guidance_scale + sp.guidance_scale_provided = False - if sp.guidance_scale_provided and sp.guidance_scale != self.dmd2_config.guidance_scale: - logger.warning( - "DMD2: ignoring guidance_scale=%.2f, forcing %.2f.", - sp.guidance_scale, - self.dmd2_config.guidance_scale, - ) - sp.guidance_scale = self.dmd2_config.guidance_scale - sp.guidance_scale_provided = False + if sp.guidance_scale_2 is not None: + logger.warning("DMD2: ignoring guidance_scale_2.") + sp.guidance_scale_2 = None - if sp.guidance_scale_2 is not None: - logger.warning("DMD2: ignoring guidance_scale_2.") - sp.guidance_scale_2 = None + if sp.true_cfg_scale is not None: + logger.warning("DMD2: ignoring true_cfg_scale.") + sp.true_cfg_scale = None - if sp.true_cfg_scale is not None: - logger.warning("DMD2: ignoring true_cfg_scale.") - sp.true_cfg_scale = None + sp.do_classifier_free_guidance = False + sp.is_cfg_negative = False - sp.do_classifier_free_guidance = False - sp.is_cfg_negative = False - - # defense: strip scheduler-override extra_args that would let the base pipeline - # (e.g. Wan22Pipeline.forward) rebuild self.scheduler mid-forward and clobber DMD2EulerScheduler. - extra_args = getattr(sp, "extra_args", None) or {} - for key in ("sample_solver", "flow_shift"): - if key in extra_args: - logger.warning("DMD2: ignoring extra_args.%s.", key) - extra_args.pop(key) + # Prevent base pipelines from replacing the DMD2 scheduler. + extra_args = getattr(sp, "extra_args", None) or {} + for key in ("sample_solver", "flow_shift"): + if key in extra_args: + logger.warning("DMD2: ignoring extra_args.%s.", key) + extra_args.pop(key) # Strip negative_prompt from each request's prompt in-place. requests = req.requests if hasattr(req, "requests") else [req] diff --git a/vllm_omni/diffusion/models/ltx2/__init__.py b/vllm_omni/diffusion/models/ltx2/__init__.py index 76a26b5b0..ca34e0390 100644 --- a/vllm_omni/diffusion/models/ltx2/__init__.py +++ b/vllm_omni/diffusion/models/ltx2/__init__.py @@ -1,23 +1,25 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from vllm_omni.diffusion.models.ltx2.ltx2_transformer import LTX2VideoTransformer3DModel -from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import ( - LTX2Pipeline, - LTX2T2VDMD2Pipeline, - LTX2TwoStagesPipeline, +from vllm_omni.diffusion.models.ltx2.ltx2_components import ( create_transformer_from_config, get_ltx2_post_process_func, load_transformer_config, ) -from vllm_omni.diffusion.models.ltx2.pipeline_ltx2_3 import LTX23Pipeline -from vllm_omni.diffusion.models.ltx2.pipeline_ltx2_3_image2video import LTX23ImageToVideoPipeline -from vllm_omni.diffusion.models.ltx2.pipeline_ltx2_image2video import ( +from vllm_omni.diffusion.models.ltx2.ltx2_transformer import LTX2VideoTransformer3DModel +from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import ( LTX2I2VDMD2Pipeline, LTX2ImageToVideoPipeline, - LTX2ImageToVideoTwoStagesPipeline, + LTX2Pipeline, + LTX2T2VDMD2Pipeline, + LTX23ImageToVideoPipeline, + LTX23Pipeline, ) from vllm_omni.diffusion.models.ltx2.pipeline_ltx2_latent_upsample import LTX2LatentUpsamplePipeline +from vllm_omni.diffusion.models.ltx2.pipeline_ltx2_two_stage import ( + LTX2ImageToVideoTwoStagesPipeline, + LTX2TwoStagesPipeline, +) __all__ = [ "LTX2Pipeline", diff --git a/vllm_omni/diffusion/models/ltx2/ltx2_components.py b/vllm_omni/diffusion/models/ltx2/ltx2_components.py new file mode 100644 index 000000000..cfe990a4e --- /dev/null +++ b/vllm_omni/diffusion/models/ltx2/ltx2_components.py @@ -0,0 +1,297 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +"""Shared component construction helpers for the LTX model family.""" + +from __future__ import annotations + +import inspect +import json +import os +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any + +import torch +from diffusers import AutoencoderKLLTX2Audio, AutoencoderKLLTX2Video, FlowMatchEulerDiscreteScheduler +from diffusers.pipelines.ltx2 import LTX2TextConnectors +from diffusers.pipelines.ltx2.vocoder import LTX2Vocoder +from diffusers.video_processor import VideoProcessor +from huggingface_hub import hf_hub_download +from transformers import AutoTokenizer, Gemma3ForConditionalGeneration + +from vllm_omni.diffusion.distributed.autoencoders.autoencoder_kl_ltx2 import DistributedAutoencoderKLLTX2Video +from vllm_omni.diffusion.distributed.utils import get_local_device +from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader +from vllm_omni.diffusion.model_loader.hub_prefetch import from_pretrained_with_prefetch, prefetch_subfolders +from vllm_omni.diffusion.offloader.module_collector import ModuleDiscovery + +if TYPE_CHECKING: + from vllm.model_executor.layers.quantization.base_config import QuantizationConfig + +from .ltx2_transformer import LTX2VideoTransformer3DModel + +try: + from diffusers.pipelines.ltx2.vocoder import LTX2VocoderWithBWE +except ImportError: + LTX2VocoderWithBWE = None + + +_LTX_COMPONENT_SUBFOLDERS = ( + "tokenizer", + "text_encoder", + "connectors", + "vae", + "audio_vae", + "vocoder", + "scheduler", +) + + +@dataclass(frozen=True) +class LTXComponentProfile: + """Component construction and discovery contract for an LTX variant.""" + + name: str + dit_modules: tuple[str, ...] + encoder_modules: tuple[str, ...] + vae_modules: tuple[str, ...] + resident_modules: tuple[str, ...] = () + video_vae_cls: type = AutoencoderKLLTX2Video + vocoder_cls: type = LTX2Vocoder + vocoder_fallback_cls: type | None = None + + +LTX2_COMPONENT_PROFILE = LTXComponentProfile( + name="ltx2", + dit_modules=("transformer",), + encoder_modules=("text_encoder", "connectors"), + vae_modules=("vae", "audio_vae"), + resident_modules=("vocoder",), + video_vae_cls=DistributedAutoencoderKLLTX2Video, +) + +LTX23_COMPONENT_PROFILE = LTXComponentProfile( + name="ltx2_3", + dit_modules=("transformer",), + encoder_modules=("text_encoder", "connectors"), + vae_modules=("vae", "audio_vae"), + resident_modules=("vocoder",), + video_vae_cls=DistributedAutoencoderKLLTX2Video, + vocoder_cls=LTX2VocoderWithBWE or LTX2Vocoder, + vocoder_fallback_cls=LTX2Vocoder, +) + + +def _detect_vocoder_output_sample_rate(model: str) -> int | None: + """Read the generated waveform sample rate from the vocoder config.""" + vocoder_config_path = os.path.join(model, "vocoder", "config.json") + if not os.path.exists(vocoder_config_path): + try: + vocoder_config_path = hf_hub_download(model, "vocoder/config.json") + except Exception: + return None + try: + with open(vocoder_config_path) as config_file: + return json.load(config_file).get("output_sampling_rate") + except Exception: + return None + + +def get_ltx2_post_process_func(od_config: Any): + """Build the common LTX engine-output adapter.""" + output_sample_rate = _detect_vocoder_output_sample_rate(od_config.model) + + def post_process_func(output: tuple[torch.Tensor, torch.Tensor] | torch.Tensor): + if not (isinstance(output, tuple) and len(output) == 2): + return output + video, audio = output + if isinstance(audio, torch.Tensor): + audio = audio.detach().cpu() + result: dict[str, Any] = {"video": video, "audio": audio} + if output_sample_rate is not None: + result["audio_sample_rate"] = output_sample_rate + return result + + return post_process_func + + +def _load_component( + component_cls: type, + model: str, + subfolder: str, + *, + local_files_only: bool, + dtype: torch.dtype, +) -> Any: + return from_pretrained_with_prefetch( + component_cls.from_pretrained, + model, + subfolder=subfolder, + prefetch_list=_LTX_COMPONENT_SUBFOLDERS, + local_files_only=local_files_only, + torch_dtype=dtype, + ) + + +def _place_aux_components(pipeline: Any) -> None: + parallel_config = getattr(pipeline.od_config, "parallel_config", None) + use_managed_placement = bool( + getattr(pipeline.od_config, "enable_cpu_offload", False) + or getattr(pipeline.od_config, "enable_layerwise_offload", False) + or getattr(parallel_config, "use_hsdp", False) + ) + if use_managed_placement: + return + + modules = ModuleDiscovery.discover(pipeline) + for module in (*modules.encoders, *modules.vaes, *modules.resident_modules): + module.to(pipeline.device) + + +def initialize_pipeline_components(pipeline: Any, od_config: Any) -> None: + """Build the common LTX component graph selected by ``component_profile``.""" + profile: LTXComponentProfile = pipeline.component_profile + pipeline.od_config = od_config + pipeline.device = get_local_device() + dtype = getattr(od_config, "dtype", torch.bfloat16) + model = od_config.model + local_files_only = os.path.exists(model) + + pipeline.weights_sources = [ + DiffusersPipelineLoader.ComponentSource( + model_or_path=model, + subfolder="transformer", + revision=None, + prefix="transformer.", + fall_back_to_pt=True, + ), + ] + prefetch_subfolders(model, _LTX_COMPONENT_SUBFOLDERS, local_files_only=local_files_only) + + pipeline.tokenizer = AutoTokenizer.from_pretrained( + model, + subfolder="tokenizer", + local_files_only=local_files_only, + ) + with torch.device("cpu"): + pipeline.text_encoder = _load_component( + Gemma3ForConditionalGeneration, + model, + "text_encoder", + local_files_only=local_files_only, + dtype=dtype, + ) + pipeline.connectors = _load_component( + LTX2TextConnectors, + model, + "connectors", + local_files_only=local_files_only, + dtype=dtype, + ) + pipeline.vae = _load_component( + profile.video_vae_cls, + model, + "vae", + local_files_only=local_files_only, + dtype=dtype, + ) + pipeline.audio_vae = _load_component( + AutoencoderKLLTX2Audio, + model, + "audio_vae", + local_files_only=local_files_only, + dtype=dtype, + ) + try: + pipeline.vocoder = _load_component( + profile.vocoder_cls, + model, + "vocoder", + local_files_only=local_files_only, + dtype=dtype, + ) + except (TypeError, OSError, ValueError): + if profile.vocoder_fallback_cls is None or profile.vocoder_fallback_cls is profile.vocoder_cls: + raise + pipeline.vocoder = _load_component( + profile.vocoder_fallback_cls, + model, + "vocoder", + local_files_only=local_files_only, + dtype=dtype, + ) + + transformer_config = load_transformer_config(model, "transformer", local_files_only) + quant_config = getattr(od_config, "quantization_config", None) + pipeline.transformer = create_transformer_from_config(transformer_config, quant_config=quant_config) + _place_aux_components(pipeline) + pipeline.scheduler = FlowMatchEulerDiscreteScheduler.from_pretrained( + model, + subfolder="scheduler", + local_files_only=local_files_only, + ) + + pipeline.vae_spatial_compression_ratio = pipeline.vae.spatial_compression_ratio + pipeline.vae_temporal_compression_ratio = pipeline.vae.temporal_compression_ratio + pipeline.audio_vae_mel_compression_ratio = pipeline.audio_vae.mel_compression_ratio + pipeline.audio_vae_temporal_compression_ratio = pipeline.audio_vae.temporal_compression_ratio + pipeline.transformer_spatial_patch_size = pipeline.transformer.config.patch_size + pipeline.transformer_temporal_patch_size = pipeline.transformer.config.patch_size_t + pipeline.audio_sampling_rate = pipeline.audio_vae.config.sample_rate + pipeline.audio_hop_length = pipeline.audio_vae.config.mel_hop_length + pipeline.video_processor = VideoProcessor(vae_scale_factor=pipeline.vae_spatial_compression_ratio) + + tokenizer_max_length = pipeline.tokenizer.model_max_length + if tokenizer_max_length is None or tokenizer_max_length > 100000: + encoder_config = getattr(pipeline.text_encoder, "config", None) + tokenizer_max_length = getattr(encoder_config, "max_position_embeddings", None) + if tokenizer_max_length is None: + tokenizer_max_length = getattr(encoder_config, "max_seq_len", None) + pipeline.tokenizer_max_length = int(tokenizer_max_length or 1024) + + pipeline._guidance_scale = None + pipeline._guidance_rescale = None + pipeline._interrupt = False + + +def load_transformer_config( + model_path: str, + subfolder: str = "transformer", + local_files_only: bool = True, +) -> dict: + """Load an LTX transformer config from a local model or the HF Hub.""" + if local_files_only: + config_path = os.path.join(model_path, subfolder, "config.json") + if os.path.exists(config_path): + with open(config_path) as config_file: + return json.load(config_file) + else: + try: + from huggingface_hub import hf_hub_download + + config_path = hf_hub_download( + repo_id=model_path, + filename=f"{subfolder}/config.json", + ) + with open(config_path) as config_file: + return json.load(config_file) + except Exception: + pass + return {} + + +def create_transformer_from_config( + config: dict, + quant_config: QuantizationConfig | None = None, +) -> LTX2VideoTransformer3DModel: + """Construct the shared LTX transformer from a Diffusers config.""" + if not config and quant_config is None: + return LTX2VideoTransformer3DModel() + + signature = inspect.signature(LTX2VideoTransformer3DModel.__init__) + allowed_keys = set(signature.parameters) + kwargs = {key: value for key, value in config.items() if key in allowed_keys} + if quant_config is not None: + kwargs["quant_config"] = quant_config + + return LTX2VideoTransformer3DModel(**kwargs) diff --git a/vllm_omni/diffusion/models/ltx2/ltx2_conditioning.py b/vllm_omni/diffusion/models/ltx2/ltx2_conditioning.py new file mode 100644 index 000000000..48d319f6d --- /dev/null +++ b/vllm_omni/diffusion/models/ltx2/ltx2_conditioning.py @@ -0,0 +1,578 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +"""Task-specific conditioning shared by LTX model versions.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any + +import PIL.Image +import torch +from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img import retrieve_latents +from diffusers.utils.torch_utils import randn_tensor +from diffusers.video_processor import VideoProcessor + +from . import ltx2_latents as latent_ops +from .ltx2_denoise import I2VVideoAudioScheduler + +if TYPE_CHECKING: + from .ltx2_denoise import LTXDenoiseContext, LTXForwardContext + from .ltx2_request import LTXRequestInputs + + +@dataclass +class LTXPromptContext: + """Connector outputs consumed by an LTX denoise phase.""" + + batch_size: int + connector_prompt_embeds: torch.Tensor + connector_audio_prompt_embeds: torch.Tensor + connector_attention_mask: torch.Tensor + positive_connector_prompt_embeds: torch.Tensor + positive_connector_audio_prompt_embeds: torch.Tensor + positive_connector_attention_mask: torch.Tensor + negative_connector_prompt_embeds: torch.Tensor | None + negative_connector_audio_prompt_embeds: torch.Tensor | None + negative_connector_attention_mask: torch.Tensor | None + + +def _repeat_prompt_tensor_for_outputs(tensor: torch.Tensor, num_outputs: int) -> torch.Tensor: + if num_outputs == 1: + return tensor + return tensor.repeat_interleave(num_outputs, dim=0) + + +class LTXTextConditioningMixin: + """Shared Gemma encoding and text connector orchestration.""" + + connector_batches_cfg = False + + def _get_gemma_prompt_embeds( + self, + prompt: str | list[str], + num_videos_per_prompt: int = 1, + max_sequence_length: int = 1024, + scale_factor: int = 8, + device: torch.device | None = None, + dtype: torch.dtype | None = None, + ) -> tuple[torch.Tensor, torch.Tensor]: + del scale_factor + device = device or self.device + dtype = dtype or self.text_encoder.dtype + prompt = [prompt] if isinstance(prompt, str) else prompt + batch_size = len(prompt) + + self.tokenizer.padding_side = "left" + if self.tokenizer.pad_token is None: + self.tokenizer.pad_token = self.tokenizer.eos_token + + text_inputs = self.tokenizer( + [text.strip() for text in prompt], + padding="max_length", + max_length=max_sequence_length, + truncation=True, + add_special_tokens=True, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids.to(device) + prompt_attention_mask = text_inputs.attention_mask.to(device) + hidden_states = self.text_encoder( + input_ids=text_input_ids, + attention_mask=prompt_attention_mask, + output_hidden_states=True, + ).hidden_states + + prompt_embeds = torch.stack(hidden_states, dim=-1).flatten(2, 3).to(dtype=dtype) + prompt_embeds = _repeat_prompt_tensor_for_outputs(prompt_embeds, num_videos_per_prompt) + prompt_attention_mask = prompt_attention_mask.view(batch_size, -1) + prompt_attention_mask = _repeat_prompt_tensor_for_outputs( + prompt_attention_mask, + num_videos_per_prompt, + ) + return prompt_embeds, prompt_attention_mask + + def encode_prompt( + self, + prompt: str | list[str] | None, + negative_prompt: str | list[str] | None = None, + do_classifier_free_guidance: bool = True, + num_videos_per_prompt: int = 1, + prompt_embeds: torch.Tensor | None = None, + negative_prompt_embeds: torch.Tensor | None = None, + prompt_attention_mask: torch.Tensor | None = None, + negative_prompt_attention_mask: torch.Tensor | None = None, + max_sequence_length: int = 1024, + scale_factor: int = 8, + device: torch.device | None = None, + dtype: torch.dtype | None = None, + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor | None, torch.Tensor | None]: + device = device or self.device + prompt = [prompt] if isinstance(prompt, str) else prompt + batch_size = len(prompt) if prompt is not None else prompt_embeds.shape[0] + negative_prompt_embeds_provided = negative_prompt_embeds is not None + + if prompt_embeds is None: + prompt_embeds, prompt_attention_mask = self._get_gemma_prompt_embeds( + prompt=prompt, + num_videos_per_prompt=num_videos_per_prompt, + max_sequence_length=max_sequence_length, + scale_factor=scale_factor, + device=device, + dtype=dtype, + ) + elif num_videos_per_prompt > 1: + prompt_embeds = _repeat_prompt_tensor_for_outputs(prompt_embeds, num_videos_per_prompt) + prompt_attention_mask = _repeat_prompt_tensor_for_outputs( + prompt_attention_mask, + num_videos_per_prompt, + ) + + if do_classifier_free_guidance and negative_prompt_embeds is None: + negative_prompt = negative_prompt or "" + negative_prompt = batch_size * [negative_prompt] if isinstance(negative_prompt, str) else negative_prompt + if prompt is not None and type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type as `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + if isinstance(negative_prompt, list) and batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt` has batch size {len(negative_prompt)}, but `prompt` has batch size" + f" {batch_size}." + ) + negative_prompt_embeds, negative_prompt_attention_mask = self._get_gemma_prompt_embeds( + prompt=negative_prompt, + num_videos_per_prompt=num_videos_per_prompt, + max_sequence_length=max_sequence_length, + scale_factor=scale_factor, + device=device, + dtype=dtype, + ) + elif do_classifier_free_guidance and negative_prompt_embeds_provided and num_videos_per_prompt > 1: + negative_prompt_embeds = _repeat_prompt_tensor_for_outputs( + negative_prompt_embeds, + num_videos_per_prompt, + ) + negative_prompt_attention_mask = _repeat_prompt_tensor_for_outputs( + negative_prompt_attention_mask, + num_videos_per_prompt, + ) + + return prompt_embeds, prompt_attention_mask, negative_prompt_embeds, negative_prompt_attention_mask + + def _prepare_prompt_context( + self, + *, + prompt: str | list[str] | None, + negative_prompt: str | list[str] | None, + prompt_embeds: torch.Tensor | None, + negative_prompt_embeds: torch.Tensor | None, + prompt_attention_mask: torch.Tensor | None, + negative_prompt_attention_mask: torch.Tensor | None, + num_videos_per_prompt: int, + max_sequence_length: int, + ) -> LTXPromptContext: + if isinstance(prompt, str): + batch_size = 1 + elif isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + prompt_embeds, prompt_attention_mask, negative_prompt_embeds, negative_prompt_attention_mask = ( + self.encode_prompt( + prompt=prompt, + negative_prompt=negative_prompt, + do_classifier_free_guidance=self.do_classifier_free_guidance, + num_videos_per_prompt=num_videos_per_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + prompt_attention_mask=prompt_attention_mask, + negative_prompt_attention_mask=negative_prompt_attention_mask, + max_sequence_length=max_sequence_length, + device=self.device, + ) + ) + padding_side = getattr(self.tokenizer, "padding_side", "left") + + if self.do_classifier_free_guidance and self.connector_batches_cfg: + prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds], dim=0) + prompt_attention_mask = torch.cat([negative_prompt_attention_mask, prompt_attention_mask], dim=0) + + video_context, audio_context, attention_mask = self.connectors( + prompt_embeds, + prompt_attention_mask, + padding_side=padding_side, + ) + positive_video_context = video_context + positive_audio_context = audio_context + positive_attention_mask = attention_mask + negative_video_context = None + negative_audio_context = None + negative_attention_mask = None + + if self.do_classifier_free_guidance and self.connector_batches_cfg: + split_batch = batch_size * num_videos_per_prompt + negative_video_context = video_context[:split_batch] + positive_video_context = video_context[split_batch:] + negative_audio_context = audio_context[:split_batch] + positive_audio_context = audio_context[split_batch:] + negative_attention_mask = attention_mask[:split_batch] + positive_attention_mask = attention_mask[split_batch:] + elif self.do_classifier_free_guidance: + negative_video_context, negative_audio_context, negative_attention_mask = self.connectors( + negative_prompt_embeds, + negative_prompt_attention_mask, + padding_side=padding_side, + ) + + return LTXPromptContext( + batch_size=batch_size, + connector_prompt_embeds=video_context, + connector_audio_prompt_embeds=audio_context, + connector_attention_mask=attention_mask, + positive_connector_prompt_embeds=positive_video_context, + positive_connector_audio_prompt_embeds=positive_audio_context, + positive_connector_attention_mask=positive_attention_mask, + negative_connector_prompt_embeds=negative_video_context, + negative_connector_audio_prompt_embeds=negative_audio_context, + negative_connector_attention_mask=negative_attention_mask, + ) + + +class LTXI2VConditioningMixin: + """First-frame conditioning behavior common to LTX2 and LTX2.3.""" + + support_image_input = True + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.video_processor = VideoProcessor( + vae_scale_factor=self.vae_spatial_compression_ratio, + resample="bilinear", + ) + + def forward(self, req: Any, image: Any | None = None, *args: Any, **kwargs: Any) -> Any: + """Preserve the public I2V positional API while sharing the T2V runner.""" + return super().forward(req, *args, image=image, **kwargs) + + @staticmethod + def _resolve_single_prompt_image(raw_image: Any) -> Any: + if isinstance(raw_image, list): + if len(raw_image) != 1: + raise ValueError( + "LTX I2V prompt dictionaries support exactly one image per prompt. " + "Pass one image per prompt for batched I2V requests." + ) + return raw_image[0] + return raw_image + + @staticmethod + def _resolve_additional_image(additional: dict[str, Any]) -> Any: + for field_name in ("preprocessed_image", "pixel_values", "image"): + raw_image = additional.get(field_name) + if raw_image is not None: + return raw_image + return None + + def _resolve_request_image( + self, + req: Any, + image: Any | None, + request_inputs: LTXRequestInputs, + ) -> Any | None: + if image is not None or not req.prompts: + return image + + raw_images = [] + for prompt_item in req.prompts: + if isinstance(prompt_item, str): + raw_image = None + else: + multi_modal_data = prompt_item.get("multi_modal_data") or {} + raw_image = multi_modal_data.get("image") + if raw_image is None: + raw_image = self._resolve_additional_image(prompt_item.get("additional_information") or {}) + raw_image = self._resolve_single_prompt_image(raw_image) + if isinstance(raw_image, str): + raw_image = PIL.Image.open(raw_image).convert("RGB") + raw_images.append(raw_image) + + if any(raw_image is None for raw_image in raw_images) and request_inputs.latents is None: + raise ValueError("Image is required for LTX I2V generation.") + if len(raw_images) == 1: + return raw_images[0] + return raw_images or image + + def _check_forward_inputs( + self, + request_inputs: LTXRequestInputs, + image: Any | None = None, + ) -> None: + if image is None and request_inputs.latents is None: + raise ValueError("Provide either `image` or `latents`. Cannot leave both undefined.") + super()._check_forward_inputs(request_inputs, image=image) + + def prepare_latents( + self, + image: torch.Tensor | None = None, + batch_size: int = 1, + num_channels_latents: int = 128, + height: int = 512, + width: int = 768, + num_frames: int = 121, + noise_scale: float = 0.0, + dtype: torch.dtype | None = None, + device: torch.device | None = None, + generator: torch.Generator | list[torch.Generator] | None = None, + latents: torch.Tensor | None = None, + ) -> tuple[torch.Tensor, torch.Tensor]: + num_frames, height, width = latent_ops.resolve_video_latent_shape( + height, + width, + num_frames, + vae_spatial_compression_ratio=self.vae_spatial_compression_ratio, + vae_temporal_compression_ratio=self.vae_temporal_compression_ratio, + ) + shape = (batch_size, num_channels_latents, num_frames, height, width) + mask_shape = (batch_size, 1, num_frames, height, width) + + if latents is not None: + if latents.ndim == 5: + batch_size, _, num_frames, height, width = latents.shape + mask_shape = (batch_size, 1, num_frames, height, width) + conditioning_mask = latents.new_zeros(mask_shape) + conditioning_mask[:, :, 0] = 1.0 + latents = latent_ops.normalize_latents( + latents, + self.vae.latents_mean, + self.vae.latents_std, + self.vae.config.scaling_factor, + ) + latents = latent_ops.create_noised_state( + latents, + noise_scale * (1 - conditioning_mask), + generator, + ) + latents = latent_ops.pack_latents( + latents, + self.transformer_spatial_patch_size, + self.transformer_temporal_patch_size, + ) + else: + conditioning_mask = latents.new_zeros(mask_shape) + conditioning_mask[:, :, 0] = 1.0 + + conditioning_mask = latent_ops.pack_latents( + conditioning_mask, + self.transformer_spatial_patch_size, + self.transformer_temporal_patch_size, + ).squeeze(-1) + if latents.ndim != 3 or latents.shape[:2] != conditioning_mask.shape: + raise ValueError( + "Provided `latents` tensor has shape" + f" {latents.shape}, but the expected shape is {conditioning_mask.shape + (num_channels_latents,)}." + ) + return latents.to(device=device, dtype=dtype), conditioning_mask + + if image is None: + raise ValueError("`image` must be provided when `latents` is None.") + + image_batch_size = image.shape[0] + if image_batch_size == 0: + raise ValueError("`image` batch is empty.") + if batch_size % image_batch_size != 0: + raise ValueError( + f"`batch_size` ({batch_size}) must be divisible by image batch size ({image_batch_size}) " + "for image-to-video outputs." + ) + num_videos_per_prompt = batch_size // image_batch_size + + if isinstance(generator, list): + if len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective" + f" batch size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + image_generators = [generator[i * num_videos_per_prompt] for i in range(image_batch_size)] + init_latents = [ + retrieve_latents( + self.vae.encode(image[i].unsqueeze(0).unsqueeze(2)), + image_generators[i], + "argmax", + ) + for i in range(image_batch_size) + ] + else: + init_latents = [ + retrieve_latents( + self.vae.encode(img.unsqueeze(0).unsqueeze(2)), + generator, + "argmax", + ) + for img in image + ] + + init_latents = torch.cat(init_latents, dim=0).to(dtype) + if num_videos_per_prompt > 1: + init_latents = init_latents.repeat_interleave(num_videos_per_prompt, dim=0) + init_latents = latent_ops.normalize_latents( + init_latents, + self.vae.latents_mean, + self.vae.latents_std, + ) + init_latents = init_latents.repeat(1, 1, num_frames, 1, 1) + + conditioning_mask = torch.zeros(mask_shape, device=device, dtype=dtype) + conditioning_mask[:, :, 0] = 1.0 + noise = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + latents = init_latents * conditioning_mask + noise * (1 - conditioning_mask) + + conditioning_mask = latent_ops.pack_latents( + conditioning_mask, + self.transformer_spatial_patch_size, + self.transformer_temporal_patch_size, + ).squeeze(-1) + latents = latent_ops.pack_latents( + latents, + self.transformer_spatial_patch_size, + self.transformer_temporal_patch_size, + ) + return latents, conditioning_mask + + def _step_video_latents_i2v( + self, + noise_pred_video: torch.Tensor, + latents: torch.Tensor, + timestep: torch.Tensor, + latent_num_frames: int, + latent_height: int, + latent_width: int, + ) -> torch.Tensor: + noise_pred_video = latent_ops.unpack_latents( + noise_pred_video, + latent_num_frames, + latent_height, + latent_width, + self.transformer_spatial_patch_size, + self.transformer_temporal_patch_size, + ) + latents_unpacked = latent_ops.unpack_latents( + latents, + latent_num_frames, + latent_height, + latent_width, + self.transformer_spatial_patch_size, + self.transformer_temporal_patch_size, + ) + noise_pred_video = noise_pred_video[:, :, 1:] + noise_latents = latents_unpacked[:, :, 1:] + pred_latents = self.scheduler.step( + noise_pred_video, + timestep, + noise_latents, + return_dict=False, + )[0] + latents_unpacked = torch.cat([latents_unpacked[:, :, :1], pred_latents], dim=2) + return latent_ops.pack_latents( + latents_unpacked, + self.transformer_spatial_patch_size, + self.transformer_temporal_patch_size, + ) + + def _prepare_video_latents_stage( + self, + request_inputs: LTXRequestInputs, + prompt_context: LTXPromptContext, + *, + device: torch.device, + noise_scale: float, + image: Any | None = None, + ) -> tuple[torch.Tensor, torch.Tensor | None]: + if request_inputs.latents is None: + if isinstance(image, torch.Tensor): + if image.ndim == 3: + image = image.unsqueeze(0) + elif isinstance(image, list) and image and isinstance(image[0], torch.Tensor): + image = torch.stack(image, dim=0) + else: + image = self.video_processor.preprocess( + image, + height=request_inputs.height, + width=request_inputs.width, + ) + image = image.to( + device=device, + dtype=prompt_context.positive_connector_prompt_embeds.dtype, + ) + + return self.prepare_latents( + image, + prompt_context.batch_size * request_inputs.num_videos_per_prompt, + self.transformer.config.in_channels, + request_inputs.height, + request_inputs.width, + request_inputs.num_frames, + noise_scale, + torch.float32, + device, + request_inputs.generator, + request_inputs.latents, + ) + + def _make_video_audio_scheduler( + self, + audio_scheduler: Any, + latent_num_frames: int, + latent_height: int, + latent_width: int, + ) -> Any: + return I2VVideoAudioScheduler( + self, + audio_scheduler, + latent_num_frames, + latent_height, + latent_width, + ) + + def _prepare_denoise_context_for_cfg( + self, + forward_ctx: LTXForwardContext, + denoise_ctx: LTXDenoiseContext, + ) -> LTXDenoiseContext: + denoise_ctx = super()._prepare_denoise_context_for_cfg(forward_ctx, denoise_ctx) + if denoise_ctx.conditioning_mask is None: + raise ValueError("LTX I2V denoising requires a conditioning mask.") + + mask_batch = denoise_ctx.conditioning_mask.shape[0] + model_batch = denoise_ctx.video_coords.shape[0] + if model_batch % mask_batch != 0: + raise ValueError( + "I2V conditioning-mask batch must divide the Transformer input batch, " + f"but got {mask_batch} and {model_batch}." + ) + repeats = model_batch // mask_batch + denoise_ctx.conditioning_mask_for_model = ( + denoise_ctx.conditioning_mask if repeats == 1 else torch.cat([denoise_ctx.conditioning_mask] * repeats) + ) + return denoise_ctx + + def _denoise_timestep_kwargs( + self, + ts: torch.Tensor, + forward_ctx: LTXForwardContext, + denoise_ctx: LTXDenoiseContext, + ) -> dict[str, torch.Tensor]: + kwargs = super()._denoise_timestep_kwargs(ts, forward_ctx, denoise_ctx) + conditioning_mask = ( + denoise_ctx.conditioning_mask if forward_ctx.cfg_parallel_ready else denoise_ctx.conditioning_mask_for_model + ) + if conditioning_mask is None: + raise ValueError("LTX I2V denoising requires a conditioning mask.") + kwargs.update( + timestep=ts.unsqueeze(-1) * (1 - conditioning_mask), + audio_timestep=ts, + ) + return kwargs diff --git a/vllm_omni/diffusion/models/ltx2/ltx2_denoise.py b/vllm_omni/diffusion/models/ltx2/ltx2_denoise.py new file mode 100644 index 000000000..af8df4b41 --- /dev/null +++ b/vllm_omni/diffusion/models/ltx2/ltx2_denoise.py @@ -0,0 +1,407 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +"""Shared denoise execution primitives for LTX pipelines.""" + +from __future__ import annotations + +import copy +from collections.abc import Callable, Iterable +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, Protocol + +import numpy as np +import torch +from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion import retrieve_timesteps + +from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch + +from .ltx2_latents import LTXAVState + +if TYPE_CHECKING: + from .ltx2_conditioning import LTXPromptContext + from .ltx2_request import LTXRequestInputs + + +@dataclass +class LTXForwardContext: + """Immutable metadata and schedulers for one LTX denoise phase.""" + + req: DiffusionRequestBatch + request_inputs: LTXRequestInputs + prompt_context: LTXPromptContext + device: torch.device + cfg_parallel_ready: bool + attention_kwargs: dict[str, Any] | None + latent_num_frames: int + latent_height: int + latent_width: int + latent_mel_bins: int + original_audio_num_frames: int + padded_audio_num_frames: int + timesteps: torch.Tensor + audio_scheduler: Any + video_audio_scheduler: Any + + @property + def batch_size(self) -> int: + return self.prompt_context.batch_size + + @property + def num_videos_per_prompt(self) -> int: + return self.request_inputs.num_videos_per_prompt + + +@dataclass +class LTXDenoiseContext: + """Mutable AV state and positional metadata for a denoise phase.""" + + latents: torch.Tensor + audio_latents: torch.Tensor + video_coords: torch.Tensor + audio_coords: torch.Tensor + conditioning_mask: torch.Tensor | None = None + conditioning_mask_for_model: torch.Tensor | None = None + + +@dataclass +class LTXPhaseResult: + """Denoised, unpacked AV latents and the context used to produce them.""" + + forward_context: LTXForwardContext + video: torch.Tensor + audio: torch.Tensor + + +class LTXDenoisePipeline(Protocol): + """Pipeline state required by :class:`LTXDenoiseExecutor`.""" + + @property + def interrupt(self) -> bool: ... + + def progress_bar(self, iterable=None, total=None): ... + + +LTXDenoiseStep = Callable[[int, torch.Tensor, LTXAVState], LTXAVState] + + +class LTXDenoiseExecutor: + """Run the one shared LTX denoise loop. + + Prediction and scheduler math remain injectable so structural refactors do + not change the existing LTX2/LTX2.3 numerical paths. Guidance will replace + that step policy independently. + """ + + @staticmethod + def run( + pipeline: LTXDenoisePipeline, + state: LTXAVState, + timesteps: Iterable[torch.Tensor], + step: LTXDenoiseStep, + ) -> LTXAVState: + timesteps = tuple(timesteps) + with pipeline.progress_bar(total=len(timesteps)) as progress_bar: + for index, timestep in enumerate(timesteps): + if pipeline.interrupt: + continue + state = step(index, timestep, state) + progress_bar.update() + return state + + +def calculate_shift( + image_seq_len: int, + base_seq_len: int = 256, + max_seq_len: int = 4096, + base_shift: float = 0.5, + max_shift: float = 1.15, +) -> float: + slope = (max_shift - base_shift) / (max_seq_len - base_seq_len) + intercept = base_shift - slope * base_seq_len + return image_seq_len * slope + intercept + + +class VideoAudioScheduler: + """Composite scheduler dispatching video and audio updates.""" + + def __init__(self, video_scheduler, audio_scheduler): + self.video_scheduler = video_scheduler + self.audio_scheduler = audio_scheduler + + def step(self, noise_pred, t, latents, return_dict=False, generator=None): + video_out = self.video_scheduler.step( + noise_pred[0], + t[0], + latents[0], + return_dict=False, + generator=generator, + )[0] + audio_out = self.audio_scheduler.step( + noise_pred[1], + t[1], + latents[1], + return_dict=False, + generator=generator, + )[0] + return ((video_out, audio_out),) + + +class I2VVideoAudioScheduler: + """Update the unconditioned video frames and the full audio state.""" + + def __init__(self, pipeline, audio_scheduler, latent_num_frames, latent_height, latent_width): + self.video_scheduler = pipeline.scheduler + self.audio_scheduler = audio_scheduler + self._pipeline = pipeline + self._latent_num_frames = latent_num_frames + self._latent_height = latent_height + self._latent_width = latent_width + + def step(self, noise_pred, t, latents, return_dict=False, generator=None): + video_out = self._pipeline._step_video_latents_i2v( + noise_pred[0], + latents[0], + t[0], + self._latent_num_frames, + self._latent_height, + self._latent_width, + ) + audio_out = self.audio_scheduler.step( + noise_pred[1], + t[1], + latents[1], + return_dict=False, + generator=generator, + )[0] + return ((video_out, audio_out),) + + +def prepare_scheduler_stage( + pipeline: Any, + request_inputs: LTXRequestInputs, + *, + device: torch.device, + sigmas: list[float] | None, + timesteps: list[int] | None, + latent_num_frames: int, + latent_height: int, + latent_width: int, +) -> tuple[Any, Any, torch.Tensor]: + sigmas = ( + np.linspace(1.0, 1 / request_inputs.num_inference_steps, request_inputs.num_inference_steps) + if sigmas is None + else sigmas + ) + mu = calculate_shift( + pipeline.scheduler.config.get("max_image_seq_len", 4096), + pipeline.scheduler.config.get("base_image_seq_len", 1024), + pipeline.scheduler.config.get("max_image_seq_len", 4096), + pipeline.scheduler.config.get("base_shift", 0.95), + pipeline.scheduler.config.get("max_shift", 2.05), + ) + audio_scheduler = copy.deepcopy(pipeline.scheduler) + video_audio_scheduler = pipeline._make_video_audio_scheduler( + audio_scheduler, + latent_num_frames, + latent_height, + latent_width, + ) + retrieve_timesteps( + audio_scheduler, + request_inputs.num_inference_steps, + device, + timesteps, + sigmas=sigmas, + mu=mu, + ) + timesteps_tensor, _ = retrieve_timesteps( + pipeline.scheduler, + request_inputs.num_inference_steps, + device, + timesteps, + sigmas=sigmas, + mu=mu, + ) + return audio_scheduler, video_audio_scheduler, timesteps_tensor + + +def prepare_rope_coords_stage( + pipeline: Any, + forward_ctx: LTXForwardContext, + latents: torch.Tensor, + audio_latents: torch.Tensor, +) -> tuple[torch.Tensor, torch.Tensor]: + video_coords = pipeline.transformer.rope.prepare_video_coords( + latents.shape[0], + forward_ctx.latent_num_frames, + forward_ctx.latent_height, + forward_ctx.latent_width, + latents.device, + fps=forward_ctx.request_inputs.frame_rate, + ) + audio_coords = pipeline.transformer.audio_rope.prepare_audio_coords( + audio_latents.shape[0], + forward_ctx.padded_audio_num_frames, + audio_latents.device, + ) + return video_coords, audio_coords + + +def build_transformer_kwargs( + pipeline: Any, + forward_ctx: LTXForwardContext, + denoise_ctx: LTXDenoiseContext, + *, + hidden_states: torch.Tensor, + audio_hidden_states: torch.Tensor, + encoder_hidden_states: torch.Tensor, + audio_encoder_hidden_states: torch.Tensor, + encoder_attention_mask: torch.Tensor | None, + audio_encoder_attention_mask: torch.Tensor | None, + ts: torch.Tensor, +) -> dict[str, Any]: + return { + "hidden_states": hidden_states, + "audio_hidden_states": audio_hidden_states, + "encoder_hidden_states": encoder_hidden_states, + "audio_encoder_hidden_states": audio_encoder_hidden_states, + **pipeline._denoise_timestep_kwargs(ts, forward_ctx, denoise_ctx), + "encoder_attention_mask": encoder_attention_mask, + "audio_encoder_attention_mask": audio_encoder_attention_mask, + "num_frames": forward_ctx.latent_num_frames, + "height": forward_ctx.latent_height, + "width": forward_ctx.latent_width, + "fps": forward_ctx.request_inputs.frame_rate, + "audio_num_frames": forward_ctx.padded_audio_num_frames, + "video_coords": denoise_ctx.video_coords, + "audio_coords": denoise_ctx.audio_coords, + "attention_kwargs": forward_ctx.attention_kwargs, + "return_dict": False, + } + + +def step_denoised_latents( + pipeline: Any, + forward_ctx: LTXForwardContext, + denoise_ctx: LTXDenoiseContext, + noise_pred_video: torch.Tensor, + noise_pred_audio: torch.Tensor, + timestep: torch.Tensor, +) -> tuple[torch.Tensor, torch.Tensor]: + latents = pipeline.scheduler_step_maybe_with_cfg( + (noise_pred_video, noise_pred_audio), + (timestep, timestep), + (denoise_ctx.latents, denoise_ctx.audio_latents), + do_true_cfg=pipeline.do_classifier_free_guidance, + per_request_scheduler=forward_ctx.video_audio_scheduler, + ) + return pipeline._synchronize_cfg_parallel_step_output( + latents, + do_true_cfg=pipeline.do_classifier_free_guidance, + ) + + +class LTXPhaseExecutor: + """Prepare and execute one LTX phase without owning model modules.""" + + @staticmethod + def run( + pipeline: Any, + req: DiffusionRequestBatch, + request_inputs: LTXRequestInputs, + *, + noise_scale: float, + sigmas: list[float] | None, + timesteps: list[int] | None, + attention_kwargs: dict[str, Any] | None, + image: Any | None = None, + prompt_context: LTXPromptContext | None = None, + ) -> LTXPhaseResult: + pipeline._check_forward_inputs(request_inputs, image=image) + cfg_parallel_ready = pipeline._setup_forward_runtime(request_inputs, attention_kwargs) + device = pipeline.device + if prompt_context is None: + prompt_context = pipeline._prepare_prompt_context( + prompt=request_inputs.prompt, + negative_prompt=request_inputs.negative_prompt, + prompt_embeds=request_inputs.prompt_embeds, + negative_prompt_embeds=request_inputs.negative_prompt_embeds, + prompt_attention_mask=request_inputs.prompt_attention_mask, + negative_prompt_attention_mask=request_inputs.negative_prompt_attention_mask, + num_videos_per_prompt=request_inputs.num_videos_per_prompt, + max_sequence_length=request_inputs.max_sequence_length, + ) + + latent_num_frames, latent_height, latent_width = pipeline._resolve_video_latent_dimensions(request_inputs) + latents, conditioning_mask = pipeline._prepare_video_latents_stage( + request_inputs, + prompt_context, + device=device, + noise_scale=noise_scale, + image=image, + ) + audio_latents, original_audio_num_frames, padded_audio_num_frames, latent_mel_bins = ( + pipeline._prepare_audio_latents_stage( + request_inputs, + prompt_context, + device=device, + noise_scale=noise_scale, + ) + ) + audio_scheduler, video_audio_scheduler, timesteps_tensor = prepare_scheduler_stage( + pipeline, + request_inputs, + device=device, + sigmas=sigmas, + timesteps=timesteps, + latent_num_frames=latent_num_frames, + latent_height=latent_height, + latent_width=latent_width, + ) + forward_ctx = LTXForwardContext( + req=req, + request_inputs=request_inputs, + prompt_context=prompt_context, + device=device, + cfg_parallel_ready=cfg_parallel_ready, + attention_kwargs=attention_kwargs, + latent_num_frames=latent_num_frames, + latent_height=latent_height, + latent_width=latent_width, + latent_mel_bins=latent_mel_bins, + original_audio_num_frames=original_audio_num_frames, + padded_audio_num_frames=padded_audio_num_frames, + timesteps=timesteps_tensor, + audio_scheduler=audio_scheduler, + video_audio_scheduler=video_audio_scheduler, + ) + video_coords, audio_coords = prepare_rope_coords_stage(pipeline, forward_ctx, latents, audio_latents) + denoise_ctx = LTXDenoiseContext( + latents=latents, + audio_latents=audio_latents, + video_coords=video_coords, + audio_coords=audio_coords, + conditioning_mask=conditioning_mask, + ) + denoise_ctx = pipeline._prepare_denoise_context_for_cfg(forward_ctx, denoise_ctx) + state = LTXDenoiseExecutor.run( + pipeline, + LTXAVState(video=denoise_ctx.latents, audio=denoise_ctx.audio_latents), + forward_ctx.timesteps, + lambda index, timestep, state: pipeline._denoise_step( + index, + timestep, + state, + forward_ctx, + denoise_ctx, + ), + ) + denoise_ctx.latents = state.video + denoise_ctx.audio_latents = state.audio + latents, audio_latents = pipeline._unpack_and_denormalize_stage( + forward_ctx, + state.video, + state.audio, + ) + return LTXPhaseResult(forward_context=forward_ctx, video=latents, audio=audio_latents) diff --git a/vllm_omni/diffusion/models/ltx2/ltx2_guidance.py b/vllm_omni/diffusion/models/ltx2/ltx2_guidance.py new file mode 100644 index 000000000..395f8e7af --- /dev/null +++ b/vllm_omni/diffusion/models/ltx2/ltx2_guidance.py @@ -0,0 +1,379 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +"""Guidance strategies for the LTX model family.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Protocol + +import torch +from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion import rescale_noise_cfg + +from vllm_omni.diffusion.distributed.cfg_parallel import CFGParallelMixin +from vllm_omni.diffusion.distributed.parallel_state import ( + get_cfg_group, + get_classifier_free_guidance_rank, + get_classifier_free_guidance_world_size, +) + +if TYPE_CHECKING: + from .ltx2_denoise import LTXDenoiseContext, LTXForwardContext + from .ltx2_latents import LTXAVState + + +class LTXGuidanceStrategy(Protocol): + """Model-version guidance behavior consumed by the shared pipeline.""" + + def validate_cfg_world_size(self, cfg_world_size: int) -> None: ... + + def prepare_denoise_context( + self, + pipeline: Any, + forward_ctx: LTXForwardContext, + denoise_ctx: LTXDenoiseContext, + ) -> LTXDenoiseContext: ... + + def timestep_kwargs( + self, + ts: torch.Tensor, + forward_ctx: LTXForwardContext, + denoise_ctx: LTXDenoiseContext, + ) -> dict[str, torch.Tensor]: ... + + def combine_cfg_noise( + self, + pipeline: Any, + positive_noise_pred: tuple[torch.Tensor, torch.Tensor], + negative_noise_pred: tuple[torch.Tensor, torch.Tensor], + true_cfg_scale: float, + cfg_normalize: bool, + context: dict[str, Any], + ) -> tuple[torch.Tensor, torch.Tensor]: ... + + def predict_noise( + self, + pipeline: Any, + index: int, + timestep: torch.Tensor, + state: LTXAVState, + forward_ctx: LTXForwardContext, + denoise_ctx: LTXDenoiseContext, + ) -> tuple[torch.Tensor, torch.Tensor]: ... + + +class _LTXGuidanceStrategyBase: + def validate_cfg_world_size(self, cfg_world_size: int) -> None: + del cfg_world_size + + def prepare_denoise_context( + self, + pipeline: Any, + forward_ctx: LTXForwardContext, + denoise_ctx: LTXDenoiseContext, + ) -> LTXDenoiseContext: + del pipeline, forward_ctx + return denoise_ctx + + def timestep_kwargs( + self, + ts: torch.Tensor, + forward_ctx: LTXForwardContext, + denoise_ctx: LTXDenoiseContext, + ) -> dict[str, torch.Tensor]: + del forward_ctx, denoise_ctx + return {"timestep": ts} + + +def combine_velocity_via_x0( + sample: torch.Tensor, + positive_velocity: torch.Tensor, + negative_velocity: torch.Tensor, + sigma: torch.Tensor, + guidance_scale: float, +) -> torch.Tensor: + """Apply CFG in x0 space and convert the result back to velocity.""" + x0_cond = sample - positive_velocity * sigma + x0_uncond = sample - negative_velocity * sigma + x0_guided = x0_cond + (guidance_scale - 1) * (x0_cond - x0_uncond) + return (sample - x0_guided) / sigma + + +class LTXLegacyVelocityGuidance(_LTXGuidanceStrategyBase): + """Existing LTX2 Diffusers-style velocity-space CFG behavior.""" + + def combine_cfg_noise( + self, + pipeline: Any, + positive_noise_pred: tuple[torch.Tensor, torch.Tensor], + negative_noise_pred: tuple[torch.Tensor, torch.Tensor], + true_cfg_scale: float, + cfg_normalize: bool, + context: dict[str, Any], + ) -> tuple[torch.Tensor, torch.Tensor]: + del context + video_pos, audio_pos = positive_noise_pred + video_neg, audio_neg = negative_noise_pred + video_combined = CFGParallelMixin.combine_cfg_noise( + pipeline, + video_pos, + video_neg, + true_cfg_scale, + cfg_normalize, + ) + audio_combined = CFGParallelMixin.combine_cfg_noise( + pipeline, + audio_pos, + audio_neg, + true_cfg_scale, + cfg_normalize, + ) + if pipeline.guidance_rescale and pipeline.guidance_rescale > 0: + video_combined = rescale_noise_cfg( + video_combined, + video_pos, + guidance_rescale=pipeline.guidance_rescale, + ) + audio_combined = rescale_noise_cfg( + audio_combined, + audio_pos, + guidance_rescale=pipeline.guidance_rescale, + ) + return video_combined, audio_combined + + def predict_noise( + self, + pipeline: Any, + index: int, + timestep: torch.Tensor, + state: LTXAVState, + forward_ctx: LTXForwardContext, + denoise_ctx: LTXDenoiseContext, + ) -> tuple[torch.Tensor, torch.Tensor]: + del index + prompt_context = forward_ctx.prompt_context + video_input = state.video.to(prompt_context.positive_connector_prompt_embeds.dtype) + audio_input = state.audio.to(prompt_context.positive_connector_prompt_embeds.dtype) + expanded_timestep = timestep.expand(video_input.shape[0]) + positive_kwargs = pipeline._build_transformer_kwargs( + forward_ctx, + denoise_ctx, + hidden_states=video_input, + audio_hidden_states=audio_input, + encoder_hidden_states=prompt_context.positive_connector_prompt_embeds, + audio_encoder_hidden_states=prompt_context.positive_connector_audio_prompt_embeds, + encoder_attention_mask=prompt_context.positive_connector_attention_mask, + audio_encoder_attention_mask=prompt_context.positive_connector_attention_mask, + ts=expanded_timestep, + ) + negative_kwargs = ( + { + **positive_kwargs, + "encoder_hidden_states": prompt_context.negative_connector_prompt_embeds, + "audio_encoder_hidden_states": prompt_context.negative_connector_audio_prompt_embeds, + "encoder_attention_mask": prompt_context.negative_connector_attention_mask, + "audio_encoder_attention_mask": prompt_context.negative_connector_attention_mask, + } + if pipeline.do_classifier_free_guidance + else None + ) + return pipeline.predict_noise_maybe_with_cfg( + do_true_cfg=pipeline.do_classifier_free_guidance, + true_cfg_scale=forward_ctx.request_inputs.guidance_scale, + positive_kwargs=positive_kwargs, + negative_kwargs=negative_kwargs, + cfg_normalize=False, + ) + + +class LTXOfficialX0Guidance(_LTXGuidanceStrategyBase): + """Official LTX x0-space CFG behavior currently used by LTX2.3.""" + + def validate_cfg_world_size(self, cfg_world_size: int) -> None: + if cfg_world_size not in (1, 2): + raise ValueError( + f"LTX x0 guidance supports CFG parallelism with cfg_parallel_size 1 or 2, but got {cfg_world_size}." + ) + + def prepare_denoise_context( + self, + pipeline: Any, + forward_ctx: LTXForwardContext, + denoise_ctx: LTXDenoiseContext, + ) -> LTXDenoiseContext: + if pipeline.do_classifier_free_guidance and not forward_ctx.cfg_parallel_ready: + denoise_ctx.video_coords = denoise_ctx.video_coords.repeat( + (2,) + (1,) * (denoise_ctx.video_coords.ndim - 1) + ) + denoise_ctx.audio_coords = denoise_ctx.audio_coords.repeat( + (2,) + (1,) * (denoise_ctx.audio_coords.ndim - 1) + ) + return denoise_ctx + + def timestep_kwargs( + self, + ts: torch.Tensor, + forward_ctx: LTXForwardContext, + denoise_ctx: LTXDenoiseContext, + ) -> dict[str, torch.Tensor]: + del forward_ctx, denoise_ctx + return {"timestep": ts, "sigma": ts} + + def combine_cfg_noise( + self, + pipeline: Any, + positive_noise_pred: tuple[torch.Tensor, torch.Tensor], + negative_noise_pred: tuple[torch.Tensor, torch.Tensor], + true_cfg_scale: float, + cfg_normalize: bool, + context: dict[str, Any], + ) -> tuple[torch.Tensor, torch.Tensor]: + required = ("video_latents", "audio_latents", "video_sigma", "audio_sigma") + if any(context.get(name) is None for name in required): + raise ValueError("LTX x0-space CFG requires video/audio latents and sigmas.") + + video_pos, audio_pos = positive_noise_pred + video_neg, audio_neg = negative_noise_pred + video_combined = combine_velocity_via_x0( + context["video_latents"], + video_pos, + video_neg, + context["video_sigma"], + true_cfg_scale, + ) + audio_combined = combine_velocity_via_x0( + context["audio_latents"], + audio_pos, + audio_neg, + context["audio_sigma"], + true_cfg_scale, + ) + if cfg_normalize: + video_combined = pipeline.cfg_normalize_function(video_pos, video_combined) + audio_combined = pipeline.cfg_normalize_function(audio_pos, audio_combined) + return video_combined, audio_combined + + def predict_parallel_cfg( + self, + pipeline: Any, + true_cfg_scale: float, + positive_kwargs: dict[str, Any], + negative_kwargs: dict[str, Any], + cfg_normalize: bool = True, + output_slice: int | None = None, + **context: Any, + ) -> tuple[torch.Tensor, torch.Tensor]: + def maybe_slice(pred: tuple[torch.Tensor, torch.Tensor]) -> tuple[torch.Tensor, torch.Tensor]: + if output_slice is None: + return pred + return pred[0][:, :output_slice], pred[1][:, :output_slice] + + cfg_world_size = get_classifier_free_guidance_world_size() + self.validate_cfg_world_size(cfg_world_size) + if cfg_world_size != 2: + raise ValueError(f"Parallel LTX x0 guidance requires cfg_parallel_size 2, but got {cfg_world_size}.") + + cfg_rank = get_classifier_free_guidance_rank() + branch_kwargs = positive_kwargs if cfg_rank == 0 else negative_kwargs + local_video_pred, local_audio_pred = maybe_slice(pipeline.predict_noise(**branch_kwargs)) + cfg_group = get_cfg_group() + gathered_video = cfg_group.all_gather(local_video_pred, separate_tensors=True) + gathered_audio = cfg_group.all_gather(local_audio_pred, separate_tensors=True) + return self.combine_cfg_noise( + pipeline, + (gathered_video[0], gathered_audio[0]), + (gathered_video[1], gathered_audio[1]), + true_cfg_scale, + cfg_normalize, + context, + ) + + def predict_noise( + self, + pipeline: Any, + index: int, + timestep: torch.Tensor, + state: LTXAVState, + forward_ctx: LTXForwardContext, + denoise_ctx: LTXDenoiseContext, + ) -> tuple[torch.Tensor, torch.Tensor]: + prompt_context = forward_ctx.prompt_context + guidance_scale = forward_ctx.request_inputs.guidance_scale + audio_scheduler = forward_ctx.audio_scheduler + if forward_ctx.cfg_parallel_ready: + video_input = state.video.to(prompt_context.positive_connector_prompt_embeds.dtype) + audio_input = state.audio.to(prompt_context.positive_connector_prompt_embeds.dtype) + ts = timestep.expand(video_input.shape[0]) + positive_kwargs = pipeline._build_transformer_kwargs( + forward_ctx, + denoise_ctx, + hidden_states=video_input, + audio_hidden_states=audio_input, + encoder_hidden_states=prompt_context.positive_connector_prompt_embeds, + audio_encoder_hidden_states=prompt_context.positive_connector_audio_prompt_embeds, + encoder_attention_mask=prompt_context.positive_connector_attention_mask, + audio_encoder_attention_mask=prompt_context.positive_connector_attention_mask, + ts=ts, + ) + negative_kwargs = { + **positive_kwargs, + "encoder_hidden_states": prompt_context.negative_connector_prompt_embeds, + "audio_encoder_hidden_states": prompt_context.negative_connector_audio_prompt_embeds, + "encoder_attention_mask": prompt_context.negative_connector_attention_mask, + "audio_encoder_attention_mask": prompt_context.negative_connector_attention_mask, + } + return self.predict_parallel_cfg( + pipeline, + true_cfg_scale=guidance_scale, + positive_kwargs=positive_kwargs, + negative_kwargs=negative_kwargs, + cfg_normalize=False, + video_latents=state.video, + audio_latents=state.audio, + video_sigma=pipeline.scheduler.sigmas[index], + audio_sigma=audio_scheduler.sigmas[index], + ) + + video_input = torch.cat([state.video] * 2) if pipeline.do_classifier_free_guidance else state.video + video_input = video_input.to(prompt_context.connector_prompt_embeds.dtype) + audio_input = torch.cat([state.audio] * 2) if pipeline.do_classifier_free_guidance else state.audio + audio_input = audio_input.to(prompt_context.connector_prompt_embeds.dtype) + ts = timestep.expand(video_input.shape[0]) + transformer_kwargs = pipeline._build_transformer_kwargs( + forward_ctx, + denoise_ctx, + hidden_states=video_input, + audio_hidden_states=audio_input, + encoder_hidden_states=prompt_context.connector_prompt_embeds, + audio_encoder_hidden_states=prompt_context.connector_audio_prompt_embeds, + encoder_attention_mask=prompt_context.connector_attention_mask, + audio_encoder_attention_mask=prompt_context.connector_attention_mask, + ts=ts, + ) + with pipeline._transformer_cache_context("cond_uncond"): + video_pred, audio_pred = pipeline.transformer(**transformer_kwargs) + video_pred = video_pred.float() + audio_pred = audio_pred.float() + + if pipeline.do_classifier_free_guidance: + video_uncond, video_cond = video_pred.chunk(2) + video_pred = combine_velocity_via_x0( + state.video, + video_cond, + video_uncond, + pipeline.scheduler.sigmas[index], + guidance_scale, + ) + audio_uncond, audio_cond = audio_pred.chunk(2) + audio_pred = combine_velocity_via_x0( + state.audio, + audio_cond, + audio_uncond, + audio_scheduler.sigmas[index], + guidance_scale, + ) + return video_pred, audio_pred + + +LTX_LEGACY_VELOCITY_GUIDANCE = LTXLegacyVelocityGuidance() +LTX_OFFICIAL_X0_GUIDANCE = LTXOfficialX0Guidance() diff --git a/vllm_omni/diffusion/models/ltx2/ltx2_latents.py b/vllm_omni/diffusion/models/ltx2/ltx2_latents.py new file mode 100644 index 000000000..4da3edf92 --- /dev/null +++ b/vllm_omni/diffusion/models/ltx2/ltx2_latents.py @@ -0,0 +1,293 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +"""Shared latent layout and normalization primitives for LTX pipelines.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + +import torch +from diffusers.utils.torch_utils import randn_tensor + + +@dataclass +class LTXAVState: + """Packed video and audio latents carried between denoise steps.""" + + video: torch.Tensor + audio: torch.Tensor + + +def pack_latents( + latents: torch.Tensor, + patch_size: int = 1, + patch_size_t: int = 1, +) -> torch.Tensor: + batch_size, _, num_frames, height, width = latents.shape + post_patch_num_frames = num_frames // patch_size_t + post_patch_height = height // patch_size + post_patch_width = width // patch_size + latents = latents.reshape( + batch_size, + -1, + post_patch_num_frames, + patch_size_t, + post_patch_height, + patch_size, + post_patch_width, + patch_size, + ) + return latents.permute(0, 2, 4, 6, 1, 3, 5, 7).flatten(4, 7).flatten(1, 3) + + +def unpack_latents( + latents: torch.Tensor, + num_frames: int, + height: int, + width: int, + patch_size: int = 1, + patch_size_t: int = 1, +) -> torch.Tensor: + batch_size = latents.size(0) + latents = latents.reshape( + batch_size, + num_frames, + height, + width, + -1, + patch_size_t, + patch_size, + patch_size, + ) + return latents.permute(0, 4, 1, 5, 2, 6, 3, 7).flatten(6, 7).flatten(4, 5).flatten(2, 3) + + +def normalize_latents( + latents: torch.Tensor, + latents_mean: torch.Tensor, + latents_std: torch.Tensor, + scaling_factor: float = 1.0, +) -> torch.Tensor: + latents_mean = latents_mean.view(1, -1, 1, 1, 1).to(latents.device, latents.dtype) + latents_std = latents_std.view(1, -1, 1, 1, 1).to(latents.device, latents.dtype) + return (latents - latents_mean) * scaling_factor / latents_std + + +def normalize_audio_latents( + latents: torch.Tensor, + latents_mean: torch.Tensor, + latents_std: torch.Tensor, +) -> torch.Tensor: + latents_mean = latents_mean.to(latents.device, latents.dtype) + latents_std = latents_std.to(latents.device, latents.dtype) + return (latents - latents_mean) / latents_std + + +def denormalize_latents( + latents: torch.Tensor, + latents_mean: torch.Tensor, + latents_std: torch.Tensor, + scaling_factor: float = 1.0, +) -> torch.Tensor: + latents_mean = latents_mean.view(1, -1, 1, 1, 1).to(latents.device, latents.dtype) + latents_std = latents_std.view(1, -1, 1, 1, 1).to(latents.device, latents.dtype) + return latents * latents_std / scaling_factor + latents_mean + + +def denormalize_audio_latents( + latents: torch.Tensor, + latents_mean: torch.Tensor, + latents_std: torch.Tensor, +) -> torch.Tensor: + latents_mean = latents_mean.to(latents.device, latents.dtype) + latents_std = latents_std.to(latents.device, latents.dtype) + return latents * latents_std + latents_mean + + +def create_noised_state( + latents: torch.Tensor, + noise_scale: float | torch.Tensor, + generator: torch.Generator | list[torch.Generator] | None = None, +) -> torch.Tensor: + noise = randn_tensor( + latents.shape, + generator=generator, + device=latents.device, + dtype=latents.dtype, + ) + return noise_scale * noise + (1 - noise_scale) * latents + + +def pack_audio_latents( + latents: torch.Tensor, + patch_size: int | None = None, + patch_size_t: int | None = None, +) -> torch.Tensor: + if patch_size is not None and patch_size_t is not None: + batch_size, _, latent_length, latent_mel_bins = latents.shape + post_patch_latent_length = latent_length / patch_size_t + post_patch_mel_bins = latent_mel_bins / patch_size + latents = latents.reshape( + batch_size, + -1, + post_patch_latent_length, + patch_size_t, + post_patch_mel_bins, + patch_size, + ) + return latents.permute(0, 2, 4, 1, 3, 5).flatten(3, 5).flatten(1, 2) + return latents.transpose(1, 2).flatten(2, 3) + + +def unpack_audio_latents( + latents: torch.Tensor, + latent_length: int, + num_mel_bins: int, + patch_size: int | None = None, + patch_size_t: int | None = None, +) -> torch.Tensor: + if patch_size is not None and patch_size_t is not None: + batch_size = latents.size(0) + latents = latents.reshape( + batch_size, + latent_length, + num_mel_bins, + -1, + patch_size_t, + patch_size, + ) + return latents.permute(0, 3, 1, 4, 2, 5).flatten(4, 5).flatten(2, 3) + return latents.unflatten(2, (-1, num_mel_bins)).transpose(1, 2) + + +def unpad_audio_latents(latents: torch.Tensor, num_frames: int) -> torch.Tensor: + return latents[:, :num_frames] + + +def get_sp_padded_audio_latent_length(audio_latent_length: int, sp_size: int) -> int: + if sp_size > 1: + audio_latent_length += (sp_size - (audio_latent_length % sp_size)) % sp_size + return audio_latent_length + + +def resolve_video_latent_shape( + height: int, + width: int, + num_frames: int, + *, + vae_spatial_compression_ratio: int, + vae_temporal_compression_ratio: int, +) -> tuple[int, int, int]: + return ( + (num_frames - 1) // vae_temporal_compression_ratio + 1, + height // vae_spatial_compression_ratio, + width // vae_spatial_compression_ratio, + ) + + +def prepare_video_latents( + pipeline: Any, + batch_size: int = 1, + num_channels_latents: int = 128, + height: int = 512, + width: int = 768, + num_frames: int = 121, + noise_scale: float = 0.0, + dtype: torch.dtype | None = None, + device: torch.device | None = None, + generator: torch.Generator | list[torch.Generator] | None = None, + latents: torch.Tensor | None = None, +) -> torch.Tensor: + if latents is not None: + if latents.ndim == 5: + latents = normalize_latents( + latents, + pipeline.vae.latents_mean, + pipeline.vae.latents_std, + pipeline.vae.config.scaling_factor, + ) + latents = pack_latents( + latents, + pipeline.transformer_spatial_patch_size, + pipeline.transformer_temporal_patch_size, + ) + if latents.ndim != 3: + raise ValueError(f"Provided `latents` has shape {latents.shape}, expected [batch, seq, features].") + return create_noised_state(latents, noise_scale, generator).to(device=device, dtype=dtype) + + num_frames, height, width = resolve_video_latent_shape( + height, + width, + num_frames, + vae_spatial_compression_ratio=pipeline.vae_spatial_compression_ratio, + vae_temporal_compression_ratio=pipeline.vae_temporal_compression_ratio, + ) + shape = (batch_size, num_channels_latents, num_frames, height, width) + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + return pack_latents( + latents, + pipeline.transformer_spatial_patch_size, + pipeline.transformer_temporal_patch_size, + ) + + +def prepare_audio_latents( + pipeline: Any, + batch_size: int = 1, + num_channels_latents: int = 8, + audio_latent_length: int = 1, + num_mel_bins: int = 64, + noise_scale: float = 0.0, + dtype: torch.dtype | None = None, + device: torch.device | None = None, + generator: torch.Generator | list[torch.Generator] | None = None, + latents: torch.Tensor | None = None, +) -> tuple[torch.Tensor, int, int]: + original_latent_length = audio_latent_length + latent_mel_bins = num_mel_bins // pipeline.audio_vae_mel_compression_ratio + sp_size = getattr(pipeline.od_config.parallel_config, "sequence_parallel_size", 1) or 1 + padded_latent_length = get_sp_padded_audio_latent_length(original_latent_length, int(sp_size)) + + if latents is not None: + if latents.ndim == 4: + latents = pack_audio_latents(latents) + if latents.ndim != 3: + raise ValueError(f"Provided `latents` has shape {latents.shape}, expected [batch, seq, features].") + latents = normalize_audio_latents( + latents, + pipeline.audio_vae.latents_mean, + pipeline.audio_vae.latents_std, + ) + latents = create_noised_state(latents, noise_scale, generator) + + if latents.shape[1] not in {original_latent_length, padded_latent_length}: + raise ValueError( + "Provided `audio_latents` has incompatible audio frame count " + f"{latents.shape[1]}; expected {original_latent_length} or {padded_latent_length}." + ) + if latents.shape[1] == original_latent_length and padded_latent_length > original_latent_length: + padding = torch.zeros( + latents.shape[0], + padded_latent_length - original_latent_length, + latents.shape[2], + dtype=latents.dtype, + device=latents.device, + ) + latents = torch.cat([latents, padding], dim=1) + return latents.to(device=device, dtype=dtype), original_latent_length, padded_latent_length + + shape = (batch_size, num_channels_latents, padded_latent_length, latent_mel_bins) + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + return pack_audio_latents(latents), original_latent_length, padded_latent_length diff --git a/vllm_omni/diffusion/models/ltx2/ltx2_pipeline_runtime.py b/vllm_omni/diffusion/models/ltx2/ltx2_pipeline_runtime.py new file mode 100644 index 000000000..08bf20330 --- /dev/null +++ b/vllm_omni/diffusion/models/ltx2/ltx2_pipeline_runtime.py @@ -0,0 +1,619 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +"""Shared runtime surface for LTX pipeline variants.""" + +from __future__ import annotations + +from collections.abc import Iterable +from contextlib import nullcontext +from typing import Any, ClassVar + +import torch +from diffusers.utils.torch_utils import randn_tensor +from torch import nn +from vllm.model_executor.models.utils import AutoWeightsLoader + +from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig +from vllm_omni.diffusion.distributed.cfg_parallel import CFGParallelMixin +from vllm_omni.diffusion.distributed.parallel_state import get_classifier_free_guidance_world_size +from vllm_omni.diffusion.models.interface import SupportsComponentDiscovery +from vllm_omni.diffusion.models.progress_bar import ProgressBarMixin +from vllm_omni.diffusion.profiler.diffusion_pipeline_profiler import DiffusionPipelineProfilerMixin +from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch, split_diffusion_output_by_request + +from . import ltx2_latents as latent_ops +from .ltx2_components import LTXComponentProfile, initialize_pipeline_components +from .ltx2_conditioning import LTXPromptContext, LTXTextConditioningMixin +from .ltx2_denoise import ( + LTXDenoiseContext, + LTXForwardContext, + LTXPhaseExecutor, + LTXPhaseResult, + VideoAudioScheduler, + build_transformer_kwargs, + step_denoised_latents, +) +from .ltx2_guidance import LTXGuidanceStrategy +from .ltx2_recipes import LTXOneStageRecipe +from .ltx2_request import LTXRequestInputs, LTXRequestMixin + + +def _expand_per_prompt_decode_value( + value: float | list[float], + *, + prompt_batch_size: int, + effective_batch_size: int, + field_name: str, +) -> list[float]: + if not isinstance(value, list): + return [value] * effective_batch_size + if len(value) == 1: + return value * effective_batch_size + if len(value) == effective_batch_size: + return value + if prompt_batch_size > 0 and len(value) == prompt_batch_size and effective_batch_size % prompt_batch_size == 0: + repeats = effective_batch_size // prompt_batch_size + return [item for item in value for _ in range(repeats)] + raise ValueError( + f"`{field_name}` must have length 1, prompt batch size ({prompt_batch_size}), or effective batch size" + f" ({effective_batch_size}); got {len(value)}." + ) + + +def _prepare_decode_timestep_conditioning( + *, + decode_timestep: float | list[float], + decode_noise_scale: float | list[float] | None, + prompt_batch_size: int, + effective_batch_size: int, + device: torch.device, + dtype: torch.dtype, +) -> tuple[torch.Tensor, torch.Tensor]: + decode_timestep_values = _expand_per_prompt_decode_value( + decode_timestep, + prompt_batch_size=prompt_batch_size, + effective_batch_size=effective_batch_size, + field_name="decode_timestep", + ) + decode_noise_scale_values = ( + decode_timestep_values + if decode_noise_scale is None + else _expand_per_prompt_decode_value( + decode_noise_scale, + prompt_batch_size=prompt_batch_size, + effective_batch_size=effective_batch_size, + field_name="decode_noise_scale", + ) + ) + return ( + torch.tensor(decode_timestep_values, device=device, dtype=dtype), + torch.tensor(decode_noise_scale_values, device=device, dtype=dtype)[:, None, None, None, None], + ) + + +class LTXPipelineRuntime( + LTXRequestMixin, + LTXTextConditioningMixin, + nn.Module, + CFGParallelMixin, + ProgressBarMixin, + SupportsComponentDiscovery, + DiffusionPipelineProfilerMixin, +): + """Shared Omni runtime for explicitly composed LTX denoise phases.""" + + component_profile: ClassVar[LTXComponentProfile] + guidance_strategy: ClassVar[LTXGuidanceStrategy] + one_stage_recipe: ClassVar[LTXOneStageRecipe] + supports_request_batch = False + connector_batches_cfg = False + distributed_video_decode = True + support_image_input = False + dummy_run_num_frames = 2 + preserve_sp_padded_audio_duration = False + reports_stage_durations = False + + def __init__(self, *, od_config: OmniDiffusionConfig, prefix: str = "") -> None: + del prefix + super().__init__() + initialize_pipeline_components(self, od_config) + self.setup_diffusion_pipeline_profiler( + enable_diffusion_pipeline_profiler=self.od_config.enable_diffusion_pipeline_profiler + ) + + def prepare_latents( + self, + batch_size: int = 1, + num_channels_latents: int = 128, + height: int = 512, + width: int = 768, + num_frames: int = 121, + noise_scale: float = 0.0, + dtype: torch.dtype | None = None, + device: torch.device | None = None, + generator: torch.Generator | list[torch.Generator] | None = None, + latents: torch.Tensor | None = None, + ) -> torch.Tensor: + return latent_ops.prepare_video_latents( + self, + batch_size, + num_channels_latents, + height, + width, + num_frames, + noise_scale, + dtype, + device, + generator, + latents, + ) + + def prepare_audio_latents( + self, + batch_size: int = 1, + num_channels_latents: int = 8, + audio_latent_length: int = 1, + num_mel_bins: int = 64, + noise_scale: float = 0.0, + dtype: torch.dtype | None = None, + device: torch.device | None = None, + generator: torch.Generator | list[torch.Generator] | None = None, + latents: torch.Tensor | None = None, + ) -> tuple[torch.Tensor, int, int]: + return latent_ops.prepare_audio_latents( + self, + batch_size, + num_channels_latents, + audio_latent_length, + num_mel_bins, + noise_scale, + dtype, + device, + generator, + latents, + ) + + @property + def guidance_scale(self): + return self._guidance_scale + + @property + def guidance_rescale(self): + return self._guidance_rescale + + @property + def do_classifier_free_guidance(self): + return self._guidance_scale is not None and self._guidance_scale > 1.0 + + @property + def interrupt(self): + return self._interrupt + + def _transformer_cache_context(self, context_name: str): + cache_context = getattr(self.transformer, "cache_context", None) + if callable(cache_context): + return cache_context(context_name) + return nullcontext() + + def predict_noise(self, **kwargs): + with self._transformer_cache_context("cond_uncond"): + noise_pred_video, noise_pred_audio = self.transformer(**kwargs) + return noise_pred_video.float(), noise_pred_audio.float() + + def combine_cfg_noise( + self, + positive_noise_pred, + negative_noise_pred, + true_cfg_scale, + cfg_normalize=False, + kwargs: dict[str, Any] | None = None, + **context: Any, + ): + if kwargs is not None: + context = {**kwargs, **context} + return self.guidance_strategy.combine_cfg_noise( + self, + positive_noise_pred, + negative_noise_pred, + true_cfg_scale, + cfg_normalize, + context, + ) + + def predict_noise_with_parallel_cfg(self, *args, **kwargs): + predict_parallel_cfg = getattr(self.guidance_strategy, "predict_parallel_cfg", None) + if predict_parallel_cfg is None: + raise NotImplementedError("The selected LTX guidance strategy does not implement parallel CFG.") + return predict_parallel_cfg(self, *args, **kwargs) + + def _synchronize_cfg_parallel_step_output( + self, + latents: tuple[torch.Tensor, torch.Tensor], + do_true_cfg: bool, + ) -> tuple[torch.Tensor, torch.Tensor]: + if not (do_true_cfg and get_classifier_free_guidance_world_size() > 1): + return latents + + # CUDA async execution otherwise permits numerical drift to accumulate + # across CFG-parallel denoise steps. + latents = tuple(tensor.contiguous() for tensor in latents) + device = next((tensor.device for tensor in latents if tensor.is_cuda), None) + if device is not None: + torch.cuda.current_stream(device).synchronize() + return latents + + def _setup_forward_runtime( + self, + request_inputs: LTXRequestInputs, + attention_kwargs: dict[str, Any] | None, + ) -> bool: + self._guidance_scale = request_inputs.guidance_scale + self._guidance_rescale = request_inputs.guidance_rescale + del attention_kwargs + self._interrupt = False + cfg_world_size = get_classifier_free_guidance_world_size() + if self.do_classifier_free_guidance: + self.guidance_strategy.validate_cfg_world_size(cfg_world_size) + return self.do_classifier_free_guidance and cfg_world_size > 1 + + def _check_forward_inputs( + self, + request_inputs: LTXRequestInputs, + image: Any | None = None, + ) -> None: + self.check_inputs( + prompt=request_inputs.prompt, + height=request_inputs.height, + width=request_inputs.width, + prompt_embeds=request_inputs.prompt_embeds, + negative_prompt_embeds=request_inputs.negative_prompt_embeds, + prompt_attention_mask=request_inputs.prompt_attention_mask, + negative_prompt_attention_mask=request_inputs.negative_prompt_attention_mask, + ) + + def _resolve_request_image( + self, + req: DiffusionRequestBatch, + image: Any | None, + request_inputs: LTXRequestInputs, + ) -> Any | None: + del req, request_inputs + return image + + def _make_output(self, output: tuple[torch.Tensor, torch.Tensor]) -> DiffusionOutput: + if self.reports_stage_durations: + return DiffusionOutput( + output=output, + stage_durations=getattr(self, "stage_durations", None), + ) + return DiffusionOutput(output=output) + + def _decode_output( + self, + *, + latents: torch.Tensor, + audio_latents: torch.Tensor, + output_type: str, + connector_prompt_embeds: torch.Tensor, + generator: torch.Generator | list[torch.Generator] | None, + device: torch.device, + decode_timestep: float | list[float], + decode_noise_scale: float | list[float] | None, + prompt_batch_size: int, + ) -> DiffusionOutput: + if output_type == "latent": + return self._make_output((latents, audio_latents)) + + latents = latents.to(connector_prompt_embeds.dtype) + if not self.vae.config.timestep_conditioning: + timestep_decode = None + else: + noise = randn_tensor(latents.shape, generator=generator, device=device, dtype=latents.dtype) + timestep_decode, decode_noise_scale_t = _prepare_decode_timestep_conditioning( + decode_timestep=decode_timestep, + decode_noise_scale=decode_noise_scale, + prompt_batch_size=prompt_batch_size, + effective_batch_size=latents.shape[0], + device=device, + dtype=latents.dtype, + ) + latents = (1 - decode_noise_scale_t) * latents + decode_noise_scale_t * noise + + dist_initialized = torch.distributed.is_initialized() + is_output_rank = not dist_initialized or torch.distributed.get_rank() == 0 + vae_decode_needs_all_ranks = False + is_distributed_vae_enabled = getattr(self.vae, "is_distributed_enabled", None) + if self.distributed_video_decode and dist_initialized and callable(is_distributed_vae_enabled): + try: + # Distributed tiled decode is collective, so every rank must enter it. + vae_decode_needs_all_ranks = bool(is_distributed_vae_enabled()) + except Exception: + pass + + should_decode_video = not self.distributed_video_decode or is_output_rank or vae_decode_needs_all_ranks + if should_decode_video: + video = self.vae.decode(latents.to(self.vae.dtype), timestep_decode, return_dict=False)[0] + else: + video = torch.empty(0, device=latents.device, dtype=latents.dtype) + + if self.distributed_video_decode and not is_output_rank: + return self._make_output( + ( + torch.empty(0, device=video.device, dtype=video.dtype), + torch.empty(0, device=audio_latents.device, dtype=audio_latents.dtype), + ) + ) + + if video.numel() > 0: + video = self.video_processor.postprocess_video(video, output_type=output_type) + generated_mel = self.audio_vae.decode(audio_latents.to(self.audio_vae.dtype), return_dict=False)[0] + audio = self.vocoder(generated_mel) + return self._make_output((video, audio)) + + def _prepare_video_latents_stage( + self, + request_inputs: LTXRequestInputs, + prompt_context: LTXPromptContext, + *, + device: torch.device, + noise_scale: float, + image: Any | None = None, + ) -> tuple[torch.Tensor, torch.Tensor | None]: + latents = self.prepare_latents( + prompt_context.batch_size * request_inputs.num_videos_per_prompt, + self.transformer.config.in_channels, + request_inputs.height, + request_inputs.width, + request_inputs.num_frames, + noise_scale, + torch.float32, + device, + request_inputs.generator, + request_inputs.latents, + ) + return latents, None + + def _resolve_video_latent_dimensions(self, request_inputs: LTXRequestInputs) -> tuple[int, int, int]: + latent_num_frames, latent_height, latent_width = latent_ops.resolve_video_latent_shape( + request_inputs.height, + request_inputs.width, + request_inputs.num_frames, + vae_spatial_compression_ratio=self.vae_spatial_compression_ratio, + vae_temporal_compression_ratio=self.vae_temporal_compression_ratio, + ) + latents = request_inputs.latents + if latents is not None: + if latents.ndim == 5: + _, _, latent_num_frames, latent_height, latent_width = latents.shape + elif latents.ndim != 3: + raise ValueError( + f"Provided `latents` tensor has shape {latents.shape}, expected a packed 3D or unpacked 5D tensor." + ) + return latent_num_frames, latent_height, latent_width + + def _prepare_audio_latents_stage( + self, + request_inputs: LTXRequestInputs, + prompt_context: LTXPromptContext, + *, + device: torch.device, + noise_scale: float, + ) -> tuple[torch.Tensor, int, int, int]: + duration_s = request_inputs.num_frames / request_inputs.frame_rate + audio_latents_per_second = ( + self.audio_sampling_rate / self.audio_hop_length / float(self.audio_vae_temporal_compression_ratio) + ) + audio_num_frames = round(duration_s * audio_latents_per_second) + audio_num_frames = self._resolve_audio_latent_length(audio_num_frames, request_inputs.audio_latents) + + num_mel_bins = self.audio_vae.config.mel_bins if self.audio_vae is not None else 64 + latent_mel_bins = num_mel_bins // self.audio_vae_mel_compression_ratio + num_channels = self.audio_vae.config.latent_channels if self.audio_vae is not None else 8 + audio_latents, original_num_frames, padded_num_frames = self.prepare_audio_latents( + prompt_context.batch_size * request_inputs.num_videos_per_prompt, + num_channels_latents=num_channels, + audio_latent_length=audio_num_frames, + num_mel_bins=num_mel_bins, + noise_scale=noise_scale, + dtype=torch.float32, + device=device, + generator=request_inputs.generator, + latents=request_inputs.audio_latents, + ) + return audio_latents, original_num_frames, padded_num_frames, latent_mel_bins + + def _resolve_audio_latent_length( + self, + requested_length: int, + audio_latents: torch.Tensor | None, + ) -> int: + if audio_latents is None or audio_latents.ndim != 4: + return requested_length + + provided_length = audio_latents.shape[2] + if not self.preserve_sp_padded_audio_duration: + return provided_length + + sp_size = getattr(self.od_config.parallel_config, "sequence_parallel_size", 1) or 1 + padded_length = latent_ops.get_sp_padded_audio_latent_length(requested_length, int(sp_size)) + return requested_length if provided_length in {requested_length, padded_length} else provided_length + + def _make_video_audio_scheduler( + self, + audio_scheduler: Any, + latent_num_frames: int, + latent_height: int, + latent_width: int, + ) -> Any: + return VideoAudioScheduler(self.scheduler, audio_scheduler) + + def _prepare_denoise_context_for_cfg( + self, + forward_ctx: LTXForwardContext, + denoise_ctx: LTXDenoiseContext, + ) -> LTXDenoiseContext: + return self.guidance_strategy.prepare_denoise_context(self, forward_ctx, denoise_ctx) + + def _denoise_timestep_kwargs( + self, + ts: torch.Tensor, + forward_ctx: LTXForwardContext, + denoise_ctx: LTXDenoiseContext, + ) -> dict[str, torch.Tensor]: + return self.guidance_strategy.timestep_kwargs(ts, forward_ctx, denoise_ctx) + + def _build_transformer_kwargs( + self, + forward_ctx: LTXForwardContext, + denoise_ctx: LTXDenoiseContext, + *, + hidden_states: torch.Tensor, + audio_hidden_states: torch.Tensor, + encoder_hidden_states: torch.Tensor, + audio_encoder_hidden_states: torch.Tensor, + encoder_attention_mask: torch.Tensor | None, + audio_encoder_attention_mask: torch.Tensor | None, + ts: torch.Tensor, + ) -> dict[str, Any]: + return build_transformer_kwargs( + self, + forward_ctx, + denoise_ctx, + hidden_states=hidden_states, + audio_hidden_states=audio_hidden_states, + encoder_hidden_states=encoder_hidden_states, + audio_encoder_hidden_states=audio_encoder_hidden_states, + encoder_attention_mask=encoder_attention_mask, + audio_encoder_attention_mask=audio_encoder_attention_mask, + ts=ts, + ) + + def _predict_noise_for_step( + self, + index: int, + timestep: torch.Tensor, + state: latent_ops.LTXAVState, + forward_ctx: LTXForwardContext, + denoise_ctx: LTXDenoiseContext, + ) -> tuple[torch.Tensor, torch.Tensor]: + return self.guidance_strategy.predict_noise( + self, + index, + timestep, + state, + forward_ctx, + denoise_ctx, + ) + + def _denoise_step( + self, + index: int, + timestep: torch.Tensor, + state: latent_ops.LTXAVState, + forward_ctx: LTXForwardContext, + denoise_ctx: LTXDenoiseContext, + ) -> latent_ops.LTXAVState: + denoise_ctx.latents = state.video + denoise_ctx.audio_latents = state.audio + noise_pred_video, noise_pred_audio = self._predict_noise_for_step( + index, + timestep, + state, + forward_ctx, + denoise_ctx, + ) + video, audio = step_denoised_latents( + self, + forward_ctx, + denoise_ctx, + noise_pred_video, + noise_pred_audio, + timestep, + ) + return latent_ops.LTXAVState(video=video, audio=audio) + + def _unpack_and_denormalize_stage( + self, + forward_ctx: LTXForwardContext, + latents: torch.Tensor, + audio_latents: torch.Tensor, + ) -> tuple[torch.Tensor, torch.Tensor]: + latents = latent_ops.unpack_latents( + latents, + forward_ctx.latent_num_frames, + forward_ctx.latent_height, + forward_ctx.latent_width, + self.transformer_spatial_patch_size, + self.transformer_temporal_patch_size, + ) + latents = latent_ops.denormalize_latents( + latents, + self.vae.latents_mean, + self.vae.latents_std, + self.vae.config.scaling_factor, + ) + + audio_latents = latent_ops.unpad_audio_latents(audio_latents, forward_ctx.original_audio_num_frames) + audio_latents = latent_ops.denormalize_audio_latents( + audio_latents, + self.audio_vae.latents_mean, + self.audio_vae.latents_std, + ) + audio_latents = latent_ops.unpack_audio_latents( + audio_latents, + forward_ctx.original_audio_num_frames, + num_mel_bins=forward_ctx.latent_mel_bins, + ) + return latents, audio_latents + + def run_phase( + self, + req: DiffusionRequestBatch, + request_inputs: LTXRequestInputs, + *, + noise_scale: float, + sigmas: list[float] | None, + timesteps: list[int] | None, + attention_kwargs: dict[str, Any] | None, + image: Any | None = None, + prompt_context: LTXPromptContext | None = None, + ) -> LTXPhaseResult: + """Prepare and execute one phase without decoding its output.""" + return LTXPhaseExecutor.run( + self, + req, + request_inputs, + noise_scale=noise_scale, + sigmas=sigmas, + timesteps=timesteps, + attention_kwargs=attention_kwargs, + image=image, + prompt_context=prompt_context, + ) + + def decode_phase(self, phase: LTXPhaseResult) -> DiffusionOutput | list[DiffusionOutput]: + """Decode one completed phase and restore per-request outputs.""" + forward_ctx = phase.forward_context + request_inputs = forward_ctx.request_inputs + output = self._decode_output( + latents=phase.video, + audio_latents=phase.audio, + output_type=request_inputs.output_type, + connector_prompt_embeds=forward_ctx.prompt_context.connector_prompt_embeds, + generator=request_inputs.generator, + device=forward_ctx.device, + decode_timestep=request_inputs.decode_timestep, + decode_noise_scale=request_inputs.decode_noise_scale, + prompt_batch_size=forward_ctx.batch_size, + ) + if not self.supports_request_batch: + return output + return split_diffusion_output_by_request( + output, + forward_ctx.req, + num_outputs_per_prompt=forward_ctx.num_videos_per_prompt, + ) + + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: + return AutoWeightsLoader(self).load_weights(weights) diff --git a/vllm_omni/diffusion/models/ltx2/ltx2_recipes.py b/vllm_omni/diffusion/models/ltx2/ltx2_recipes.py new file mode 100644 index 000000000..6491b4a5f --- /dev/null +++ b/vllm_omni/diffusion/models/ltx2/ltx2_recipes.py @@ -0,0 +1,20 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +"""User-visible one-stage defaults for the LTX model family.""" + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class LTXOneStageRecipe: + height: int = 512 + width: int = 768 + num_frames: int = 121 + frame_rate: float = 24.0 + num_inference_steps: int = 40 + guidance_scale: float = 4.0 + + +LTX2_ONE_STAGE_RECIPE = LTXOneStageRecipe() +LTX23_ONE_STAGE_RECIPE = LTXOneStageRecipe() diff --git a/vllm_omni/diffusion/models/ltx2/ltx2_request.py b/vllm_omni/diffusion/models/ltx2/ltx2_request.py new file mode 100644 index 000000000..02cab34d2 --- /dev/null +++ b/vllm_omni/diffusion/models/ltx2/ltx2_request.py @@ -0,0 +1,257 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +"""Request normalization shared by LTX pipeline variants.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + +import torch + +from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch + + +@dataclass +class LTXRequestInputs: + """Resolved request values shared by all LTX execution variants.""" + + prompt: str | list[str] | None + negative_prompt: str | list[str] | None + height: int + width: int + num_frames: int + frame_rate: float + num_inference_steps: int + guidance_scale: float + guidance_rescale: float + num_videos_per_prompt: int + generator: torch.Generator | list[torch.Generator] | None + latents: torch.Tensor | None + audio_latents: torch.Tensor | None + prompt_embeds: torch.Tensor | None + negative_prompt_embeds: torch.Tensor | None + prompt_attention_mask: torch.Tensor | None + negative_prompt_attention_mask: torch.Tensor | None + decode_timestep: float | list[float] + decode_noise_scale: float | list[float] | None + output_type: str + max_sequence_length: int + + +def _unwrap_request_tensor(value: Any) -> Any: + if isinstance(value, list): + return value[0] if value else None + return value + + +def _get_prompt_field(prompt: Any, *keys: str) -> Any: + if isinstance(prompt, str): + return None + additional = prompt.get("additional_information") + for key in keys: + value = prompt.get(key) + if value is None and isinstance(additional, dict): + value = additional.get(key) + if value is not None: + return _unwrap_request_tensor(value) + return None + + +def _get_audio_latents_from_sampling(sampling: Any) -> torch.Tensor | None: + if sampling.audio_latents is not None: + return sampling.audio_latents + return sampling.extra_args.get("audio_latents") + + +class LTXRequestMixin: + """Normalize serving requests without coupling them to a model version.""" + + supports_request_batch = False + supports_guidance_rescale = False + + def check_inputs( + self, + prompt: str | list[str] | None, + height: int, + width: int, + prompt_embeds: torch.Tensor | None = None, + negative_prompt_embeds: torch.Tensor | None = None, + prompt_attention_mask: torch.Tensor | None = None, + negative_prompt_attention_mask: torch.Tensor | None = None, + ) -> None: + if height % 32 != 0 or width % 32 != 0: + raise ValueError(f"`height` and `width` must be divisible by 32 but are {height} and {width}.") + if prompt is not None and prompt_embeds is not None: + raise ValueError("Cannot forward both `prompt` and `prompt_embeds`.") + if prompt is None and prompt_embeds is None: + raise ValueError("Provide either `prompt` or `prompt_embeds`.") + if prompt is not None and not isinstance(prompt, (str, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + if prompt_embeds is not None and prompt_attention_mask is None: + raise ValueError("Must provide `prompt_attention_mask` when specifying `prompt_embeds`.") + if negative_prompt_embeds is not None and negative_prompt_attention_mask is None: + raise ValueError("Must provide `negative_prompt_attention_mask` when specifying `negative_prompt_embeds`.") + if prompt_embeds is not None and negative_prompt_embeds is not None: + if prompt_embeds.shape != negative_prompt_embeds.shape: + raise ValueError( + "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but" + f" got {prompt_embeds.shape} and {negative_prompt_embeds.shape}." + ) + if prompt_attention_mask.shape != negative_prompt_attention_mask.shape: + raise ValueError( + "`prompt_attention_mask` and `negative_prompt_attention_mask` must have the same shape when" + f" passed directly, but got {prompt_attention_mask.shape} and" + f" {negative_prompt_attention_mask.shape}." + ) + + def _resolve_request_inputs( + self, + req: DiffusionRequestBatch, + *, + prompt: str | list[str] | None, + negative_prompt: str | list[str] | None, + height: int | None, + width: int | None, + num_frames: int | None, + frame_rate: float | None, + num_inference_steps: int | None, + timesteps: list[int] | None, + guidance_scale: float, + num_videos_per_prompt: int | None, + generator: torch.Generator | list[torch.Generator] | None, + latents: torch.Tensor | None, + audio_latents: torch.Tensor | None, + prompt_embeds: torch.Tensor | None, + negative_prompt_embeds: torch.Tensor | None, + prompt_attention_mask: torch.Tensor | None, + negative_prompt_attention_mask: torch.Tensor | None, + decode_timestep: float | list[float], + decode_noise_scale: float | list[float] | None, + output_type: str, + max_sequence_length: int | None, + guidance_rescale: float = 0.0, + ) -> LTXRequestInputs: + sampling_params_list = req.sampling_params_list + sampling = sampling_params_list[0] + prompt = [item if isinstance(item, str) else (item.get("prompt") or "") for item in req.prompts] or prompt + if all(isinstance(item, str) or item.get("negative_prompt") is None for item in req.prompts): + negative_prompt = None + elif req.prompts: + negative_prompt = [ + "" if isinstance(item, str) else (item.get("negative_prompt") or "") for item in req.prompts + ] + + height = sampling.height or height or self.one_stage_recipe.height + width = sampling.width or width or self.one_stage_recipe.width + num_frames = sampling.num_frames or num_frames or self.one_stage_recipe.num_frames + frame_rate = sampling.resolved_frame_rate or frame_rate or self.one_stage_recipe.frame_rate + num_inference_steps = ( + sampling.num_inference_steps or num_inference_steps or self.one_stage_recipe.num_inference_steps + ) + if timesteps is None: + num_inference_steps = max(int(num_inference_steps), 2) + elif len(timesteps) < 2: + raise ValueError("`timesteps` must contain at least 2 values for FlowMatchEulerDiscreteScheduler.") + + num_videos_per_prompt = ( + sampling.num_outputs_per_prompt if sampling.num_outputs_per_prompt > 0 else num_videos_per_prompt or 1 + ) + max_sequence_length = sampling.max_sequence_length or max_sequence_length or self.tokenizer_max_length + if sampling.guidance_scale_provided: + guidance_scale = sampling.guidance_scale + if self.supports_guidance_rescale and sampling.guidance_rescale is not None: + guidance_rescale = sampling.guidance_rescale + + if self.supports_request_batch: + if generator is None: + generator = req.collate_request_generators(num_videos_per_prompt, generator) + latents = req.collate_request_tensors("latents", latents) + audio_latents = DiffusionRequestBatch.collate_tensors( + [_get_audio_latents_from_sampling(item) for item in sampling_params_list], + "audio_latents", + audio_latents, + ) + prompt_fields = DiffusionRequestBatch.collate_prompt_field_map( + req.prompts, + { + "prompt_embeds": prompt_embeds, + "negative_prompt_embeds": negative_prompt_embeds, + "prompt_attention_mask": prompt_attention_mask, + "negative_prompt_attention_mask": negative_prompt_attention_mask, + }, + field_aliases={ + "prompt_attention_mask": ("prompt_attention_mask", "attention_mask"), + "negative_prompt_attention_mask": ( + "negative_prompt_attention_mask", + "negative_attention_mask", + ), + }, + ) + prompt_embeds = prompt_fields["prompt_embeds"] + negative_prompt_embeds = prompt_fields["negative_prompt_embeds"] + prompt_attention_mask = prompt_fields["prompt_attention_mask"] + negative_prompt_attention_mask = prompt_fields["negative_prompt_attention_mask"] + if prompt_embeds is not None: + prompt = None + if negative_prompt_embeds is not None: + negative_prompt = None + else: + if generator is None: + generator = sampling.generator + if generator is None and sampling.seed is not None: + generator = torch.Generator(device=self.device).manual_seed(sampling.seed) + latents = sampling.latents if sampling.latents is not None else latents + sampling_audio_latents = _get_audio_latents_from_sampling(sampling) + if sampling_audio_latents is not None: + audio_latents = sampling_audio_latents + + request_prompt_embeds = [_get_prompt_field(item, "prompt_embeds") for item in req.prompts] + if any(value is not None for value in request_prompt_embeds): + prompt_embeds = torch.stack(request_prompt_embeds) # type: ignore[arg-type] + request_negative_embeds = [_get_prompt_field(item, "negative_prompt_embeds") for item in req.prompts] + if any(value is not None for value in request_negative_embeds): + negative_prompt_embeds = torch.stack(request_negative_embeds) # type: ignore[arg-type] + request_prompt_masks = [ + _get_prompt_field(item, "prompt_attention_mask", "attention_mask") for item in req.prompts + ] + if any(value is not None for value in request_prompt_masks): + prompt_attention_mask = torch.stack(request_prompt_masks) # type: ignore[arg-type] + request_negative_masks = [ + _get_prompt_field(item, "negative_prompt_attention_mask", "negative_attention_mask") + for item in req.prompts + ] + if any(value is not None for value in request_negative_masks): + negative_prompt_attention_mask = torch.stack(request_negative_masks) # type: ignore[arg-type] + + if sampling.decode_timestep is not None: + decode_timestep = sampling.decode_timestep + if sampling.decode_noise_scale is not None: + decode_noise_scale = sampling.decode_noise_scale + if sampling.output_type is not None: + output_type = sampling.output_type + + return LTXRequestInputs( + prompt=prompt, + negative_prompt=negative_prompt, + height=int(height), + width=int(width), + num_frames=int(num_frames), + frame_rate=float(frame_rate), + num_inference_steps=int(num_inference_steps), + guidance_scale=guidance_scale, + guidance_rescale=guidance_rescale, + num_videos_per_prompt=int(num_videos_per_prompt), + generator=generator, + latents=latents, + audio_latents=audio_latents, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + prompt_attention_mask=prompt_attention_mask, + negative_prompt_attention_mask=negative_prompt_attention_mask, + decode_timestep=decode_timestep, + decode_noise_scale=decode_noise_scale, + output_type=output_type, + max_sequence_length=int(max_sequence_length), + ) diff --git a/vllm_omni/diffusion/models/ltx2/pipeline_ltx2.py b/vllm_omni/diffusion/models/ltx2/pipeline_ltx2.py index bce0813e9..c32a6918a 100644 --- a/vllm_omni/diffusion/models/ltx2/pipeline_ltx2.py +++ b/vllm_omni/diffusion/models/ltx2/pipeline_ltx2.py @@ -1,749 +1,66 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""One-stage entry points for the LTX model family.""" + from __future__ import annotations -import copy -import inspect -import json -import os -from collections.abc import Iterable -from contextlib import nullcontext -from typing import TYPE_CHECKING, Any, ClassVar +from typing import Any, ClassVar -import numpy as np import torch -if TYPE_CHECKING: - from vllm.model_executor.layers.quantization.base_config import QuantizationConfig -from diffusers import AutoencoderKLLTX2Audio, AutoencoderKLLTX2Video, FlowMatchEulerDiscreteScheduler -from diffusers.pipelines.ltx2 import LTX2TextConnectors -from diffusers.pipelines.ltx2.utils import DISTILLED_SIGMA_VALUES, STAGE_2_DISTILLED_SIGMA_VALUES -from diffusers.pipelines.ltx2.vocoder import LTX2Vocoder -from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion import rescale_noise_cfg, retrieve_timesteps -from diffusers.utils.torch_utils import randn_tensor -from diffusers.video_processor import VideoProcessor -from torch import nn -from transformers import AutoTokenizer, Gemma3ForConditionalGeneration -from vllm.logger import init_logger -from vllm.model_executor.models.utils import AutoWeightsLoader - from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig -from vllm_omni.diffusion.distributed.cfg_parallel import CFGParallelMixin -from vllm_omni.diffusion.distributed.parallel_state import ( - get_classifier_free_guidance_world_size, -) -from vllm_omni.diffusion.distributed.utils import get_local_device -from vllm_omni.diffusion.lora.manager import DiffusionLoRAManager -from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader -from vllm_omni.diffusion.model_loader.hub_prefetch import from_pretrained_with_prefetch, prefetch_subfolders from vllm_omni.diffusion.models.dmd2 import DMD2PipelineMixin -from vllm_omni.diffusion.models.interface import SupportsComponentDiscovery -from vllm_omni.diffusion.models.progress_bar import ProgressBarMixin from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch -from vllm_omni.lora.request import LoRARequest -from .ltx2_transformer import LTX2VideoTransformer3DModel -from .pipeline_ltx2_latent_upsample import LTX2LatentUpsamplePipeline - -logger = init_logger(__name__) +from .ltx2_components import ( + LTX2_COMPONENT_PROFILE, + LTX23_COMPONENT_PROFILE, + LTXComponentProfile, +) +from .ltx2_components import ( + get_ltx2_post_process_func as get_ltx2_post_process_func, # noqa: F401 +) +from .ltx2_conditioning import LTXI2VConditioningMixin +from .ltx2_guidance import ( + LTX_LEGACY_VELOCITY_GUIDANCE, + LTX_OFFICIAL_X0_GUIDANCE, +) +from .ltx2_pipeline_runtime import LTXPipelineRuntime +from .ltx2_recipes import LTX2_ONE_STAGE_RECIPE, LTX23_ONE_STAGE_RECIPE, LTXOneStageRecipe +from .ltx2_request import LTXRequestInputs -def load_transformer_config(model_path: str, subfolder: str = "transformer", local_files_only: bool = True) -> dict: - """Load transformer config from model directory or HF Hub.""" - if local_files_only: - config_path = os.path.join(model_path, subfolder, "config.json") - if os.path.exists(config_path): - with open(config_path) as f: - return json.load(f) - else: - try: - from huggingface_hub import hf_hub_download +class LTXOneStagePipeline(LTXPipelineRuntime): + """Single execution path configured by model-version and task entries.""" - config_path = hf_hub_download( - repo_id=model_path, - filename=f"{subfolder}/config.json", - ) - with open(config_path) as f: - return json.load(f) - except Exception: - pass - return {} + component_profile: ClassVar[LTXComponentProfile] + one_stage_recipe: ClassVar[LTXOneStageRecipe] + supports_request_batch = True + supports_guidance_rescale = False -def create_transformer_from_config( - config: dict, - quant_config: QuantizationConfig | None = None, -) -> LTX2VideoTransformer3DModel: - """Create LTX2VideoTransformer3DModel from config dict.""" - if not config and quant_config is None: - return LTX2VideoTransformer3DModel() - - signature = inspect.signature(LTX2VideoTransformer3DModel.__init__) - allowed_keys = set(signature.parameters.keys()) - kwargs = {k: v for k, v in config.items() if k in allowed_keys} - if quant_config is not None: - kwargs["quant_config"] = quant_config - - return LTX2VideoTransformer3DModel(**kwargs) - - -def get_ltx2_post_process_func( - od_config: OmniDiffusionConfig, -): - def post_process_func(output: tuple[torch.Tensor, torch.Tensor] | torch.Tensor): - if isinstance(output, tuple) and len(output) == 2: - video, audio = output - if isinstance(audio, torch.Tensor): - audio = audio.detach().cpu() - return {"video": video, "audio": audio} - return output - - return post_process_func - - -def _unwrap_request_tensor(value: Any) -> Any: - if isinstance(value, list): - return value[0] if value else None - return value - - -def _get_prompt_field(prompt: Any, key: str) -> Any: - if isinstance(prompt, str): - return None - value = prompt.get(key) - if value is None: - additional = prompt.get("additional_information") - if isinstance(additional, dict): - value = additional.get(key) - return _unwrap_request_tensor(value) - - -def calculate_shift( - image_seq_len, - base_seq_len: int = 256, - max_seq_len: int = 4096, - base_shift: float = 0.5, - max_shift: float = 1.15, -): - m = (max_shift - base_shift) / (max_seq_len - base_seq_len) - b = base_shift - m * base_seq_len - mu = image_seq_len * m + b - return mu - - -class _VideoAudioScheduler: - """Composite scheduler dispatching to video and audio schedulers.""" - - def __init__(self, video_scheduler, audio_scheduler): - self.video_scheduler = video_scheduler - self.audio_scheduler = audio_scheduler - - def step(self, noise_pred, t, latents, return_dict=False, generator=None): - video_out = self.video_scheduler.step( - noise_pred[0], - t[0], - latents[0], - return_dict=False, - generator=generator, - )[0] - audio_out = self.audio_scheduler.step( - noise_pred[1], - t[1], - latents[1], - return_dict=False, - generator=generator, - )[0] - return ((video_out, audio_out),) - - -class LTX2Pipeline(nn.Module, CFGParallelMixin, ProgressBarMixin, SupportsComponentDiscovery): - supports_request_batch = False - - _dit_modules: ClassVar[list[str]] = ["transformer"] - _encoder_modules: ClassVar[list[str]] = ["text_encoder"] - _vae_modules: ClassVar[list[str]] = ["vae", "audio_vae"] - - # Audio is diffused jointly with video; warmup must size audio tokens. - dummy_run_num_frames = 2 - - def __init__( + def _forward_impl( self, + req: DiffusionRequestBatch, + request_inputs: LTXRequestInputs, *, - od_config: OmniDiffusionConfig, - prefix: str = "", - ): - super().__init__() - self.od_config = od_config - - self.device = get_local_device() - dtype = getattr(od_config, "dtype", torch.bfloat16) - - model = od_config.model - local_files_only = os.path.exists(model) - - self.weights_sources = [ - DiffusersPipelineLoader.ComponentSource( - model_or_path=od_config.model, - subfolder="transformer", - revision=None, - prefix="transformer.", - fall_back_to_pt=True, - ), - ] - - # See ``hub_prefetch.py`` for the transformers v5 multi-worker subfolder - # race; prefetch the whole component set before any from_pretrained. - ltx2_subfolders = [ - "tokenizer", - "text_encoder", - "connectors", - "vae", - "audio_vae", - "vocoder", - "scheduler", - ] - prefetch_subfolders(model, ltx2_subfolders, local_files_only=local_files_only) - - self.tokenizer = AutoTokenizer.from_pretrained( - model, - subfolder="tokenizer", - local_files_only=local_files_only, + noise_scale: float, + sigmas: list[float] | None, + timesteps: list[int] | None, + attention_kwargs: dict[str, Any] | None, + image: Any | None = None, + ) -> DiffusionOutput | list[DiffusionOutput]: + phase = self.run_phase( + req, + request_inputs, + noise_scale=noise_scale, + sigmas=sigmas, + timesteps=timesteps, + attention_kwargs=attention_kwargs, + image=image, ) - # prefer mmap loading as default device is cuda, and the output of text encoder - # could be deterministic. - with torch.device("cpu"): - self.text_encoder = from_pretrained_with_prefetch( - Gemma3ForConditionalGeneration.from_pretrained, - model, - subfolder="text_encoder", - prefetch_list=ltx2_subfolders, - local_files_only=local_files_only, - torch_dtype=dtype, - ).to(self.device) - self.connectors = from_pretrained_with_prefetch( - LTX2TextConnectors.from_pretrained, - model, - subfolder="connectors", - prefetch_list=ltx2_subfolders, - local_files_only=local_files_only, - torch_dtype=dtype, - ).to(self.device) - - self.vae = from_pretrained_with_prefetch( - AutoencoderKLLTX2Video.from_pretrained, - model, - subfolder="vae", - prefetch_list=ltx2_subfolders, - local_files_only=local_files_only, - torch_dtype=dtype, - ).to(self.device) - self.audio_vae = from_pretrained_with_prefetch( - AutoencoderKLLTX2Audio.from_pretrained, - model, - subfolder="audio_vae", - prefetch_list=ltx2_subfolders, - local_files_only=local_files_only, - torch_dtype=dtype, - ).to(self.device) - self.vocoder = from_pretrained_with_prefetch( - LTX2Vocoder.from_pretrained, - model, - subfolder="vocoder", - prefetch_list=ltx2_subfolders, - local_files_only=local_files_only, - torch_dtype=dtype, - ).to(self.device) - - transformer_config = load_transformer_config(model, "transformer", local_files_only) - quant_config = getattr(self.od_config, "quantization_config", None) - self.transformer = create_transformer_from_config(transformer_config, quant_config=quant_config) - - self.scheduler = FlowMatchEulerDiscreteScheduler.from_pretrained( - model, subfolder="scheduler", local_files_only=local_files_only - ) - - self.vae_spatial_compression_ratio = ( - self.vae.spatial_compression_ratio if getattr(self, "vae", None) is not None else 32 - ) - self.vae_temporal_compression_ratio = ( - self.vae.temporal_compression_ratio if getattr(self, "vae", None) is not None else 8 - ) - self.audio_vae_mel_compression_ratio = ( - self.audio_vae.mel_compression_ratio if getattr(self, "audio_vae", None) is not None else 4 - ) - self.audio_vae_temporal_compression_ratio = ( - self.audio_vae.temporal_compression_ratio if getattr(self, "audio_vae", None) is not None else 4 - ) - self.transformer_spatial_patch_size = ( - self.transformer.config.patch_size if getattr(self, "transformer", None) is not None else 1 - ) - self.transformer_temporal_patch_size = ( - self.transformer.config.patch_size_t if getattr(self, "transformer", None) is not None else 1 - ) - - self.audio_sampling_rate = ( - self.audio_vae.config.sample_rate if getattr(self, "audio_vae", None) is not None else 16000 - ) - self.audio_hop_length = ( - self.audio_vae.config.mel_hop_length if getattr(self, "audio_vae", None) is not None else 160 - ) - - self.video_processor = VideoProcessor(vae_scale_factor=self.vae_spatial_compression_ratio) - tokenizer_max_length = 1024 - if getattr(self, "tokenizer", None) is not None: - tokenizer_max_length = self.tokenizer.model_max_length - if tokenizer_max_length is None or tokenizer_max_length > 100000: - encoder_config = getattr(self.text_encoder, "config", None) - config_max_len = getattr(encoder_config, "max_position_embeddings", None) - if config_max_len is None: - config_max_len = getattr(encoder_config, "max_seq_len", None) - tokenizer_max_length = config_max_len or 1024 - self.tokenizer_max_length = int(tokenizer_max_length) - - self._guidance_scale = None - self._guidance_rescale = None - self._attention_kwargs = None - self._interrupt = False - self._num_timesteps = None - self._current_timestep = None - - def _get_gemma_prompt_embeds( - self, - prompt: str | list[str], - num_videos_per_prompt: int = 1, - max_sequence_length: int = 1024, - scale_factor: int = 8, - device: torch.device | None = None, - dtype: torch.dtype | None = None, - ): - device = device or self.device - dtype = dtype or self.text_encoder.dtype - - prompt = [prompt] if isinstance(prompt, str) else prompt - batch_size = len(prompt) - - if getattr(self, "tokenizer", None) is not None: - self.tokenizer.padding_side = "left" - if self.tokenizer.pad_token is None: - self.tokenizer.pad_token = self.tokenizer.eos_token - - prompt = [p.strip() for p in prompt] - text_inputs = self.tokenizer( - prompt, - padding="max_length", - max_length=max_sequence_length, - truncation=True, - add_special_tokens=True, - return_tensors="pt", - ) - text_input_ids = text_inputs.input_ids - prompt_attention_mask = text_inputs.attention_mask - text_input_ids = text_input_ids.to(device) - prompt_attention_mask = prompt_attention_mask.to(device) - - text_encoder_outputs = self.text_encoder( - input_ids=text_input_ids, attention_mask=prompt_attention_mask, output_hidden_states=True - ) - text_encoder_hidden_states = text_encoder_outputs.hidden_states - text_encoder_hidden_states = torch.stack(text_encoder_hidden_states, dim=-1) - # `diffusers>=0.38` moved per_layer_masked_mean_norm inside the connector, - # so pack to 3D here and let the connector handle normalization. - prompt_embeds = text_encoder_hidden_states.flatten(2, 3).to(dtype=dtype) - - _, seq_len, _ = prompt_embeds.shape - prompt_embeds = prompt_embeds.repeat(1, num_videos_per_prompt, 1) - prompt_embeds = prompt_embeds.view(batch_size * num_videos_per_prompt, seq_len, -1) - - prompt_attention_mask = prompt_attention_mask.view(batch_size, -1) - prompt_attention_mask = prompt_attention_mask.repeat(num_videos_per_prompt, 1) - - return prompt_embeds, prompt_attention_mask - - def encode_prompt( - self, - prompt: str | list[str], - negative_prompt: str | list[str] | None = None, - do_classifier_free_guidance: bool = True, - num_videos_per_prompt: int = 1, - prompt_embeds: torch.Tensor | None = None, - negative_prompt_embeds: torch.Tensor | None = None, - prompt_attention_mask: torch.Tensor | None = None, - negative_prompt_attention_mask: torch.Tensor | None = None, - max_sequence_length: int = 1024, - scale_factor: int = 8, - device: torch.device | None = None, - dtype: torch.dtype | None = None, - ): - device = device or self.device - - prompt = [prompt] if isinstance(prompt, str) else prompt - if prompt is not None: - batch_size = len(prompt) - else: - batch_size = prompt_embeds.shape[0] - - if prompt_embeds is None: - prompt_embeds, prompt_attention_mask = self._get_gemma_prompt_embeds( - prompt=prompt, - num_videos_per_prompt=num_videos_per_prompt, - max_sequence_length=max_sequence_length, - scale_factor=scale_factor, - device=device, - dtype=dtype, - ) - - if do_classifier_free_guidance and negative_prompt_embeds is None: - negative_prompt = negative_prompt or "" - negative_prompt = batch_size * [negative_prompt] if isinstance(negative_prompt, str) else negative_prompt - - if prompt is not None and type(prompt) is not type(negative_prompt): - raise TypeError( - f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" - f" {type(prompt)}." - ) - elif batch_size != len(negative_prompt): - raise ValueError( - f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" - f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" - " the batch size of `prompt`." - ) - - negative_prompt_embeds, negative_prompt_attention_mask = self._get_gemma_prompt_embeds( - prompt=negative_prompt, - num_videos_per_prompt=num_videos_per_prompt, - max_sequence_length=max_sequence_length, - scale_factor=scale_factor, - device=device, - dtype=dtype, - ) - - return prompt_embeds, prompt_attention_mask, negative_prompt_embeds, negative_prompt_attention_mask - - def check_inputs( - self, - prompt, - height, - width, - prompt_embeds=None, - negative_prompt_embeds=None, - prompt_attention_mask=None, - negative_prompt_attention_mask=None, - ): - if height % 32 != 0 or width % 32 != 0: - raise ValueError(f"`height` and `width` have to be divisible by 32 but are {height} and {width}.") - - if prompt is not None and prompt_embeds is not None: - raise ValueError( - f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" - " only forward one of the two." - ) - elif prompt is None and prompt_embeds is None: - raise ValueError( - "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." - ) - elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): - raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") - - if prompt_embeds is not None and prompt_attention_mask is None: - raise ValueError("Must provide `prompt_attention_mask` when specifying `prompt_embeds`.") - - if negative_prompt_embeds is not None and negative_prompt_attention_mask is None: - raise ValueError("Must provide `negative_prompt_attention_mask` when specifying `negative_prompt_embeds`.") - - if prompt_embeds is not None and negative_prompt_embeds is not None: - if prompt_embeds.shape != negative_prompt_embeds.shape: - raise ValueError( - "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but" - f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`" - f" {negative_prompt_embeds.shape}." - ) - if prompt_attention_mask.shape != negative_prompt_attention_mask.shape: - raise ValueError( - "`prompt_attention_mask` and `negative_prompt_attention_mask` must have the same shape when " - "passed directly, but got: `prompt_attention_mask` " - f"{prompt_attention_mask.shape} != `negative_prompt_attention_mask` " - f"{negative_prompt_attention_mask.shape}." - ) - - @staticmethod - def _pack_latents(latents: torch.Tensor, patch_size: int = 1, patch_size_t: int = 1) -> torch.Tensor: - batch_size, num_channels, num_frames, height, width = latents.shape - post_patch_num_frames = num_frames // patch_size_t - post_patch_height = height // patch_size - post_patch_width = width // patch_size - latents = latents.reshape( - batch_size, - -1, - post_patch_num_frames, - patch_size_t, - post_patch_height, - patch_size, - post_patch_width, - patch_size, - ) - latents = latents.permute(0, 2, 4, 6, 1, 3, 5, 7).flatten(4, 7).flatten(1, 3) - return latents - - @staticmethod - def _unpack_latents( - latents: torch.Tensor, num_frames: int, height: int, width: int, patch_size: int = 1, patch_size_t: int = 1 - ) -> torch.Tensor: - batch_size = latents.size(0) - latents = latents.reshape(batch_size, num_frames, height, width, -1, patch_size_t, patch_size, patch_size) - latents = latents.permute(0, 4, 1, 5, 2, 6, 3, 7).flatten(6, 7).flatten(4, 5).flatten(2, 3) - return latents - - @staticmethod - def _normalize_latents( - latents: torch.Tensor, latents_mean: torch.Tensor, latents_std: torch.Tensor, scaling_factor: float = 1.0 - ) -> torch.Tensor: - # Normalize latents across the channel dimension [B, C, F, H, W] - latents_mean = latents_mean.view(1, -1, 1, 1, 1).to(latents.device, latents.dtype) - latents_std = latents_std.view(1, -1, 1, 1, 1).to(latents.device, latents.dtype) - latents = (latents - latents_mean) * scaling_factor / latents_std - return latents - - @staticmethod - def _normalize_audio_latents(latents: torch.Tensor, latents_mean: torch.Tensor, latents_std: torch.Tensor): - latents_mean = latents_mean.to(latents.device, latents.dtype) - latents_std = latents_std.to(latents.device, latents.dtype) - return (latents - latents_mean) / latents_std - - @staticmethod - def _denormalize_latents( - latents: torch.Tensor, latents_mean: torch.Tensor, latents_std: torch.Tensor, scaling_factor: float = 1.0 - ) -> torch.Tensor: - latents_mean = latents_mean.view(1, -1, 1, 1, 1).to(latents.device, latents.dtype) - latents_std = latents_std.view(1, -1, 1, 1, 1).to(latents.device, latents.dtype) - latents = latents * latents_std / scaling_factor + latents_mean - return latents - - @staticmethod - def _denormalize_audio_latents(latents: torch.Tensor, latents_mean: torch.Tensor, latents_std: torch.Tensor): - latents_mean = latents_mean.to(latents.device, latents.dtype) - latents_std = latents_std.to(latents.device, latents.dtype) - return (latents * latents_std) + latents_mean - - @staticmethod - def _create_noised_state( - latents: torch.Tensor, noise_scale: float | torch.Tensor, generator: torch.Generator | None = None - ): - noise = randn_tensor(latents.shape, generator=generator, device=latents.device, dtype=latents.dtype) - noised_latents = noise_scale * noise + (1 - noise_scale) * latents - return noised_latents - - @staticmethod - def _pack_audio_latents( - latents: torch.Tensor, patch_size: int | None = None, patch_size_t: int | None = None - ) -> torch.Tensor: - if patch_size is not None and patch_size_t is not None: - batch_size, num_channels, latent_length, latent_mel_bins = latents.shape - post_patch_latent_length = latent_length / patch_size_t - post_patch_mel_bins = latent_mel_bins / patch_size - latents = latents.reshape( - batch_size, -1, post_patch_latent_length, patch_size_t, post_patch_mel_bins, patch_size - ) - latents = latents.permute(0, 2, 4, 1, 3, 5).flatten(3, 5).flatten(1, 2) - else: - latents = latents.transpose(1, 2).flatten(2, 3) - return latents - - @staticmethod - def _unpack_audio_latents( - latents: torch.Tensor, - latent_length: int, - num_mel_bins: int, - patch_size: int | None = None, - patch_size_t: int | None = None, - ) -> torch.Tensor: - if patch_size is not None and patch_size_t is not None: - batch_size = latents.size(0) - latents = latents.reshape(batch_size, latent_length, num_mel_bins, -1, patch_size_t, patch_size) - latents = latents.permute(0, 3, 1, 4, 2, 5).flatten(4, 5).flatten(2, 3) - else: - latents = latents.unflatten(2, (-1, num_mel_bins)).transpose(1, 2) - return latents - - @staticmethod - def _unpad_audio_latents(latents: torch.Tensor, num_frames: int) -> torch.Tensor: - return latents[:, :num_frames] - - def prepare_latents( - self, - batch_size: int = 1, - num_channels_latents: int = 128, - height: int = 512, - width: int = 768, - num_frames: int = 121, - noise_scale: float = 0.0, - dtype: torch.dtype | None = None, - device: torch.device | None = None, - generator: torch.Generator | None = None, - latents: torch.Tensor | None = None, - ) -> torch.Tensor: - if latents is not None: - if latents.ndim == 5: - latents = self._normalize_latents( - latents, self.vae.latents_mean, self.vae.latents_std, self.vae.config.scaling_factor - ) - # latents are of shape [B, C, F, H, W], need to be packed - latents = self._pack_latents( - latents, self.transformer_spatial_patch_size, self.transformer_temporal_patch_size - ) - if latents.ndim != 3: - raise ValueError( - f"Provided `latents` tensor has shape {latents.shape}, but the expected shape is [batch_size, num_seq, num_features]." # noqa - ) - latents = self._create_noised_state(latents, noise_scale, generator) - return latents.to(device=device, dtype=dtype) - - height = height // self.vae_spatial_compression_ratio - width = width // self.vae_spatial_compression_ratio - num_frames = (num_frames - 1) // self.vae_temporal_compression_ratio + 1 - - shape = (batch_size, num_channels_latents, num_frames, height, width) - - if isinstance(generator, list) and len(generator) != batch_size: - raise ValueError( - f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" - f" size of {batch_size}. Make sure the batch size matches the length of the generators." - ) - - latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) - latents = self._pack_latents(latents, self.transformer_spatial_patch_size, self.transformer_temporal_patch_size) - - return latents - - def prepare_audio_latents( - self, - batch_size: int = 1, - num_channels_latents: int = 8, - audio_latent_length: int = 1, # 1 is just a dummy value - num_mel_bins: int = 64, - noise_scale: float = 0.0, - dtype: torch.dtype | None = None, - device: torch.device | None = None, - generator: torch.Generator | list[torch.Generator] | None = None, - latents: torch.Tensor | None = None, - ) -> tuple[torch.Tensor, int, int]: - original_latent_length = audio_latent_length - padded_latent_length = original_latent_length - - latent_mel_bins = num_mel_bins // self.audio_vae_mel_compression_ratio - - sp_size = getattr(self.od_config.parallel_config, "sequence_parallel_size", 1) - if sp_size > 1: - padded_latent_length += (sp_size - (original_latent_length % sp_size)) % sp_size - - if latents is not None: - if latents.ndim == 4: - # latents are of shape [B, C, L, M], need to be packed - latents = self._pack_audio_latents(latents) - if latents.ndim != 3: - raise ValueError( - f"Provided `latents` tensor has shape {latents.shape}, but the expected shape is " - "[batch_size, num_seq, num_features] or [batch_size, num_channels, audio_length, mel_bins]." - ) - latents = self._normalize_audio_latents(latents, self.audio_vae.latents_mean, self.audio_vae.latents_std) - latents = self._create_noised_state(latents, noise_scale, generator) - - if latents.shape[1] not in {original_latent_length, padded_latent_length}: - raise ValueError( - "Provided `audio_latents` has incompatible audio frame count " - f"{latents.shape[1]}; expected {original_latent_length} or {padded_latent_length}." - ) - - if latents.shape[1] == original_latent_length and padded_latent_length > original_latent_length: - padding = torch.zeros( - latents.shape[0], - padded_latent_length - original_latent_length, - latents.shape[2], - dtype=latents.dtype, - device=latents.device, - ) - latents = torch.cat([latents, padding], dim=1) - - return latents.to(device=device, dtype=dtype), original_latent_length, padded_latent_length - - shape = (batch_size, num_channels_latents, padded_latent_length, latent_mel_bins) - - if isinstance(generator, list) and len(generator) != batch_size: - raise ValueError( - f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" - f" size of {batch_size}. Make sure the batch size matches the length of the generators." - ) - - latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) - latents = self._pack_audio_latents(latents) - return latents, original_latent_length, padded_latent_length - - @property - def guidance_scale(self): - return self._guidance_scale - - @property - def guidance_rescale(self): - return self._guidance_rescale - - @property - def do_classifier_free_guidance(self): - return self._guidance_scale > 1.0 - - @property - def num_timesteps(self): - return self._num_timesteps - - @property - def current_timestep(self): - return self._current_timestep - - @property - def attention_kwargs(self): - return self._attention_kwargs - - @property - def interrupt(self): - return self._interrupt - - def _transformer_cache_context(self, context_name: str): - cache_context = getattr(self.transformer, "cache_context", None) - if callable(cache_context): - return cache_context(context_name) - return nullcontext() - - def predict_noise(self, **kwargs): - with self._transformer_cache_context("cond_uncond"): - noise_pred_video, noise_pred_audio = self.transformer(**kwargs) - return noise_pred_video.float(), noise_pred_audio.float() - - def combine_cfg_noise(self, positive_noise_pred, negative_noise_pred, true_cfg_scale, cfg_normalize=False): - """Per-element CFG combine with guidance_rescale support.""" - (video_pos, audio_pos) = positive_noise_pred - (video_neg, audio_neg) = negative_noise_pred - video_combined = super().combine_cfg_noise(video_pos, video_neg, true_cfg_scale, cfg_normalize) - audio_combined = super().combine_cfg_noise(audio_pos, audio_neg, true_cfg_scale, cfg_normalize) - if self._guidance_rescale and self._guidance_rescale > 0: - video_combined = rescale_noise_cfg(video_combined, video_pos, guidance_rescale=self._guidance_rescale) - audio_combined = rescale_noise_cfg(audio_combined, audio_pos, guidance_rescale=self._guidance_rescale) - return (video_combined, audio_combined) - - def _synchronize_cfg_parallel_step_output( - self, - latents: tuple[torch.Tensor, torch.Tensor], - do_true_cfg: bool, - ) -> tuple[torch.Tensor, torch.Tensor]: - if not (do_true_cfg and get_classifier_free_guidance_world_size() > 1): - return latents - - # Without this sync, CUDA async execution causes non-deterministic - # numerical drift across denoising steps in CFG parallel mode, - # producing different video outputs across runs. - latents = tuple(tensor.contiguous() for tensor in latents) - device = next((tensor.device for tensor in latents if tensor.is_cuda), None) - if device is not None: - torch.cuda.current_stream(device).synchronize() - return latents + return self.decode_phase(phase) @torch.no_grad() def forward( @@ -775,428 +92,79 @@ class LTX2Pipeline(nn.Module, CFGParallelMixin, ProgressBarMixin, SupportsCompon return_dict: bool = True, attention_kwargs: dict[str, Any] | None = None, max_sequence_length: int | None = None, - ) -> DiffusionOutput: - # Extract prompt/negative_prompt from request. - # Input format: req.prompts is a list of str or dict with "prompt"/"negative_prompt" keys. - prompt = [p if isinstance(p, str) else (p.get("prompt") or "") for p in req.prompts] or prompt - if all(isinstance(p, str) or p.get("negative_prompt") is None for p in req.prompts): - negative_prompt = None - elif req.prompts: - negative_prompt = ["" if isinstance(p, str) else (p.get("negative_prompt") or "") for p in req.prompts] - - height = req.sampling_params.height or height or 512 - width = req.sampling_params.width or width or 768 - num_frames = req.sampling_params.num_frames or num_frames or 121 - frame_rate = req.sampling_params.resolved_frame_rate or frame_rate or 24.0 - num_inference_steps = req.sampling_params.num_inference_steps or num_inference_steps or 40 - if timesteps is None: - num_inference_steps = max(int(num_inference_steps), 2) - elif len(timesteps) < 2: - raise ValueError("`timesteps` must contain at least 2 values for FlowMatchEulerDiscreteScheduler.") - num_videos_per_prompt = ( - req.sampling_params.num_outputs_per_prompt - if req.sampling_params.num_outputs_per_prompt > 0 - else num_videos_per_prompt or 1 - ) - max_sequence_length = ( - req.sampling_params.max_sequence_length or max_sequence_length or self.tokenizer_max_length - ) - - if req.sampling_params.guidance_scale_provided: - guidance_scale = req.sampling_params.guidance_scale - if req.sampling_params.guidance_rescale is not None: - guidance_rescale = req.sampling_params.guidance_rescale - - if generator is None: - generator = req.sampling_params.generator - if generator is None and req.sampling_params.seed is not None: - generator = torch.Generator(device=self.device).manual_seed(req.sampling_params.seed) - - latents = req.sampling_params.latents if req.sampling_params.latents is not None else latents - audio_latents = ( - req.sampling_params.audio_latents - if req.sampling_params.audio_latents is not None - else req.sampling_params.extra_args.get("audio_latents", audio_latents) - ) - - # Override with pre-computed embeddings if provided in request. - req_prompt_embeds = [_get_prompt_field(p, "prompt_embeds") for p in req.prompts] - if any(p is not None for p in req_prompt_embeds): - prompt_embeds = torch.stack(req_prompt_embeds) # type: ignore[arg-type] - - req_negative_prompt_embeds = [_get_prompt_field(p, "negative_prompt_embeds") for p in req.prompts] - if any(p is not None for p in req_negative_prompt_embeds): - negative_prompt_embeds = torch.stack(req_negative_prompt_embeds) # type: ignore[arg-type] - - req_prompt_attention_masks = [ - _get_prompt_field(p, "prompt_attention_mask") or _get_prompt_field(p, "attention_mask") for p in req.prompts - ] - if any(m is not None for m in req_prompt_attention_masks): - prompt_attention_mask = torch.stack(req_prompt_attention_masks) # type: ignore[arg-type] - - req_negative_attention_masks = [ - _get_prompt_field(p, "negative_prompt_attention_mask") or _get_prompt_field(p, "negative_attention_mask") - for p in req.prompts - ] - if any(m is not None for m in req_negative_attention_masks): - negative_prompt_attention_mask = torch.stack(req_negative_attention_masks) # type: ignore[arg-type] - - if req.sampling_params.decode_timestep is not None: - decode_timestep = req.sampling_params.decode_timestep - if req.sampling_params.decode_noise_scale is not None: - decode_noise_scale = req.sampling_params.decode_noise_scale - if req.sampling_params.output_type is not None: - output_type = req.sampling_params.output_type - - self.check_inputs( - prompt=prompt, - height=height, - width=width, - prompt_embeds=prompt_embeds, - negative_prompt_embeds=negative_prompt_embeds, - prompt_attention_mask=prompt_attention_mask, - negative_prompt_attention_mask=negative_prompt_attention_mask, - ) - - self._guidance_scale = guidance_scale - self._guidance_rescale = guidance_rescale - self._attention_kwargs = attention_kwargs - self._interrupt = False - self._current_timestep = None - - if prompt is not None and isinstance(prompt, str): - batch_size = 1 - elif prompt is not None and isinstance(prompt, list): - batch_size = len(prompt) - else: - batch_size = prompt_embeds.shape[0] - - device = self.device - - ( - prompt_embeds, - prompt_attention_mask, - negative_prompt_embeds, - negative_prompt_attention_mask, - ) = self.encode_prompt( + *, + image: Any | None = None, + ) -> DiffusionOutput | list[DiffusionOutput]: + del return_dict + request_inputs = self._resolve_request_inputs( + req, prompt=prompt, negative_prompt=negative_prompt, - do_classifier_free_guidance=self.do_classifier_free_guidance, + height=height, + width=width, + num_frames=num_frames, + frame_rate=frame_rate, + num_inference_steps=num_inference_steps, + timesteps=timesteps, + guidance_scale=self.one_stage_recipe.guidance_scale if guidance_scale is None else guidance_scale, + guidance_rescale=guidance_rescale, num_videos_per_prompt=num_videos_per_prompt, + generator=generator, + latents=latents, + audio_latents=audio_latents, prompt_embeds=prompt_embeds, negative_prompt_embeds=negative_prompt_embeds, prompt_attention_mask=prompt_attention_mask, negative_prompt_attention_mask=negative_prompt_attention_mask, + decode_timestep=decode_timestep, + decode_noise_scale=decode_noise_scale, + output_type=output_type, max_sequence_length=max_sequence_length, - device=device, ) - # Compute positive prompt connectors - tokenizer_padding_side = getattr(self.tokenizer, "padding_side", "left") - connector_prompt_embeds, connector_audio_prompt_embeds, connector_attention_mask = self.connectors( - prompt_embeds, prompt_attention_mask, padding_side=tokenizer_padding_side + image = self._resolve_request_image(req, image, request_inputs) + forward_kwargs = { + "noise_scale": noise_scale, + "sigmas": sigmas, + "timesteps": timesteps, + "attention_kwargs": attention_kwargs, + } + if self.support_image_input: + forward_kwargs["image"] = image + return self._forward_impl( + req, + request_inputs, + **forward_kwargs, ) - # Compute negative prompt connectors when CFG is enabled - negative_connector_prompt_embeds = None - negative_connector_audio_prompt_embeds = None - negative_connector_attention_mask = None - if self.do_classifier_free_guidance: - ( - negative_connector_prompt_embeds, - negative_connector_audio_prompt_embeds, - negative_connector_attention_mask, - ) = self.connectors( - negative_prompt_embeds, - negative_prompt_attention_mask, - padding_side=tokenizer_padding_side, - ) - latent_num_frames = (num_frames - 1) // self.vae_temporal_compression_ratio + 1 - latent_height = height // self.vae_spatial_compression_ratio - latent_width = width // self.vae_spatial_compression_ratio - if latents is not None: - if latents.ndim == 5: - logger.info( - "Got latents of shape [batch_size, latent_dim, latent_frames, latent_height, latent_width], `latent_num_frames`, `latent_height`, `latent_width` will be inferred." # noqa - ) - _, _, latent_num_frames, latent_height, latent_width = latents.shape # [B, C, F, H, W] - elif latents.ndim == 3: - logger.warning( - f"You have supplied packed `latents` of shape {latents.shape}, so the latent dims cannot be" - f" inferred. Make sure the supplied `height`, `width`, and `num_frames` are correct." - ) - else: - raise ValueError( - f"Provided `latents` tensor has shape {latents.shape}, but the expected shape is either [batch_size, seq_len, num_features] or [batch_size, latent_dim, latent_frames, latent_height, latent_width]." # noqa - ) - video_sequence_length = latent_num_frames * latent_height * latent_width +class LTX2Pipeline(LTXOneStagePipeline): + """LTX2 one-stage text-to-video entry.""" - num_channels_latents = self.transformer.config.in_channels - latents = self.prepare_latents( - batch_size * num_videos_per_prompt, - num_channels_latents, - height, - width, - num_frames, - noise_scale, - torch.float32, - device, - generator, - latents, - ) - - duration_s = num_frames / frame_rate - audio_latents_per_second = ( - self.audio_sampling_rate / self.audio_hop_length / float(self.audio_vae_temporal_compression_ratio) - ) - audio_num_frames = round(duration_s * audio_latents_per_second) - if audio_latents is not None: - if audio_latents.ndim == 4: - logger.info( - "Got audio_latents of shape [batch_size, num_channels, audio_length, mel_bins], `audio_num_frames` will be inferred." # noqa - ) - _, _, audio_num_frames, _ = audio_latents.shape # [B, C, L, M] - elif audio_latents.ndim == 3: - logger.warning( - f"You have supplied packed `audio_latents` of shape {audio_latents.shape}, so the latent dims" - f" cannot be inferred. Make sure the supplied `num_frames` and `frame_rate` are correct." - ) - else: - raise ValueError( - f"Provided `audio_latents` tensor has shape {audio_latents.shape}, but the expected shape is either [batch_size, seq_len, num_features] or [batch_size, num_channels, audio_length, mel_bins]." # noqa - ) - - num_mel_bins = self.audio_vae.config.mel_bins if getattr(self, "audio_vae", None) is not None else 64 - latent_mel_bins = num_mel_bins // self.audio_vae_mel_compression_ratio - - num_channels_latents_audio = ( - self.audio_vae.config.latent_channels if getattr(self, "audio_vae", None) is not None else 8 - ) - audio_latents, original_audio_num_frames, padded_audio_num_frames = self.prepare_audio_latents( - batch_size * num_videos_per_prompt, - num_channels_latents=num_channels_latents_audio, - audio_latent_length=audio_num_frames, - num_mel_bins=num_mel_bins, - noise_scale=noise_scale, - dtype=torch.float32, - device=device, - generator=generator, - latents=audio_latents, - ) - - sigmas = np.linspace(1.0, 1 / num_inference_steps, num_inference_steps) if sigmas is None else sigmas - mu = calculate_shift( - video_sequence_length, - self.scheduler.config.get("base_image_seq_len", 1024), - self.scheduler.config.get("max_image_seq_len", 4096), - self.scheduler.config.get("base_shift", 0.95), - self.scheduler.config.get("max_shift", 2.05), - ) - audio_scheduler = copy.deepcopy(self.scheduler) - video_audio_scheduler = _VideoAudioScheduler(self.scheduler, audio_scheduler) - _ = retrieve_timesteps( - audio_scheduler, - num_inference_steps, - device, - timesteps, - sigmas=sigmas, - mu=mu, - ) - timesteps, num_inference_steps = retrieve_timesteps( - self.scheduler, - num_inference_steps, - device, - timesteps, - sigmas=sigmas, - mu=mu, - ) - self._num_timesteps = len(timesteps) - - video_coords = self.transformer.rope.prepare_video_coords( - latents.shape[0], latent_num_frames, latent_height, latent_width, latents.device, fps=frame_rate - ) - audio_coords = self.transformer.audio_rope.prepare_audio_coords( - audio_latents.shape[0], padded_audio_num_frames, audio_latents.device - ) - # No coord duplication needed: mixin handles CFG via separate forward calls, - # not batch=2. Each forward gets batch=1 coords directly. - - with self.progress_bar(total=len(timesteps)) as pbar: - for i, t in enumerate(timesteps): - if self.interrupt: - continue - - self._current_timestep = t - - latent_model_input = latents.to(prompt_embeds.dtype) - audio_latent_model_input = audio_latents.to(prompt_embeds.dtype) - timestep = t.expand(latent_model_input.shape[0]) - do_true_cfg = self.do_classifier_free_guidance - - positive_kwargs = { - "hidden_states": latent_model_input, - "audio_hidden_states": audio_latent_model_input, - "encoder_hidden_states": connector_prompt_embeds, - "audio_encoder_hidden_states": connector_audio_prompt_embeds, - "timestep": timestep, - "encoder_attention_mask": connector_attention_mask, - "audio_encoder_attention_mask": connector_attention_mask, - "num_frames": latent_num_frames, - "height": latent_height, - "width": latent_width, - "fps": frame_rate, - "audio_num_frames": padded_audio_num_frames, - "video_coords": video_coords, - "audio_coords": audio_coords, - "attention_kwargs": attention_kwargs, - "return_dict": False, - } - negative_kwargs = ( - { - **positive_kwargs, - "encoder_hidden_states": negative_connector_prompt_embeds, - "audio_encoder_hidden_states": negative_connector_audio_prompt_embeds, - "encoder_attention_mask": negative_connector_attention_mask, - "audio_encoder_attention_mask": negative_connector_attention_mask, - } - if do_true_cfg - else None - ) - - noise_pred_video, noise_pred_audio = self.predict_noise_maybe_with_cfg( - do_true_cfg=do_true_cfg, - true_cfg_scale=guidance_scale, - positive_kwargs=positive_kwargs, - negative_kwargs=negative_kwargs, - cfg_normalize=False, - ) - - latents, audio_latents = self.scheduler_step_maybe_with_cfg( - (noise_pred_video, noise_pred_audio), - (t, t), - (latents, audio_latents), - do_true_cfg=do_true_cfg, - per_request_scheduler=video_audio_scheduler, - ) - latents, audio_latents = self._synchronize_cfg_parallel_step_output( - (latents, audio_latents), - do_true_cfg=do_true_cfg, - ) - - pbar.update() - - latents = self._unpack_latents( - latents, - latent_num_frames, - latent_height, - latent_width, - self.transformer_spatial_patch_size, - self.transformer_temporal_patch_size, - ) - latents = self._denormalize_latents( - latents, self.vae.latents_mean, self.vae.latents_std, self.vae.config.scaling_factor - ) - - audio_latents = self._unpad_audio_latents(audio_latents, original_audio_num_frames) - audio_latents = self._denormalize_audio_latents( - audio_latents, self.audio_vae.latents_mean, self.audio_vae.latents_std - ) - audio_latents = self._unpack_audio_latents( - audio_latents, - original_audio_num_frames, - num_mel_bins=latent_mel_bins, - ) - - if output_type == "latent": - video = latents - audio = audio_latents - else: - latents = latents.to(prompt_embeds.dtype) - - if not self.vae.config.timestep_conditioning: - timestep = None - else: - noise = randn_tensor(latents.shape, generator=generator, device=device, dtype=latents.dtype) - if not isinstance(decode_timestep, list): - decode_timestep = [decode_timestep] * batch_size - if decode_noise_scale is None: - decode_noise_scale = decode_timestep - elif not isinstance(decode_noise_scale, list): - decode_noise_scale = [decode_noise_scale] * batch_size - - timestep = torch.tensor(decode_timestep, device=device, dtype=latents.dtype) - decode_noise_scale = torch.tensor(decode_noise_scale, device=device, dtype=latents.dtype)[ - :, None, None, None, None - ] - latents = (1 - decode_noise_scale) * latents + decode_noise_scale * noise - - latents = latents.to(self.vae.dtype) - video = self.vae.decode(latents, timestep, return_dict=False)[0] - video = self.video_processor.postprocess_video(video, output_type=output_type) - - audio_latents = audio_latents.to(self.audio_vae.dtype) - generated_mel_spectrograms = self.audio_vae.decode(audio_latents, return_dict=False)[0] - audio = self.vocoder(generated_mel_spectrograms) - - return DiffusionOutput(output=(video, audio)) - - def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - loader = AutoWeightsLoader(self) - return loader.load_weights(weights) + supports_guidance_rescale = True + component_profile = LTX2_COMPONENT_PROFILE + guidance_strategy = LTX_LEGACY_VELOCITY_GUIDANCE + one_stage_recipe = LTX2_ONE_STAGE_RECIPE + _dit_modules: ClassVar[list[str]] = list(component_profile.dit_modules) + _encoder_modules: ClassVar[list[str]] = list(component_profile.encoder_modules) + _vae_modules: ClassVar[list[str]] = list(component_profile.vae_modules) + _resident_modules: ClassVar[list[str]] = list(component_profile.resident_modules) -class LTX2TwoStagesPipeline(nn.Module, SupportsComponentDiscovery): - """LTX2TwoStagesPipeline is for two stages image to video generation""" +class LTX23Pipeline(LTXOneStagePipeline): + """LTX2.3 one-stage text-to-video entry.""" - dummy_run_num_frames = 2 - supports_request_batch = False - - _dit_modules: ClassVar[list[str]] = ["pipe.transformer"] - _encoder_modules: ClassVar[list[str]] = ["pipe.text_encoder"] - _vae_modules: ClassVar[list[str]] = ["pipe.vae", "pipe.audio_vae"] - - def __init__( - self, - *, - od_config: OmniDiffusionConfig, - prefix: str = "", - ): - super().__init__() - - self.device = get_local_device() - self.dtype = getattr(od_config, "dtype", torch.bfloat16) - self.model_path = od_config.model - self.distilled = False - # User provided model path may contain '/' in the end and basename function - # will not return the expected directory name, so we need to remove it by normpath - if "distilled" in os.path.basename(os.path.normpath(self.model_path)): - self.distilled = True - else: - raise NotImplementedError(f"{self.model_path} is not supported for {self.__class__.__name__}.") - - self.pipe = LTX2Pipeline(od_config=od_config, prefix=prefix) - self.upsample_pipe = LTX2LatentUpsamplePipeline( - vae=self.pipe.vae, - od_config=od_config, - ) - - self.lora_manager = DiffusionLoRAManager( - pipeline=self.pipe, - device=self.device, - dtype=self.dtype, - max_cached_adapters=od_config.max_cpu_loras, - ) - - self.weights_sources = [ - DiffusersPipelineLoader.ComponentSource( - model_or_path=od_config.model, - subfolder="transformer", - revision=None, - prefix="pipe.transformer.", - fall_back_to_pt=True, - ), - ] + connector_batches_cfg = True + preserve_sp_padded_audio_duration = True + reports_stage_durations = True + component_profile = LTX23_COMPONENT_PROFILE + guidance_strategy = LTX_OFFICIAL_X0_GUIDANCE + one_stage_recipe = LTX23_ONE_STAGE_RECIPE + _dit_modules: ClassVar[list[str]] = list(component_profile.dit_modules) + _encoder_modules: ClassVar[list[str]] = list(component_profile.encoder_modules) + _vae_modules: ClassVar[list[str]] = list(component_profile.vae_modules) + _resident_modules: ClassVar[list[str]] = list(component_profile.resident_modules) + @torch.no_grad() def forward( self, req: DiffusionRequestBatch, @@ -1207,9 +175,9 @@ class LTX2TwoStagesPipeline(nn.Module, SupportsComponentDiscovery): num_frames: int | None = None, frame_rate: float | None = None, num_inference_steps: int | None = None, + sigmas: list[float] | None = None, timesteps: list[int] | None = None, guidance_scale: float = 4.0, - guidance_rescale: float = 0.0, noise_scale: float = 0.0, num_videos_per_prompt: int | None = 1, generator: torch.Generator | list[torch.Generator] | None = None, @@ -1225,9 +193,12 @@ class LTX2TwoStagesPipeline(nn.Module, SupportsComponentDiscovery): return_dict: bool = True, attention_kwargs: dict[str, Any] | None = None, max_sequence_length: int | None = None, - ) -> DiffusionOutput: - stage1_output = self.pipe( - req=req, + *, + image: Any | None = None, + ) -> DiffusionOutput | list[DiffusionOutput]: + """Preserve the pre-refactor LTX2.3 positional argument order.""" + return super().forward( + req, prompt=prompt, negative_prompt=negative_prompt, height=height, @@ -1235,10 +206,10 @@ class LTX2TwoStagesPipeline(nn.Module, SupportsComponentDiscovery): num_frames=num_frames, frame_rate=frame_rate, num_inference_steps=num_inference_steps, - sigmas=DISTILLED_SIGMA_VALUES if self.distilled else None, + sigmas=sigmas, timesteps=timesteps, guidance_scale=guidance_scale, - guidance_rescale=guidance_rescale, + guidance_rescale=0.0, noise_scale=noise_scale, num_videos_per_prompt=num_videos_per_prompt, generator=generator, @@ -1250,71 +221,32 @@ class LTX2TwoStagesPipeline(nn.Module, SupportsComponentDiscovery): negative_prompt_attention_mask=negative_prompt_attention_mask, decode_timestep=decode_timestep, decode_noise_scale=decode_noise_scale, - output_type="latent", + output_type=output_type, return_dict=return_dict, attention_kwargs=attention_kwargs, max_sequence_length=max_sequence_length, + image=image, ) - video_latent, audio_latent = stage1_output.output - upscaled_video_latent = self.upsample_pipe( - latents=video_latent, - output_type="latent", - return_dict=False, - )[0] - if not self.distilled: - # Load Stage 2 distilled LoRA - lora_path = f"{self.model_path}/ltx-2-19b-distilled-lora-384.safetensors" - lora_request = LoRARequest( - lora_name="stage_2_distilled", - lora_int_id=1, - lora_path=lora_path, - ) - self.lora_manager.set_active_adapter(lora_request, lora_scale=1.0) +class LTX2ImageToVideoPipeline(LTXI2VConditioningMixin, LTX2Pipeline): + """LTX2 one-stage image-to-video entry.""" - # Change scheduler to use Stage 2 distilled sigmas as is - new_scheduler = FlowMatchEulerDiscreteScheduler.from_config( - self.pipe.scheduler.config, - use_dynamic_shifting=False, - shift_terminal=None, - ) - self.pipe.scheduler = new_scheduler - # We only want to change num_inference_steps here, so no need - # to deep copy the whole request. `req` is a DiffusionRequestBatch - # whose `sampling_params` is a read-only property, so clone the - # underlying request(s) and override their sampling params instead. - stage_2_req = copy.copy(req) - stage_2_req.requests = [copy.copy(r) for r in req.requests] - for stage_2_request in stage_2_req.requests: - stage_2_request.sampling_params = stage_2_request.sampling_params.clone() - stage_2_request.sampling_params.num_inference_steps = 3 - - stage2_output = self.pipe( - req=stage_2_req, - latents=upscaled_video_latent, - audio_latents=audio_latent, - prompt=prompt, - negative_prompt=negative_prompt, - noise_scale=STAGE_2_DISTILLED_SIGMA_VALUES[0], - sigmas=STAGE_2_DISTILLED_SIGMA_VALUES, - guidance_scale=1.0, - generator=generator, - output_type="np", - return_dict=False, - ) - video, audio = stage2_output.output - - return DiffusionOutput(output=(video, audio)) - - def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - loader = AutoWeightsLoader(self) - return loader.load_weights(weights) +class LTX23ImageToVideoPipeline(LTXI2VConditioningMixin, LTX23Pipeline): + """LTX2.3 one-stage image-to-video entry.""" class LTX2T2VDMD2Pipeline(DMD2PipelineMixin, LTX2Pipeline): - """LTX-2 T2V pipeline for FastGen DMD2-distilled models.""" + """LTX2 T2V entry for FastGen DMD2-distilled models.""" + + def __init__(self, *, od_config: OmniDiffusionConfig, prefix: str = ""): + super().__init__(od_config=od_config, prefix=prefix) + self.__init_dmd2__() + + +class LTX2I2VDMD2Pipeline(DMD2PipelineMixin, LTX2ImageToVideoPipeline): + """LTX2 I2V entry for FastGen DMD2-distilled models.""" def __init__(self, *, od_config: OmniDiffusionConfig, prefix: str = ""): super().__init__(od_config=od_config, prefix=prefix) diff --git a/vllm_omni/diffusion/models/ltx2/pipeline_ltx2_3.py b/vllm_omni/diffusion/models/ltx2/pipeline_ltx2_3.py deleted file mode 100644 index 45be23296..000000000 --- a/vllm_omni/diffusion/models/ltx2/pipeline_ltx2_3.py +++ /dev/null @@ -1,1832 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -""" -Fully independent LTX-2.3 pipeline for vLLM-Omni. - -This pipeline does NOT inherit from LTX2Pipeline because: -- LTX-2.3 connectors run per_token_rms_norm + per-modality video/audio - projection internally (per_modality_projections=True), - versus LTX-2's per_layer_masked_mean_norm + shared projection path -- LTX-2.3 uses a BWE vocoder outputting 48kHz audio (not 16kHz) -- LTX-2.3 transformer requires the sigma parameter for prompt modulation -""" - -from __future__ import annotations - -import copy -import json -import os -from collections.abc import Iterable -from contextlib import nullcontext -from dataclasses import dataclass -from typing import Any, ClassVar - -import numpy as np -import torch -from diffusers import AutoencoderKLLTX2Audio, FlowMatchEulerDiscreteScheduler -from diffusers.pipelines.ltx2 import LTX2TextConnectors -from diffusers.pipelines.ltx2.vocoder import LTX2Vocoder -from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion import retrieve_timesteps -from diffusers.utils.torch_utils import randn_tensor -from diffusers.video_processor import VideoProcessor -from huggingface_hub import hf_hub_download -from torch import nn -from transformers import AutoTokenizer, Gemma3ForConditionalGeneration -from vllm.logger import init_logger -from vllm.model_executor.models.utils import AutoWeightsLoader - -from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig -from vllm_omni.diffusion.distributed.autoencoders.autoencoder_kl_ltx2 import DistributedAutoencoderKLLTX2Video -from vllm_omni.diffusion.distributed.cfg_parallel import CFGParallelMixin -from vllm_omni.diffusion.distributed.parallel_state import ( - get_cfg_group, - get_classifier_free_guidance_rank, - get_classifier_free_guidance_world_size, -) -from vllm_omni.diffusion.distributed.utils import get_local_device -from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader -from vllm_omni.diffusion.model_loader.hub_prefetch import from_pretrained_with_prefetch, prefetch_subfolders -from vllm_omni.diffusion.models.interface import SupportsComponentDiscovery -from vllm_omni.diffusion.models.progress_bar import ProgressBarMixin -from vllm_omni.diffusion.offloader.module_collector import ModuleDiscovery -from vllm_omni.diffusion.profiler.diffusion_pipeline_profiler import DiffusionPipelineProfilerMixin -from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch, split_diffusion_output_by_request - -from .pipeline_ltx2 import ( - _VideoAudioScheduler, - calculate_shift, - create_transformer_from_config, - load_transformer_config, -) - -logger = init_logger(__name__) - - -def _get_audio_latents_from_sampling(sampling: Any) -> torch.Tensor | None: - if sampling.audio_latents is not None: - return sampling.audio_latents - return sampling.extra_args.get("audio_latents") - - -def _is_output_rank() -> bool: - return not torch.distributed.is_initialized() or torch.distributed.get_rank() == 0 - - -def _vae_decode_needs_all_ranks(vae: Any) -> bool: - if not torch.distributed.is_initialized(): - return False - is_distributed_enabled = getattr(vae, "is_distributed_enabled", None) - if not callable(is_distributed_enabled): - return False - try: - return bool(is_distributed_enabled()) - except Exception: - return False - - -def _should_decode_video_on_rank(vae: Any) -> bool: - return _is_output_rank() or _vae_decode_needs_all_ranks(vae) - - -# Try to import LTX2VocoderWithBWE (diffusers >= 0.38.0) -try: - from diffusers.pipelines.ltx2.vocoder import LTX2VocoderWithBWE -except ImportError: - LTX2VocoderWithBWE = None - - -@dataclass -class _LTX23RequestInputs: - prompt: str | list[str] | None - negative_prompt: str | list[str] | None - height: int - width: int - num_frames: int - frame_rate: float - num_inference_steps: int - guidance_scale: float - num_videos_per_prompt: int - generator: torch.Generator | list[torch.Generator] | None - latents: torch.Tensor | None - audio_latents: torch.Tensor | None - prompt_embeds: torch.Tensor | None - negative_prompt_embeds: torch.Tensor | None - prompt_attention_mask: torch.Tensor | None - negative_prompt_attention_mask: torch.Tensor | None - decode_timestep: float | list[float] - decode_noise_scale: float | list[float] | None - output_type: str - max_sequence_length: int - - -@dataclass -class _LTX23PromptContext: - batch_size: int - connector_prompt_embeds: torch.Tensor - connector_audio_prompt_embeds: torch.Tensor - connector_attention_mask: torch.Tensor - positive_connector_prompt_embeds: torch.Tensor - positive_connector_audio_prompt_embeds: torch.Tensor - positive_connector_attention_mask: torch.Tensor - negative_connector_prompt_embeds: torch.Tensor | None - negative_connector_audio_prompt_embeds: torch.Tensor | None - negative_connector_attention_mask: torch.Tensor | None - - -@dataclass -class _LTX23ForwardContext: - req: DiffusionRequestBatch - request_inputs: _LTX23RequestInputs - prompt_context: _LTX23PromptContext - device: torch.device - cfg_parallel_ready: bool - attention_kwargs: dict[str, Any] | None - latent_num_frames: int - latent_height: int - latent_width: int - latent_mel_bins: int - original_audio_num_frames: int - padded_audio_num_frames: int - timesteps: torch.Tensor - audio_scheduler: Any - video_audio_scheduler: Any - - @property - def batch_size(self) -> int: - return self.prompt_context.batch_size - - @property - def num_videos_per_prompt(self) -> int: - return self.request_inputs.num_videos_per_prompt - - -@dataclass -class _LTX23DenoiseContext: - latents: torch.Tensor - audio_latents: torch.Tensor - video_coords: torch.Tensor - audio_coords: torch.Tensor - conditioning_mask: torch.Tensor | None = None - conditioning_mask_for_model: torch.Tensor | None = None - - -def _repeat_prompt_tensor_for_outputs(tensor: torch.Tensor, num_videos_per_prompt: int) -> torch.Tensor: - if num_videos_per_prompt == 1: - return tensor - return tensor.repeat_interleave(num_videos_per_prompt, dim=0) - - -def _detect_vocoder_output_sample_rate(model: str) -> int | None: - """Detect the vocoder output sample rate from vocoder/config.json. - - This runs at factory time (engine process) so the rate is captured in - the post-process closure and doesn't need cross-process communication. - - Returns: - Output sample rate (e.g. 48000 for LTX-2.3 BWE vocoder) or None. - """ - vocoder_config_path = os.path.join(model, "vocoder", "config.json") - if not os.path.exists(vocoder_config_path): - try: - vocoder_config_path = hf_hub_download(model, "vocoder/config.json") - except Exception: - return None - try: - with open(vocoder_config_path) as f: - cfg = json.load(f) - return cfg.get("output_sampling_rate") - except Exception: - return None - - -def get_ltx2_post_process_func(od_config: OmniDiffusionConfig): - """Factory for the LTX-2.3 post-process function. - - Detects the vocoder output sample rate at factory time and captures it - in the closure so that the audio_sample_rate flows through - DiffusionEngine -> OmniRequestOutput -> serving_video. - """ - output_sr = _detect_vocoder_output_sample_rate(od_config.model) - - def post_process_func(output: tuple[torch.Tensor, torch.Tensor] | torch.Tensor): - if isinstance(output, tuple) and len(output) == 2: - video, audio = output - if isinstance(audio, torch.Tensor): - audio = audio.detach().cpu() - result: dict[str, Any] = {"video": video, "audio": audio} - if output_sr is not None: - result["audio_sample_rate"] = output_sr - return result - return output - - return post_process_func - - -def _expand_per_prompt_decode_value( - value: float | list[float], - *, - prompt_batch_size: int, - effective_batch_size: int, - field_name: str, -) -> list[float]: - if not isinstance(value, list): - return [value] * effective_batch_size - if len(value) == 1: - return value * effective_batch_size - if len(value) == effective_batch_size: - return value - if prompt_batch_size > 0 and len(value) == prompt_batch_size and effective_batch_size % prompt_batch_size == 0: - repeats = effective_batch_size // prompt_batch_size - return [item for item in value for _ in range(repeats)] - raise ValueError( - f"`{field_name}` must have length 1, prompt batch size ({prompt_batch_size}), or effective batch size" - f" ({effective_batch_size}); got {len(value)}." - ) - - -def _prepare_decode_timestep_conditioning( - *, - decode_timestep: float | list[float], - decode_noise_scale: float | list[float] | None, - prompt_batch_size: int, - effective_batch_size: int, - device: torch.device, - dtype: torch.dtype, -) -> tuple[torch.Tensor, torch.Tensor]: - decode_timestep_values = _expand_per_prompt_decode_value( - decode_timestep, - prompt_batch_size=prompt_batch_size, - effective_batch_size=effective_batch_size, - field_name="decode_timestep", - ) - if decode_noise_scale is None: - decode_noise_scale_values = decode_timestep_values - else: - decode_noise_scale_values = _expand_per_prompt_decode_value( - decode_noise_scale, - prompt_batch_size=prompt_batch_size, - effective_batch_size=effective_batch_size, - field_name="decode_noise_scale", - ) - return ( - torch.tensor(decode_timestep_values, device=device, dtype=dtype), - torch.tensor(decode_noise_scale_values, device=device, dtype=dtype)[:, None, None, None, None], - ) - - -class LTX23Pipeline( - nn.Module, - CFGParallelMixin, - ProgressBarMixin, - SupportsComponentDiscovery, - DiffusionPipelineProfilerMixin, -): - """Fully independent LTX-2.3 pipeline. - - Key differences from LTX2Pipeline: - - Text encoding: uses ALL 49 hidden states from Gemma-3-12B, flattened - - Connectors: uses padding_side API (not additive_mask) - - Vocoder: uses LTX2VocoderWithBWE (48kHz output) - - Transformer: passes sigma for prompt_adaln - """ - - supports_request_batch = True - # Audio is diffused jointly with video; warmup must size audio tokens. - dummy_run_num_frames = 2 - _dit_modules: ClassVar[list[str]] = ["transformer"] - _encoder_modules: ClassVar[list[str]] = ["text_encoder", "connectors"] - _vae_modules: ClassVar[list[str]] = ["vae", "audio_vae"] - _resident_modules: ClassVar[list[str]] = ["vocoder"] - - def __init__( - self, - *, - od_config: OmniDiffusionConfig, - prefix: str = "", - ): - super().__init__() - self.od_config = od_config - self.device = get_local_device() - dtype = getattr(od_config, "dtype", torch.bfloat16) - model = od_config.model - local_files_only = os.path.exists(model) - - # Weight sources for transformer (loaded via AutoWeightsLoader) - self.weights_sources = [ - DiffusersPipelineLoader.ComponentSource( - model_or_path=od_config.model, - subfolder="transformer", - revision=None, - prefix="transformer.", - fall_back_to_pt=True, - ), - ] - - # See ``hub_prefetch.py`` for the transformers v5 multi-worker subfolder - # race; prefetch the whole component set before any from_pretrained. - ltx2_subfolders = [ - "tokenizer", - "text_encoder", - "connectors", - "vae", - "audio_vae", - "vocoder", - "scheduler", - ] - prefetch_subfolders(model, ltx2_subfolders, local_files_only=local_files_only) - - # --- Tokenizer (lightweight, stays wherever) --- - self.tokenizer = AutoTokenizer.from_pretrained(model, subfolder="tokenizer", local_files_only=local_files_only) - - # --- Text encoder --- - with torch.device("cpu"): - self.text_encoder = from_pretrained_with_prefetch( - Gemma3ForConditionalGeneration.from_pretrained, - model, - subfolder="text_encoder", - prefetch_list=ltx2_subfolders, - local_files_only=local_files_only, - torch_dtype=dtype, - ) - - # --- Connectors (LTX-2.3 connectors include caption projection) --- - self.connectors = from_pretrained_with_prefetch( - LTX2TextConnectors.from_pretrained, - model, - subfolder="connectors", - prefetch_list=ltx2_subfolders, - local_files_only=local_files_only, - torch_dtype=dtype, - ) - - # --- VAE, Audio VAE --- - self.vae = from_pretrained_with_prefetch( - DistributedAutoencoderKLLTX2Video.from_pretrained, - model, - subfolder="vae", - prefetch_list=ltx2_subfolders, - local_files_only=local_files_only, - torch_dtype=dtype, - ) - self.audio_vae = from_pretrained_with_prefetch( - AutoencoderKLLTX2Audio.from_pretrained, - model, - subfolder="audio_vae", - prefetch_list=ltx2_subfolders, - local_files_only=local_files_only, - torch_dtype=dtype, - ) - - # --- Vocoder: prefer BWE vocoder (48kHz) for LTX-2.3 --- - vocoder_cls = LTX2VocoderWithBWE or LTX2Vocoder - try: - self.vocoder = vocoder_cls.from_pretrained( - model, subfolder="vocoder", torch_dtype=dtype, local_files_only=local_files_only - ) - except (TypeError, OSError, ValueError): - self.vocoder = LTX2Vocoder.from_pretrained( - model, subfolder="vocoder", torch_dtype=dtype, local_files_only=local_files_only - ) - - # --- Transformer: created empty, weights loaded via AutoWeightsLoader --- - transformer_config = load_transformer_config(model, "transformer", local_files_only) - quant_config = getattr(self.od_config, "quantization_config", None) - self.transformer = create_transformer_from_config(transformer_config, quant_config=quant_config) - self._place_aux_components() - - # --- Scheduler --- - self.scheduler = FlowMatchEulerDiscreteScheduler.from_pretrained( - model, subfolder="scheduler", local_files_only=local_files_only - ) - - # --- Derived compression ratios --- - self.vae_spatial_compression_ratio = self.vae.spatial_compression_ratio if self.vae is not None else 32 - self.vae_temporal_compression_ratio = self.vae.temporal_compression_ratio if self.vae is not None else 8 - self.audio_vae_mel_compression_ratio = self.audio_vae.mel_compression_ratio if self.audio_vae is not None else 4 - self.audio_vae_temporal_compression_ratio = ( - self.audio_vae.temporal_compression_ratio if self.audio_vae is not None else 4 - ) - self.transformer_spatial_patch_size = self.transformer.config.patch_size if self.transformer is not None else 1 - self.transformer_temporal_patch_size = ( - self.transformer.config.patch_size_t if self.transformer is not None else 1 - ) - self.audio_sampling_rate = self.audio_vae.config.sample_rate if self.audio_vae is not None else 16000 - self.audio_hop_length = self.audio_vae.config.mel_hop_length if self.audio_vae is not None else 160 - - self.video_processor = VideoProcessor(vae_scale_factor=self.vae_spatial_compression_ratio) - - # Tokenizer max length - tokenizer_max_length = 1024 - if self.tokenizer is not None: - tokenizer_max_length = self.tokenizer.model_max_length - if tokenizer_max_length is None or tokenizer_max_length > 100000: - encoder_config = getattr(self.text_encoder, "config", None) - config_max_len = getattr(encoder_config, "max_position_embeddings", None) - if config_max_len is None: - config_max_len = getattr(encoder_config, "max_seq_len", None) - tokenizer_max_length = config_max_len or 1024 - self.tokenizer_max_length = int(tokenizer_max_length) - - # Pipeline state - self._guidance_scale = None - self._attention_kwargs = None - self._interrupt = False - self._num_timesteps = None - self._current_timestep = None - - self.setup_diffusion_pipeline_profiler( - enable_diffusion_pipeline_profiler=self.od_config.enable_diffusion_pipeline_profiler - ) - - def _place_aux_components(self) -> None: - parallel_config = getattr(self.od_config, "parallel_config", None) - use_managed_placement = bool( - getattr(self.od_config, "enable_cpu_offload", False) - or getattr(self.od_config, "enable_layerwise_offload", False) - or getattr(parallel_config, "use_hsdp", False) - ) - if use_managed_placement: - return - - modules = ModuleDiscovery.discover(self) - for module in (*modules.encoders, *modules.vaes, *modules.resident_modules): - module.to(self.device) - - # ------------------------------------------------------------------ - # Text Encoding (LTX-2.3 specific) - # ------------------------------------------------------------------ - - def _get_gemma_prompt_embeds( - self, - prompt: str | list[str], - num_videos_per_prompt: int = 1, - max_sequence_length: int = 1024, - device: torch.device | None = None, - dtype: torch.dtype | None = None, - ): - """Encode prompts using Gemma-3-12B, returning ALL 49 hidden states flattened. - - Stacks all 49 hidden states and flattens to [B, seq, hidden * 49]. The - connectors unflatten, apply per_token_rms_norm, and project internally - (same shape contract as LTX-2 since the `diffusers==0.38` connector - migration; the two differ only in the connector's internal norm path). - """ - device = device or self.device - dtype = dtype or self.text_encoder.dtype - - prompt = [prompt] if isinstance(prompt, str) else prompt - batch_size = len(prompt) - - if self.tokenizer is not None: - self.tokenizer.padding_side = "left" - if self.tokenizer.pad_token is None: - self.tokenizer.pad_token = self.tokenizer.eos_token - - prompt = [p.strip() for p in prompt] - text_inputs = self.tokenizer( - prompt, - padding="max_length", - max_length=max_sequence_length, - truncation=True, - add_special_tokens=True, - return_tensors="pt", - ) - text_input_ids = text_inputs.input_ids.to(device) - prompt_attention_mask = text_inputs.attention_mask.to(device) - - text_encoder_outputs = self.text_encoder( - input_ids=text_input_ids, - attention_mask=prompt_attention_mask, - output_hidden_states=True, - ) - - hidden_states = text_encoder_outputs.hidden_states - - # LTX-2.3: Stack ALL 49 hidden states and flatten - # [49 x (B, seq, 3840)] -> [B, seq, 3840, 49] -> [B, seq, 188160] - prompt_embeds = torch.stack(hidden_states, dim=-1).flatten(2, 3).to(dtype=dtype) - - prompt_attention_mask = prompt_attention_mask.view(batch_size, -1) - prompt_embeds = _repeat_prompt_tensor_for_outputs(prompt_embeds, num_videos_per_prompt) - prompt_attention_mask = _repeat_prompt_tensor_for_outputs(prompt_attention_mask, num_videos_per_prompt) - - return prompt_embeds, prompt_attention_mask - - def encode_prompt( - self, - prompt: str | list[str], - negative_prompt: str | list[str] | None = None, - do_classifier_free_guidance: bool = True, - num_videos_per_prompt: int = 1, - prompt_embeds: torch.Tensor | None = None, - negative_prompt_embeds: torch.Tensor | None = None, - prompt_attention_mask: torch.Tensor | None = None, - negative_prompt_attention_mask: torch.Tensor | None = None, - max_sequence_length: int = 1024, - device: torch.device | None = None, - dtype: torch.dtype | None = None, - ): - device = device or self.device - - prompt = [prompt] if isinstance(prompt, str) else prompt - if prompt is not None: - batch_size = len(prompt) - else: - batch_size = prompt_embeds.shape[0] - - negative_prompt_embeds_provided = negative_prompt_embeds is not None - - if prompt_embeds is None: - prompt_embeds, prompt_attention_mask = self._get_gemma_prompt_embeds( - prompt=prompt, - num_videos_per_prompt=num_videos_per_prompt, - max_sequence_length=max_sequence_length, - device=device, - dtype=dtype, - ) - elif num_videos_per_prompt > 1: - prompt_embeds = _repeat_prompt_tensor_for_outputs(prompt_embeds, num_videos_per_prompt) - prompt_attention_mask = _repeat_prompt_tensor_for_outputs(prompt_attention_mask, num_videos_per_prompt) - - if do_classifier_free_guidance and negative_prompt_embeds is None: - negative_prompt = negative_prompt or "" - negative_prompt = batch_size * [negative_prompt] if isinstance(negative_prompt, str) else negative_prompt - - if prompt is not None and type(prompt) is not type(negative_prompt): - raise TypeError( - f"`negative_prompt` should be the same type as `prompt`, but got {type(negative_prompt)} !=" - f" {type(prompt)}." - ) - if isinstance(negative_prompt, list) and batch_size != len(negative_prompt): - raise ValueError( - f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" - f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" - " the batch size of `prompt`." - ) - - negative_prompt_embeds, negative_prompt_attention_mask = self._get_gemma_prompt_embeds( - prompt=negative_prompt, - num_videos_per_prompt=num_videos_per_prompt, - max_sequence_length=max_sequence_length, - device=device, - dtype=dtype, - ) - elif do_classifier_free_guidance and negative_prompt_embeds_provided and num_videos_per_prompt > 1: - negative_prompt_embeds = _repeat_prompt_tensor_for_outputs(negative_prompt_embeds, num_videos_per_prompt) - negative_prompt_attention_mask = _repeat_prompt_tensor_for_outputs( - negative_prompt_attention_mask, - num_videos_per_prompt, - ) - - return prompt_embeds, prompt_attention_mask, negative_prompt_embeds, negative_prompt_attention_mask - - # ------------------------------------------------------------------ - # Latent utilities (shared with LTX2Pipeline) - # ------------------------------------------------------------------ - - @staticmethod - def _pack_latents(latents: torch.Tensor, patch_size: int = 1, patch_size_t: int = 1) -> torch.Tensor: - batch_size, num_channels, num_frames, height, width = latents.shape - post_patch_num_frames = num_frames // patch_size_t - post_patch_height = height // patch_size - post_patch_width = width // patch_size - latents = latents.reshape( - batch_size, - -1, - post_patch_num_frames, - patch_size_t, - post_patch_height, - patch_size, - post_patch_width, - patch_size, - ) - latents = latents.permute(0, 2, 4, 6, 1, 3, 5, 7).flatten(4, 7).flatten(1, 3) - return latents - - @staticmethod - def _unpack_latents( - latents: torch.Tensor, - num_frames: int, - height: int, - width: int, - patch_size: int = 1, - patch_size_t: int = 1, - ) -> torch.Tensor: - batch_size = latents.size(0) - latents = latents.reshape(batch_size, num_frames, height, width, -1, patch_size_t, patch_size, patch_size) - latents = latents.permute(0, 4, 1, 5, 2, 6, 3, 7).flatten(6, 7).flatten(4, 5).flatten(2, 3) - return latents - - @staticmethod - def _normalize_latents( - latents: torch.Tensor, latents_mean: torch.Tensor, latents_std: torch.Tensor, scaling_factor: float = 1.0 - ) -> torch.Tensor: - latents_mean = latents_mean.view(1, -1, 1, 1, 1).to(latents.device, latents.dtype) - latents_std = latents_std.view(1, -1, 1, 1, 1).to(latents.device, latents.dtype) - latents = (latents - latents_mean) * scaling_factor / latents_std - return latents - - @staticmethod - def _normalize_audio_latents(latents: torch.Tensor, latents_mean: torch.Tensor, latents_std: torch.Tensor): - latents_mean = latents_mean.to(latents.device, latents.dtype) - latents_std = latents_std.to(latents.device, latents.dtype) - return (latents - latents_mean) / latents_std - - @staticmethod - def _denormalize_latents( - latents: torch.Tensor, latents_mean: torch.Tensor, latents_std: torch.Tensor, scaling_factor: float = 1.0 - ) -> torch.Tensor: - latents_mean = latents_mean.view(1, -1, 1, 1, 1).to(latents.device, latents.dtype) - latents_std = latents_std.view(1, -1, 1, 1, 1).to(latents.device, latents.dtype) - latents = latents * latents_std / scaling_factor + latents_mean - return latents - - @staticmethod - def _denormalize_audio_latents(latents: torch.Tensor, latents_mean: torch.Tensor, latents_std: torch.Tensor): - latents_mean = latents_mean.to(latents.device, latents.dtype) - latents_std = latents_std.to(latents.device, latents.dtype) - return (latents * latents_std) + latents_mean - - @staticmethod - def _pack_audio_latents( - latents: torch.Tensor, patch_size: int | None = None, patch_size_t: int | None = None - ) -> torch.Tensor: - if patch_size is not None and patch_size_t is not None: - batch_size, num_channels, latent_length, latent_mel_bins = latents.shape - post_patch_latent_length = latent_length / patch_size_t - post_patch_mel_bins = latent_mel_bins / patch_size - latents = latents.reshape( - batch_size, -1, post_patch_latent_length, patch_size_t, post_patch_mel_bins, patch_size - ) - latents = latents.permute(0, 2, 4, 1, 3, 5).flatten(3, 5).flatten(1, 2) - else: - latents = latents.transpose(1, 2).flatten(2, 3) - return latents - - @staticmethod - def _unpack_audio_latents( - latents: torch.Tensor, - latent_length: int, - num_mel_bins: int, - patch_size: int | None = None, - patch_size_t: int | None = None, - ) -> torch.Tensor: - if patch_size is not None and patch_size_t is not None: - batch_size = latents.size(0) - latents = latents.reshape(batch_size, latent_length, num_mel_bins, -1, patch_size_t, patch_size) - latents = latents.permute(0, 3, 1, 4, 2, 5).flatten(4, 5).flatten(2, 3) - else: - latents = latents.unflatten(2, (-1, num_mel_bins)).transpose(1, 2) - return latents - - @staticmethod - def _unpad_audio_latents(latents: torch.Tensor, num_frames: int) -> torch.Tensor: - return latents[:, :num_frames] - - @staticmethod - def _get_sp_padded_audio_latent_length(audio_latent_length: int, sp_size: int) -> int: - if sp_size > 1: - audio_latent_length += (sp_size - (audio_latent_length % sp_size)) % sp_size - return audio_latent_length - - def _resolve_audio_latent_length(self, audio_latent_length: int, audio_latents: torch.Tensor | None) -> int: - if audio_latents is None or audio_latents.ndim != 4: - return audio_latent_length - - provided_latent_length = audio_latents.shape[2] - sp_size = getattr(self.od_config.parallel_config, "sequence_parallel_size", 1) or 1 - padded_latent_length = self._get_sp_padded_audio_latent_length(audio_latent_length, int(sp_size)) - - # Keep requested duration semantics when callers pass 4D latents that - # are already padded for SP; other 4D lengths retain shape inference. - if provided_latent_length in {audio_latent_length, padded_latent_length}: - return audio_latent_length - return provided_latent_length - - def _decode_output( - self, - *, - latents: torch.Tensor, - audio_latents: torch.Tensor, - output_type: str, - connector_prompt_embeds: torch.Tensor, - generator: torch.Generator | list[torch.Generator] | None, - device: torch.device, - decode_timestep: float | list[float], - decode_noise_scale: float | list[float] | None, - prompt_batch_size: int, - ) -> DiffusionOutput: - if output_type == "latent": - return DiffusionOutput( - output=(latents, audio_latents), - stage_durations=self.stage_durations if hasattr(self, "stage_durations") else None, - ) - - latents = latents.to(connector_prompt_embeds.dtype) - if not self.vae.config.timestep_conditioning: - timestep_decode = None - else: - noise = randn_tensor(latents.shape, generator=generator, device=device, dtype=latents.dtype) - timestep_decode, decode_noise_scale_t = _prepare_decode_timestep_conditioning( - decode_timestep=decode_timestep, - decode_noise_scale=decode_noise_scale, - prompt_batch_size=prompt_batch_size, - effective_batch_size=latents.shape[0], - device=device, - dtype=latents.dtype, - ) - latents = (1 - decode_noise_scale_t) * latents + decode_noise_scale_t * noise - - if _should_decode_video_on_rank(self.vae): - latents = latents.to(self.vae.dtype) - video = self.vae.decode(latents, timestep_decode, return_dict=False)[0] - else: - video = torch.empty(0, device=latents.device, dtype=latents.dtype) - - if not _is_output_rank(): - return DiffusionOutput( - output=( - torch.empty(0, device=video.device, dtype=video.dtype), - torch.empty(0, device=audio_latents.device, dtype=audio_latents.dtype), - ), - stage_durations=self.stage_durations if hasattr(self, "stage_durations") else None, - ) - - if video.numel() > 0: - video = self.video_processor.postprocess_video(video, output_type=output_type) - - audio_latents = audio_latents.to(self.audio_vae.dtype) - generated_mel_spectrograms = self.audio_vae.decode(audio_latents, return_dict=False)[0] - audio = self.vocoder(generated_mel_spectrograms) - - return DiffusionOutput( - output=(video, audio), - stage_durations=self.stage_durations if hasattr(self, "stage_durations") else None, - ) - - # ------------------------------------------------------------------ - # Latent preparation - # ------------------------------------------------------------------ - - def prepare_latents( - self, - batch_size: int = 1, - num_channels_latents: int = 128, - height: int = 512, - width: int = 768, - num_frames: int = 121, - noise_scale: float = 0.0, - dtype: torch.dtype | None = None, - device: torch.device | None = None, - generator: torch.Generator | None = None, - latents: torch.Tensor | None = None, - ) -> torch.Tensor: - if latents is not None: - if latents.ndim == 5: - latents = self._normalize_latents( - latents, self.vae.latents_mean, self.vae.latents_std, self.vae.config.scaling_factor - ) - latents = self._pack_latents( - latents, self.transformer_spatial_patch_size, self.transformer_temporal_patch_size - ) - if latents.ndim != 3: - raise ValueError(f"Provided `latents` has shape {latents.shape}, expected [batch, seq, features].") - noise = randn_tensor(latents.shape, generator=generator, device=latents.device, dtype=latents.dtype) - latents = noise_scale * noise + (1 - noise_scale) * latents - return latents.to(device=device, dtype=dtype) - - height = height // self.vae_spatial_compression_ratio - width = width // self.vae_spatial_compression_ratio - num_frames = (num_frames - 1) // self.vae_temporal_compression_ratio + 1 - shape = (batch_size, num_channels_latents, num_frames, height, width) - latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) - latents = self._pack_latents(latents, self.transformer_spatial_patch_size, self.transformer_temporal_patch_size) - return latents - - def prepare_audio_latents( - self, - batch_size: int = 1, - num_channels_latents: int = 8, - audio_latent_length: int = 1, - num_mel_bins: int = 64, - noise_scale: float = 0.0, - dtype: torch.dtype | None = None, - device: torch.device | None = None, - generator: torch.Generator | list[torch.Generator] | None = None, - latents: torch.Tensor | None = None, - ) -> tuple[torch.Tensor, int, int]: - original_latent_length = audio_latent_length - latent_mel_bins = num_mel_bins // self.audio_vae_mel_compression_ratio - - sp_size = getattr(self.od_config.parallel_config, "sequence_parallel_size", 1) or 1 - padded_latent_length = self._get_sp_padded_audio_latent_length(original_latent_length, int(sp_size)) - - if latents is not None: - if latents.ndim == 4: - latents = self._pack_audio_latents(latents) - if latents.ndim != 3: - raise ValueError(f"Provided `latents` has shape {latents.shape}, expected [batch, seq, features].") - latents = self._normalize_audio_latents(latents, self.audio_vae.latents_mean, self.audio_vae.latents_std) - noise = randn_tensor(latents.shape, generator=generator, device=latents.device, dtype=latents.dtype) - latents = noise_scale * noise + (1 - noise_scale) * latents - - if latents.shape[1] not in {original_latent_length, padded_latent_length}: - raise ValueError( - "Provided `audio_latents` has incompatible audio frame count " - f"{latents.shape[1]}; expected {original_latent_length} or {padded_latent_length}." - ) - - if latents.shape[1] == original_latent_length and padded_latent_length > original_latent_length: - padding = torch.zeros( - latents.shape[0], - padded_latent_length - original_latent_length, - latents.shape[2], - dtype=latents.dtype, - device=latents.device, - ) - latents = torch.cat([latents, padding], dim=1) - - return latents.to(device=device, dtype=dtype), original_latent_length, padded_latent_length - - shape = (batch_size, num_channels_latents, padded_latent_length, latent_mel_bins) - if isinstance(generator, list) and len(generator) != batch_size: - raise ValueError( - f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" - f" size of {batch_size}. Make sure the batch size matches the length of the generators." - ) - - latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) - latents = self._pack_audio_latents(latents) - return latents, original_latent_length, padded_latent_length - - # ------------------------------------------------------------------ - # Properties - # ------------------------------------------------------------------ - - @property - def guidance_scale(self): - return self._guidance_scale - - @property - def do_classifier_free_guidance(self): - return self._guidance_scale is not None and self._guidance_scale > 1.0 - - @property - def num_timesteps(self): - return self._num_timesteps - - @property - def current_timestep(self): - return self._current_timestep - - @property - def interrupt(self): - return self._interrupt - - # ------------------------------------------------------------------ - # Input validation - # ------------------------------------------------------------------ - - def check_inputs( - self, - prompt, - height, - width, - prompt_embeds=None, - negative_prompt_embeds=None, - prompt_attention_mask=None, - negative_prompt_attention_mask=None, - ): - if height % 32 != 0 or width % 32 != 0: - raise ValueError(f"`height` and `width` must be divisible by 32 but are {height} and {width}.") - if prompt is not None and prompt_embeds is not None: - raise ValueError("Cannot forward both `prompt` and `prompt_embeds`.") - elif prompt is None and prompt_embeds is None: - raise ValueError("Provide either `prompt` or `prompt_embeds`.") - elif prompt is not None and not isinstance(prompt, (str, list)): - raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") - - if prompt_embeds is not None and prompt_attention_mask is None: - raise ValueError("Must provide `prompt_attention_mask` when specifying `prompt_embeds`.") - - if negative_prompt_embeds is not None and negative_prompt_attention_mask is None: - raise ValueError("Must provide `negative_prompt_attention_mask` when specifying `negative_prompt_embeds`.") - - if prompt_embeds is not None and negative_prompt_embeds is not None: - if prompt_embeds.shape != negative_prompt_embeds.shape: - raise ValueError( - "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but" - f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`" - f" {negative_prompt_embeds.shape}." - ) - if prompt_attention_mask.shape != negative_prompt_attention_mask.shape: - raise ValueError( - "`prompt_attention_mask` and `negative_prompt_attention_mask` must have the same shape when " - "passed directly, but got: `prompt_attention_mask` " - f"{prompt_attention_mask.shape} != `negative_prompt_attention_mask` " - f"{negative_prompt_attention_mask.shape}." - ) - - # ------------------------------------------------------------------ - # Cache context - # ------------------------------------------------------------------ - - def _transformer_cache_context(self, context_name: str): - cache_context = getattr(self.transformer, "cache_context", None) - if callable(cache_context): - return cache_context(context_name) - return nullcontext() - - # ------------------------------------------------------------------ - # CFG helpers - # ------------------------------------------------------------------ - - @staticmethod - def _combine_x0_space_cfg( - sample: torch.Tensor, - positive_noise_pred: torch.Tensor, - negative_noise_pred: torch.Tensor, - sigma: torch.Tensor, - guidance_scale: float, - ) -> torch.Tensor: - x0_cond = sample - positive_noise_pred * sigma - x0_uncond = sample - negative_noise_pred * sigma - x0_guided = x0_cond + (guidance_scale - 1) * (x0_cond - x0_uncond) - return (sample - x0_guided) / sigma - - def predict_noise(self, **kwargs): - with self._transformer_cache_context("cond_uncond"): - noise_pred_video, noise_pred_audio = self.transformer(**kwargs) - return noise_pred_video.float(), noise_pred_audio.float() - - def combine_cfg_noise( - self, - positive_noise_pred, - negative_noise_pred, - true_cfg_scale, - cfg_normalize=False, - *, - video_latents: torch.Tensor | None = None, - audio_latents: torch.Tensor | None = None, - video_sigma: torch.Tensor | None = None, - audio_sigma: torch.Tensor | None = None, - ): - if video_latents is None or audio_latents is None or video_sigma is None or audio_sigma is None: - raise ValueError("LTX23Pipeline applies CFG in x0-space and requires video/audio latents and sigmas.") - - video_pos, audio_pos = positive_noise_pred - video_neg, audio_neg = negative_noise_pred - video_combined = self._combine_x0_space_cfg( - video_latents, - video_pos, - video_neg, - video_sigma, - true_cfg_scale, - ) - audio_combined = self._combine_x0_space_cfg( - audio_latents, - audio_pos, - audio_neg, - audio_sigma, - true_cfg_scale, - ) - if cfg_normalize: - video_combined = self.cfg_normalize_function(video_pos, video_combined) - audio_combined = self.cfg_normalize_function(audio_pos, audio_combined) - return video_combined, audio_combined - - def predict_noise_with_parallel_cfg( - self, - true_cfg_scale: float, - positive_kwargs: dict[str, Any], - negative_kwargs: dict[str, Any], - cfg_normalize: bool = True, - output_slice: int | None = None, - *, - video_latents: torch.Tensor | None = None, - audio_latents: torch.Tensor | None = None, - video_sigma: torch.Tensor | None = None, - audio_sigma: torch.Tensor | None = None, - ) -> tuple[torch.Tensor, torch.Tensor]: - def maybe_slice(pred: tuple[torch.Tensor, torch.Tensor]) -> tuple[torch.Tensor, torch.Tensor]: - if output_slice is None: - return pred - return pred[0][:, :output_slice], pred[1][:, :output_slice] - - cfg_world_size = get_classifier_free_guidance_world_size() - if cfg_world_size != 2: - raise ValueError(f"LTX23Pipeline parallel CFG requires cfg_parallel_size 2, but got {cfg_world_size}.") - - cfg_group = get_cfg_group() - cfg_rank = get_classifier_free_guidance_rank() - branch_kwargs = positive_kwargs if cfg_rank == 0 else negative_kwargs - local_video_pred, local_audio_pred = maybe_slice(self.predict_noise(**branch_kwargs)) - - gathered_video = cfg_group.all_gather(local_video_pred, separate_tensors=True) - gathered_audio = cfg_group.all_gather(local_audio_pred, separate_tensors=True) - positive_noise_pred = (gathered_video[0], gathered_audio[0]) - negative_noise_pred = (gathered_video[1], gathered_audio[1]) - - return self.combine_cfg_noise( - positive_noise_pred, - negative_noise_pred, - true_cfg_scale, - cfg_normalize, - video_latents=video_latents, - audio_latents=audio_latents, - video_sigma=video_sigma, - audio_sigma=audio_sigma, - ) - - def _synchronize_cfg_parallel_step_output( - self, - latents: tuple[torch.Tensor, torch.Tensor], - do_true_cfg: bool, - ) -> tuple[torch.Tensor, torch.Tensor]: - if not (do_true_cfg and get_classifier_free_guidance_world_size() > 1): - return latents - - latents = tuple(tensor.contiguous() for tensor in latents) - device = next((tensor.device for tensor in latents if tensor.is_cuda), None) - if device is not None: - torch.cuda.current_stream(device).synchronize() - return latents - - def _resolve_request_inputs( - self, - req: DiffusionRequestBatch, - *, - prompt: str | list[str] | None, - negative_prompt: str | list[str] | None, - height: int | None, - width: int | None, - num_frames: int | None, - frame_rate: float | None, - num_inference_steps: int | None, - timesteps: list[int] | None, - guidance_scale: float, - num_videos_per_prompt: int | None, - generator: torch.Generator | list[torch.Generator] | None, - latents: torch.Tensor | None, - audio_latents: torch.Tensor | None, - prompt_embeds: torch.Tensor | None, - negative_prompt_embeds: torch.Tensor | None, - prompt_attention_mask: torch.Tensor | None, - negative_prompt_attention_mask: torch.Tensor | None, - decode_timestep: float | list[float], - decode_noise_scale: float | list[float] | None, - output_type: str, - max_sequence_length: int | None, - ) -> _LTX23RequestInputs: - sampling_params_list = req.sampling_params_list - common_sampling_params = sampling_params_list[0] - prompt = [p if isinstance(p, str) else (p.get("prompt") or "") for p in req.prompts] or prompt - if all(isinstance(p, str) or p.get("negative_prompt") is None for p in req.prompts): - negative_prompt = None - elif req.prompts: - negative_prompt = ["" if isinstance(p, str) else (p.get("negative_prompt") or "") for p in req.prompts] - - height = common_sampling_params.height or height or 512 - width = common_sampling_params.width or width or 768 - num_frames = common_sampling_params.num_frames or num_frames or 121 - frame_rate = common_sampling_params.resolved_frame_rate or frame_rate or 24.0 - num_inference_steps = common_sampling_params.num_inference_steps or num_inference_steps or 40 - if timesteps is None: - num_inference_steps = max(int(num_inference_steps), 2) - elif len(timesteps) < 2: - raise ValueError("`timesteps` must contain at least 2 values for FlowMatchEulerDiscreteScheduler.") - - num_videos_per_prompt = ( - common_sampling_params.num_outputs_per_prompt - if common_sampling_params.num_outputs_per_prompt > 0 - else num_videos_per_prompt or 1 - ) - max_sequence_length = ( - common_sampling_params.max_sequence_length or max_sequence_length or self.tokenizer_max_length - ) - - if common_sampling_params.guidance_scale_provided: - guidance_scale = common_sampling_params.guidance_scale - - if generator is None: - generator = req.collate_request_generators(num_videos_per_prompt, generator) - - latents = req.collate_request_tensors("latents", latents) - audio_latents = DiffusionRequestBatch.collate_tensors( - [_get_audio_latents_from_sampling(sampling) for sampling in sampling_params_list], - "audio_latents", - audio_latents, - ) - - prompt_fields = DiffusionRequestBatch.collate_prompt_field_map( - req.prompts, - { - "prompt_embeds": prompt_embeds, - "negative_prompt_embeds": negative_prompt_embeds, - "prompt_attention_mask": prompt_attention_mask, - "negative_prompt_attention_mask": negative_prompt_attention_mask, - }, - field_aliases={ - "prompt_attention_mask": ("prompt_attention_mask", "attention_mask"), - "negative_prompt_attention_mask": ( - "negative_prompt_attention_mask", - "negative_attention_mask", - ), - }, - ) - prompt_embeds = prompt_fields["prompt_embeds"] - negative_prompt_embeds = prompt_fields["negative_prompt_embeds"] - prompt_attention_mask = prompt_fields["prompt_attention_mask"] - negative_prompt_attention_mask = prompt_fields["negative_prompt_attention_mask"] - if prompt_embeds is not None: - prompt = None - if negative_prompt_embeds is not None: - negative_prompt = None - - if common_sampling_params.decode_timestep is not None: - decode_timestep = common_sampling_params.decode_timestep - if common_sampling_params.decode_noise_scale is not None: - decode_noise_scale = common_sampling_params.decode_noise_scale - if common_sampling_params.output_type is not None: - output_type = common_sampling_params.output_type - - return _LTX23RequestInputs( - prompt=prompt, - negative_prompt=negative_prompt, - height=int(height), - width=int(width), - num_frames=int(num_frames), - frame_rate=float(frame_rate), - num_inference_steps=int(num_inference_steps), - guidance_scale=guidance_scale, - num_videos_per_prompt=int(num_videos_per_prompt), - generator=generator, - latents=latents, - audio_latents=audio_latents, - prompt_embeds=prompt_embeds, - negative_prompt_embeds=negative_prompt_embeds, - prompt_attention_mask=prompt_attention_mask, - negative_prompt_attention_mask=negative_prompt_attention_mask, - decode_timestep=decode_timestep, - decode_noise_scale=decode_noise_scale, - output_type=output_type, - max_sequence_length=int(max_sequence_length), - ) - - def _prepare_prompt_context( - self, - *, - prompt: str | list[str] | None, - negative_prompt: str | list[str] | None, - prompt_embeds: torch.Tensor | None, - negative_prompt_embeds: torch.Tensor | None, - prompt_attention_mask: torch.Tensor | None, - negative_prompt_attention_mask: torch.Tensor | None, - num_videos_per_prompt: int, - max_sequence_length: int, - ) -> _LTX23PromptContext: - if prompt is not None and isinstance(prompt, str): - batch_size = 1 - elif prompt is not None and isinstance(prompt, list): - batch_size = len(prompt) - else: - batch_size = prompt_embeds.shape[0] - - prompt_embeds, prompt_attention_mask, negative_prompt_embeds, negative_prompt_attention_mask = ( - self.encode_prompt( - prompt=prompt, - negative_prompt=negative_prompt, - do_classifier_free_guidance=self.do_classifier_free_guidance, - num_videos_per_prompt=num_videos_per_prompt, - prompt_embeds=prompt_embeds, - negative_prompt_embeds=negative_prompt_embeds, - prompt_attention_mask=prompt_attention_mask, - negative_prompt_attention_mask=negative_prompt_attention_mask, - max_sequence_length=max_sequence_length, - device=self.device, - ) - ) - - if self.do_classifier_free_guidance: - prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds], dim=0) - prompt_attention_mask = torch.cat([negative_prompt_attention_mask, prompt_attention_mask], dim=0) - - connector_prompt_embeds, connector_audio_prompt_embeds, connector_attention_mask = self.connectors( - prompt_embeds, - prompt_attention_mask, - padding_side=getattr(self.tokenizer, "padding_side", "left"), - ) - - positive_connector_prompt_embeds = connector_prompt_embeds - positive_connector_audio_prompt_embeds = connector_audio_prompt_embeds - positive_connector_attention_mask = connector_attention_mask - negative_connector_prompt_embeds = None - negative_connector_audio_prompt_embeds = None - negative_connector_attention_mask = None - if self.do_classifier_free_guidance: - split_batch = batch_size * num_videos_per_prompt - negative_connector_prompt_embeds = connector_prompt_embeds[:split_batch] - positive_connector_prompt_embeds = connector_prompt_embeds[split_batch:] - negative_connector_audio_prompt_embeds = connector_audio_prompt_embeds[:split_batch] - positive_connector_audio_prompt_embeds = connector_audio_prompt_embeds[split_batch:] - negative_connector_attention_mask = connector_attention_mask[:split_batch] - positive_connector_attention_mask = connector_attention_mask[split_batch:] - - return _LTX23PromptContext( - batch_size=batch_size, - connector_prompt_embeds=connector_prompt_embeds, - connector_audio_prompt_embeds=connector_audio_prompt_embeds, - connector_attention_mask=connector_attention_mask, - positive_connector_prompt_embeds=positive_connector_prompt_embeds, - positive_connector_audio_prompt_embeds=positive_connector_audio_prompt_embeds, - positive_connector_attention_mask=positive_connector_attention_mask, - negative_connector_prompt_embeds=negative_connector_prompt_embeds, - negative_connector_audio_prompt_embeds=negative_connector_audio_prompt_embeds, - negative_connector_attention_mask=negative_connector_attention_mask, - ) - - def _setup_forward_runtime( - self, - request_inputs: _LTX23RequestInputs, - attention_kwargs: dict[str, Any] | None, - ) -> bool: - self._guidance_scale = request_inputs.guidance_scale - self._attention_kwargs = attention_kwargs - self._interrupt = False - self._current_timestep = None - cfg_world_size = get_classifier_free_guidance_world_size() - if self.do_classifier_free_guidance and cfg_world_size not in (1, 2): - raise ValueError( - f"LTX23Pipeline supports CFG parallelism with cfg_parallel_size 1 or 2, but got {cfg_world_size}." - ) - return self.do_classifier_free_guidance and cfg_world_size > 1 - - def _check_forward_inputs( - self, - request_inputs: _LTX23RequestInputs, - image: Any | None = None, - ) -> None: - self.check_inputs( - prompt=request_inputs.prompt, - height=request_inputs.height, - width=request_inputs.width, - prompt_embeds=request_inputs.prompt_embeds, - negative_prompt_embeds=request_inputs.negative_prompt_embeds, - prompt_attention_mask=request_inputs.prompt_attention_mask, - negative_prompt_attention_mask=request_inputs.negative_prompt_attention_mask, - ) - - def _resolve_video_latent_dimensions( - self, - request_inputs: _LTX23RequestInputs, - ) -> tuple[int, int, int]: - latent_num_frames = (request_inputs.num_frames - 1) // self.vae_temporal_compression_ratio + 1 - latent_height = request_inputs.height // self.vae_spatial_compression_ratio - latent_width = request_inputs.width // self.vae_spatial_compression_ratio - if request_inputs.latents is not None and request_inputs.latents.ndim == 5: - _, _, latent_num_frames, latent_height, latent_width = request_inputs.latents.shape - return latent_num_frames, latent_height, latent_width - - def _prepare_video_latents_stage( - self, - request_inputs: _LTX23RequestInputs, - prompt_context: _LTX23PromptContext, - *, - device: torch.device, - noise_scale: float, - image: Any | None = None, - ) -> tuple[torch.Tensor, torch.Tensor | None]: - num_channels_latents = self.transformer.config.in_channels - latents = self.prepare_latents( - prompt_context.batch_size * request_inputs.num_videos_per_prompt, - num_channels_latents, - request_inputs.height, - request_inputs.width, - request_inputs.num_frames, - noise_scale, - torch.float32, - device, - request_inputs.generator, - request_inputs.latents, - ) - return latents, None - - def _prepare_audio_latents_stage( - self, - request_inputs: _LTX23RequestInputs, - prompt_context: _LTX23PromptContext, - *, - device: torch.device, - noise_scale: float, - ) -> tuple[torch.Tensor, int, int, int]: - duration_s = request_inputs.num_frames / request_inputs.frame_rate - audio_latents_per_second = ( - self.audio_sampling_rate / self.audio_hop_length / float(self.audio_vae_temporal_compression_ratio) - ) - audio_num_frames = round(duration_s * audio_latents_per_second) - audio_num_frames = self._resolve_audio_latent_length(audio_num_frames, request_inputs.audio_latents) - - num_mel_bins = self.audio_vae.config.mel_bins if self.audio_vae is not None else 64 - latent_mel_bins = num_mel_bins // self.audio_vae_mel_compression_ratio - num_channels_latents_audio = self.audio_vae.config.latent_channels if self.audio_vae is not None else 8 - audio_latents, original_audio_num_frames, padded_audio_num_frames = self.prepare_audio_latents( - prompt_context.batch_size * request_inputs.num_videos_per_prompt, - num_channels_latents=num_channels_latents_audio, - audio_latent_length=audio_num_frames, - num_mel_bins=num_mel_bins, - noise_scale=noise_scale, - dtype=torch.float32, - device=device, - generator=request_inputs.generator, - latents=request_inputs.audio_latents, - ) - return audio_latents, original_audio_num_frames, padded_audio_num_frames, latent_mel_bins - - def _make_video_audio_scheduler( - self, - audio_scheduler: Any, - latent_num_frames: int, - latent_height: int, - latent_width: int, - ) -> Any: - return _VideoAudioScheduler(self.scheduler, audio_scheduler) - - def _prepare_scheduler_stage( - self, - request_inputs: _LTX23RequestInputs, - *, - device: torch.device, - sigmas: list[float] | None, - timesteps: list[int] | None, - latent_num_frames: int, - latent_height: int, - latent_width: int, - ) -> tuple[Any, Any, torch.Tensor]: - sigmas = ( - np.linspace(1.0, 1 / request_inputs.num_inference_steps, request_inputs.num_inference_steps) - if sigmas is None - else sigmas - ) - # Use max_image_seq_len (not actual video_sequence_length) for mu calculation, - # matching diffusers' LTX2Pipeline which hardcodes this value. - mu = calculate_shift( - self.scheduler.config.get("max_image_seq_len", 4096), - self.scheduler.config.get("base_image_seq_len", 1024), - self.scheduler.config.get("max_image_seq_len", 4096), - self.scheduler.config.get("base_shift", 0.95), - self.scheduler.config.get("max_shift", 2.05), - ) - audio_scheduler = copy.deepcopy(self.scheduler) - video_audio_scheduler = self._make_video_audio_scheduler( - audio_scheduler, - latent_num_frames, - latent_height, - latent_width, - ) - _ = retrieve_timesteps( - audio_scheduler, - request_inputs.num_inference_steps, - device, - timesteps, - sigmas=sigmas, - mu=mu, - ) - timesteps, _ = retrieve_timesteps( - self.scheduler, - request_inputs.num_inference_steps, - device, - timesteps, - sigmas=sigmas, - mu=mu, - ) - self._num_timesteps = len(timesteps) - return audio_scheduler, video_audio_scheduler, timesteps - - def _prepare_rope_coords_stage( - self, - forward_ctx: _LTX23ForwardContext, - latents: torch.Tensor, - audio_latents: torch.Tensor, - ) -> tuple[torch.Tensor, torch.Tensor]: - request_inputs = forward_ctx.request_inputs - video_coords = self.transformer.rope.prepare_video_coords( - latents.shape[0], - forward_ctx.latent_num_frames, - forward_ctx.latent_height, - forward_ctx.latent_width, - latents.device, - fps=request_inputs.frame_rate, - ) - audio_coords = self.transformer.audio_rope.prepare_audio_coords( - audio_latents.shape[0], - forward_ctx.padded_audio_num_frames, - audio_latents.device, - ) - return video_coords, audio_coords - - def _prepare_denoise_context_for_cfg( - self, - forward_ctx: _LTX23ForwardContext, - denoise_ctx: _LTX23DenoiseContext, - ) -> _LTX23DenoiseContext: - if self.do_classifier_free_guidance and not forward_ctx.cfg_parallel_ready: - denoise_ctx.video_coords = denoise_ctx.video_coords.repeat( - (2,) + (1,) * (denoise_ctx.video_coords.ndim - 1) - ) - denoise_ctx.audio_coords = denoise_ctx.audio_coords.repeat( - (2,) + (1,) * (denoise_ctx.audio_coords.ndim - 1) - ) - return denoise_ctx - - def _denoise_timestep_kwargs( - self, - ts: torch.Tensor, - forward_ctx: _LTX23ForwardContext, - denoise_ctx: _LTX23DenoiseContext, - ) -> dict[str, torch.Tensor]: - return {"timestep": ts, "sigma": ts} - - def _build_transformer_kwargs( - self, - forward_ctx: _LTX23ForwardContext, - denoise_ctx: _LTX23DenoiseContext, - *, - hidden_states: torch.Tensor, - audio_hidden_states: torch.Tensor, - encoder_hidden_states: torch.Tensor, - audio_encoder_hidden_states: torch.Tensor, - encoder_attention_mask: torch.Tensor, - audio_encoder_attention_mask: torch.Tensor, - ts: torch.Tensor, - ) -> dict[str, Any]: - request_inputs = forward_ctx.request_inputs - return { - "hidden_states": hidden_states, - "audio_hidden_states": audio_hidden_states, - "encoder_hidden_states": encoder_hidden_states, - "audio_encoder_hidden_states": audio_encoder_hidden_states, - **self._denoise_timestep_kwargs(ts, forward_ctx, denoise_ctx), - "encoder_attention_mask": encoder_attention_mask, - "audio_encoder_attention_mask": audio_encoder_attention_mask, - "num_frames": forward_ctx.latent_num_frames, - "height": forward_ctx.latent_height, - "width": forward_ctx.latent_width, - "fps": request_inputs.frame_rate, - "audio_num_frames": forward_ctx.padded_audio_num_frames, - "video_coords": denoise_ctx.video_coords, - "audio_coords": denoise_ctx.audio_coords, - "attention_kwargs": forward_ctx.attention_kwargs, - "return_dict": False, - } - - def _step_denoised_latents( - self, - forward_ctx: _LTX23ForwardContext, - denoise_ctx: _LTX23DenoiseContext, - noise_pred_video: torch.Tensor, - noise_pred_audio: torch.Tensor, - t: torch.Tensor, - ) -> tuple[torch.Tensor, torch.Tensor]: - if forward_ctx.cfg_parallel_ready: - latents, audio_latents = self.scheduler_step_maybe_with_cfg( - (noise_pred_video, noise_pred_audio), - (t, t), - (denoise_ctx.latents, denoise_ctx.audio_latents), - do_true_cfg=self.do_classifier_free_guidance, - per_request_scheduler=forward_ctx.video_audio_scheduler, - ) - return self._synchronize_cfg_parallel_step_output( - (latents, audio_latents), - do_true_cfg=self.do_classifier_free_guidance, - ) - - latents = self.scheduler.step(noise_pred_video, t, denoise_ctx.latents, return_dict=False)[0] - audio_latents = forward_ctx.audio_scheduler.step( - noise_pred_audio, - t, - denoise_ctx.audio_latents, - return_dict=False, - )[0] - return latents, audio_latents - - def _denoise_loop( - self, - forward_ctx: _LTX23ForwardContext, - denoise_ctx: _LTX23DenoiseContext, - ) -> tuple[torch.Tensor, torch.Tensor]: - request_inputs = forward_ctx.request_inputs - prompt_context = forward_ctx.prompt_context - guidance_scale = request_inputs.guidance_scale - audio_scheduler = forward_ctx.audio_scheduler - - with self.progress_bar(total=len(forward_ctx.timesteps)) as pbar: - for i, t in enumerate(forward_ctx.timesteps): - if self.interrupt: - continue - - self._current_timestep = t - - if forward_ctx.cfg_parallel_ready: - latent_model_input = denoise_ctx.latents.to(prompt_context.positive_connector_prompt_embeds.dtype) - audio_latent_model_input = denoise_ctx.audio_latents.to( - prompt_context.positive_connector_prompt_embeds.dtype - ) - ts = t.expand(latent_model_input.shape[0]) - positive_kwargs = self._build_transformer_kwargs( - forward_ctx, - denoise_ctx, - hidden_states=latent_model_input, - audio_hidden_states=audio_latent_model_input, - encoder_hidden_states=prompt_context.positive_connector_prompt_embeds, - audio_encoder_hidden_states=prompt_context.positive_connector_audio_prompt_embeds, - encoder_attention_mask=prompt_context.positive_connector_attention_mask, - audio_encoder_attention_mask=prompt_context.positive_connector_attention_mask, - ts=ts, - ) - negative_kwargs = { - **positive_kwargs, - "encoder_hidden_states": prompt_context.negative_connector_prompt_embeds, - "audio_encoder_hidden_states": prompt_context.negative_connector_audio_prompt_embeds, - "encoder_attention_mask": prompt_context.negative_connector_attention_mask, - "audio_encoder_attention_mask": prompt_context.negative_connector_attention_mask, - } - noise_pred_video, noise_pred_audio = self.predict_noise_with_parallel_cfg( - true_cfg_scale=guidance_scale, - positive_kwargs=positive_kwargs, - negative_kwargs=negative_kwargs, - cfg_normalize=False, - video_latents=denoise_ctx.latents, - audio_latents=denoise_ctx.audio_latents, - video_sigma=self.scheduler.sigmas[i], - audio_sigma=audio_scheduler.sigmas[i], - ) - else: - latent_model_input = ( - torch.cat([denoise_ctx.latents] * 2) - if self.do_classifier_free_guidance - else denoise_ctx.latents - ) - latent_model_input = latent_model_input.to(prompt_context.connector_prompt_embeds.dtype) - audio_latent_model_input = ( - torch.cat([denoise_ctx.audio_latents] * 2) - if self.do_classifier_free_guidance - else denoise_ctx.audio_latents - ) - audio_latent_model_input = audio_latent_model_input.to(prompt_context.connector_prompt_embeds.dtype) - ts = t.expand(latent_model_input.shape[0]) - - transformer_kwargs = self._build_transformer_kwargs( - forward_ctx, - denoise_ctx, - hidden_states=latent_model_input, - audio_hidden_states=audio_latent_model_input, - encoder_hidden_states=prompt_context.connector_prompt_embeds, - audio_encoder_hidden_states=prompt_context.connector_audio_prompt_embeds, - encoder_attention_mask=prompt_context.connector_attention_mask, - audio_encoder_attention_mask=prompt_context.connector_attention_mask, - ts=ts, - ) - with self._transformer_cache_context("cond_uncond"): - noise_pred_video, noise_pred_audio = self.transformer(**transformer_kwargs) - - noise_pred_video = noise_pred_video.float() - noise_pred_audio = noise_pred_audio.float() - - if self.do_classifier_free_guidance: - noise_pred_video_uncond, noise_pred_video_cond = noise_pred_video.chunk(2) - noise_pred_video = self._combine_x0_space_cfg( - denoise_ctx.latents, - noise_pred_video_cond, - noise_pred_video_uncond, - self.scheduler.sigmas[i], - guidance_scale, - ) - - noise_pred_audio_uncond, noise_pred_audio_cond = noise_pred_audio.chunk(2) - noise_pred_audio = self._combine_x0_space_cfg( - denoise_ctx.audio_latents, - noise_pred_audio_cond, - noise_pred_audio_uncond, - audio_scheduler.sigmas[i], - guidance_scale, - ) - - denoise_ctx.latents, denoise_ctx.audio_latents = self._step_denoised_latents( - forward_ctx, - denoise_ctx, - noise_pred_video, - noise_pred_audio, - t, - ) - pbar.update() - - return denoise_ctx.latents, denoise_ctx.audio_latents - - def _unpack_and_denormalize_stage( - self, - forward_ctx: _LTX23ForwardContext, - latents: torch.Tensor, - audio_latents: torch.Tensor, - ) -> tuple[torch.Tensor, torch.Tensor]: - latents = self._unpack_latents( - latents, - forward_ctx.latent_num_frames, - forward_ctx.latent_height, - forward_ctx.latent_width, - self.transformer_spatial_patch_size, - self.transformer_temporal_patch_size, - ) - latents = self._denormalize_latents( - latents, - self.vae.latents_mean, - self.vae.latents_std, - self.vae.config.scaling_factor, - ) - - audio_latents = self._unpad_audio_latents(audio_latents, forward_ctx.original_audio_num_frames) - audio_latents = self._denormalize_audio_latents( - audio_latents, - self.audio_vae.latents_mean, - self.audio_vae.latents_std, - ) - audio_latents = self._unpack_audio_latents( - audio_latents, - forward_ctx.original_audio_num_frames, - num_mel_bins=forward_ctx.latent_mel_bins, - ) - return latents, audio_latents - - def _decode_and_split( - self, - forward_ctx: _LTX23ForwardContext, - latents: torch.Tensor, - audio_latents: torch.Tensor, - ) -> list[DiffusionOutput]: - request_inputs = forward_ctx.request_inputs - return split_diffusion_output_by_request( - self._decode_output( - latents=latents, - audio_latents=audio_latents, - output_type=request_inputs.output_type, - connector_prompt_embeds=forward_ctx.prompt_context.connector_prompt_embeds, - generator=request_inputs.generator, - device=forward_ctx.device, - decode_timestep=request_inputs.decode_timestep, - decode_noise_scale=request_inputs.decode_noise_scale, - prompt_batch_size=forward_ctx.batch_size, - ), - forward_ctx.req, - num_outputs_per_prompt=forward_ctx.num_videos_per_prompt, - ) - - def _forward_impl( - self, - req: DiffusionRequestBatch, - request_inputs: _LTX23RequestInputs, - *, - noise_scale: float, - sigmas: list[float] | None, - timesteps: list[int] | None, - attention_kwargs: dict[str, Any] | None, - image: Any | None = None, - ) -> list[DiffusionOutput]: - self._check_forward_inputs(request_inputs, image=image) - cfg_parallel_ready = self._setup_forward_runtime(request_inputs, attention_kwargs) - device = self.device - prompt_context = self._prepare_prompt_context( - prompt=request_inputs.prompt, - negative_prompt=request_inputs.negative_prompt, - prompt_embeds=request_inputs.prompt_embeds, - negative_prompt_embeds=request_inputs.negative_prompt_embeds, - prompt_attention_mask=request_inputs.prompt_attention_mask, - negative_prompt_attention_mask=request_inputs.negative_prompt_attention_mask, - num_videos_per_prompt=request_inputs.num_videos_per_prompt, - max_sequence_length=request_inputs.max_sequence_length, - ) - - latent_num_frames, latent_height, latent_width = self._resolve_video_latent_dimensions(request_inputs) - latents, conditioning_mask = self._prepare_video_latents_stage( - request_inputs, - prompt_context, - device=device, - noise_scale=noise_scale, - image=image, - ) - audio_latents, original_audio_num_frames, padded_audio_num_frames, latent_mel_bins = ( - self._prepare_audio_latents_stage( - request_inputs, - prompt_context, - device=device, - noise_scale=noise_scale, - ) - ) - audio_scheduler, video_audio_scheduler, timesteps_tensor = self._prepare_scheduler_stage( - request_inputs, - device=device, - sigmas=sigmas, - timesteps=timesteps, - latent_num_frames=latent_num_frames, - latent_height=latent_height, - latent_width=latent_width, - ) - forward_ctx = _LTX23ForwardContext( - req=req, - request_inputs=request_inputs, - prompt_context=prompt_context, - device=device, - cfg_parallel_ready=cfg_parallel_ready, - attention_kwargs=attention_kwargs, - latent_num_frames=latent_num_frames, - latent_height=latent_height, - latent_width=latent_width, - latent_mel_bins=latent_mel_bins, - original_audio_num_frames=original_audio_num_frames, - padded_audio_num_frames=padded_audio_num_frames, - timesteps=timesteps_tensor, - audio_scheduler=audio_scheduler, - video_audio_scheduler=video_audio_scheduler, - ) - video_coords, audio_coords = self._prepare_rope_coords_stage(forward_ctx, latents, audio_latents) - denoise_ctx = _LTX23DenoiseContext( - latents=latents, - audio_latents=audio_latents, - video_coords=video_coords, - audio_coords=audio_coords, - conditioning_mask=conditioning_mask, - ) - denoise_ctx = self._prepare_denoise_context_for_cfg(forward_ctx, denoise_ctx) - latents, audio_latents = self._denoise_loop(forward_ctx, denoise_ctx) - latents, audio_latents = self._unpack_and_denormalize_stage(forward_ctx, latents, audio_latents) - return self._decode_and_split(forward_ctx, latents, audio_latents) - - # ------------------------------------------------------------------ - # Forward pass - # ------------------------------------------------------------------ - - @torch.no_grad() - def forward( - self, - req: DiffusionRequestBatch, - prompt: str | list[str] | None = None, - negative_prompt: str | list[str] | None = None, - height: int | None = None, - width: int | None = None, - num_frames: int | None = None, - frame_rate: float | None = None, - num_inference_steps: int | None = None, - sigmas: list[float] | None = None, - timesteps: list[int] | None = None, - guidance_scale: float = 4.0, - noise_scale: float = 0.0, - num_videos_per_prompt: int | None = 1, - generator: torch.Generator | list[torch.Generator] | None = None, - latents: torch.Tensor | None = None, - audio_latents: torch.Tensor | None = None, - prompt_embeds: torch.Tensor | None = None, - negative_prompt_embeds: torch.Tensor | None = None, - prompt_attention_mask: torch.Tensor | None = None, - negative_prompt_attention_mask: torch.Tensor | None = None, - decode_timestep: float | list[float] = 0.0, - decode_noise_scale: float | list[float] | None = None, - output_type: str = "np", - return_dict: bool = True, - attention_kwargs: dict[str, Any] | None = None, - max_sequence_length: int | None = None, - ) -> list[DiffusionOutput]: - request_inputs = self._resolve_request_inputs( - req, - prompt=prompt, - negative_prompt=negative_prompt, - height=height, - width=width, - num_frames=num_frames, - frame_rate=frame_rate, - num_inference_steps=num_inference_steps, - timesteps=timesteps, - guidance_scale=guidance_scale, - num_videos_per_prompt=num_videos_per_prompt, - generator=generator, - latents=latents, - audio_latents=audio_latents, - prompt_embeds=prompt_embeds, - negative_prompt_embeds=negative_prompt_embeds, - prompt_attention_mask=prompt_attention_mask, - negative_prompt_attention_mask=negative_prompt_attention_mask, - decode_timestep=decode_timestep, - decode_noise_scale=decode_noise_scale, - output_type=output_type, - max_sequence_length=max_sequence_length, - ) - return self._forward_impl( - req, - request_inputs, - noise_scale=noise_scale, - sigmas=sigmas, - timesteps=timesteps, - attention_kwargs=attention_kwargs, - ) - - def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - loader = AutoWeightsLoader(self) - return loader.load_weights(weights) diff --git a/vllm_omni/diffusion/models/ltx2/pipeline_ltx2_3_image2video.py b/vllm_omni/diffusion/models/ltx2/pipeline_ltx2_3_image2video.py deleted file mode 100644 index bcccd7596..000000000 --- a/vllm_omni/diffusion/models/ltx2/pipeline_ltx2_3_image2video.py +++ /dev/null @@ -1,454 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -"""LTX-2.3 image-to-video pipeline.""" - -from __future__ import annotations - -from typing import Any - -import PIL.Image -import torch -from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img import retrieve_latents -from diffusers.utils.torch_utils import randn_tensor -from diffusers.video_processor import VideoProcessor - -from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig -from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch - -from .pipeline_ltx2_3 import ( - LTX23Pipeline, - _LTX23DenoiseContext, - _LTX23ForwardContext, - _LTX23PromptContext, - _LTX23RequestInputs, - get_ltx2_post_process_func, -) -from .pipeline_ltx2_image2video import LTX2ImageToVideoPipeline, _I2VVideoAudioScheduler - - -class LTX23ImageToVideoPipeline(LTX23Pipeline): - """LTX-2.3 image-to-video pipeline. - - This keeps the LTX-2.3 prompt connector, x0-space CFG, sigma prompt - modulation, and audio branch semantics from ``LTX23Pipeline`` while - reusing the existing LTX image-conditioning contract: the first video - latent frame is encoded from the input image and remains fixed during - denoising. - """ - - def __init__(self, *, od_config: OmniDiffusionConfig, prefix: str = ""): - super().__init__(od_config=od_config, prefix=prefix) - self.video_processor = VideoProcessor(vae_scale_factor=self.vae_spatial_compression_ratio, resample="bilinear") - - support_image_input = True - - _normalize_latents = staticmethod(LTX2ImageToVideoPipeline._normalize_latents) - _create_noised_state = staticmethod(LTX2ImageToVideoPipeline._create_noised_state) - - @staticmethod - def _resolve_single_prompt_image(raw_image: Any) -> Any: - if isinstance(raw_image, list): - if len(raw_image) != 1: - raise ValueError( - "LTX-2.3 I2V prompt dictionaries support exactly one image per prompt. " - "Pass one image per prompt for batched I2V requests." - ) - return raw_image[0] - return raw_image - - @staticmethod - def _resolve_additional_image(additional: dict[str, Any]) -> Any: - raw_image = additional.get("preprocessed_image") - if raw_image is None: - raw_image = additional.get("pixel_values") - if raw_image is None: - raw_image = additional.get("image") - return raw_image - - def prepare_latents( - self, - image: torch.Tensor | None = None, - batch_size: int = 1, - num_channels_latents: int = 128, - height: int = 512, - width: int = 768, - num_frames: int = 121, - noise_scale: float = 0.0, - dtype: torch.dtype | None = None, - device: torch.device | None = None, - generator: torch.Generator | list[torch.Generator] | None = None, - latents: torch.Tensor | None = None, - ) -> tuple[torch.Tensor, torch.Tensor]: - """Prepare I2V latents and the first-frame conditioning mask. - - If caller-provided latents are used without an image, the latents must - already represent the full video state including the conditioning first - frame. Packed 3D latents are assumed to be in transformer token layout. - """ - height = height // self.vae_spatial_compression_ratio - width = width // self.vae_spatial_compression_ratio - num_frames = (num_frames - 1) // self.vae_temporal_compression_ratio + 1 - - shape = (batch_size, num_channels_latents, num_frames, height, width) - mask_shape = (batch_size, 1, num_frames, height, width) - - if latents is not None: - if latents.ndim == 5: - batch_size, _, num_frames, height, width = latents.shape - mask_shape = (batch_size, 1, num_frames, height, width) - conditioning_mask = latents.new_zeros(mask_shape) - conditioning_mask[:, :, 0] = 1.0 - - latents = self._normalize_latents( - latents, - self.vae.latents_mean, - self.vae.latents_std, - self.vae.config.scaling_factor, - ) - latents = self._create_noised_state(latents, noise_scale * (1 - conditioning_mask), generator) - latents = self._pack_latents( - latents, - self.transformer_spatial_patch_size, - self.transformer_temporal_patch_size, - ) - else: - conditioning_mask = latents.new_zeros(mask_shape) - conditioning_mask[:, :, 0] = 1.0 - - conditioning_mask = self._pack_latents( - conditioning_mask, - self.transformer_spatial_patch_size, - self.transformer_temporal_patch_size, - ).squeeze(-1) - if latents.ndim != 3 or latents.shape[:2] != conditioning_mask.shape: - raise ValueError( - "Provided `latents` tensor has shape" - f" {latents.shape}, but the expected shape is {conditioning_mask.shape + (num_channels_latents,)}." - ) - return latents.to(device=device, dtype=dtype), conditioning_mask - - if image is None: - raise ValueError("`image` must be provided when `latents` is None.") - - image_batch_size = image.shape[0] - if image_batch_size == 0: - raise ValueError("`image` batch is empty.") - if batch_size % image_batch_size != 0: - raise ValueError( - f"`batch_size` ({batch_size}) must be divisible by image batch size ({image_batch_size}) " - "for image-to-video outputs." - ) - num_videos_per_prompt = batch_size // image_batch_size - - if isinstance(generator, list): - if len(generator) != batch_size: - raise ValueError( - f"You have passed a list of generators of length {len(generator)}, but requested an effective" - f" batch size of {batch_size}. Make sure the batch size matches the length of the generators." - ) - image_generators = [generator[i * num_videos_per_prompt] for i in range(image_batch_size)] - init_latents = [ - retrieve_latents(self.vae.encode(image[i].unsqueeze(0).unsqueeze(2)), image_generators[i], "argmax") - for i in range(image_batch_size) - ] - else: - init_latents = [ - retrieve_latents(self.vae.encode(img.unsqueeze(0).unsqueeze(2)), generator, "argmax") for img in image - ] - - init_latents = torch.cat(init_latents, dim=0).to(dtype) - if num_videos_per_prompt > 1: - init_latents = init_latents.repeat_interleave(num_videos_per_prompt, dim=0) - init_latents = self._normalize_latents( - init_latents, - self.vae.latents_mean, - self.vae.latents_std, - ) - init_latents = init_latents.repeat(1, 1, num_frames, 1, 1) - - conditioning_mask = torch.zeros(mask_shape, device=device, dtype=dtype) - conditioning_mask[:, :, 0] = 1.0 - - noise = randn_tensor(shape, generator=generator, device=device, dtype=dtype) - latents = init_latents * conditioning_mask + noise * (1 - conditioning_mask) - - conditioning_mask = self._pack_latents( - conditioning_mask, - self.transformer_spatial_patch_size, - self.transformer_temporal_patch_size, - ).squeeze(-1) - latents = self._pack_latents(latents, self.transformer_spatial_patch_size, self.transformer_temporal_patch_size) - - return latents, conditioning_mask - - def check_inputs( - self, - image, - height, - width, - prompt, - latents=None, - prompt_embeds=None, - negative_prompt_embeds=None, - prompt_attention_mask=None, - negative_prompt_attention_mask=None, - ): - if image is None and latents is None: - raise ValueError("Provide either `image` or `latents`. Cannot leave both undefined.") - super().check_inputs( - prompt=prompt, - height=height, - width=width, - prompt_embeds=prompt_embeds, - negative_prompt_embeds=negative_prompt_embeds, - prompt_attention_mask=prompt_attention_mask, - negative_prompt_attention_mask=negative_prompt_attention_mask, - ) - - _step_video_latents_i2v = LTX2ImageToVideoPipeline._step_video_latents_i2v - - def _resolve_request_image( - self, - req: DiffusionRequestBatch, - image: PIL.Image.Image | torch.Tensor | list[PIL.Image.Image | torch.Tensor] | None, - request_inputs: _LTX23RequestInputs, - ) -> PIL.Image.Image | torch.Tensor | list[PIL.Image.Image | torch.Tensor] | None: - if image is not None or not req.prompts: - return image - - raw_images = [] - for prompt_item in req.prompts: - if isinstance(prompt_item, str): - raw_image = None - else: - multi_modal_data = prompt_item.get("multi_modal_data") or {} - raw_image = multi_modal_data.get("image") - if raw_image is None: - additional = prompt_item.get("additional_information") or {} - raw_image = self._resolve_additional_image(additional) - raw_image = self._resolve_single_prompt_image(raw_image) - if isinstance(raw_image, str): - raw_image = PIL.Image.open(raw_image).convert("RGB") - raw_images.append(raw_image) - - if any(img is None for img in raw_images) and request_inputs.latents is None: - raise ValueError("Image is required for LTX-2.3 I2V generation.") - if len(raw_images) == 1: - return raw_images[0] - if raw_images: - return raw_images - return image - - def _check_forward_inputs( - self, - request_inputs: _LTX23RequestInputs, - image: Any | None = None, - ) -> None: - self.check_inputs( - image=image, - height=request_inputs.height, - width=request_inputs.width, - prompt=request_inputs.prompt, - latents=request_inputs.latents, - prompt_embeds=request_inputs.prompt_embeds, - negative_prompt_embeds=request_inputs.negative_prompt_embeds, - prompt_attention_mask=request_inputs.prompt_attention_mask, - negative_prompt_attention_mask=request_inputs.negative_prompt_attention_mask, - ) - - def _resolve_video_latent_dimensions( - self, - request_inputs: _LTX23RequestInputs, - ) -> tuple[int, int, int]: - latent_num_frames = (request_inputs.num_frames - 1) // self.vae_temporal_compression_ratio + 1 - latent_height = request_inputs.height // self.vae_spatial_compression_ratio - latent_width = request_inputs.width // self.vae_spatial_compression_ratio - if request_inputs.latents is not None: - if request_inputs.latents.ndim == 5: - _, _, latent_num_frames, latent_height, latent_width = request_inputs.latents.shape - elif request_inputs.latents.ndim != 3: - raise ValueError( - f"Provided `latents` tensor has shape {request_inputs.latents.shape}, but the expected shape is " - "either [batch_size, seq_len, num_features] or " - "[batch_size, latent_dim, latent_frames, latent_height, latent_width]." - ) - return latent_num_frames, latent_height, latent_width - - def _prepare_video_latents_stage( - self, - request_inputs: _LTX23RequestInputs, - prompt_context: _LTX23PromptContext, - *, - device: torch.device, - noise_scale: float, - image: Any | None = None, - ) -> tuple[torch.Tensor, torch.Tensor | None]: - if request_inputs.latents is None: - if isinstance(image, torch.Tensor): - if image.ndim == 3: - image = image.unsqueeze(0) - elif isinstance(image, list) and image and isinstance(image[0], torch.Tensor): - image = torch.stack(image, dim=0) - else: - image = self.video_processor.preprocess( - image, - height=request_inputs.height, - width=request_inputs.width, - ) - image = image.to(device=device, dtype=prompt_context.positive_connector_prompt_embeds.dtype) - - num_channels_latents = self.transformer.config.in_channels - return self.prepare_latents( - image, - prompt_context.batch_size * request_inputs.num_videos_per_prompt, - num_channels_latents, - request_inputs.height, - request_inputs.width, - request_inputs.num_frames, - noise_scale, - torch.float32, - device, - request_inputs.generator, - request_inputs.latents, - ) - - def _make_video_audio_scheduler( - self, - audio_scheduler: Any, - latent_num_frames: int, - latent_height: int, - latent_width: int, - ) -> Any: - return _I2VVideoAudioScheduler( - self, - audio_scheduler, - latent_num_frames, - latent_height, - latent_width, - ) - - def _prepare_denoise_context_for_cfg( - self, - forward_ctx: _LTX23ForwardContext, - denoise_ctx: _LTX23DenoiseContext, - ) -> _LTX23DenoiseContext: - denoise_ctx = super()._prepare_denoise_context_for_cfg(forward_ctx, denoise_ctx) - if denoise_ctx.conditioning_mask is None: - raise ValueError("I2V denoising requires a conditioning mask.") - if self.do_classifier_free_guidance and not forward_ctx.cfg_parallel_ready: - denoise_ctx.conditioning_mask_for_model = torch.cat( - [denoise_ctx.conditioning_mask, denoise_ctx.conditioning_mask] - ) - else: - denoise_ctx.conditioning_mask_for_model = denoise_ctx.conditioning_mask - return denoise_ctx - - def _denoise_timestep_kwargs( - self, - ts: torch.Tensor, - forward_ctx: _LTX23ForwardContext, - denoise_ctx: _LTX23DenoiseContext, - ) -> dict[str, torch.Tensor]: - conditioning_mask = ( - denoise_ctx.conditioning_mask if forward_ctx.cfg_parallel_ready else denoise_ctx.conditioning_mask_for_model - ) - if conditioning_mask is None: - raise ValueError("I2V denoising requires a conditioning mask.") - return { - "timestep": ts.unsqueeze(-1) * (1 - conditioning_mask), - "audio_timestep": ts, - "sigma": ts, - } - - def _step_denoised_latents( - self, - forward_ctx: _LTX23ForwardContext, - denoise_ctx: _LTX23DenoiseContext, - noise_pred_video: torch.Tensor, - noise_pred_audio: torch.Tensor, - t: torch.Tensor, - ) -> tuple[torch.Tensor, torch.Tensor]: - latents, audio_latents = self.scheduler_step_maybe_with_cfg( - (noise_pred_video, noise_pred_audio), - (t, t), - (denoise_ctx.latents, denoise_ctx.audio_latents), - do_true_cfg=self.do_classifier_free_guidance, - per_request_scheduler=forward_ctx.video_audio_scheduler, - ) - return self._synchronize_cfg_parallel_step_output( - (latents, audio_latents), - do_true_cfg=self.do_classifier_free_guidance, - ) - - @torch.no_grad() - def forward( - self, - req: DiffusionRequestBatch, - image: PIL.Image.Image | torch.Tensor | list[PIL.Image.Image | torch.Tensor] | None = None, - prompt: str | list[str] | None = None, - negative_prompt: str | list[str] | None = None, - height: int | None = None, - width: int | None = None, - num_frames: int | None = None, - frame_rate: float | None = None, - num_inference_steps: int | None = None, - sigmas: list[float] | None = None, - timesteps: list[int] | None = None, - guidance_scale: float = 4.0, - noise_scale: float = 0.0, - num_videos_per_prompt: int | None = 1, - generator: torch.Generator | list[torch.Generator] | None = None, - latents: torch.Tensor | None = None, - audio_latents: torch.Tensor | None = None, - prompt_embeds: torch.Tensor | None = None, - negative_prompt_embeds: torch.Tensor | None = None, - prompt_attention_mask: torch.Tensor | None = None, - negative_prompt_attention_mask: torch.Tensor | None = None, - decode_timestep: float | list[float] = 0.0, - decode_noise_scale: float | list[float] | None = None, - output_type: str = "np", - return_dict: bool = True, - attention_kwargs: dict[str, Any] | None = None, - max_sequence_length: int | None = None, - ) -> list[DiffusionOutput]: - request_inputs = self._resolve_request_inputs( - req, - prompt=prompt, - negative_prompt=negative_prompt, - height=height, - width=width, - num_frames=num_frames, - frame_rate=frame_rate, - num_inference_steps=num_inference_steps, - timesteps=timesteps, - guidance_scale=guidance_scale, - num_videos_per_prompt=num_videos_per_prompt, - generator=generator, - latents=latents, - audio_latents=audio_latents, - prompt_embeds=prompt_embeds, - negative_prompt_embeds=negative_prompt_embeds, - prompt_attention_mask=prompt_attention_mask, - negative_prompt_attention_mask=negative_prompt_attention_mask, - decode_timestep=decode_timestep, - decode_noise_scale=decode_noise_scale, - output_type=output_type, - max_sequence_length=max_sequence_length, - ) - image = self._resolve_request_image(req, image, request_inputs) - return self._forward_impl( - req, - request_inputs, - noise_scale=noise_scale, - sigmas=sigmas, - timesteps=timesteps, - attention_kwargs=attention_kwargs, - image=image, - ) - - -__all__ = [ - "LTX23ImageToVideoPipeline", - "get_ltx2_post_process_func", -] diff --git a/vllm_omni/diffusion/models/ltx2/pipeline_ltx2_image2video.py b/vllm_omni/diffusion/models/ltx2/pipeline_ltx2_image2video.py deleted file mode 100644 index a1caf6ce8..000000000 --- a/vllm_omni/diffusion/models/ltx2/pipeline_ltx2_image2video.py +++ /dev/null @@ -1,909 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from __future__ import annotations - -import copy -import os -from collections.abc import Iterable -from typing import Any, ClassVar - -import numpy as np -import PIL.Image -import torch -import torch.nn as nn -from diffusers import FlowMatchEulerDiscreteScheduler -from diffusers.pipelines.ltx2.utils import DISTILLED_SIGMA_VALUES, STAGE_2_DISTILLED_SIGMA_VALUES -from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion import retrieve_timesteps -from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img import retrieve_latents -from diffusers.utils.torch_utils import randn_tensor -from diffusers.video_processor import VideoProcessor -from vllm.logger import init_logger -from vllm.model_executor.models.utils import AutoWeightsLoader - -from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig -from vllm_omni.diffusion.distributed.utils import get_local_device -from vllm_omni.diffusion.lora.manager import DiffusionLoRAManager -from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader -from vllm_omni.diffusion.models.dmd2 import DMD2PipelineMixin -from vllm_omni.diffusion.models.interface import SupportsComponentDiscovery -from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch -from vllm_omni.lora.request import LoRARequest - -from .pipeline_ltx2 import ( - LTX2Pipeline, - _get_prompt_field, - calculate_shift, -) -from .pipeline_ltx2 import ( - get_ltx2_post_process_func as _get_ltx2_post_process_func, -) -from .pipeline_ltx2_latent_upsample import LTX2LatentUpsamplePipeline - -logger = init_logger(__name__) - - -def get_ltx2_post_process_func(od_config: OmniDiffusionConfig): - return _get_ltx2_post_process_func(od_config) - - -class _I2VVideoAudioScheduler: - """Composite scheduler for I2V: uses _step_video_latents_i2v for video, standard step for audio.""" - - def __init__(self, pipeline, audio_scheduler, latent_num_frames, latent_height, latent_width): - self.video_scheduler = pipeline.scheduler - self.audio_scheduler = audio_scheduler - self._pipeline = pipeline - self._latent_num_frames = latent_num_frames - self._latent_height = latent_height - self._latent_width = latent_width - - def step(self, noise_pred, t, latents, return_dict=False, generator=None): - video_out = self._pipeline._step_video_latents_i2v( - noise_pred[0], - latents[0], - t[0], - self._latent_num_frames, - self._latent_height, - self._latent_width, - ) - audio_out = self.audio_scheduler.step(noise_pred[1], t[1], latents[1], return_dict=False, generator=generator)[ - 0 - ] - return ((video_out, audio_out),) - - -class LTX2ImageToVideoPipeline(LTX2Pipeline): - supports_request_batch = False - support_image_input = True - - def __init__( - self, - *, - od_config: OmniDiffusionConfig, - prefix: str = "", - ): - super().__init__(od_config=od_config, prefix=prefix) - self.video_processor = VideoProcessor(vae_scale_factor=self.vae_spatial_compression_ratio, resample="bilinear") - - @staticmethod - def _normalize_latents( - latents: torch.Tensor, latents_mean: torch.Tensor, latents_std: torch.Tensor, scaling_factor: float = 1.0 - ) -> torch.Tensor: - latents_mean = latents_mean.view(1, -1, 1, 1, 1).to(latents.device, latents.dtype) - latents_std = latents_std.view(1, -1, 1, 1, 1).to(latents.device, latents.dtype) - latents = (latents - latents_mean) * scaling_factor / latents_std - return latents - - def prepare_latents( - self, - image: torch.Tensor | None = None, - batch_size: int = 1, - num_channels_latents: int = 128, - height: int = 512, - width: int = 768, - num_frames: int = 121, - noise_scale: float = 0.0, - dtype: torch.dtype | None = None, - device: torch.device | None = None, - generator: torch.Generator | list[torch.Generator] | None = None, - latents: torch.Tensor | None = None, - ) -> tuple[torch.Tensor, torch.Tensor]: - height = height // self.vae_spatial_compression_ratio - width = width // self.vae_spatial_compression_ratio - num_frames = (num_frames - 1) // self.vae_temporal_compression_ratio + 1 - - shape = (batch_size, num_channels_latents, num_frames, height, width) - mask_shape = (batch_size, 1, num_frames, height, width) - - if latents is not None: - if latents.ndim == 5: - # conditioning_mask needs to the same shape as latents in two stages generation. - batch_size, _, num_frames, height, width = latents.shape - mask_shape = (batch_size, 1, num_frames, height, width) - conditioning_mask = latents.new_zeros(mask_shape) - conditioning_mask[:, :, 0] = 1.0 - - latents = self._normalize_latents( - latents, self.vae.latents_mean, self.vae.latents_std, self.vae.config.scaling_factor - ) - latents = self._create_noised_state(latents, noise_scale * (1 - conditioning_mask), generator) - # latents are of shape [B, C, F, H, W], need to be packed - latents = self._pack_latents( - latents, self.transformer_spatial_patch_size, self.transformer_temporal_patch_size - ) - else: - conditioning_mask = latents.new_zeros(mask_shape) - conditioning_mask[:, :, 0] = 1.0 - - conditioning_mask = self._pack_latents( - conditioning_mask, self.transformer_spatial_patch_size, self.transformer_temporal_patch_size - ).squeeze(-1) - - if latents.ndim != 3 or latents.shape[:2] != conditioning_mask.shape: - raise ValueError( - "Provided `latents` tensor has shape" - f" {latents.shape}, but the expected shape is {conditioning_mask.shape + (num_channels_latents,)}." - ) - return latents.to(device=device, dtype=dtype), conditioning_mask - - if image is None: - raise ValueError("`image` must be provided when `latents` is None.") - - image_batch_size = image.shape[0] - if image_batch_size == 0: - raise ValueError("`image` batch is empty.") - if batch_size % image_batch_size != 0: - raise ValueError( - f"`batch_size` ({batch_size}) must be divisible by image batch size ({image_batch_size}) " - "for image-to-video outputs." - ) - num_videos_per_prompt = batch_size // image_batch_size - - if isinstance(generator, list): - if len(generator) != batch_size: - raise ValueError( - f"You have passed a list of generators of length {len(generator)}, but requested an effective" - f" batch size of {batch_size}. Make sure the batch size matches the length of the generators." - ) - - image_generators = [generator[i * num_videos_per_prompt] for i in range(image_batch_size)] - init_latents = [ - retrieve_latents(self.vae.encode(image[i].unsqueeze(0).unsqueeze(2)), image_generators[i], "argmax") - for i in range(image_batch_size) - ] - else: - init_latents = [ - retrieve_latents(self.vae.encode(img.unsqueeze(0).unsqueeze(2)), generator, "argmax") for img in image - ] - - init_latents = torch.cat(init_latents, dim=0).to(dtype) - if num_videos_per_prompt > 1: - init_latents = init_latents.repeat_interleave(num_videos_per_prompt, dim=0) - init_latents = self._normalize_latents(init_latents, self.vae.latents_mean, self.vae.latents_std) - init_latents = init_latents.repeat(1, 1, num_frames, 1, 1) - - conditioning_mask = torch.zeros(mask_shape, device=device, dtype=dtype) - conditioning_mask[:, :, 0] = 1.0 - - noise = randn_tensor(shape, generator=generator, device=device, dtype=dtype) - latents = init_latents * conditioning_mask + noise * (1 - conditioning_mask) - - conditioning_mask = self._pack_latents( - conditioning_mask, self.transformer_spatial_patch_size, self.transformer_temporal_patch_size - ).squeeze(-1) - latents = self._pack_latents(latents, self.transformer_spatial_patch_size, self.transformer_temporal_patch_size) - - return latents, conditioning_mask - - def check_inputs( - self, - image, - height, - width, - prompt, - latents=None, - prompt_embeds=None, - negative_prompt_embeds=None, - prompt_attention_mask=None, - negative_prompt_attention_mask=None, - ): - if height % 32 != 0 or width % 32 != 0: - raise ValueError(f"`height` and `width` have to be divisible by 32 but are {height} and {width}.") - - if image is None and latents is None: - raise ValueError("Provide either `image` or `latents`. Cannot leave both undefined.") - - if prompt is not None and prompt_embeds is not None: - raise ValueError( - f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" - " only forward one of the two." - ) - if prompt is None and prompt_embeds is None: - raise ValueError( - "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." - ) - if prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): - raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") - - if prompt_embeds is not None and prompt_attention_mask is None: - raise ValueError("Must provide `prompt_attention_mask` when specifying `prompt_embeds`.") - - if negative_prompt_embeds is not None and negative_prompt_attention_mask is None: - raise ValueError("Must provide `negative_prompt_attention_mask` when specifying `negative_prompt_embeds`.") - - if prompt_embeds is not None and negative_prompt_embeds is not None: - if prompt_embeds.shape != negative_prompt_embeds.shape: - raise ValueError( - "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but" - f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`" - f" {negative_prompt_embeds.shape}." - ) - if prompt_attention_mask.shape != negative_prompt_attention_mask.shape: - raise ValueError( - "`prompt_attention_mask` and `negative_prompt_attention_mask` must have the same shape when" - " passed directly, but" - f" got: `prompt_attention_mask` {prompt_attention_mask.shape} != `negative_prompt_attention_mask`" - f" {negative_prompt_attention_mask.shape}." - ) - - def _step_video_latents_i2v( - self, - noise_pred_video: torch.Tensor, - latents: torch.Tensor, - t: torch.Tensor, - latent_num_frames: int, - latent_height: int, - latent_width: int, - ) -> torch.Tensor: - noise_pred_video = self._unpack_latents( - noise_pred_video, - latent_num_frames, - latent_height, - latent_width, - self.transformer_spatial_patch_size, - self.transformer_temporal_patch_size, - ) - latents_unpacked = self._unpack_latents( - latents, - latent_num_frames, - latent_height, - latent_width, - self.transformer_spatial_patch_size, - self.transformer_temporal_patch_size, - ) - - noise_pred_video = noise_pred_video[:, :, 1:] - noise_latents = latents_unpacked[:, :, 1:] - pred_latents = self.scheduler.step(noise_pred_video, t, noise_latents, return_dict=False)[0] - - latents_unpacked = torch.cat([latents_unpacked[:, :, :1], pred_latents], dim=2) - latents = self._pack_latents( - latents_unpacked, self.transformer_spatial_patch_size, self.transformer_temporal_patch_size - ) - return latents - - @torch.no_grad() - def forward( - self, - req: DiffusionRequestBatch, - image: PIL.Image.Image | torch.Tensor | None = None, - prompt: str | list[str] | None = None, - negative_prompt: str | list[str] | None = None, - height: int | None = None, - width: int | None = None, - num_frames: int | None = None, - frame_rate: float | None = None, - num_inference_steps: int | None = None, - sigmas: list[float] | None = None, - timesteps: list[int] | None = None, - guidance_scale: float = 4.0, - guidance_rescale: float = 0.0, - noise_scale: float = 0.0, - num_videos_per_prompt: int | None = 1, - generator: torch.Generator | list[torch.Generator] | None = None, - latents: torch.Tensor | None = None, - audio_latents: torch.Tensor | None = None, - prompt_embeds: torch.Tensor | None = None, - negative_prompt_embeds: torch.Tensor | None = None, - prompt_attention_mask: torch.Tensor | None = None, - negative_prompt_attention_mask: torch.Tensor | None = None, - decode_timestep: float | list[float] = 0.0, - decode_noise_scale: float | list[float] | None = None, - output_type: str = "np", - return_dict: bool = True, - attention_kwargs: dict[str, Any] | None = None, - max_sequence_length: int | None = None, - ) -> DiffusionOutput: - # Extract prompt/negative_prompt from request. - # Input format: req.prompts is a list of str or dict with "prompt"/"negative_prompt" keys. - prompt = [p if isinstance(p, str) else (p.get("prompt") or "") for p in req.prompts] or prompt - if all(isinstance(p, str) or p.get("negative_prompt") is None for p in req.prompts): - negative_prompt = None - elif req.prompts: - negative_prompt = ["" if isinstance(p, str) else (p.get("negative_prompt") or "") for p in req.prompts] - - height = req.sampling_params.height or height or 512 - width = req.sampling_params.width or width or 768 - num_frames = req.sampling_params.num_frames or num_frames or 121 - frame_rate = req.sampling_params.resolved_frame_rate or frame_rate or 24.0 - num_inference_steps = req.sampling_params.num_inference_steps or num_inference_steps or 40 - if timesteps is None: - num_inference_steps = max(int(num_inference_steps), 2) - elif len(timesteps) < 2: - raise ValueError("`timesteps` must contain at least 2 values for FlowMatchEulerDiscreteScheduler.") - num_videos_per_prompt = ( - req.sampling_params.num_outputs_per_prompt - if req.sampling_params.num_outputs_per_prompt > 0 - else num_videos_per_prompt or 1 - ) - max_sequence_length = ( - req.sampling_params.max_sequence_length or max_sequence_length or self.tokenizer_max_length - ) - - if req.sampling_params.guidance_scale_provided: - guidance_scale = req.sampling_params.guidance_scale - if req.sampling_params.guidance_rescale is not None: - guidance_rescale = req.sampling_params.guidance_rescale - - if generator is None: - generator = req.sampling_params.generator - if generator is None and req.sampling_params.seed is not None: - generator = torch.Generator(device=self.device).manual_seed(req.sampling_params.seed) - - latents = req.sampling_params.latents if req.sampling_params.latents is not None else latents - audio_latents = ( - req.sampling_params.audio_latents - if req.sampling_params.audio_latents is not None - else req.sampling_params.extra_args.get("audio_latents", audio_latents) - ) - - # Override with pre-computed embeddings if provided in request. - req_prompt_embeds = [_get_prompt_field(p, "prompt_embeds") for p in req.prompts] - if any(p is not None for p in req_prompt_embeds): - prompt_embeds = torch.stack(req_prompt_embeds) # type: ignore[arg-type] - - req_negative_prompt_embeds = [_get_prompt_field(p, "negative_prompt_embeds") for p in req.prompts] - if any(p is not None for p in req_negative_prompt_embeds): - negative_prompt_embeds = torch.stack(req_negative_prompt_embeds) # type: ignore[arg-type] - - req_prompt_attention_masks = [ - _get_prompt_field(p, "prompt_attention_mask") or _get_prompt_field(p, "attention_mask") for p in req.prompts - ] - if any(m is not None for m in req_prompt_attention_masks): - prompt_attention_mask = torch.stack(req_prompt_attention_masks) # type: ignore[arg-type] - - req_negative_attention_masks = [ - _get_prompt_field(p, "negative_prompt_attention_mask") or _get_prompt_field(p, "negative_attention_mask") - for p in req.prompts - ] - if any(m is not None for m in req_negative_attention_masks): - negative_prompt_attention_mask = torch.stack(req_negative_attention_masks) # type: ignore[arg-type] - - if req.sampling_params.decode_timestep is not None: - decode_timestep = req.sampling_params.decode_timestep - if req.sampling_params.decode_noise_scale is not None: - decode_noise_scale = req.sampling_params.decode_noise_scale - if req.sampling_params.output_type is not None: - output_type = req.sampling_params.output_type - - if image is None and req.prompts: - raw_images = [] - for prompt_item in req.prompts: - if isinstance(prompt_item, str): - raw_image = None - else: - multi_modal_data = prompt_item.get("multi_modal_data") or {} - raw_image = multi_modal_data.get("image") - if raw_image is None: - additional = prompt_item.get("additional_information") or {} - raw_image = ( - additional.get("preprocessed_image") - or additional.get("pixel_values") - or additional.get("image") - ) - if isinstance(raw_image, list): - raw_image = raw_image[0] if raw_image else None - if isinstance(raw_image, str): - raw_image = PIL.Image.open(raw_image).convert("RGB") - raw_images.append(raw_image) - - if any(img is None for img in raw_images): - if latents is None: - raise ValueError("Image is required for LTX2 I2V generation.") - if len(raw_images) == 1: - image = raw_images[0] - elif raw_images: - image = raw_images - - self.check_inputs( - image=image, - height=height, - width=width, - prompt=prompt, - latents=latents, - prompt_embeds=prompt_embeds, - negative_prompt_embeds=negative_prompt_embeds, - prompt_attention_mask=prompt_attention_mask, - negative_prompt_attention_mask=negative_prompt_attention_mask, - ) - - self._guidance_scale = guidance_scale - self._guidance_rescale = guidance_rescale - self._attention_kwargs = attention_kwargs - self._interrupt = False - self._current_timestep = None - - if prompt is not None and isinstance(prompt, str): - batch_size = 1 - elif prompt is not None and isinstance(prompt, list): - batch_size = len(prompt) - else: - batch_size = prompt_embeds.shape[0] - - device = self.device - - ( - prompt_embeds, - prompt_attention_mask, - negative_prompt_embeds, - negative_prompt_attention_mask, - ) = self.encode_prompt( - prompt=prompt, - negative_prompt=negative_prompt, - do_classifier_free_guidance=self.do_classifier_free_guidance, - num_videos_per_prompt=num_videos_per_prompt, - prompt_embeds=prompt_embeds, - negative_prompt_embeds=negative_prompt_embeds, - prompt_attention_mask=prompt_attention_mask, - negative_prompt_attention_mask=negative_prompt_attention_mask, - max_sequence_length=max_sequence_length, - device=device, - ) - # Compute positive prompt connectors - tokenizer_padding_side = getattr(self.tokenizer, "padding_side", "left") - connector_prompt_embeds, connector_audio_prompt_embeds, connector_attention_mask = self.connectors( - prompt_embeds, prompt_attention_mask, padding_side=tokenizer_padding_side - ) - - # Compute negative prompt connectors when CFG is enabled - negative_connector_prompt_embeds = None - negative_connector_audio_prompt_embeds = None - negative_connector_attention_mask = None - if self.do_classifier_free_guidance: - ( - negative_connector_prompt_embeds, - negative_connector_audio_prompt_embeds, - negative_connector_attention_mask, - ) = self.connectors( - negative_prompt_embeds, - negative_prompt_attention_mask, - padding_side=tokenizer_padding_side, - ) - - latent_num_frames = (num_frames - 1) // self.vae_temporal_compression_ratio + 1 - latent_height = height // self.vae_spatial_compression_ratio - latent_width = width // self.vae_spatial_compression_ratio - if latents is not None: - if latents.ndim == 5: - logger.info( - "Got latents of shape [batch_size, latent_dim, latent_frames, latent_height, latent_width], `latent_num_frames`, `latent_height`, `latent_width` will be inferred." # noqa - ) - _, _, latent_num_frames, latent_height, latent_width = latents.shape # [B, C, F, H, W] - elif latents.ndim == 3: - logger.warning( - f"You have supplied packed `latents` of shape {latents.shape}, so the latent dims cannot be" - f" inferred. Make sure the supplied `height`, `width`, and `num_frames` are correct." - ) - else: - raise ValueError( - f"Provided `latents` tensor has shape {latents.shape}, but the expected shape is either [batch_size, seq_len, num_features] or [batch_size, latent_dim, latent_frames, latent_height, latent_width]." # noqa - ) - video_sequence_length = latent_num_frames * latent_height * latent_width - - if latents is None: - if isinstance(image, torch.Tensor): - if image.ndim == 3: - image = image.unsqueeze(0) - elif isinstance(image, list) and image and isinstance(image[0], torch.Tensor): - image = torch.stack(image, dim=0) - else: - image = self.video_processor.preprocess(image, height=height, width=width) - image = image.to(device=device, dtype=prompt_embeds.dtype) - - num_channels_latents = self.transformer.config.in_channels - latents, conditioning_mask = self.prepare_latents( - image, - batch_size * num_videos_per_prompt, - num_channels_latents, - height, - width, - num_frames, - noise_scale, - torch.float32, - device, - generator, - latents, - ) - - duration_s = num_frames / frame_rate - audio_latents_per_second = ( - self.audio_sampling_rate / self.audio_hop_length / float(self.audio_vae_temporal_compression_ratio) - ) - audio_num_frames = round(duration_s * audio_latents_per_second) - if audio_latents is not None: - if audio_latents.ndim == 4: - logger.info( - "Got audio_latents of shape [batch_size, num_channels, audio_length, mel_bins], `audio_num_frames` will be inferred." # noqa - ) - _, _, audio_num_frames, _ = audio_latents.shape # [B, C, L, M] - elif audio_latents.ndim == 3: - logger.warning( - f"You have supplied packed `audio_latents` of shape {audio_latents.shape}, so the latent dims" - f" cannot be inferred. Make sure the supplied `num_frames` and `frame_rate` are correct." - ) - else: - raise ValueError( - f"Provided `audio_latents` tensor has shape {audio_latents.shape}, but the expected shape is either [batch_size, seq_len, num_features] or [batch_size, num_channels, audio_length, mel_bins]." # noqa - ) - - num_mel_bins = self.audio_vae.config.mel_bins if getattr(self, "audio_vae", None) is not None else 64 - latent_mel_bins = num_mel_bins // self.audio_vae_mel_compression_ratio - num_channels_latents_audio = ( - self.audio_vae.config.latent_channels if getattr(self, "audio_vae", None) is not None else 8 - ) - audio_latents, original_audio_num_frames, padded_audio_num_frames = self.prepare_audio_latents( - batch_size * num_videos_per_prompt, - num_channels_latents=num_channels_latents_audio, - audio_latent_length=audio_num_frames, - num_mel_bins=num_mel_bins, - noise_scale=noise_scale, - dtype=torch.float32, - device=device, - generator=generator, - latents=audio_latents, - ) - - sigmas = np.linspace(1.0, 1 / num_inference_steps, num_inference_steps) if sigmas is None else sigmas - mu = calculate_shift( - video_sequence_length, - self.scheduler.config.get("base_image_seq_len", 1024), - self.scheduler.config.get("max_image_seq_len", 4096), - self.scheduler.config.get("base_shift", 0.95), - self.scheduler.config.get("max_shift", 2.05), - ) - audio_scheduler = copy.deepcopy(self.scheduler) - _ = retrieve_timesteps( - audio_scheduler, - num_inference_steps, - device, - timesteps, - sigmas=sigmas, - mu=mu, - ) - timesteps, num_inference_steps = retrieve_timesteps( - self.scheduler, - num_inference_steps, - device, - timesteps, - sigmas=sigmas, - mu=mu, - ) - self._num_timesteps = len(timesteps) - - video_coords = self.transformer.rope.prepare_video_coords( - latents.shape[0], latent_num_frames, latent_height, latent_width, latents.device, fps=frame_rate - ) - audio_coords = self.transformer.audio_rope.prepare_audio_coords( - audio_latents.shape[0], padded_audio_num_frames, audio_latents.device - ) - - i2v_scheduler = _I2VVideoAudioScheduler( - pipeline=self, - audio_scheduler=audio_scheduler, - latent_num_frames=latent_num_frames, - latent_height=latent_height, - latent_width=latent_width, - ) - # No coord duplication needed: mixin handles CFG via separate forward calls. - - with self.progress_bar(total=len(timesteps)) as pbar: - for i, t in enumerate(timesteps): - if self.interrupt: - continue - - self._current_timestep = t - - latent_model_input = latents.to(prompt_embeds.dtype) - audio_latent_model_input = audio_latents.to(prompt_embeds.dtype) - timestep = t.expand(latent_model_input.shape[0]) - video_timestep = timestep.unsqueeze(-1) * (1 - conditioning_mask) - do_true_cfg = self.do_classifier_free_guidance - - positive_kwargs = { - "hidden_states": latent_model_input, - "audio_hidden_states": audio_latent_model_input, - "encoder_hidden_states": connector_prompt_embeds, - "audio_encoder_hidden_states": connector_audio_prompt_embeds, - "timestep": video_timestep, - "audio_timestep": timestep, - "encoder_attention_mask": connector_attention_mask, - "audio_encoder_attention_mask": connector_attention_mask, - "num_frames": latent_num_frames, - "height": latent_height, - "width": latent_width, - "fps": frame_rate, - "audio_num_frames": padded_audio_num_frames, - "video_coords": video_coords, - "audio_coords": audio_coords, - "attention_kwargs": attention_kwargs, - "return_dict": False, - } - negative_kwargs = ( - { - **positive_kwargs, - "encoder_hidden_states": negative_connector_prompt_embeds, - "audio_encoder_hidden_states": negative_connector_audio_prompt_embeds, - "encoder_attention_mask": negative_connector_attention_mask, - "audio_encoder_attention_mask": negative_connector_attention_mask, - } - if do_true_cfg - else None - ) - - noise_pred_video, noise_pred_audio = self.predict_noise_maybe_with_cfg( - do_true_cfg=do_true_cfg, - true_cfg_scale=guidance_scale, - positive_kwargs=positive_kwargs, - negative_kwargs=negative_kwargs, - cfg_normalize=False, - ) - - latents, audio_latents = self.scheduler_step_maybe_with_cfg( - (noise_pred_video, noise_pred_audio), - (t, t), - (latents, audio_latents), - do_true_cfg=do_true_cfg, - per_request_scheduler=i2v_scheduler, - ) - latents, audio_latents = self._synchronize_cfg_parallel_step_output( - (latents, audio_latents), - do_true_cfg=do_true_cfg, - ) - - pbar.update() - - latents = self._unpack_latents( - latents, - latent_num_frames, - latent_height, - latent_width, - self.transformer_spatial_patch_size, - self.transformer_temporal_patch_size, - ) - latents = self._denormalize_latents( - latents, self.vae.latents_mean, self.vae.latents_std, self.vae.config.scaling_factor - ) - - audio_latents = self._unpad_audio_latents(audio_latents, original_audio_num_frames) - audio_latents = self._denormalize_audio_latents( - audio_latents, self.audio_vae.latents_mean, self.audio_vae.latents_std - ) - audio_latents = self._unpack_audio_latents( - audio_latents, - original_audio_num_frames, - num_mel_bins=latent_mel_bins, - ) - - if output_type == "latent": - video = latents - audio = audio_latents - else: - latents = latents.to(prompt_embeds.dtype) - - if not self.vae.config.timestep_conditioning: - timestep = None - else: - noise = randn_tensor(latents.shape, generator=generator, device=device, dtype=latents.dtype) - if not isinstance(decode_timestep, list): - decode_timestep = [decode_timestep] * batch_size - if decode_noise_scale is None: - decode_noise_scale = decode_timestep - elif not isinstance(decode_noise_scale, list): - decode_noise_scale = [decode_noise_scale] * batch_size - - timestep = torch.tensor(decode_timestep, device=device, dtype=latents.dtype) - decode_noise_scale = torch.tensor(decode_noise_scale, device=device, dtype=latents.dtype)[ - :, None, None, None, None - ] - latents = (1 - decode_noise_scale) * latents + decode_noise_scale * noise - - latents = latents.to(self.vae.dtype) - video = self.vae.decode(latents, timestep, return_dict=False)[0] - video = self.video_processor.postprocess_video(video, output_type=output_type) - - audio_latents = audio_latents.to(self.audio_vae.dtype) - generated_mel_spectrograms = self.audio_vae.decode(audio_latents, return_dict=False)[0] - audio = self.vocoder(generated_mel_spectrograms) - - return DiffusionOutput(output=(video, audio)) - - -class LTX2ImageToVideoTwoStagesPipeline(nn.Module, SupportsComponentDiscovery): - """LTXImageToVideoTwoStagesPipeline is for two stages image to video generation""" - - supports_request_batch = False - support_image_input = True - dummy_run_num_frames = 2 - - _dit_modules: ClassVar[list[str]] = ["pipe.transformer"] - _encoder_modules: ClassVar[list[str]] = ["pipe.text_encoder"] - _vae_modules: ClassVar[list[str]] = ["pipe.vae", "pipe.audio_vae"] - - def __init__( - self, - *, - od_config: OmniDiffusionConfig, - prefix: str = "", - ): - super().__init__() - - self.device = get_local_device() - self.dtype = getattr(od_config, "dtype", torch.bfloat16) - self.model_path = od_config.model - self.distilled = False - # User provided model path may contain '/' in the end and basename function - # will not return the expected directory name, so we need to remove it by normpath - if "distilled" in os.path.basename(os.path.normpath(self.model_path)): - self.distilled = True - else: - raise NotImplementedError(f"{self.model_path} is not supported for {self.__class__.__name__}.") - - self.pipe = LTX2ImageToVideoPipeline(od_config=od_config, prefix=prefix) - self.upsample_pipe = LTX2LatentUpsamplePipeline( - vae=self.pipe.vae, - od_config=od_config, - ) - - self.lora_manager = DiffusionLoRAManager( - pipeline=self.pipe, - device=self.device, - dtype=self.dtype, - max_cached_adapters=od_config.max_cpu_loras, - ) - - self.weights_sources = [ - DiffusersPipelineLoader.ComponentSource( - model_or_path=od_config.model, - subfolder="transformer", - revision=None, - prefix="pipe.transformer.", - fall_back_to_pt=True, - ), - ] - - @torch.no_grad() - def forward( - self, - req: DiffusionRequestBatch, - image: PIL.Image.Image | torch.Tensor | None = None, - prompt: str | list[str] | None = None, - negative_prompt: str | list[str] | None = None, - height: int | None = None, - width: int | None = None, - num_frames: int | None = None, - frame_rate: float | None = None, - num_inference_steps: int | None = None, - sigmas: list[float] | None = None, - timesteps: list[int] | None = None, - guidance_scale: float = 4.0, - guidance_rescale: float = 0.0, - noise_scale: float = 0.0, - num_videos_per_prompt: int | None = 1, - generator: torch.Generator | list[torch.Generator] | None = None, - latents: torch.Tensor | None = None, - audio_latents: torch.Tensor | None = None, - prompt_embeds: torch.Tensor | None = None, - negative_prompt_embeds: torch.Tensor | None = None, - prompt_attention_mask: torch.Tensor | None = None, - negative_prompt_attention_mask: torch.Tensor | None = None, - decode_timestep: float | list[float] = 0.0, - decode_noise_scale: float | list[float] | None = None, - output_type: str = "np", - return_dict: bool = True, - attention_kwargs: dict[str, Any] | None = None, - max_sequence_length: int | None = None, - ) -> DiffusionOutput: - stage1_output = self.pipe( - req=req, - image=image, - prompt=prompt, - negative_prompt=negative_prompt, - height=height, - width=width, - num_frames=num_frames, - frame_rate=frame_rate, - num_inference_steps=num_inference_steps, - sigmas=DISTILLED_SIGMA_VALUES if self.distilled else None, - timesteps=timesteps, - guidance_scale=guidance_scale, - guidance_rescale=guidance_rescale, - noise_scale=noise_scale, - num_videos_per_prompt=num_videos_per_prompt, - generator=generator, - latents=latents, - audio_latents=audio_latents, - prompt_embeds=prompt_embeds, - negative_prompt_embeds=negative_prompt_embeds, - prompt_attention_mask=prompt_attention_mask, - negative_prompt_attention_mask=negative_prompt_attention_mask, - decode_timestep=decode_timestep, - decode_noise_scale=decode_noise_scale, - output_type="latent", - return_dict=return_dict, - attention_kwargs=attention_kwargs, - max_sequence_length=max_sequence_length, - ) - video_latent, audio_latent = stage1_output.output - - upscaled_video_latent = self.upsample_pipe( - latents=video_latent, - output_type="latent", - return_dict=False, - )[0] - - if not self.distilled: - # Load Stage 2 distilled LoRA - lora_path = f"{self.model_path}/ltx-2-19b-distilled-lora-384.safetensors" - lora_request = LoRARequest( - lora_name="stage_2_distilled", - lora_int_id=1, - lora_path=lora_path, - ) - self.lora_manager.set_active_adapter(lora_request, lora_scale=1.0) - - # Change scheduler to use Stage 2 distilled sigmas as is - new_scheduler = FlowMatchEulerDiscreteScheduler.from_config( - self.pipe.scheduler.config, - use_dynamic_shifting=False, - shift_terminal=None, - ) - self.pipe.scheduler = new_scheduler - - # `req` is a DiffusionRequestBatch whose `sampling_params` is a - # read-only property; clone the underlying request(s) and override - # their sampling params instead of assigning to the batch. - stage_2_req = copy.copy(req) - stage_2_req.requests = [copy.copy(r) for r in req.requests] - for stage_2_request in stage_2_req.requests: - stage_2_request.sampling_params = stage_2_request.sampling_params.clone() - stage_2_request.sampling_params.num_inference_steps = 3 - - stage2_output = self.pipe( - req=stage_2_req, - latents=upscaled_video_latent, - audio_latents=audio_latent, - prompt=prompt, - negative_prompt=negative_prompt, - noise_scale=STAGE_2_DISTILLED_SIGMA_VALUES[0], - sigmas=STAGE_2_DISTILLED_SIGMA_VALUES, - guidance_scale=1.0, - generator=generator, - output_type="np", - return_dict=False, - ) - video, audio = stage2_output.output - - return DiffusionOutput(output=(video, audio)) - - def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - loader = AutoWeightsLoader(self) - return loader.load_weights(weights) - - -class LTX2I2VDMD2Pipeline(DMD2PipelineMixin, LTX2ImageToVideoPipeline): - """LTX-2 I2V pipeline for FastGen DMD2-distilled models.""" - - def __init__(self, *, od_config: OmniDiffusionConfig, prefix: str = ""): - super().__init__(od_config=od_config, prefix=prefix) - self.__init_dmd2__() diff --git a/vllm_omni/diffusion/models/ltx2/pipeline_ltx2_two_stage.py b/vllm_omni/diffusion/models/ltx2/pipeline_ltx2_two_stage.py new file mode 100644 index 000000000..8b786b817 --- /dev/null +++ b/vllm_omni/diffusion/models/ltx2/pipeline_ltx2_two_stage.py @@ -0,0 +1,207 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +"""Two-stage entry points for the LTX model family.""" + +from __future__ import annotations + +import os +from dataclasses import replace +from typing import Any, ClassVar + +import torch +from diffusers.pipelines.ltx2.utils import DISTILLED_SIGMA_VALUES, STAGE_2_DISTILLED_SIGMA_VALUES + +from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig +from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch + +from .ltx2_components import ( + LTX2_COMPONENT_PROFILE, +) +from .ltx2_components import ( + get_ltx2_post_process_func as get_ltx2_post_process_func, # noqa: F401 +) +from .ltx2_conditioning import LTXI2VConditioningMixin +from .ltx2_guidance import LTX_LEGACY_VELOCITY_GUIDANCE +from .ltx2_pipeline_runtime import LTXPipelineRuntime +from .ltx2_recipes import LTX2_ONE_STAGE_RECIPE +from .ltx2_request import LTXRequestInputs +from .pipeline_ltx2_latent_upsample import LTX2LatentUpsamplePipeline + + +class LTX2TwoStagesPipeline(LTXPipelineRuntime): + """Legacy distilled-only LTX2 two-stage compatibility entry.""" + + component_profile = LTX2_COMPONENT_PROFILE + guidance_strategy = LTX_LEGACY_VELOCITY_GUIDANCE + one_stage_recipe = LTX2_ONE_STAGE_RECIPE + supports_request_batch = False + support_image_input = False + + _dit_modules: ClassVar[list[str]] = list(component_profile.dit_modules) + _encoder_modules: ClassVar[list[str]] = list(component_profile.encoder_modules) + _vae_modules: ClassVar[list[str]] = list(component_profile.vae_modules) + _resident_modules: ClassVar[list[str]] = list(component_profile.resident_modules) + + def __init__(self, *, od_config: OmniDiffusionConfig, prefix: str = ""): + model_path = od_config.model + self.distilled = "distilled" in os.path.basename(os.path.normpath(model_path)) + if not self.distilled: + raise NotImplementedError(f"{model_path} is not supported for {self.__class__.__name__}.") + + super().__init__(od_config=od_config, prefix=prefix) + self.upsample_pipe = LTX2LatentUpsamplePipeline(vae=self.vae, od_config=od_config) + + def _run_two_stage( + self, + req: DiffusionRequestBatch, + request_inputs: LTXRequestInputs, + *, + noise_scale: float, + timesteps: list[int] | None, + attention_kwargs: dict[str, Any] | None, + image: Any | None = None, + ) -> DiffusionOutput | list[DiffusionOutput]: + stage1 = self.run_phase( + req, + request_inputs, + noise_scale=noise_scale, + sigmas=DISTILLED_SIGMA_VALUES if self.distilled else None, + timesteps=timesteps, + attention_kwargs=attention_kwargs, + image=image, + ) + upscaled_video_latent = self.upsample_pipe( + latents=stage1.video, + output_type="latent", + return_dict=False, + )[0] + + stage2_inputs = replace( + request_inputs, + num_inference_steps=3, + guidance_scale=1.0, + latents=upscaled_video_latent, + audio_latents=stage1.audio, + decode_timestep=0.0, + decode_noise_scale=None, + output_type="np", + ) + stage2 = self.run_phase( + req, + stage2_inputs, + noise_scale=STAGE_2_DISTILLED_SIGMA_VALUES[0], + sigmas=STAGE_2_DISTILLED_SIGMA_VALUES, + timesteps=None, + attention_kwargs=attention_kwargs, + prompt_context=stage1.forward_context.prompt_context, + ) + return self.decode_phase(stage2) + + @torch.no_grad() + def forward( + self, + req: DiffusionRequestBatch, + prompt: str | list[str] | None = None, + negative_prompt: str | list[str] | None = None, + height: int | None = None, + width: int | None = None, + num_frames: int | None = None, + frame_rate: float | None = None, + num_inference_steps: int | None = None, + timesteps: list[int] | None = None, + guidance_scale: float = 4.0, + guidance_rescale: float = 0.0, + noise_scale: float = 0.0, + num_videos_per_prompt: int | None = 1, + generator: torch.Generator | list[torch.Generator] | None = None, + latents: torch.Tensor | None = None, + audio_latents: torch.Tensor | None = None, + prompt_embeds: torch.Tensor | None = None, + negative_prompt_embeds: torch.Tensor | None = None, + prompt_attention_mask: torch.Tensor | None = None, + negative_prompt_attention_mask: torch.Tensor | None = None, + decode_timestep: float | list[float] = 0.0, + decode_noise_scale: float | list[float] | None = None, + output_type: str = "np", + return_dict: bool = True, + attention_kwargs: dict[str, Any] | None = None, + max_sequence_length: int | None = None, + *, + image: Any | None = None, + ) -> DiffusionOutput | list[DiffusionOutput]: + del return_dict + request_inputs = self._resolve_request_inputs( + req, + prompt=prompt, + negative_prompt=negative_prompt, + height=height, + width=width, + num_frames=num_frames, + frame_rate=frame_rate, + num_inference_steps=num_inference_steps, + timesteps=timesteps, + guidance_scale=( + getattr(getattr(self, "one_stage_recipe", None), "guidance_scale", 4.0) + if guidance_scale is None + else guidance_scale + ), + guidance_rescale=guidance_rescale, + num_videos_per_prompt=num_videos_per_prompt, + generator=generator, + latents=latents, + audio_latents=audio_latents, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + prompt_attention_mask=prompt_attention_mask, + negative_prompt_attention_mask=negative_prompt_attention_mask, + decode_timestep=decode_timestep, + decode_noise_scale=decode_noise_scale, + output_type=output_type, + max_sequence_length=max_sequence_length, + ) + if self.support_image_input: + image = self._resolve_request_image(req, image, request_inputs) + return self._run_two_stage( + req, + request_inputs, + noise_scale=noise_scale, + timesteps=timesteps, + attention_kwargs=attention_kwargs, + image=image, + ) + + +class LTX2ImageToVideoTwoStagesPipeline(LTXI2VConditioningMixin, LTX2TwoStagesPipeline): + """LTX2 two-stage image-to-video entry.""" + + def forward( + self, + req: DiffusionRequestBatch, + image: Any | None = None, + prompt: str | list[str] | None = None, + negative_prompt: str | list[str] | None = None, + height: int | None = None, + width: int | None = None, + num_frames: int | None = None, + frame_rate: float | None = None, + num_inference_steps: int | None = None, + sigmas: list[float] | None = None, + *args: Any, + **kwargs: Any, + ) -> DiffusionOutput | list[DiffusionOutput]: + """Preserve the legacy I2V sigma slot, which was ignored by this pipeline.""" + del sigmas + return super().forward( + req, + image, + prompt, + negative_prompt, + height, + width, + num_frames, + frame_rate, + num_inference_steps, + *args, + **kwargs, + ) diff --git a/vllm_omni/diffusion/registry.py b/vllm_omni/diffusion/registry.py index 5b8b2fba1..22c36f32b 100644 --- a/vllm_omni/diffusion/registry.py +++ b/vllm_omni/diffusion/registry.py @@ -73,17 +73,17 @@ _DIFFUSION_MODELS = { ), "LTX2ImageToVideoPipeline": ( "ltx2", - "pipeline_ltx2_image2video", + "pipeline_ltx2", "LTX2ImageToVideoPipeline", ), "LTX2TwoStagesPipeline": ( "ltx2", - "pipeline_ltx2", + "pipeline_ltx2_two_stage", "LTX2TwoStagesPipeline", ), "LTX2ImageToVideoTwoStagesPipeline": ( "ltx2", - "pipeline_ltx2_image2video", + "pipeline_ltx2_two_stage", "LTX2ImageToVideoTwoStagesPipeline", ), "LTX2T2VDMD2Pipeline": ( @@ -93,17 +93,17 @@ _DIFFUSION_MODELS = { ), "LTX2I2VDMD2Pipeline": ( "ltx2", - "pipeline_ltx2_image2video", + "pipeline_ltx2", "LTX2I2VDMD2Pipeline", ), "LTX23Pipeline": ( "ltx2", - "pipeline_ltx2_3", + "pipeline_ltx2", "LTX23Pipeline", ), "LTX23ImageToVideoPipeline": ( "ltx2", - "pipeline_ltx2_3_image2video", + "pipeline_ltx2", "LTX23ImageToVideoPipeline", ), "StableAudioPipeline": ( From 74f2742ac2773a0c2205a725dbfcb62ae00c6e6a Mon Sep 17 00:00:00 2001 From: amy-why-3459 Date: Mon, 20 Jul 2026 21:45:28 +0800 Subject: [PATCH 028/240] =?UTF-8?q?[Hardware][Ascend][Model]=20Add=20NPU?= =?UTF-8?q?=20support=20for=20MiniCPM-o=204.5=20talker/vo=E2=80=A6=20(#511?= =?UTF-8?q?7)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: amy-why-3459 --- examples/online_serving/minicpmo/README.md | 9 +- recipes/OpenBMB/MiniCPM-o-4_5.md | 72 +++++--- vllm_omni/deploy/minicpmo_4_5.yaml | 36 ++-- vllm_omni/deploy/minicpmo_4_5_2gpu.yaml | 64 +++++++ vllm_omni/deploy/minicpmo_4_5_3gpu.yaml | 6 +- .../minicpmo_4_5/minicpmo_4_5_omni_llm.py | 7 +- .../minicpmo_4_5/minicpmo_4_5_omni_tts.py | 122 +++++++++++-- .../minicpmo_4_5/minicpmo_4_5_token2wav.py | 136 ++++++++++++++ .../step_audio2/step_audio2_token2wav.py | 16 ++ .../npu/models/cosyvoice2_dit_attn.py | 169 ++++++++++++++++++ .../npu/models/step_audio2_token2wav.py | 149 +++++++++++++++ 11 files changed, 724 insertions(+), 62 deletions(-) create mode 100644 vllm_omni/deploy/minicpmo_4_5_2gpu.yaml create mode 100644 vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_token2wav.py create mode 100644 vllm_omni/platforms/npu/models/cosyvoice2_dit_attn.py create mode 100644 vllm_omni/platforms/npu/models/step_audio2_token2wav.py diff --git a/examples/online_serving/minicpmo/README.md b/examples/online_serving/minicpmo/README.md index a34764652..334b5ed11 100644 --- a/examples/online_serving/minicpmo/README.md +++ b/examples/online_serving/minicpmo/README.md @@ -11,17 +11,18 @@ The UI supports: ## 1. Start the backend server The deploy config auto-loads via `--omni`; the default -`vllm_omni/deploy/minicpmo_4_5.yaml` targets a 2-GPU layout (thinker on -GPU 0, talker + t2w sharing GPU 1). For other hardware layouts pick +`vllm_omni/deploy/minicpmo_4_5.yaml` targets a single-GPU layout (thinker +and talker + t2w co-located on GPU 0). For other hardware layouts pick one of the deploy variants below. | deploy config | GPUs | Notes | |---|---|---| -| `minicpmo_4_5.yaml` (default) | 2 | Thinker on GPU0, talker+t2w on GPU1. | +| `minicpmo_4_5.yaml` (default) | 1 | Thinker and talker+t2w co-located on GPU0. | +| `minicpmo_4_5_2gpu.yaml` | 2 | Thinker on GPU0, talker+t2w on GPU1. | | `minicpmo_4_5_3gpu.yaml` | 3 | Thinker 2-way TP on GPU0/1, talker+t2w share GPU2. | | `minicpmo_4_5_8x4090.yaml` | 8 | Full 8x4090 layout. | -Default (2-GPU): +Default (single-GPU): ```bash vllm serve openbmb/MiniCPM-o-4_5 --omni \ diff --git a/recipes/OpenBMB/MiniCPM-o-4_5.md b/recipes/OpenBMB/MiniCPM-o-4_5.md index 8b3d2909d..4bbf416ba 100644 --- a/recipes/OpenBMB/MiniCPM-o-4_5.md +++ b/recipes/OpenBMB/MiniCPM-o-4_5.md @@ -19,15 +19,18 @@ Use this recipe as a known-good starting point for serving of the MiniCPM-o family — it pairs a multimodal-understanding thinker LLM with a streaming `MiniCPMTTS + Token2Wav` talker so a single `/v1/chat/completions` call can return text and 24 kHz speech in one -shot. The recipe covers three shipped GPU layouts (2 / 3 / 8 GPUs) -selected via `--deploy-config`. +shot. The recipe covers the shipped GPU layouts (single / 2 / 3 / 8 GPUs): + the default co-locates both stages on one GPU, and the larger scale-out + layouts (2 / 3 / 8 GPUs) are selected via `--deploy-config`. ## References - Default deploy configs (auto-loaded by HF `model_type=minicpmo` + `hf_config.version="4.5"`): - - 2-GPU layout (default): + - Single-GPU layout (default): [`vllm_omni/deploy/minicpmo_4_5.yaml`](../../vllm_omni/deploy/minicpmo_4_5.yaml) + - 2-GPU layout (thinker on GPU 0, talker on GPU 1): + [`vllm_omni/deploy/minicpmo_4_5_2gpu.yaml`](../../vllm_omni/deploy/minicpmo_4_5_2gpu.yaml) - 3-GPU layout (thinker TP=2): [`vllm_omni/deploy/minicpmo_4_5_3gpu.yaml`](../../vllm_omni/deploy/minicpmo_4_5_3gpu.yaml) - 8x RTX 4090 layout: @@ -45,28 +48,30 @@ selected via `--deploy-config`. ## Hardware Support -Three GPU layouts ship with default deploy configs. Pick the layout that -matches your hardware and pass it via `--deploy-config`; the talker -(`MiniCPMTTS + Token2Wav`) always lives on its own GPU because of the -in-process vocoder, and the thinker is the part that scales out via TP. +Four GPU layouts ship with default deploy configs. Pick the layout that +matches your hardware and pass it via `--deploy-config`. The default +co-locates the thinker and the talker (`MiniCPMTTS + Token2Wav`, with its +in-process vocoder) on a single GPU; the multi-GPU layouts split the +talker onto its own GPU and scale the thinker out via TP. | Layout | Thinker | Talker + Token2Wav | Typical hardware | | --- | --- | --- | --- | -| 2-GPU (default) | GPU 0 | GPU 1 | 2x A100/H100/H200 80GB | +| Single-GPU (default) | GPU 0 | GPU 0 | 1x A100/H100/H200 80GB | +| 2-GPU | GPU 0 | GPU 1 | 2x A100/H100/H200 80GB | | 3-GPU (thinker TP=2) | GPU 0,1 (TP=2) | GPU 2 | 3x mid-tier GPUs | | 8x RTX 4090 24GB | GPU 0–3 (TP=4) | GPU 4 | 8x RTX 4090 consumer | ## GPU -### 2 x GPU (default — single command) +### 1 x GPU (default — single command) The default [`vllm_omni/deploy/minicpmo_4_5.yaml`](../../vllm_omni/deploy/minicpmo_4_5.yaml) -puts the thinker on GPU 0 (`~70 %` memory, `enforce_eager: true`, -`max_num_seqs: 1`) and the talker + Token2Wav vocoder on GPU 1 -(`~75 %` memory). This is the recommended starting layout — works on -any pair of 80GB-class GPUs (A100, H100, H200) and on most 40GB+ -pairs as long as the thinker model weights fit. +co-locates both stages on GPU 0: the thinker (`~80 %` memory) and the +talker + Token2Wav vocoder (`~10 %` memory, `max_num_seqs: 1`). This is +the recommended starting layout — works on a single 80GB-class GPU +(A100, H100, H200) as long as the thinker model weights and the talker's +in-process vocoder fit together. #### Environment @@ -85,7 +90,7 @@ vllm serve openbmb/MiniCPM-o-4_5 --omni \ ``` The deploy config is auto-loaded by the model registry — no -`--deploy-config` flag needed for this default 2-GPU layout. +`--deploy-config` flag needed for this default single-GPU layout. #### Verification @@ -142,17 +147,40 @@ speech output (TTS)"** checkbox on / off. #### Notes -- Memory budget: thinker weights occupy GPU 0 at `gpu_memory_utilization: - 0.7`; talker + Token2Wav vocoder share GPU 1 at `0.75`. +- Memory budget: both stages share GPU 0 — the thinker at + `gpu_memory_utilization: 0.8`, the talker + Token2Wav vocoder at `0.1`. - `--trust-remote-code` is required — the HF repo ships a custom `MiniCPMO` config / model class. -- Pin: `enforce_eager: true` on both stages (CUDA graph capture is off - by design for the talker's Token2Wav path). - Stage 1 (talker) is hard-capped to `max_num_seqs: 1`: the talker only consumes `runtime_additional_information[0]`, so any value > 1 makes concurrent requests share request-0's audio. This is the same cap baked into the deploy config. +### 2 x GPU (talker on its own GPU) + +Use +[`vllm_omni/deploy/minicpmo_4_5_2gpu.yaml`](../../vllm_omni/deploy/minicpmo_4_5_2gpu.yaml) +when you have two GPUs and want to give the talker + Token2Wav vocoder a +dedicated card instead of sharing GPU 0 with the thinker. The thinker +runs on GPU 0 (`~90 %` mem, TP=1) and the talker on GPU 1 (`~75 %` mem, +`max_num_seqs: 1`). This relieves the memory pressure of the default +single-GPU co-located layout and is the recommended step up when a +second 80GB-class card is available but full 3-way TP scale-out is not +needed. + +#### Command + +```bash +vllm serve openbmb/MiniCPM-o-4_5 --omni \ + --deploy-config vllm_omni/deploy/minicpmo_4_5_2gpu.yaml \ + --trust-remote-code \ + --host 0.0.0.0 --port 8099 +``` + +Verification and Notes mirror the single-GPU section; the only +difference is that the talker no longer competes with the thinker for +GPU 0 memory. + ### 3 x GPU (thinker TP=2) Use @@ -171,7 +199,7 @@ vllm serve openbmb/MiniCPM-o-4_5 --omni \ --host 0.0.0.0 --port 8099 ``` -Verification and Notes mirror the 2-GPU section; thinker latency +Verification and Notes mirror the single-GPU section; thinker latency roughly halves under load thanks to TP=2. ### 8 x RTX 4090 24GB (consumer-GPU layout) @@ -197,8 +225,8 @@ vllm serve openbmb/MiniCPM-o-4_5 --omni \ 4090s. Raise it if your cards have more headroom (e.g. 4090 D / custom 32 GB SKUs), but verify with a long-prompt run before promoting. -- All other knobs match the 2-GPU section; the only difference is the - per-card memory pressure on the thinker shards. +- All other knobs match the single-GPU section; the only difference is + the per-card memory pressure on the thinker shards. ## Notes (applies to all layouts) diff --git a/vllm_omni/deploy/minicpmo_4_5.yaml b/vllm_omni/deploy/minicpmo_4_5.yaml index 980615479..d191dddcc 100644 --- a/vllm_omni/deploy/minicpmo_4_5.yaml +++ b/vllm_omni/deploy/minicpmo_4_5.yaml @@ -1,9 +1,9 @@ -# MiniCPM-o 4.5 default deploy: 2-GPU layout. +# MiniCPM-o 4.5 default deploy: single-GPU layout. # -# Stage 0 (thinker): GPU 0 — multimodal understanding, ~70% mem. -# Stage 1 (talker) : GPU 1 — MiniCPMTTS + Token2Wav, ~75% mem -# (covers both the talker LM and the in-process -# Token2Wav vocoder that emits the waveform). +# Both stages are co-located on GPU 0: +# Stage 0 (thinker): multimodal understanding, ~80% mem. +# Stage 1 (talker) : MiniCPMTTS + in-process Token2Wav vocoder, ~10% mem +# (talker LM plus the vocoder that emits the waveform). # # Other GPU layouts: minicpmo_4_5_3gpu.yaml, minicpmo_4_5_8x4090.yaml. async_chunk: false @@ -13,9 +13,9 @@ enable_prefix_caching: false stages: - stage_id: 0 - max_num_seqs: 1 - gpu_memory_utilization: 0.7 - enforce_eager: true + max_num_seqs: 16 + gpu_memory_utilization: 0.8 + enforce_eager: false tensor_parallel_size: 1 max_num_batched_tokens: 16384 devices: "0" @@ -34,10 +34,10 @@ stages: # Talker only consumes runtime_additional_information[0]; any value > 1 # makes concurrent requests share request-0's audio (mm_outputs fallback). max_num_seqs: 1 - gpu_memory_utilization: 0.75 - enforce_eager: true + gpu_memory_utilization: 0.1 + enforce_eager: false max_num_batched_tokens: 8192 - devices: "1" + devices: "0" default_sampling_params: temperature: 0.0 top_p: 1.0 @@ -46,6 +46,14 @@ stages: seed: 42 detokenize: false -# MiniCPM-o 4.5 has only been verified on CUDA devices. Add per-platform -# overrides here when other backends (ROCm/XPU/NPU) are verified end-to-end. -platforms: {} +platforms: + npu: + stages: + - stage_id: 0 + max_num_batched_tokens: 8192 + compilation_config: + cudagraph_mode: PIECEWISE + - stage_id: 1 + max_num_batched_tokens: 8192 + compilation_config: + cudagraph_mode: PIECEWISE diff --git a/vllm_omni/deploy/minicpmo_4_5_2gpu.yaml b/vllm_omni/deploy/minicpmo_4_5_2gpu.yaml new file mode 100644 index 000000000..7a9c400cc --- /dev/null +++ b/vllm_omni/deploy/minicpmo_4_5_2gpu.yaml @@ -0,0 +1,64 @@ +# MiniCPM-o 4.5 deploy: 2-GPU layout (thinker on GPU 0, talker on GPU 1). +# +# Stage 0 (thinker): GPU 0 — multimodal understanding, ~90% mem (thinker +# gets a dedicated card, so it can take more of +# the GPU than the single-GPU co-located layout). +# Stage 1 (talker) : GPU 1 — MiniCPMTTS + Token2Wav, ~75% mem +# (covers both the talker LM and the in-process +# Token2Wav vocoder that emits the waveform). +# +# Use this when you have two GPUs and want the talker + Token2Wav vocoder +# on its own card instead of sharing GPU 0 with the thinker (the default +# single-GPU layout co-locates both and can OOM under load). +pipeline: minicpmo_4_5 +async_chunk: false + +trust_remote_code: true +enable_prefix_caching: false + +stages: + - stage_id: 0 + max_num_seqs: 16 + gpu_memory_utilization: 0.9 + enforce_eager: false + tensor_parallel_size: 1 + max_num_batched_tokens: 16384 + devices: "0" + limit_mm_per_prompt: + video: 1 + default_sampling_params: + temperature: 0.0 + top_p: 1.0 + top_k: -1 + max_tokens: 2048 + seed: 42 + detokenize: true + repetition_penalty: 1.1 + + - stage_id: 1 + # Talker only consumes runtime_additional_information[0]; any value > 1 + # makes concurrent requests share request-0's audio (mm_outputs fallback). + max_num_seqs: 1 + gpu_memory_utilization: 0.75 + enforce_eager: false + max_num_batched_tokens: 8192 + devices: "1" + default_sampling_params: + temperature: 0.0 + top_p: 1.0 + top_k: -1 + max_tokens: 1 + seed: 42 + detokenize: false + +platforms: + npu: + stages: + - stage_id: 0 + max_num_batched_tokens: 8192 + compilation_config: + cudagraph_mode: PIECEWISE + - stage_id: 1 + max_num_batched_tokens: 8192 + compilation_config: + cudagraph_mode: PIECEWISE diff --git a/vllm_omni/deploy/minicpmo_4_5_3gpu.yaml b/vllm_omni/deploy/minicpmo_4_5_3gpu.yaml index f5c0b03a6..69f77eb6d 100644 --- a/vllm_omni/deploy/minicpmo_4_5_3gpu.yaml +++ b/vllm_omni/deploy/minicpmo_4_5_3gpu.yaml @@ -12,9 +12,9 @@ enable_prefix_caching: false stages: - stage_id: 0 - max_num_seqs: 1 + max_num_seqs: 16 gpu_memory_utilization: 0.7 - enforce_eager: true + enforce_eager: false tensor_parallel_size: 2 max_num_batched_tokens: 16384 devices: "0,1" @@ -34,7 +34,7 @@ stages: # makes concurrent requests share request-0's audio (mm_outputs fallback). max_num_seqs: 1 gpu_memory_utilization: 0.75 - enforce_eager: true + enforce_eager: false max_num_batched_tokens: 8192 devices: "2" default_sampling_params: diff --git a/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py b/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py index b930578d6..3019a5b3d 100644 --- a/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py +++ b/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py @@ -760,7 +760,12 @@ class MiniCPMVImageProcessor(BaseImageProcessor): ) -AutoImageProcessor.register("MiniCPMVImageProcessor", MiniCPMVImageProcessor) +# transformers >= 5.x requires the config *class* (not a string) as the +# first arg to AutoImageProcessor.register — it does `key.__module__` +# internally. Passing "MiniCPMVImageProcessor" raises: +# AttributeError: 'str' object has no attribute '__module__' +# See also processing_gr00t_n1d7.py for the same transformers API change. +AutoImageProcessor.register(MiniCPMOConfig, MiniCPMVImageProcessor, exist_ok=True) # ============== SigLIP Vision Transformer Classes ============== diff --git a/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_tts.py b/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_tts.py index 083f54afe..d14fe14cf 100644 --- a/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_tts.py +++ b/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_tts.py @@ -15,6 +15,7 @@ Pipeline: import io import logging import os +import sys from collections.abc import Iterable import numpy as np @@ -27,13 +28,33 @@ from vllm.model_executor.models.interfaces import SupportsPP from vllm_omni.platforms import current_omni_platform -try: - from stepaudio2 import Token2wav as _Token2wav +# Preserve the established external vocoder on CUDA. Ascend uses the in-tree +# adapter because ``stepaudio2-minicpmo`` hard-codes CUDA device placement. +if current_omni_platform.is_npu(): + try: + from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_token2wav import ( + MiniCPMO45Token2wav as _Token2wav, + ) - _stepaudio2_available = True -except ImportError: - _Token2wav = None - _stepaudio2_available = False + _token2wav_backend = "step_audio2_core" + except ImportError: + try: + from stepaudio2 import Token2wav as _Token2wav + + _token2wav_backend = "stepaudio2_pkg" + except ImportError: + _Token2wav = None + _token2wav_backend = None +else: + try: + from stepaudio2 import Token2wav as _Token2wav + + _token2wav_backend = "stepaudio2_pkg" + except ImportError: + _Token2wav = None + _token2wav_backend = None + +_stepaudio2_available = _Token2wav is not None logger = logging.getLogger(__name__) @@ -101,14 +122,25 @@ class MiniCPMO45OmniTTSForConditionalGeneration(nn.Module, SupportsPP): return try: model_path = self.vllm_config.model_config.model - import os - import sys if model_path not in sys.path: sys.path.insert(0, model_path) + from transformers import AutoImageProcessor from transformers.dynamic_module_utils import get_class_from_dynamic_module - MiniCPMTTS = get_class_from_dynamic_module("modeling_minicpmo.MiniCPMTTS", model_path) + # openbmb/MiniCPM-o-4_5/processing_minicpmo.py registers via a + # string: AutoImageProcessor.register("MiniCPMVImageProcessor", ...), + # which crashes on transformers>=5 (register reads key.__module__). + # Loading MiniCPMTTS imports that module, so no-op the string form + # (unused by the standalone talker) while it runs, then restore. + original_register = AutoImageProcessor.register + AutoImageProcessor.register = ( # type: ignore[method-assign] + lambda key, *a, **k: None if isinstance(key, str) else original_register(key, *a, **k) + ) + try: + MiniCPMTTS = get_class_from_dynamic_module("modeling_minicpmo.MiniCPMTTS", model_path) + finally: + AutoImageProcessor.register = original_register # type: ignore[method-assign] # MiniCPMTTS.__init__ reads `config.top_p / top_k / repetition_penalty` # directly (modeling_minicpmo.py L4112-4114), but the model repo's @@ -121,6 +153,11 @@ class MiniCPMO45OmniTTSForConditionalGeneration(nn.Module, SupportsPP): if not hasattr(self._tts_config, _attr): setattr(self._tts_config, _attr, _default) + # The copied Hugging Face flash_attention_2 setting is not valid + # for this standalone MiniCPMTTS path. Use PyTorch SDPA on every + # backend until a dedicated flash-attention implementation exists. + self._tts_config.attn_implementation = "sdpa" + prev_dtype = torch.get_default_dtype() torch.set_default_dtype(torch.float32) try: @@ -159,7 +196,11 @@ class MiniCPMO45OmniTTSForConditionalGeneration(nn.Module, SupportsPP): finally: torch.set_default_dtype(prev_dtype2) self.tts_obj.audio_tokenizer = self.audio_tokenizer - logger.info("Loaded Token2wav from %s", token2wav_dir) + logger.info( + "Loaded Token2wav from %s (backend=%s)", + token2wav_dir, + _token2wav_backend, + ) # Only mark init as complete after every step succeeds, so a # partial failure leaves the next call free to retry the full # init instead of short-circuiting back to a silent empty path. @@ -189,19 +230,25 @@ class MiniCPMO45OmniTTSForConditionalGeneration(nn.Module, SupportsPP): tts = self.tts_obj device = tts.emb_text.weight.device - dtype = tts.emb_text.weight.dtype + # MiniCPMTTS AR backbone uses FlashAttention (fp16/bf16 only). The + # submodule is constructed under float32 default dtype during lazy init, + # so pin the condition embeddings to bfloat16 explicitly rather than + # inheriting the (float32) parameter dtype — a float32 condition breaks + # the CUDA FA2 path and wastes memory on the NPU sdpa path. + ar_dtype = torch.bfloat16 llm_embeds = tts.emb_text(tts_token_ids.to(device)) - hidden_embeds = tts.projector_semantic(tts_hidden_states.to(device=device, dtype=dtype)) + hidden_embeds = tts.projector_semantic(tts_hidden_states.to(device=device, dtype=ar_dtype)) if getattr(tts.config, "normalize_projected_hidden", False): hidden_embeds = F.normalize(hidden_embeds, p=2, dim=-1) - tts_embeds = llm_embeds + hidden_embeds + tts_embeds = (llm_embeds + hidden_embeds).to(dtype=ar_dtype) text_eos = tts.emb_text(torch.tensor([tts.config.text_eos_token_id], device=device, dtype=torch.long)) audio_bos = tts.emb_text(torch.tensor([tts.audio_bos_token_id], device=device, dtype=torch.long)) - spk_embeds = torch.zeros(0, tts.config.hidden_size, device=device, dtype=tts_embeds.dtype) + spk_embeds = torch.zeros(0, tts.config.hidden_size, device=device, dtype=ar_dtype) inputs_embeds = torch.cat([spk_embeds, tts_embeds, text_eos, audio_bos], dim=0).unsqueeze(0) + inputs_embeds = inputs_embeds.to(dtype=ar_dtype) logger.info("generate_speech: inputs_embeds shape=%s", list(inputs_embeds.shape)) # Scale max_new_token with input text length to avoid mid-stream truncation on long @@ -249,7 +296,15 @@ class MiniCPMO45OmniTTSForConditionalGeneration(nn.Module, SupportsPP): prev_dtype = torch.get_default_dtype() torch.set_default_dtype(torch.float32) try: - with torch.amp.autocast("cuda", enabled=False): + # Vocoder path is float32; use the platform abstraction because + # torch.amp.autocast validates unsupported device types even when + # autocast is disabled. + autocast_device = device.type if isinstance(device, torch.device) else str(device) + with current_omni_platform.create_autocast_context( + device_type=autocast_device, + dtype=torch.float32, + enabled=False, + ): token_list = generated_tokens.squeeze(0).tolist() num_tokens = len(token_list) @@ -418,7 +473,17 @@ class MiniCPMO45OmniTTSForConditionalGeneration(nn.Module, SupportsPP): return None, None def compute_logits(self, hidden_states, *args, **kwargs): - return torch.zeros(1, 2, device=hidden_states.device if isinstance(hidden_states, torch.Tensor) else "cuda") + # Placeholder logits: one row per sampled request (the scheduler + # indexes sampled_token_ids by req_index). Hardcoding a single row + # breaks batched/concurrent decoding with IndexError. The values are + # discarded — real output is the waveform via multimodal_outputs. + if isinstance(hidden_states, torch.Tensor): + device = hidden_states.device + num_reqs = hidden_states.shape[0] if hidden_states.ndim >= 1 else 1 + else: + device = current_omni_platform.get_torch_device() + num_reqs = 1 + return torch.zeros(num_reqs, 2, device=device) def sample(self, logits, sampling_metadata): return None @@ -442,10 +507,31 @@ class MiniCPMO45OmniTTSForConditionalGeneration(nn.Module, SupportsPP): logger.warning("TTS missing keys (%d): %s", len(missing), missing[:5]) if unexpected: logger.warning("TTS unexpected keys (%d): %s", len(unexpected), unexpected[:5]) - self.tts_obj = self.tts_obj.to("cuda") + # Move the AR backbone to the active device (cuda / npu / …) and + # cast to bfloat16: MiniCPMTTS AR uses FlashAttention (fp16/bf16 + # only) and is built under a float32 default dtype during lazy + # init, so an uncast float32 backbone breaks CUDA FA2 and wastes + # memory on the NPU sdpa path. Detach the Token2wav vocoder first + # so the cast does not drag it onto the accelerator or downcast + # its float32 flow/HiFT weights: it manages its own device + # placement (e.g. CPU HiFT on NPU) and may not be an nn.Module. + device = current_omni_platform.get_torch_device() + audio_tok = getattr(self.tts_obj, "audio_tokenizer", None) + if audio_tok is not None: + self.tts_obj.audio_tokenizer = None + try: + self.tts_obj = self.tts_obj.to(device=device, dtype=torch.bfloat16) + finally: + if audio_tok is not None: + self.tts_obj.audio_tokenizer = audio_tok + self.audio_tokenizer = audio_tok self.emb_text = self.tts_obj.emb_text self.projector_semantic = self.tts_obj.projector_semantic - logger.info("Loaded %d TTS weights, moved to cuda", len(tts_weights)) + logger.info( + "Loaded %d TTS weights, moved to %s (bfloat16)", + len(tts_weights), + device, + ) return loaded diff --git a/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_token2wav.py b/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_token2wav.py new file mode 100644 index 000000000..c069b8d09 --- /dev/null +++ b/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_token2wav.py @@ -0,0 +1,136 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""MiniCPM-o 4.5 Token2wav adapter over in-tree ``StepAudio2Token2WavCore``. + +``minicpmo_4_5_omni_tts`` historically depended on the external +``from stepaudio2 import Token2wav`` entry point (``stepaudio2-minicpmo``). +That package hard-codes ``.cuda()`` and duplicates the flow/HiFT stack that +vLLM-Omni already vendors for Step-Audio2 (which also carries the Ascend/NPU +fixes: CPU HiFT, DiT mask expand, MATH SDPA, compile disable, PE buffer +extension). + +This module exposes the same call surface MiniCPM expects +(``__call__`` / ``set_stream_cache`` / ``stream``) while delegating the +actual vocoder work to +``vllm_omni.model_executor.models.step_audio2.step_audio2_token2wav.StepAudio2Token2WavCore``. +""" + +from __future__ import annotations + +from typing import Any + +import numpy as np +import torch + +from vllm_omni.model_executor.models.step_audio2.step_audio2_token2wav import ( + StepAudio2Token2WavCore, + _StreamState, +) + + +def _resolve_device(device: str | torch.device | None) -> str: + if device is not None: + return str(device) + try: + from vllm_omni.platforms import current_omni_platform + + plat_dev = getattr(current_omni_platform, "device_type", None) + if plat_dev: + return str(plat_dev) + except Exception: + pass + if torch.cuda.is_available(): + return "cuda" + if hasattr(torch, "npu") and torch.npu.is_available(): + return "npu" + return "cpu" + + +class MiniCPMO45Token2wav: + """Token2wav-compatible facade around ``StepAudio2Token2WavCore``. + + Matches the MiniCPM-o / ``stepaudio2-minicpmo`` vocoder API used by + ``MiniCPMO45OmniTTSForConditionalGeneration.generate_speech``: + + * ``Token2wav(model_path, float16=False, n_timesteps=10)`` + * ``__(tokens, prompt_wav) -> wav_bytes`` + * ``set_stream_cache(prompt_wav) -> (stream_cache, hift_cache_dict)`` + * ``stream(tokens, prompt_wav, last_chunk=..., return_waveform=...)`` + """ + + def __init__( + self, + model_path: str, + float16: bool = False, + n_timesteps: int = 10, + device: str | torch.device | None = None, + ): + self.float16 = float16 + self.n_timesteps = n_timesteps + self.device = _resolve_device(device) + self._core = StepAudio2Token2WavCore( + model_path=model_path, + float16=float16, + device=self.device, + n_timesteps=n_timesteps, + ) + # Eager-load so construction failures surface at init time (same as + # the external Token2wav package), not on the first request. + self._core._ensure_models_loaded() + + # Mutable streaming fields expected by MiniCPM's long-form path. + self.stream_cache: Any | None = None + self.hift_cache_dict: dict[str, torch.Tensor] = {} + + def __call__(self, generated_speech_tokens, prompt_wav) -> bytes: + """One-shot tokens → 24 kHz WAV bytes.""" + return self._core.forward( + generated_speech_tokens, + prompt_wav, + return_bytes=True, + ) + + def set_stream_cache(self, prompt_wav: str): + """Initialise flow + HiFT caches for chunked vocoding. + + Returns ``(stream_cache, hift_cache_dict)`` so callers can assign them + back onto this object the way ``stepaudio2.Token2wav`` does. + """ + state = _StreamState() + self._core.setup_stream_for(prompt_wav, state) + self.stream_cache = state.stream_cache + self.hift_cache_dict = state.hift_cache_dict + return self.stream_cache, self.hift_cache_dict + + def stream( + self, + generated_speech_tokens, + prompt_wav: str, + last_chunk: bool = False, + return_waveform: bool = False, + ): + """Process one streaming chunk; updates ``stream_cache`` / ``hift_cache_dict``.""" + if self.stream_cache is None: + raise ValueError("stream_cache is not set") + + state = _StreamState() + state.setup_done = True + state.stream_cache = self.stream_cache + state.hift_cache_dict = self.hift_cache_dict + + speech = self._core.stream_chunk_for( + list(generated_speech_tokens), + prompt_wav, + last_chunk, + state, + ) + self.stream_cache = state.stream_cache + self.hift_cache_dict = state.hift_cache_dict + + wav_np = speech.detach().float().cpu().numpy() + if return_waveform: + return wav_np + + wav_np = np.clip(wav_np, -1.0, 1.0) + wav_int16 = (wav_np * 32767.0).astype(" torch.Tensor | None: + """Expand CosyVoice key-padding masks to Ascend FA-compatible shapes.""" + if attn_mask is None: + return None + kv_len = kv_len if kv_len is not None else q_len + + # (B, 1, S) key-padding -> (B, S_q, S_kv) then unsqueeze heads below. + if attn_mask.dim() == 3 and attn_mask.shape[-2] == 1: + attn_mask = attn_mask.expand(-1, q_len, -1) + if attn_mask.dim() == 3: + # (B, S_q, S_kv) -> (B, 1, S_q, S_kv) + attn_mask = attn_mask.unsqueeze(1) + if attn_mask.dim() == 4 and attn_mask.shape[-2] == 1 and kv_len > 1: + # (B, 1, 1, S) -> (B, 1, S_q, S) + attn_mask = attn_mask.expand(-1, -1, q_len, -1) + return attn_mask + + +def _patched_attention_forward(self, x: torch.Tensor, attn_mask: torch.Tensor) -> torch.Tensor: + b, t, c = x.shape + + q = self.to_heads(self.to_q(x)) + k = self.to_heads(self.to_k(x)) + v = self.to_heads(self.to_v(x)) + + q = self.q_norm(q) + k = self.k_norm(k) + + attn_mask = _expand_attn_mask_for_npu(attn_mask, q_len=t, kv_len=t) + x = F.scaled_dot_product_attention( + q, + k, + v, + attn_mask=attn_mask, + dropout_p=self.attn_drop.p if self.training else 0.0, + ) + x = x.transpose(1, 2).reshape(b, t, -1) + x = self.proj(x) + x = self.proj_drop(x) + return x + + +def _patched_attention_forward_chunk( + self, + x: torch.Tensor, + att_cache: torch.Tensor | None = None, + attn_mask: torch.Tensor | None = None, +): + b, t, c = x.shape + + q = self.to_heads(self.to_q(x)) + k = self.to_heads(self.to_k(x)) + v = self.to_heads(self.to_v(x)) + + q = self.q_norm(q) + k = self.k_norm(k) + + if att_cache is not None: + k_cache, v_cache = att_cache.chunk(2, dim=3) + k = torch.cat([k, k_cache], dim=2) + v = torch.cat([v, v_cache], dim=2) + + new_att_cache = torch.cat([k, v], dim=3) + kv_len = k.shape[2] + if attn_mask is not None: + attn_mask = _expand_attn_mask_for_npu(attn_mask, q_len=t, kv_len=kv_len) + x = F.scaled_dot_product_attention(q, k, v, attn_mask=attn_mask) + x = x.transpose(1, 2).reshape(b, t, -1) + x = self.proj(x) + x = self.proj_drop(x) + return x, new_att_cache + + +@contextmanager +def npu_math_sdpa_context() -> Iterator[None]: + """Force SDPA MATH backend so Ascend does not call fused FA.""" + try: + from torch.nn.attention import SDPBackend, sdpa_kernel + + with sdpa_kernel(SDPBackend.MATH): + yield + except Exception: + # Older torch / missing backend enum — just run as-is. + with nullcontext(): + yield + + +def _disable_upsample_encoder_compile() -> None: + """Run CosyVoice2 UpsampleConformerEncoderV2 eagerly on Ascend. + + ``forward_chunk`` is decorated with ``@torch.compile(dynamic=True, + backend="eager")`` upstream. The backend gives no speedup (still eager), + but Dynamo tracing of the relative-position attention with a KV cache + creates a symbolic add ``matrix_ac(..., chunk+cache) + matrix_bd(..., + (pos//2)+1)`` whose sizes Ascend torch's fake-tensor ``infer_size`` + cannot reconcile, crashing on the 2nd+ streaming chunk. Unwrapping the + compile and disabling Dynamo keeps concrete shapes (always equal). + """ + try: + from cosyvoice2.transformer import upsample_encoder_v2 + except ImportError: + return + + enc_cls = getattr(upsample_encoder_v2, "UpsampleConformerEncoderV2", None) + if enc_cls is None: + return + + fn = enc_cls.forward_chunk + orig = getattr(fn, "_torchdynamo_orig_callable", None) or getattr(fn, "__wrapped__", fn) + enc_cls.forward_chunk = torch._dynamo.disable(orig) # type: ignore[method-assign] + logger.info("Disabled torch.compile on CosyVoice2 UpsampleConformerEncoderV2.forward_chunk (Ascend eager)") + + +def apply_cosyvoice2_dit_attn_npu_patch() -> None: + """Monkey-patch CosyVoice2 DiT Attention for Ascend FA mask constraints.""" + global _PATCHED + if _PATCHED: + return + + try: + from cosyvoice2.flow import decoder_dit + except ImportError: + logger.debug("cosyvoice2 not installed; skip DiT attn NPU patch") + return + + attn_cls = getattr(decoder_dit, "Attention", None) + if attn_cls is None: + return + + attn_cls.forward = _patched_attention_forward # type: ignore[method-assign] + attn_cls.forward_chunk = _patched_attention_forward_chunk # type: ignore[method-assign] + _disable_upsample_encoder_compile() + _PATCHED = True + logger.info("Applied CosyVoice2 DiT Attention NPU patch (expand attn_mask to Bx1xSxS for aclnnFlashAttentionScore)") diff --git a/vllm_omni/platforms/npu/models/step_audio2_token2wav.py b/vllm_omni/platforms/npu/models/step_audio2_token2wav.py new file mode 100644 index 000000000..04768d04d --- /dev/null +++ b/vllm_omni/platforms/npu/models/step_audio2_token2wav.py @@ -0,0 +1,149 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""NPU patches for Step-Audio2 / MiniCPM Token2Wav. + +Ascend-specific workarounds that must not live in the shared GPU model file: + +1. HiFT vocoder CPU offload — STFT/ISTFT and sine-source generation are + unstable on Ascend (aicore 507015). HiFT is tiny, so running it on CPU + is acceptable; inputs/outputs are moved transparently. +2. CosyVoice2 DiT SDPA — force MATH backend (+ DiT attn mask expand) to + avoid fused FA rejecting CosyVoice ``(B,1,1,S)`` masks (error 161001). +""" + +from __future__ import annotations + +from collections.abc import Iterator +from contextlib import contextmanager, nullcontext +from types import MethodType + +import torch +from vllm.logger import init_logger + +logger = init_logger(__name__) + +_PATCHED = False +_original_ensure_models_loaded = None +_original_forward = None +_original_stream_chunk_for = None + + +def patch_hift_module_for_npu(hift: torch.nn.Module) -> None: + """Run the entire HiFT vocoder on CPU when on Ascend NPU. + + HiFT is tiny, so keeping it on NPU yields negligible speedup while + several of its operators are unstable on Ascend and trigger aicore + exceptions (runtime error 507015): the STFT/ISTFT as well as the + sine-source generation (``f0_upsamp`` + ``m_source`` cumsum/sin over + the full waveform length). Running the whole module on CPU sidesteps + all of them. Inputs are copied to CPU and outputs are moved back to + the original device transparently, so callers stay unchanged. + """ + if getattr(hift, "_npu_cpu_offload_patched", False): + return + + hift.to("cpu") + original_forward = hift.forward + + def _forward_on_cpu(_module, *args, **kwargs): + output_device: torch.device | None = None + + def to_cpu(value): + nonlocal output_device + if isinstance(value, torch.Tensor): + if output_device is None and value.device.type != "cpu": + output_device = value.device + return value.cpu() + return value + + cpu_args = tuple(to_cpu(a) for a in args) + cpu_kwargs = {k: to_cpu(v) for k, v in kwargs.items()} + output = original_forward(*cpu_args, **cpu_kwargs) + + if output_device is None: + return output + + def to_device(value): + if isinstance(value, torch.Tensor): + return value.to(output_device) + return value + + if isinstance(output, tuple): + return tuple(to_device(v) for v in output) + return to_device(output) + + hift.forward = MethodType(_forward_on_cpu, hift) + hift._npu_cpu_offload_patched = True + + +@contextmanager +def npu_token2wav_sdpa_context() -> Iterator[None]: + """Expand CosyVoice masks + force MATH SDPA to avoid FA 161001.""" + try: + from vllm_omni.platforms.npu.models.cosyvoice2_dit_attn import ( + apply_cosyvoice2_dit_attn_npu_patch, + npu_math_sdpa_context, + ) + + apply_cosyvoice2_dit_attn_npu_patch() + with npu_math_sdpa_context(): + yield + except Exception: + with nullcontext(): + yield + + +def _patched_ensure_models_loaded(self) -> None: + assert _original_ensure_models_loaded is not None + was_loaded = self._models_loaded + _original_ensure_models_loaded(self) + if was_loaded or self.device.type != "npu" or self._hift is None: + return + patch_hift_module_for_npu(self._hift) + + +def _patched_forward(self, generated_speech_tokens, prompt_wav, return_bytes=True): + assert _original_forward is not None + if self.device.type != "npu": + return _original_forward(self, generated_speech_tokens, prompt_wav, return_bytes) + with npu_token2wav_sdpa_context(): + return _original_forward(self, generated_speech_tokens, prompt_wav, return_bytes) + + +def _patched_stream_chunk_for(self, audio_tokens, prompt_wav, last_chunk, state): + assert _original_stream_chunk_for is not None + if self.device.type != "npu": + return _original_stream_chunk_for(self, audio_tokens, prompt_wav, last_chunk, state) + with npu_token2wav_sdpa_context(): + return _original_stream_chunk_for(self, audio_tokens, prompt_wav, last_chunk, state) + + +def apply_step_audio2_token2wav_npu_patch() -> None: + """Monkey-patch StepAudio2Token2WavCore for Ascend NPU. + + Import is deferred and optional: platform bootstrap (e.g. resolving + ``current_omni_platform`` from rotary embedding) must not require + Token2Wav optional deps such as ``librosa``. + """ + global _PATCHED, _original_ensure_models_loaded, _original_forward, _original_stream_chunk_for + if _PATCHED: + return + + try: + from vllm_omni.model_executor.models.step_audio2.step_audio2_token2wav import ( + StepAudio2Token2WavCore, + ) + except ImportError as e: + logger.debug("step_audio2 token2wav deps unavailable; skip NPU patch: %s", e) + return + + _original_ensure_models_loaded = StepAudio2Token2WavCore._ensure_models_loaded + _original_forward = StepAudio2Token2WavCore.forward + _original_stream_chunk_for = StepAudio2Token2WavCore.stream_chunk_for + + StepAudio2Token2WavCore._ensure_models_loaded = _patched_ensure_models_loaded # type: ignore[method-assign] + StepAudio2Token2WavCore.forward = _patched_forward # type: ignore[method-assign] + StepAudio2Token2WavCore.stream_chunk_for = _patched_stream_chunk_for # type: ignore[method-assign] + + _PATCHED = True + logger.debug("Applied NPU patch for StepAudio2Token2WavCore") From 7d75095b842a352031528536b90d8c01b81e2e0a Mon Sep 17 00:00:00 2001 From: Yukim1 <121286183+zwhzzz0821@users.noreply.github.com> Date: Mon, 20 Jul 2026 22:34:42 +0800 Subject: [PATCH 029/240] [BugFix] Fix SoulX-Singer deploy pipeline registration (#5210) Signed-off-by: zwhzzz0821 <2831474076@qq.com> --- tests/test_config_factory.py | 10 +++++ vllm_omni/config/pipeline_registry.py | 6 +++ .../models/soulx_singer/__init__.py | 5 +++ .../models/soulx_singer/pipeline.py | 45 +++++++++++++++++++ 4 files changed, 66 insertions(+) create mode 100644 vllm_omni/model_executor/models/soulx_singer/__init__.py create mode 100644 vllm_omni/model_executor/models/soulx_singer/pipeline.py diff --git a/tests/test_config_factory.py b/tests/test_config_factory.py index b8777dcce..e08349140 100644 --- a/tests/test_config_factory.py +++ b/tests/test_config_factory.py @@ -573,6 +573,16 @@ class TestPipelineDiscovery: return assert Path(get_deploy_config_path(pipeline.default_deploy_config_name)).is_file() + @pytest.mark.parametrize( + "deploy_path", + sorted(_DEPLOY_DIR.glob("*.yaml")), + ids=lambda deploy_path: deploy_path.name, + ) + def test_explicit_deploy_pipeline_is_registered(self, deploy_path): + deploy = load_deploy_config(deploy_path) + if deploy.pipeline is not None: + assert deploy.pipeline in OMNI_PIPELINES + def test_registry_returns_none_for_unknown(self): """Unknown model_types aren't found and resolve to `None`.""" assert "definitely_not_a_real_model" not in OMNI_PIPELINES diff --git a/vllm_omni/config/pipeline_registry.py b/vllm_omni/config/pipeline_registry.py index 941319c9c..0ee96eca4 100644 --- a/vllm_omni/config/pipeline_registry.py +++ b/vllm_omni/config/pipeline_registry.py @@ -86,6 +86,10 @@ from vllm_omni.model_executor.models.qwen2_5_omni.pipeline import ( ) from vllm_omni.model_executor.models.qwen3_omni.pipeline import resolve_qwen3_omni_pipeline from vllm_omni.model_executor.models.qwen3_tts.pipeline import QWEN3_TTS_PIPELINE +from vllm_omni.model_executor.models.soulx_singer.pipeline import ( + SOULXSINGER_SVC_PIPELINE, + SOULXSINGER_SVS_PIPELINE, +) from vllm_omni.model_executor.models.step_audio2.pipeline import ( STEP_AUDIO2_ASR_PIPELINE, STEP_AUDIO2_PIPELINE, @@ -144,6 +148,8 @@ OMNI_PIPELINES: dict[str, PipelineConfig | PipelineResolverFunc] = { "higgs_multimodal_qwen3": HIGGS_AUDIO_V3_PIPELINE, "dynin_omni": DYNIN_OMNI_PIPELINE, "indextts2": INDEXTTS2_PIPELINE, + "soulxsinger_svc": SOULXSINGER_SVC_PIPELINE, + "soulxsinger_svs": SOULXSINGER_SVS_PIPELINE, } diff --git a/vllm_omni/model_executor/models/soulx_singer/__init__.py b/vllm_omni/model_executor/models/soulx_singer/__init__.py new file mode 100644 index 000000000..1ab85a8d3 --- /dev/null +++ b/vllm_omni/model_executor/models/soulx_singer/__init__.py @@ -0,0 +1,5 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +# Keep this package lightweight. The pipeline registry imports pipeline.py +# directly, while diffusion model classes are loaded lazily from their registry. diff --git a/vllm_omni/model_executor/models/soulx_singer/pipeline.py b/vllm_omni/model_executor/models/soulx_singer/pipeline.py new file mode 100644 index 000000000..4d2420528 --- /dev/null +++ b/vllm_omni/model_executor/models/soulx_singer/pipeline.py @@ -0,0 +1,45 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""SoulX-Singer pipeline topologies.""" + +from vllm_omni.config.stage_config import ( + PipelineConfig, + StageExecutionType, + StagePipelineConfig, +) + +SOULXSINGER_SVS_PIPELINE = PipelineConfig( + model_type="soulxsinger_svs", + default_deploy_config_name="soulxsinger_svs.yaml", + model_arch="SoulXSingerPipeline", + hf_architectures=("SoulXSingerPipeline",), + stages=( + StagePipelineConfig( + stage_id=0, + model_stage="dit", + execution_type=StageExecutionType.DIFFUSION, + input_sources=(), + final_output=True, + final_output_type="audio", + model_arch="SoulXSingerPipeline", + ), + ), +) + +SOULXSINGER_SVC_PIPELINE = PipelineConfig( + model_type="soulxsinger_svc", + default_deploy_config_name="soulxsinger_svc.yaml", + model_arch="SoulXSingerSVCPipeline", + hf_architectures=("SoulXSingerSVCPipeline",), + stages=( + StagePipelineConfig( + stage_id=0, + model_stage="dit", + execution_type=StageExecutionType.DIFFUSION, + input_sources=(), + final_output=True, + final_output_type="audio", + model_arch="SoulXSingerSVCPipeline", + ), + ), +) From 4d799997d33f6ff9942c824e0faf0d36be328e31 Mon Sep 17 00:00:00 2001 From: zhumingjue138 Date: Mon, 20 Jul 2026 23:20:53 +0800 Subject: [PATCH 030/240] [Bugfix] Fix issue 5205, add trust-remote-code (#5213) Signed-off-by: zhumingjue --- .buildkite/cuda/test-nightly.yml | 41 ++++++++++++++++ .../reliability/test_reliability_voxcpm2.py | 3 ++ .../distributed/test_wan_spatial_shard.py | 49 +++++++++++++++---- tests/e2e/accuracy/test_hunyuan_image3.py | 1 + .../test_dynin_omni_expansion.py | 9 +++- 5 files changed, 92 insertions(+), 11 deletions(-) diff --git a/.buildkite/cuda/test-nightly.yml b/.buildkite/cuda/test-nightly.yml index 0a23e5825..dd62ee68b 100644 --- a/.buildkite/cuda/test-nightly.yml +++ b/.buildkite/cuda/test-nightly.yml @@ -1604,6 +1604,47 @@ steps: volumes: - "/fsx/hf_cache:/fsx/hf_cache" + - label: ":full_moon: Diffusion · Wan VAE Spatial Shard Correctness Test" + timeout_in_minutes: 60 + commands: + - >- + pytest -sv tests/diffusion/distributed/test_wan_spatial_shard.py + -m "full_model and diffusion and H100 and distributed_cuda" + --run-level "full_model" + agents: + queue: "mithril-h100-pool" + plugins: + - kubernetes: + podSpec: + containers: + - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT + resources: + limits: + nvidia.com/gpu: 2 + volumeMounts: + - name: devshm + mountPath: /dev/shm + - name: hf-cache + mountPath: /root/.cache/huggingface + env: + - name: HF_HOME + value: /root/.cache/huggingface + - name: HF_TOKEN + valueFrom: + secretKeyRef: + name: hf-token-secret + key: token + nodeSelector: + node.kubernetes.io/instance-type: gpu-h100-sxm + volumes: + - name: devshm + emptyDir: + medium: Memory + - name: hf-cache + hostPath: + path: /mnt/hf-cache + type: DirectoryOrCreate + - label: ":bar_chart: Testcase Statistics" key: nightly-testcase-statistics timeout_in_minutes: 120 diff --git a/tests/dfx/reliability/test_reliability_voxcpm2.py b/tests/dfx/reliability/test_reliability_voxcpm2.py index 014bb4e89..632c51081 100644 --- a/tests/dfx/reliability/test_reliability_voxcpm2.py +++ b/tests/dfx/reliability/test_reliability_voxcpm2.py @@ -43,6 +43,9 @@ RELIABILITY_SCENARIOS: list[dict[str, Any]] = [ "server_params": { "model": "openbmb/VoxCPM2", "stage_config_name": "voxcpm2.yaml", + # VoxCPM2 uses a custom HF tokenizer; CLI default trust_remote_code=False + # would otherwise override deploy YAML settings during server startup. + "server_args": ["--trust-remote-code", "--disable-log-stats"], }, }, ] diff --git a/tests/diffusion/distributed/test_wan_spatial_shard.py b/tests/diffusion/distributed/test_wan_spatial_shard.py index 6d668df3a..72a2a45f5 100644 --- a/tests/diffusion/distributed/test_wan_spatial_shard.py +++ b/tests/diffusion/distributed/test_wan_spatial_shard.py @@ -6,18 +6,19 @@ import torch import torch.distributed as dist import torch.multiprocessing as mp +from tests.helpers.mark import hardware_test from vllm_omni.diffusion.distributed.autoencoders import wan_spatial_shard from vllm_omni.diffusion.distributed.autoencoders.autoencoder_kl_wan import ( DistributedAutoencoderKLWan, ) from vllm_omni.platforms import current_omni_platform -# Module-level markers apply to the CPU unit tests below. The multi-GPU -# correctness test at the end of the file adds the nightly/distributed markers -# and a skip guard so it only runs where >= 2 CUDA devices are available. -pytestmark = [pytest.mark.core_model, pytest.mark.cpu] +# CPU unit tests are marked core_model + cpu. The multi-GPU correctness test at +# the end uses hardware_test (H100 x2) plus full_model / diffusion / parallel. +@pytest.mark.core_model +@pytest.mark.cpu def test_split_for_parallel_decode_pads_uneven_height(): x = torch.arange(1 * 1 * 1 * 5 * 2, dtype=torch.float32).reshape(1, 1, 1, 5, 2) @@ -34,6 +35,8 @@ def test_split_for_parallel_decode_pads_uneven_height(): assert torch.equal(local[..., 1, :], torch.zeros_like(local[..., 1, :])) +@pytest.mark.core_model +@pytest.mark.cpu def test_split_for_parallel_decode_pads_uneven_width(): x = torch.arange(1 * 1 * 1 * 2 * 5, dtype=torch.float32).reshape(1, 1, 1, 2, 5) @@ -51,6 +54,8 @@ def test_split_for_parallel_decode_pads_uneven_width(): assert torch.equal(local[..., :, 1], torch.zeros_like(local[..., :, 1])) +@pytest.mark.core_model +@pytest.mark.cpu def test_split_for_parallel_decode_rejects_invalid_split_dim(): x = torch.zeros((1, 1, 1, 4, 4), dtype=torch.float32) @@ -64,6 +69,8 @@ def test_split_for_parallel_decode_rejects_invalid_split_dim(): ) +@pytest.mark.core_model +@pytest.mark.cpu def test_split_for_parallel_decode_rejects_zero_world_size(): x = torch.zeros((1, 1, 1, 4, 4), dtype=torch.float32) @@ -76,6 +83,8 @@ def test_split_for_parallel_decode_rejects_zero_world_size(): ) +@pytest.mark.core_model +@pytest.mark.cpu def test_split_for_parallel_decode_rejects_rank_out_of_range(): x = torch.zeros((1, 1, 1, 4, 4), dtype=torch.float32) @@ -88,6 +97,8 @@ def test_split_for_parallel_decode_rejects_rank_out_of_range(): ) +@pytest.mark.core_model +@pytest.mark.cpu def test_gather_and_trim_height(monkeypatch: pytest.MonkeyPatch): monkeypatch.setattr(wan_spatial_shard, "_rank_world", lambda group: (0, 3)) @@ -104,6 +115,8 @@ def test_gather_and_trim_height(monkeypatch: pytest.MonkeyPatch): assert torch.equal(out.flatten(), torch.tensor([0.0, 0.0, 1.0, 1.0, 2.0])) +@pytest.mark.core_model +@pytest.mark.cpu def test_gather_and_trim_width(monkeypatch: pytest.MonkeyPatch): monkeypatch.setattr(wan_spatial_shard, "_rank_world", lambda group: (0, 3)) @@ -120,6 +133,8 @@ def test_gather_and_trim_width(monkeypatch: pytest.MonkeyPatch): assert torch.equal(out.flatten(), torch.tensor([0.0, 0.0, 1.0, 1.0, 2.0])) +@pytest.mark.core_model +@pytest.mark.cpu def test_gather_and_trim_rank0_only_assembles_on_rank0(monkeypatch: pytest.MonkeyPatch): monkeypatch.setattr(wan_spatial_shard, "_rank_world", lambda group: (0, 3)) @@ -136,6 +151,8 @@ def test_gather_and_trim_rank0_only_assembles_on_rank0(monkeypatch: pytest.Monke assert torch.equal(out.flatten(), torch.tensor([0.0, 0.0, 1.0, 1.0, 2.0])) +@pytest.mark.core_model +@pytest.mark.cpu def test_gather_and_trim_rank0_only_returns_empty_on_non_zero_rank(monkeypatch: pytest.MonkeyPatch): monkeypatch.setattr(wan_spatial_shard, "_rank_world", lambda group: (1, 3)) @@ -156,6 +173,8 @@ def test_gather_and_trim_rank0_only_returns_empty_on_non_zero_rank(monkeypatch: assert out.numel() == 0 +@pytest.mark.core_model +@pytest.mark.cpu def test_reshard_from_trimmed_height_pads_invalid_rows(monkeypatch: pytest.MonkeyPatch): monkeypatch.setattr(wan_spatial_shard, "_rank_world", lambda group: (2, 3)) @@ -183,6 +202,8 @@ def test_reshard_from_trimmed_height_pads_invalid_rows(monkeypatch: pytest.Monke assert torch.equal(out.flatten(), torch.tensor([4.0, 0.0])) +@pytest.mark.core_model +@pytest.mark.cpu def test_reshard_from_trimmed_width_pads_invalid_columns(monkeypatch: pytest.MonkeyPatch): monkeypatch.setattr(wan_spatial_shard, "_rank_world", lambda group: (2, 3)) @@ -210,6 +231,8 @@ def test_reshard_from_trimmed_width_pads_invalid_columns(monkeypatch: pytest.Mon assert torch.equal(out.flatten(), torch.tensor([4.0, 0.0])) +@pytest.mark.core_model +@pytest.mark.cpu def test_halo_exchange_single_rank_noop(monkeypatch: pytest.MonkeyPatch): monkeypatch.setattr(wan_spatial_shard, "_rank_world", lambda group: (0, 1)) @@ -225,6 +248,8 @@ def test_halo_exchange_single_rank_noop(monkeypatch: pytest.MonkeyPatch): assert recv_bottom is None +@pytest.mark.core_model +@pytest.mark.cpu def test_dist_zero_pad_only_applies_global_height_edges(monkeypatch: pytest.MonkeyPatch): x = torch.ones((1, 1, 2, 2)) @@ -239,6 +264,8 @@ def test_dist_zero_pad_only_applies_global_height_edges(monkeypatch: pytest.Monk assert last.shape == (1, 1, 3, 3) +@pytest.mark.core_model +@pytest.mark.cpu def test_dist_zero_pad_only_applies_global_width_edges(monkeypatch: pytest.MonkeyPatch): x = torch.ones((1, 1, 2, 2)) @@ -253,6 +280,8 @@ def test_dist_zero_pad_only_applies_global_width_edges(monkeypatch: pytest.Monke assert last.shape == (1, 1, 2, 3) +@pytest.mark.core_model +@pytest.mark.cpu def test_spatial_shard_height_gate_falls_back_for_partial_group(monkeypatch: pytest.MonkeyPatch): monkeypatch.setattr( "vllm_omni.diffusion.distributed.autoencoders.autoencoder_kl_wan.dist.get_world_size", @@ -270,6 +299,8 @@ def test_spatial_shard_height_gate_falls_back_for_partial_group(monkeypatch: pyt assert vae._spatial_shard_decode_enabled(z) is False +@pytest.mark.core_model +@pytest.mark.cpu def test_spatial_shard_width_gate_selects_width(monkeypatch: pytest.MonkeyPatch): monkeypatch.setattr( "vllm_omni.diffusion.distributed.autoencoders.autoencoder_kl_wan.dist.get_world_size", @@ -286,6 +317,8 @@ def test_spatial_shard_width_gate_selects_width(monkeypatch: pytest.MonkeyPatch) assert vae._spatial_shard_decode_enabled(z) is True +@pytest.mark.core_model +@pytest.mark.cpu def test_tile_mode_disables_spatial_shard_decode(): vae = DistributedAutoencoderKLWan.__new__(DistributedAutoencoderKLWan) vae.distributed_executor = SimpleNamespace(group=object(), parallel_size=2, parallel_mode="tile") @@ -298,7 +331,7 @@ def test_tile_mode_disables_spatial_shard_decode(): # ============================================================================= -# Multi-GPU numerical-correctness test (nightly / full_model, distributed CUDA) +# Multi-GPU numerical-correctness test (nightly Diffusion Test group, H100 x2) # # Spawns a small process group and verifies that spatial_shard_height/spatial_shard_width decode match # a single-process (non-distributed) reference decode of the same latent within @@ -383,11 +416,7 @@ def _spatial_shard_decode_worker(rank: int, split_dim: str, return_dict, master_ @pytest.mark.full_model @pytest.mark.diffusion @pytest.mark.parallel -@pytest.mark.distributed_cuda -@pytest.mark.skipif( - current_omni_platform.get_device_count() < _SPATIAL_SHARD_WORLD_SIZE, - reason="Requires >= 2 accelerator devices", -) +@hardware_test(res={"cuda": "H100"}, num_cards=_SPATIAL_SHARD_WORLD_SIZE) @pytest.mark.parametrize("split_dim", ["height", "width"]) def test_spatial_shard_decode_matches_reference(split_dim: str): manager = mp.get_context("spawn").Manager() diff --git a/tests/e2e/accuracy/test_hunyuan_image3.py b/tests/e2e/accuracy/test_hunyuan_image3.py index 88c7e255d..2214b734d 100644 --- a/tests/e2e/accuracy/test_hunyuan_image3.py +++ b/tests/e2e/accuracy/test_hunyuan_image3.py @@ -368,6 +368,7 @@ def _run_online(stage_configs_path: str, output_path: Path) -> tuple[Image.Image "300", "--init-timeout", "900", + "--trust-remote-code", ] try: with OmniServer(MODEL_PATH, server_args, use_omni=True) as omni_server: diff --git a/tests/e2e/online_serving/test_dynin_omni_expansion.py b/tests/e2e/online_serving/test_dynin_omni_expansion.py index 348191b6e..1f16a584b 100644 --- a/tests/e2e/online_serving/test_dynin_omni_expansion.py +++ b/tests/e2e/online_serving/test_dynin_omni_expansion.py @@ -26,7 +26,14 @@ T2I_PROMPT = "A high quality detailed living room interior photo." T2S_PROMPT = "Please read this sentence naturally: Hello from online serving." I2I_PROMPT = "Transform this outdoor nature boardwalk scene into a painting style with vivid colors." -TEST_PARAMS = [OmniServerParams(model=MODEL, stage_config_path=STAGE_CONFIG, stage_init_timeout=600)] +TEST_PARAMS = [ + OmniServerParams( + model=MODEL, + stage_config_path=STAGE_CONFIG, + stage_init_timeout=600, + server_args=["--trust-remote-code"], + ) +] _STAGE_COUNT = 3 _I2I_STAGE_SAMPLING = {"max_tokens": 1, "temperature": 0.0, "top_p": 1.0, "detokenize": False} From e24bddfb22f1c04ab28470043ac9a3585db95f64 Mon Sep 17 00:00:00 2001 From: leo <92263211+napleon-liu@users.noreply.github.com> Date: Mon, 20 Jul 2026 23:52:21 +0800 Subject: [PATCH 031/240] Fix Qwen3-Omni thinker fused MoE LoRA (#5191) Signed-off-by: leo --- .../qwen3_omni/test_qwen3_omni_moe_lora.py | 165 ++++++++++++++++++ .../qwen3_omni/qwen3_omni_moe_thinker.py | 12 ++ 2 files changed, 177 insertions(+) create mode 100644 tests/model_executor/models/qwen3_omni/test_qwen3_omni_moe_lora.py diff --git a/tests/model_executor/models/qwen3_omni/test_qwen3_omni_moe_lora.py b/tests/model_executor/models/qwen3_omni/test_qwen3_omni_moe_lora.py new file mode 100644 index 000000000..e5fa38a6b --- /dev/null +++ b/tests/model_executor/models/qwen3_omni/test_qwen3_omni_moe_lora.py @@ -0,0 +1,165 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Regression tests for Qwen3-Omni Thinker fused-MoE LoRA setup.""" + +from pathlib import Path +from types import SimpleNamespace + +import pytest +import torch +from peft import LoraConfig as PeftLoraConfig +from peft import get_peft_model +from torch import nn +from transformers import PretrainedConfig +from vllm.config.lora import LoRAConfig +from vllm.lora.layers.fused_moe import FusedMoE3DWithLoRA +from vllm.lora.lora_model import LoRAModel +from vllm.lora.model_manager import LoRAModelManager +from vllm.lora.peft_helper import PEFTHelper + +from vllm_omni.model_executor.models.qwen3_omni.qwen3_omni_moe_thinker import ( + Qwen3OmniMoeThinkerForConditionalGeneration, + _ensure_thinker_architecture, +) + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + +_NUM_EXPERTS = 2 +_HIDDEN_SIZE = 4 +_INTERMEDIATE_SIZE = 4 +_LORA_RANK = 1 +_EXPERTS_MODULE = "language_model.model.layers.0.mlp.experts" +_PEFT_TARGET_PARAMETERS = [ + "thinker.model.layers.0.mlp.experts.gate_up_proj", + "thinker.model.layers.0.mlp.experts.down_proj", +] + + +class _TinyExperts(nn.Module): + """Expert parameters with the same 3-D layout as Qwen3-Omni.""" + + def __init__(self): + super().__init__() + self.gate_up_proj = nn.Parameter(torch.zeros(_NUM_EXPERTS, 2 * _INTERMEDIATE_SIZE, _HIDDEN_SIZE)) + self.down_proj = nn.Parameter(torch.zeros(_NUM_EXPERTS, _HIDDEN_SIZE, _INTERMEDIATE_SIZE)) + + +class _TinyQwen3OmniMoe(nn.Module): + """Minimal module tree needed for PEFT to save expert LoRA weights.""" + + def __init__(self, model_path: Path): + super().__init__() + self.thinker = nn.ModuleDict( + { + "model": nn.ModuleDict( + {"layers": nn.ModuleList([nn.ModuleDict({"mlp": nn.ModuleDict({"experts": _TinyExperts()})})])} + ) + } + ) + self.name_or_path = str(model_path) + self.config = {"model_type": "qwen3_omni_moe"} + + def forward(self, inputs: torch.Tensor) -> torch.Tensor: + return inputs + + +class _CpuFusedMoE3DWithLoRA(FusedMoE3DWithLoRA): + """CPU harness for the real 3-D MoE LoRA weight methods.""" + + def __init__(self): + nn.Module.__init__(self) + self.moe_config = SimpleNamespace( + hidden_dim=_HIDDEN_SIZE, + num_local_experts=_NUM_EXPERTS, + num_experts=_NUM_EXPERTS, + intermediate_size_per_partition=_INTERMEDIATE_SIZE, + moe_parallel_config=SimpleNamespace( + tp_size=1, + tp_rank=0, + ep_rank=0, + use_ep=False, + ), + ) + self.device = torch.device("cpu") + self.tp_size = 1 + self.tp_rank = 0 + self._w13_slices = 1 + + +def _save_peft_adapter(tmp_path: Path) -> Path: + base_model_dir = tmp_path / "base_model" + base_model_dir.mkdir() + (base_model_dir / "config.json").write_text('{"model_type": "qwen3_omni_moe"}', encoding="utf-8") + + peft_model = get_peft_model( + _TinyQwen3OmniMoe(base_model_dir), + PeftLoraConfig( + r=_LORA_RANK, + lora_alpha=_LORA_RANK, + target_modules=[], + target_parameters=_PEFT_TARGET_PARAMETERS, + ), + ) + with torch.no_grad(): + for value, parameter in enumerate( + (parameter for parameter in peft_model.parameters() if parameter.requires_grad), + start=1, + ): + parameter.fill_(value) + + adapter_dir = tmp_path / "adapter" + peft_model.save_pretrained(adapter_dir) + return adapter_dir + + +def test_peft_expert_lora_loads_into_thinker(tmp_path: Path): + adapter_dir = _save_peft_adapter(tmp_path) + peft_helper = PEFTHelper.from_local_dir(str(adapter_dir), max_position_embeddings=128) + lora_model = LoRAModel.from_local_checkpoint( + str(adapter_dir), + {"experts"}, + peft_helper=peft_helper, + lora_model_id=1, + device="cpu", + dtype=torch.float32, + weights_mapper=Qwen3OmniMoeThinkerForConditionalGeneration.hf_to_vllm_mapper, + ) + + thinker_config = PretrainedConfig() + _ensure_thinker_architecture(thinker_config) + fused_moe = _CpuFusedMoE3DWithLoRA() + fused_moe.create_lora_weights( + max_loras=1, + lora_config=LoRAConfig( + max_lora_rank=_LORA_RANK, + max_loras=1, + lora_dtype=torch.float32, + ), + model_config=thinker_config, + ) + + manager = object.__new__(LoRAModelManager) + manager.is_pooling_model = False + manager._is_3d_moe_model = Qwen3OmniMoeThinkerForConditionalGeneration.is_3d_moe_weight + manager._stack_moe_lora_weights(lora_model, fused_moe, _EXPERTS_MODULE) + + expert_lora = lora_model.get_lora(_EXPERTS_MODULE) + assert expert_lora is not None + fused_moe.set_lora(0, expert_lora.lora_a, expert_lora.lora_b) + + assert fused_moe.adapter_enabled.tolist() == [1, 0] + for weights in ( + fused_moe.w13_lora_a_stacked[0], + fused_moe.w13_lora_b_stacked[0], + fused_moe.w2_lora_a_stacked[0], + fused_moe.w2_lora_b_stacked[0], + ): + assert torch.count_nonzero(weights[0]) == weights[0].numel() + + +def test_existing_thinker_architecture_is_preserved(): + config = PretrainedConfig(architectures=["CustomThinkerArchitecture"]) + + _ensure_thinker_architecture(config) + + assert config.architectures == ["CustomThinkerArchitecture"] diff --git a/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py b/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py index fde493deb..65ec102a1 100644 --- a/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py +++ b/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py @@ -137,6 +137,14 @@ except (ImportError, ModuleNotFoundError): logger = init_logger(__name__) +_THINKER_ARCHITECTURE = "Qwen3OmniMoeThinkerForConditionalGeneration" + + +def _ensure_thinker_architecture(config: PretrainedConfig) -> None: + """Give the nested Thinker config the architecture required by MoE LoRA.""" + if not config.architectures: + config.architectures = [_THINKER_ARCHITECTURE] + class Qwen3Omni_VisionTransformer(_Qwen3Omni_VisionTransformer): """Subclass that fixes Qwen2_5_VisionAttention.forward() compatibility. @@ -1121,6 +1129,9 @@ class Qwen3OmniMoeThinkerForConditionalGeneration( Qwen3OmniMoeConditionalGenerationMixin, SupportsTranscription, ): + # PEFT stores the expert LoRA matrices with an explicit expert dimension. + is_3d_moe_weight: bool = True + hf_to_vllm_mapper = WeightsMapper( orig_to_new_prefix={ "thinker.lm_head.": "language_model.lm_head.", @@ -1164,6 +1175,7 @@ class Qwen3OmniMoeThinkerForConditionalGeneration( self.vllm_config = vllm_config # needed for torch compile forward context hf_config = vllm_config.model_config.hf_config thinker_config: Qwen3OmniMoeThinkerConfig = getattr(hf_config, "thinker_config", None) or hf_config + _ensure_thinker_architecture(thinker_config) quant_config = vllm_config.quant_config multimodal_config = vllm_config.model_config.multimodal_config self.config = thinker_config From dacb8e6f10f06ffee05f6705f61f084e186470a4 Mon Sep 17 00:00:00 2001 From: Juan Pablo Zuluaga <46724788+JuanPZuluaga@users.noreply.github.com> Date: Mon, 20 Jul 2026 19:22:45 +0200 Subject: [PATCH 032/240] [Bugfix][Perf] mimo_audio: fix per-request speech-code routing under continuous batching (#5070) Signed-off-by: JuanPZuluaga Co-authored-by: Yueqian Lin <70319226+linyueqian@users.noreply.github.com> --- .../test_mimo_audio_per_request_codes.py | 120 ++++++++++++++++++ vllm_omni/entrypoints/openai/serving_chat.py | 5 +- .../models/mimo_audio/mimo_audio.py | 14 +- .../models/mimo_audio/mimo_audio_llm.py | 53 +++++--- 4 files changed, 172 insertions(+), 20 deletions(-) create mode 100644 tests/model_executor/models/mimo_audio/test_mimo_audio_per_request_codes.py diff --git a/tests/model_executor/models/mimo_audio/test_mimo_audio_per_request_codes.py b/tests/model_executor/models/mimo_audio/test_mimo_audio_per_request_codes.py new file mode 100644 index 000000000..4dc365ef0 --- /dev/null +++ b/tests/model_executor/models/mimo_audio/test_mimo_audio_per_request_codes.py @@ -0,0 +1,120 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Regression tests for MiMo-Audio per-request speech-code routing. + +Guards the cross-request audio-corruption class (same class as RFC #4316 +Issue 1 for MOSS-TTS): the fused thinker+talker emits one speech-code row per +request, so ``codes["audio"]`` must be a batch-ordered **list** for the +generic splitter (``to_payload_element``) to route ``element[idx]`` to +request ``idx``. A batched ``(num_reqs, 1, 8, 4)`` tensor is only split +correctly by the token-offset path when every request has exactly one +scheduled token; in a mixed prefill+decode step the tensor's first dim +matches neither token length, the splitter falls through to ``clone()``, and +every request receives the whole batch — leaking codes across requests and +crashing the async-chunk ragged-list conversion +(``expected sequence of length 36 at dim 1 (got 144)``). + +CPU-only and weight-free: ``forward`` is driven on a bare model with +``generate_codes`` stubbed to return row-indexed codes, so assertions on the +payload prove which request got which row. +""" + +from __future__ import annotations + +import functools + +import pytest + +torch = pytest.importorskip("torch") + +from vllm_omni.utils.mm_outputs import to_payload_element # noqa: E402 + +NUM_CODEBOOKS = 8 +GROUP_SIZE = 4 + + +@functools.lru_cache(maxsize=1) +def _model_cls(): + """Defer the import (pulls vLLM model_executor) until first use.""" + from vllm_omni.model_executor.models.mimo_audio.mimo_audio import ( + MiMoAudioForConditionalGeneration, + ) + + return MiMoAudioForConditionalGeneration + + +def _row_indexed_codes(num_reqs: int) -> torch.Tensor: + """(num_reqs, 1, 8, 4) tensor where request r's codes all hold r.""" + codes = torch.empty((num_reqs, 1, NUM_CODEBOOKS, GROUP_SIZE), dtype=torch.long) + for r in range(num_reqs): + codes[r] = r + return codes + + +def _make_bare_model(next_speech_tokens, num_tokens: int): + """A model whose only behaviour is the emit logic under test.""" + cls = _model_cls() + model = cls.__new__(cls) + model.model_stage = "fused_thinker_talker" + hidden = torch.zeros((num_tokens, 16), dtype=torch.float32) + model.generate_codes = lambda **kwargs: (next_speech_tokens, hidden) + return model + + +def _emit(next_speech_tokens, num_tokens: int): + model = _make_bare_model(next_speech_tokens, num_tokens) + result = model.forward( + input_ids=torch.zeros((num_tokens,), dtype=torch.long), + positions=torch.zeros((num_tokens,), dtype=torch.long), + ) + return result.multimodal_outputs["codes"]["audio"] + + +class TestPerRequestCodeRouting: + def test_emit_is_batch_aligned_list(self): + audio = _emit(_row_indexed_codes(3), num_tokens=3) + + assert isinstance(audio, list) + assert len(audio) == 3 # one entry per request, batch-aligned + for r, codes in enumerate(audio): + assert codes.shape == (1, 1, NUM_CODEBOOKS, GROUP_SIZE) + assert torch.all(codes == r) + + def test_no_codes_step_emits_none(self): + audio = _emit(None, num_tokens=2) + assert audio is None + + def test_mixed_prefill_decode_routes_by_request_index(self): + """req 1 prefills 7 tokens while reqs 0 and 2 decode: the splitter + must hand each request its own row, not the whole batch.""" + audio = _emit(_row_indexed_codes(3), num_tokens=9) + + seq_len = 9 # 1 (decode) + 7 (prefill) + 1 (decode) scheduled tokens + spans = [(0, 1), (1, 8), (8, 9)] + for idx, (start, end) in enumerate(spans): + routed = to_payload_element( + audio, + idx, + start, + end, + seq_len=seq_len, + scheduled_seq_len=seq_len, + ) + assert routed.shape == (1, 1, NUM_CODEBOOKS, GROUP_SIZE) + assert torch.all(routed == idx), f"req {idx} received another request's codes" + + def test_pure_decode_routes_by_request_index(self): + """One token per request (the only case the old tensor emit handled).""" + audio = _emit(_row_indexed_codes(4), num_tokens=4) + + for idx in range(4): + routed = to_payload_element(audio, idx, idx, idx + 1, seq_len=4, scheduled_seq_len=4) + assert torch.all(routed == idx) + + def test_routed_codes_do_not_alias_the_emit(self): + """The splitter clones list elements; mutating the routed payload must + not corrupt another request's view of the emit.""" + audio = _emit(_row_indexed_codes(2), num_tokens=5) + routed = to_payload_element(audio, 0, 0, 4, seq_len=5, scheduled_seq_len=5) + routed.fill_(99) + assert torch.all(audio[0] == 0) diff --git a/vllm_omni/entrypoints/openai/serving_chat.py b/vllm_omni/entrypoints/openai/serving_chat.py index eec6a03b9..3f6250be8 100755 --- a/vllm_omni/entrypoints/openai/serving_chat.py +++ b/vllm_omni/entrypoints/openai/serving_chat.py @@ -2539,7 +2539,10 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin): final_res = omni_outputs.request_output # OMNI: Access multimodal_output from CompletionOutput (outputs[0]), not from RequestOutput # Reference: examples/offline_inference/qwen3_omni/end2end.py line 421 - mm_output = final_res.outputs[0].multimodal_output + # The attribute is attached dynamically when stage audio arrives; fall + # back to the no-audio error response instead of an AttributeError 500 + # when the pipeline produced no audio for this request. + mm_output = getattr(final_res.outputs[0], "multimodal_output", None) or {} audio_data = mm_output.get("audio") if isinstance(audio_data, list): if not audio_data: diff --git a/vllm_omni/model_executor/models/mimo_audio/mimo_audio.py b/vllm_omni/model_executor/models/mimo_audio/mimo_audio.py index ecf094c87..2af6abd42 100644 --- a/vllm_omni/model_executor/models/mimo_audio/mimo_audio.py +++ b/vllm_omni/model_executor/models/mimo_audio/mimo_audio.py @@ -793,9 +793,21 @@ class MiMoAudioForConditionalGeneration( **kwargs, ) + # next_speech_tokens is request-indexed (num_reqs, 1, 8, 4), not + # token-indexed, so ship it as a per-request list. The payload + # builder slices batched tensors by token offsets (start:end), + # which only coincides with the request index in pure-decode + # steps; in a mixed prefill+decode batch every request would + # receive the whole batch tensor, leaking codes across requests + # and crashing the downstream ragged-list conversion. + if next_speech_tokens is not None: + speech_token_payload = list(next_speech_tokens.split(1, dim=0)) + else: + speech_token_payload = None + return OmniOutput( text_hidden_states=text_hidden_states.reshape(-1, text_hidden_states.shape[-1]), - multimodal_outputs={"codes": {"audio": next_speech_tokens}}, + multimodal_outputs={"codes": {"audio": speech_token_payload}}, ) if self.model_stage == "code2wav": diff --git a/vllm_omni/model_executor/models/mimo_audio/mimo_audio_llm.py b/vllm_omni/model_executor/models/mimo_audio/mimo_audio_llm.py index 48715fb98..19a339056 100644 --- a/vllm_omni/model_executor/models/mimo_audio/mimo_audio_llm.py +++ b/vllm_omni/model_executor/models/mimo_audio/mimo_audio_llm.py @@ -897,20 +897,28 @@ class MiMoAudioLLMForConditionalGeneration(nn.Module, SupportsMultiModal, Suppor input_ids: torch.Tensor, *, request_ids: list[str], - query_start_loc: torch.Tensor, + query_start_loc_list: list[int], kwargs: dict, ) -> tuple[dict[str, any], dict]: has_merge_mm_embedding = False merge_mm_embedding_info: dict[str, any] = {} seq_len = input_ids.shape[1] if input_ids.ndim == 2 else input_ids.shape[0] - for req_idx, req_id in enumerate(request_ids): - query_start_loc_by_req = int(query_start_loc[req_idx].item()) - query_end_loc_by_req = int(query_start_loc[req_idx + 1].item()) - input_ids_by_req = input_ids[query_start_loc_by_req:query_end_loc_by_req] - seq_len_by_req = input_ids_by_req.shape[1] if input_ids_by_req.ndim == 2 else input_ids_by_req.shape[0] + # Only each request's start token is ever read below, and only for + # single-token requests. Gather just those start offsets (num_reqs elements) + # to host instead of copying the whole flattened sequence every forward. + # query_start_loc entries are flat token offsets, so we index the flattened + # ids by the same offsets and derive per-request length from the boundaries. + flat_input_ids = input_ids.reshape(-1) + start_positions = query_start_loc_list[:-1] + start_is_empty_cpu = (flat_input_ids[start_positions] == self.empty_token_id).cpu().tolist() - if seq_len_by_req == 1 and bool(input_ids_by_req == self.empty_token_id): + for req_idx, req_id in enumerate(request_ids): + query_start_loc_by_req = query_start_loc_list[req_idx] + query_end_loc_by_req = query_start_loc_list[req_idx + 1] + seq_len_by_req = query_end_loc_by_req - query_start_loc_by_req + + if seq_len_by_req == 1 and start_is_empty_cpu[req_idx]: merge_mm_embedding_info[req_id] = { "query_start_loc": query_start_loc_by_req, "query_end_loc": query_end_loc_by_req, @@ -1035,13 +1043,19 @@ class MiMoAudioLLMForConditionalGeneration(nn.Module, SupportsMultiModal, Suppor inputs_embeds: torch.Tensor | None = None, **kwargs: object, ) -> torch.Tensor | IntermediateTensors: + # Materialize the request boundaries on the host once instead of one + # `.item()` sync per boundary in the loops below. _forward_context = get_forward_context() - _default_query_start_loc = torch.tensor([0, input_ids.shape[-1]], device=input_ids.device) - query_start_loc = ( - next(iter(_forward_context.attn_metadata.values())).query_start_loc - if _forward_context.attn_metadata is not None - else _default_query_start_loc - ) + if _forward_context.attn_metadata is not None: + query_start_loc = next(iter(_forward_context.attn_metadata.values())).query_start_loc + query_start_loc_list = query_start_loc.tolist() + else: + # Single implicit request over all tokens. Build the host boundaries + # directly (no .tolist() D2H); the small device tensor is still needed + # for logits_indices below. + n_tokens = int(input_ids.shape[-1]) + query_start_loc_list = [0, n_tokens] + query_start_loc = torch.tensor([0, n_tokens], device=input_ids.device) runtime_additional_information = kwargs.get("runtime_additional_information", []) if runtime_additional_information: @@ -1054,7 +1068,7 @@ class MiMoAudioLLMForConditionalGeneration(nn.Module, SupportsMultiModal, Suppor merge_mm_embedding_info, has_merge_mm_embedding, kwargs = self._collect_merge_mm_embedding_info( input_ids, request_ids=request_ids, - query_start_loc=query_start_loc, + query_start_loc_list=query_start_loc_list, kwargs=kwargs, ) @@ -1077,15 +1091,18 @@ class MiMoAudioLLMForConditionalGeneration(nn.Module, SupportsMultiModal, Suppor batch_next_speech_tokens: torch.Tensor | None = None if not is_capturing and next_ids is not None and num_reqs > 0: - if (next_ids == self.empty_token_id).any(): + # Copy the empty-token mask to host once; the `.any()` gate and the + # per-request checks below then avoid a GPU sync each. + next_is_empty_cpu = (next_ids == self.empty_token_id).cpu() + if bool(next_is_empty_cpu.any()): batch_hs_list = [] valid_mask = [] for req_idx in range(num_reqs): - start = int(query_start_loc[req_idx].item()) - end = int(query_start_loc[req_idx + 1].item()) + start = query_start_loc_list[req_idx] + end = query_start_loc_list[req_idx + 1] hs_req = hidden_states[start:end][-1:, :] - is_empty = bool(next_ids[req_idx] == self.empty_token_id) + is_empty = bool(next_is_empty_cpu[req_idx]) valid_mask.append(is_empty) if not is_empty: From 64bcdc2186837533bf6572953a25625187ef043a Mon Sep 17 00:00:00 2001 From: amy-why-3459 Date: Tue, 21 Jul 2026 11:37:15 +0800 Subject: [PATCH 033/240] [Example][MiniCPM-o] Add offline/online MiniCPM-o 4.5 examples (#5222) Signed-off-by: amy-why-3459 Co-authored-by: Cursor --- examples/offline_inference/minicpmo/README.md | 91 +++ .../offline_inference/minicpmo/end2end.py | 543 ++++++++++++++++++ .../minicpmo/run_multiple_prompts.sh | 9 + .../minicpmo/run_single_prompt.sh | 7 + .../minicpmo/run_single_prompt_tp.sh | 12 + examples/online_serving/minicpmo/README.md | 139 ++++- ...letion_client_for_multimodal_generation.py | 237 ++++++++ .../run_curl_multimodal_generation.sh | 154 +++++ recipes/OpenBMB/MiniCPM-o-4_5.md | 7 +- 9 files changed, 1174 insertions(+), 25 deletions(-) create mode 100644 examples/offline_inference/minicpmo/README.md create mode 100644 examples/offline_inference/minicpmo/end2end.py create mode 100755 examples/offline_inference/minicpmo/run_multiple_prompts.sh create mode 100755 examples/offline_inference/minicpmo/run_single_prompt.sh create mode 100755 examples/offline_inference/minicpmo/run_single_prompt_tp.sh create mode 100644 examples/online_serving/minicpmo/openai_chat_completion_client_for_multimodal_generation.py create mode 100755 examples/online_serving/minicpmo/run_curl_multimodal_generation.sh diff --git a/examples/offline_inference/minicpmo/README.md b/examples/offline_inference/minicpmo/README.md new file mode 100644 index 000000000..bb6ce2568 --- /dev/null +++ b/examples/offline_inference/minicpmo/README.md @@ -0,0 +1,91 @@ +# MiniCPM-o 4.5: Offline inference + +Two-stage pipeline: **thinker** (multimodal understanding) → **talker + Token2Wav** +(24 kHz speech). Deploy config auto-loads from +`vllm_omni/deploy/minicpmo_4_5.yaml` (2-GPU default). + +## Setup + +- Install talker deps: `pip install 'vllm-omni[minicpmo]'` (or `stepaudio2-minicpmo`) +- `--trust-remote-code` is always passed by `end2end.py` (required for MiniCPMO) +- See [stage configuration docs](https://docs.vllm.ai/projects/vllm-omni/en/latest/configuration/stage_configs/) for memory tuning + +## Run examples + +### Single prompt + +```bash +cd examples/offline_inference/minicpmo +bash run_single_prompt.sh +``` + +### Multiple prompts + +```bash +bash run_multiple_prompts.sh +``` + +### Thinker tensor parallel (3-GPU) + +```bash +bash run_single_prompt_tp.sh +``` + +### Modality control + +Text-only (skip talker / no `<|tts_bos|>`): + +```bash +python end2end.py --query-type use_audio --modalities text +``` + +Text + speech (default — appends `<|tts_bos|>`): + +```bash +python end2end.py --query-type text --modalities text,audio +``` + +### Local media files + +```bash +python end2end.py --query-type use_video --video-path /path/to/video.mp4 +python end2end.py --query-type use_image --image-path /path/to/image.jpg +python end2end.py --query-type use_audio --audio-path /path/to/audio.wav +python end2end.py --query-type use_mixed_modalities \ + --video-path /path/to/video.mp4 \ + --image-path /path/to/image.jpg \ + --audio-path /path/to/audio.wav +``` + +Supported `--query-type` values: + +| Query type | Inputs | +|---|---| +| `text` | Text only | +| `use_image` | Image + text | +| `use_audio` | Audio + text | +| `use_video` | Video + text | +| `use_multi_audios` | Two audio clips | +| `use_mixed_modalities` | Audio + image + video | + +### Custom deploy config + +```bash +python end2end.py --query-type text \ + --deploy-config /path/to/vllm_omni/deploy/minicpmo_4_5_8x4090.yaml +``` + +## Notes + +- Speech requires `<|tts_bos|>` on the assistant prefix (offline equivalent of + online `chat_template_kwargs.use_tts_template=true`). Without it, the talker + gets an empty TTS span. +- Output WAV is **24 kHz mono**. +- Placeholders in the prompt are MiniCPM-style: + `(./)`, `()`, `()`. +- Default layout needs **2 GPUs**. Async chunking is off in the bundled YAMLs. + +## Online serving + +See [`examples/online_serving/minicpmo/`](../../online_serving/minicpmo/) and +the recipe [`recipes/OpenBMB/MiniCPM-o-4_5.md`](../../../recipes/OpenBMB/MiniCPM-o-4_5.md). diff --git a/examples/offline_inference/minicpmo/end2end.py b/examples/offline_inference/minicpmo/end2end.py new file mode 100644 index 000000000..e9c9019c0 --- /dev/null +++ b/examples/offline_inference/minicpmo/end2end.py @@ -0,0 +1,543 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +""" +Offline end-to-end inference for MiniCPM-o 4.5 (thinker + talker/Token2Wav). + +MiniCPM-o uses placeholders ``(./)``, ``()``, +and ``()``. Speech output requires a ``<|tts_bos|>`` suffix on +the assistant prefix (equivalent to online ``chat_template_kwargs.use_tts_template``). +""" + +import os +import time +from typing import NamedTuple + +import numpy as np +import soundfile as sf +import vllm +from PIL import Image +from vllm import SamplingParams +from vllm.assets.audio import AudioAsset +from vllm.assets.image import ImageAsset +from vllm.assets.video import VideoAsset, video_to_ndarrays +from vllm.multimodal.image import convert_image_mode +from vllm.multimodal.media.audio import load_audio + +from vllm_omni.entrypoints.omni import Omni +from vllm_omni.utils.tracking_parser import TrackingArgumentParser + +SEED = 42 + +default_system = ( + "You are MiniCPM-o, a helpful multimodal assistant that can " + "understand images, audio and video, and respond in text and speech." +) + + +class QueryResult(NamedTuple): + inputs: dict + limit_mm_per_prompt: dict[str, int] + + +def _assistant_prefix(use_tts: bool) -> str: + # Matches HF chat_template: assistant header, optional empty , then TTS bos. + prefix = "<|im_start|>assistant\n" + if use_tts: + prefix += "<|tts_bos|>" + return prefix + + +def _build_prompt(user_body: str, use_tts: bool) -> str: + return ( + f"<|im_start|>system\n{default_system}<|im_end|>\n" + f"<|im_start|>user\n{user_body}<|im_end|>\n" + f"{_assistant_prefix(use_tts)}" + ) + + +def get_text_query(question: str | None = None, use_tts: bool = True) -> QueryResult: + if question is None: + question = "Say hello, then introduce vLLM-Omni in one sentence." + return QueryResult( + inputs={"prompt": _build_prompt(question, use_tts=use_tts)}, + limit_mm_per_prompt={}, + ) + + +def get_image_query( + question: str | None = None, + image_path: str | None = None, + use_tts: bool = True, +) -> QueryResult: + if question is None: + question = "What is the content of this image?" + user_body = f"(./)\n{question}" + + if image_path: + if not os.path.exists(image_path): + raise FileNotFoundError(f"Image file not found: {image_path}") + image_data = convert_image_mode(Image.open(image_path), "RGB") + else: + image_data = convert_image_mode(ImageAsset("cherry_blossom").pil_image, "RGB") + + return QueryResult( + inputs={ + "prompt": _build_prompt(user_body, use_tts=use_tts), + "multi_modal_data": {"image": image_data}, + }, + limit_mm_per_prompt={"image": 1}, + ) + + +def get_audio_query( + question: str | None = None, + audio_path: str | None = None, + sampling_rate: int = 16000, + use_tts: bool = True, +) -> QueryResult: + if question is None: + question = "What is the content of this audio?" + user_body = f"()\n{question}" + + if audio_path: + if not os.path.exists(audio_path): + raise FileNotFoundError(f"Audio file not found: {audio_path}") + audio_signal, sr = load_audio(audio_path, sr=sampling_rate) + audio_data = (audio_signal.astype(np.float32), sr) + else: + audio_data = AudioAsset("mary_had_lamb").audio_and_sample_rate + + return QueryResult( + inputs={ + "prompt": _build_prompt(user_body, use_tts=use_tts), + "multi_modal_data": {"audio": audio_data}, + }, + limit_mm_per_prompt={"audio": 1}, + ) + + +def get_video_query( + question: str | None = None, + video_path: str | None = None, + num_frames: int = 16, + use_tts: bool = True, +) -> QueryResult: + if question is None: + question = "Why is this video funny?" + user_body = f"()\n{question}" + + if video_path: + if not os.path.exists(video_path): + raise FileNotFoundError(f"Video file not found: {video_path}") + video_frames = video_to_ndarrays(video_path, num_frames=num_frames) + else: + video_frames = VideoAsset(name="baby_reading", num_frames=num_frames).np_ndarrays + + return QueryResult( + inputs={ + "prompt": _build_prompt(user_body, use_tts=use_tts), + "multi_modal_data": {"video": video_frames}, + }, + limit_mm_per_prompt={"video": 1}, + ) + + +def get_mixed_modalities_query( + video_path: str | None = None, + image_path: str | None = None, + audio_path: str | None = None, + num_frames: int = 16, + sampling_rate: int = 16000, + use_tts: bool = True, +) -> QueryResult: + question = "What is recited in the audio? What is the content of this image? Why is this video funny?" + user_body = f"()\n(./)\n()\n{question}" + + if video_path: + if not os.path.exists(video_path): + raise FileNotFoundError(f"Video file not found: {video_path}") + video_frames = video_to_ndarrays(video_path, num_frames=num_frames) + else: + video_frames = VideoAsset(name="baby_reading", num_frames=num_frames).np_ndarrays + + if image_path: + if not os.path.exists(image_path): + raise FileNotFoundError(f"Image file not found: {image_path}") + image_data = convert_image_mode(Image.open(image_path), "RGB") + else: + image_data = convert_image_mode(ImageAsset("cherry_blossom").pil_image, "RGB") + + if audio_path: + if not os.path.exists(audio_path): + raise FileNotFoundError(f"Audio file not found: {audio_path}") + audio_signal, sr = load_audio(audio_path, sr=sampling_rate) + audio_data = (audio_signal.astype(np.float32), sr) + else: + audio_data = AudioAsset("mary_had_lamb").audio_and_sample_rate + + return QueryResult( + inputs={ + "prompt": _build_prompt(user_body, use_tts=use_tts), + "multi_modal_data": { + "audio": audio_data, + "image": image_data, + "video": video_frames, + }, + }, + limit_mm_per_prompt={"audio": 1, "image": 1, "video": 1}, + ) + + +def get_multi_audios_query(use_tts: bool = True) -> QueryResult: + question = "Are these two audio clips the same?" + user_body = f"()\n()\n{question}" + return QueryResult( + inputs={ + "prompt": _build_prompt(user_body, use_tts=use_tts), + "multi_modal_data": { + "audio": [ + AudioAsset("winning_call").audio_and_sample_rate, + AudioAsset("mary_had_lamb").audio_and_sample_rate, + ], + }, + }, + limit_mm_per_prompt={"audio": 2}, + ) + + +query_map = { + "text": get_text_query, + "use_audio": get_audio_query, + "use_image": get_image_query, + "use_video": get_video_query, + "use_multi_audios": get_multi_audios_query, + "use_mixed_modalities": get_mixed_modalities_query, +} + + +def _wants_tts(modalities: str | None) -> bool: + """TTS bos is needed whenever audio is among the requested output modalities.""" + if modalities is None: + return True + parts = {p.strip() for p in modalities.split(",") if p.strip()} + return "audio" in parts + + +def main(args): + model_name = args.model + print("=" * 20, "\n", f"vllm version: {vllm.__version__}", "\n", "=" * 20) + + video_path = getattr(args, "video_path", None) + image_path = getattr(args, "image_path", None) + audio_path = getattr(args, "audio_path", None) + use_tts = _wants_tts(args.modalities) + + query_func = query_map[args.query_type] + if args.query_type == "use_video": + query_result = query_func( + video_path=video_path, + num_frames=getattr(args, "num_frames", 16), + use_tts=use_tts, + ) + elif args.query_type == "use_image": + query_result = query_func(image_path=image_path, use_tts=use_tts) + elif args.query_type == "use_audio": + query_result = query_func( + audio_path=audio_path, + sampling_rate=getattr(args, "sampling_rate", 16000), + use_tts=use_tts, + ) + elif args.query_type == "use_mixed_modalities": + query_result = query_func( + video_path=video_path, + image_path=image_path, + audio_path=audio_path, + num_frames=getattr(args, "num_frames", 16), + sampling_rate=getattr(args, "sampling_rate", 16000), + use_tts=use_tts, + ) + elif args.query_type == "use_multi_audios": + query_result = query_func(use_tts=use_tts) + else: + query_result = query_func(use_tts=use_tts) + + omni_kwargs = vars(args).copy() + # Drop example-only CLI keys that are not Omni / engine args. + for key in ( + "query_type", + "output_wav", + "output_dir", + "num_prompts", + "txt_prompts", + "video_path", + "image_path", + "audio_path", + "num_frames", + "sampling_rate", + "modalities", + "py_generator", + "enable_profiler", + "profiler_stages", + "log_dir", + ): + omni_kwargs.pop(key, None) + omni_kwargs["model"] = model_name + omni_kwargs["trust_remote_code"] = True + omni = Omni(**omni_kwargs) + + # Stage 0 (thinker): multimodal understanding → text (+ TTS span when enabled). + thinker_sampling_params = SamplingParams( + temperature=0.0, + top_p=1.0, + top_k=-1, + max_tokens=2048, + seed=SEED, + detokenize=True, + repetition_penalty=1.1, + ) + # Stage 1 (talker + Token2Wav): max_tokens=1 satisfies the scheduler; + # waveform is produced in-process by Token2Wav. + talker_sampling_params = SamplingParams( + temperature=0.0, + top_p=1.0, + top_k=-1, + max_tokens=1, + seed=SEED, + detokenize=False, + ) + sampling_params_list = [thinker_sampling_params, talker_sampling_params][: omni.num_stages] + + if args.txt_prompts is None: + prompts = [query_result.inputs for _ in range(args.num_prompts)] + else: + assert args.query_type == "text", "txt-prompts is only supported for text query type" + with open(args.txt_prompts, encoding="utf-8") as f: + lines = [ln.strip() for ln in f.readlines()] + prompts = [get_text_query(ln, use_tts=use_tts).inputs for ln in lines if ln != ""] + print(f"[Info] Loaded {len(prompts)} prompts from {args.txt_prompts}") + + if args.modalities is not None: + output_modalities = args.modalities.split(",") + for prompt in prompts: + prompt["modalities"] = output_modalities + + profiler_enabled = args.enable_profiler + if profiler_enabled: + omni.start_profile(stages=args.profiler_stages) + + omni_generator = omni.generate(prompts, sampling_params_list, py_generator=args.py_generator) + output_dir = args.output_dir if getattr(args, "output_dir", None) else args.output_wav + os.makedirs(output_dir, exist_ok=True) + + total_requests = len(prompts) + processed_count = 0 + print(f"query type: {args.query_type}") + print(f"use_tts (tts_bos): {use_tts}") + + for stage_outputs in omni_generator: + output = stage_outputs.request_output + if stage_outputs.final_output_type == "text": + request_id = output.request_id + text_output = output.outputs[0].text + out_txt = os.path.join(output_dir, f"{request_id}.txt") + lines = [ + "Prompt:\n", + str(output.prompt) + "\n", + "vllm_text_output:\n", + str(text_output).strip() + "\n", + ] + try: + with open(out_txt, "w", encoding="utf-8") as f: + f.writelines(lines) + except Exception as e: + print(f"[Warn] Failed writing text file {out_txt}: {e}") + print(f"Request ID: {request_id}, Text saved to {out_txt}") + elif stage_outputs.final_output_type == "audio": + request_id = output.request_id + audio_tensor = output.outputs[0].multimodal_output["audio"] + output_wav = os.path.join(output_dir, f"output_{request_id}.wav") + + if isinstance(audio_tensor, list): + import torch + + audio_tensor = torch.cat( + [(t if isinstance(t, torch.Tensor) else torch.tensor(t)).flatten() for t in audio_tensor] + ) + audio_numpy = audio_tensor.float().detach().cpu().numpy() + if audio_numpy.ndim > 1: + audio_numpy = audio_numpy.flatten() + + # MiniCPM-o 4.5 Token2Wav emits 24 kHz mono. + sf.write(output_wav, audio_numpy, samplerate=24000, format="WAV") + print(f"Request ID: {request_id}, Saved audio to {output_wav}") + + processed_count += 1 + if profiler_enabled and processed_count >= total_requests: + print(f"[Info] Processed {processed_count}/{total_requests}. Stopping profiler inside active loop...") + omni.stop_profile(stages=args.profiler_stages) + print("[Info] Waiting 30s for workers to write trace files to disk...") + time.sleep(30) + print("[Info] Trace export wait time finished.") + omni.close() + + +def parse_args(): + parser = TrackingArgumentParser( + description="Offline inference demo for MiniCPM-o 4.5 (text / image / audio / video → text + speech)" + ) + parser.add_argument( + "--model", + type=str, + default="openbmb/MiniCPM-o-4_5", + help="Model name or path.", + ) + parser.add_argument( + "--query-type", + "-q", + type=str, + default="text", + choices=query_map.keys(), + help="Query type.", + ) + parser.add_argument( + "--log-stats", + action="store_true", + default=False, + help="Enable writing detailed statistics (default: disabled)", + ) + parser.add_argument( + "--stage-init-timeout", + type=int, + default=300, + help="Timeout for initializing a single stage in seconds (default: 300)", + ) + parser.add_argument( + "--batch-timeout", + type=int, + default=5, + help="Timeout for batching in seconds (default: 5)", + ) + parser.add_argument( + "--init-timeout", + type=int, + default=300, + help="Timeout for initializing stages in seconds (default: 300)", + ) + parser.add_argument( + "--shm-threshold-bytes", + type=int, + default=65536, + help="Threshold for using shared memory in bytes (default: 65536)", + ) + parser.add_argument( + "--output-wav", + default="output_audio", + help="[Deprecated] Output wav directory (use --output-dir).", + ) + parser.add_argument( + "--output-dir", + type=str, + default=None, + help="Output directory for text/audio results.", + ) + parser.add_argument( + "--num-prompts", + type=int, + default=1, + help="Number of prompts to generate.", + ) + parser.add_argument( + "--txt-prompts", + type=str, + default=None, + help="Path to a .txt file with one prompt per line (preferred).", + ) + parser.add_argument( + "--stage-configs-path", + type=str, + default=None, + help="Path to a stage configs file (deprecated; prefer --deploy-config).", + ) + parser.add_argument( + "--deploy-config", + type=str, + default=None, + help="Path to a deploy YAML (default: auto-load minicpmo_4_5.yaml).", + ) + parser.add_argument( + "--video-path", + "-v", + type=str, + default=None, + help="Path to local video file. If not provided, uses default video asset.", + ) + parser.add_argument( + "--image-path", + "-i", + type=str, + default=None, + help="Path to local image file. If not provided, uses default image asset.", + ) + parser.add_argument( + "--audio-path", + "-a", + type=str, + default=None, + help="Path to local audio file. If not provided, uses default audio asset.", + ) + parser.add_argument( + "--num-frames", + type=int, + default=16, + help="Number of frames to extract from video (default: 16).", + ) + parser.add_argument( + "--sampling-rate", + type=int, + default=16000, + help="Sampling rate for audio loading (default: 16000).", + ) + parser.add_argument( + "--log-dir", + type=str, + default="logs", + help="Log directory (default: logs).", + ) + parser.add_argument( + "--modalities", + type=str, + default=None, + help="Output modalities (comma-separated), e.g. text or text,audio.", + ) + parser.add_argument( + "--py-generator", + action="store_true", + default=False, + help="Use py_generator mode. The returned type of Omni.generate() is a Python Generator object.", + ) + parser.add_argument( + "--enable-profiler", + action="store_true", + default=False, + help="Enables profiling when set.", + ) + parser.add_argument( + "--profiler-stages", + type=int, + nargs="*", + default=None, + help="List of stage IDs to profile. If not set, profiles all stages.", + ) + parser.add_argument( + "--dtype", + type=str, + default="auto", + help="Model dtype (auto, half, float16, bfloat16, float, float32).", + ) + + return parser.parse_args() + + +if __name__ == "__main__": + args = parse_args() + main(args) diff --git a/examples/offline_inference/minicpmo/run_multiple_prompts.sh b/examples/offline_inference/minicpmo/run_multiple_prompts.sh new file mode 100755 index 000000000..f130fece3 --- /dev/null +++ b/examples/offline_inference/minicpmo/run_multiple_prompts.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +# Multi-prompt offline batch (py_generator mode). +set -euo pipefail +cd "$(dirname "$0")" + +python end2end.py --output-wav output_audio \ + --query-type text \ + --txt-prompts ../qwen3_omni/text_prompts_10.txt \ + --py-generator diff --git a/examples/offline_inference/minicpmo/run_single_prompt.sh b/examples/offline_inference/minicpmo/run_single_prompt.sh new file mode 100755 index 000000000..3a43ddda9 --- /dev/null +++ b/examples/offline_inference/minicpmo/run_single_prompt.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +# Single-prompt offline smoke test (text → text + 24 kHz speech). +set -euo pipefail +cd "$(dirname "$0")" + +python end2end.py --output-wav output_audio \ + --query-type text diff --git a/examples/offline_inference/minicpmo/run_single_prompt_tp.sh b/examples/offline_inference/minicpmo/run_single_prompt_tp.sh new file mode 100755 index 000000000..88f488d2c --- /dev/null +++ b/examples/offline_inference/minicpmo/run_single_prompt_tp.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +# Single-prompt offline run with thinker tensor-parallel (3-GPU layout). +# Thinker on GPU 0,1 (TP=2); talker + Token2Wav on GPU 2. +set -euo pipefail +cd "$(dirname "$0")" + +REPO_ROOT="$(cd ../../.. && pwd)" + +python end2end.py --output-wav output_audio \ + --query-type use_audio \ + --deploy-config "${REPO_ROOT}/vllm_omni/deploy/minicpmo_4_5_3gpu.yaml" \ + --stage-init-timeout 300 diff --git a/examples/online_serving/minicpmo/README.md b/examples/online_serving/minicpmo/README.md index 334b5ed11..e1c084224 100644 --- a/examples/online_serving/minicpmo/README.md +++ b/examples/online_serving/minicpmo/README.md @@ -1,14 +1,19 @@ -# vLLM-Omni · MiniCPM-o 4.5 Online Demo +# MiniCPM-o 4.5: Online serving -Gradio-based web UI for **MiniCPM-o 4.5** served via `vllm-omni`'s -OpenAI-compatible endpoints. +OpenAI-compatible `/v1/chat/completions` serving for **MiniCPM-o 4.5**, plus a +Gradio UI and curl / Python clients. -The UI supports: +Inputs: text / image / audio / video. Outputs: text and optional **24 kHz** speech. -- **Inputs**: text prompt + optional image, audio (file or mic), video. -- **Outputs**: text + speech (WAV player). +## Installation -## 1. Start the backend server +Please refer to [README.md](../../../README.md). Install the talker extra: + +```bash +pip install 'vllm-omni[minicpmo]' +``` + +## Launch the server The deploy config auto-loads via `--omni`; the default `vllm_omni/deploy/minicpmo_4_5.yaml` targets a single-GPU layout (thinker @@ -30,37 +35,125 @@ vllm serve openbmb/MiniCPM-o-4_5 --omni \ --host 0.0.0.0 --port 8099 ``` -Other layouts via `--deploy-config`: +Other layouts: ```bash vllm serve openbmb/MiniCPM-o-4_5 --omni \ - --deploy-config vllm_omni/deploy/minicpmo_4_5_8x4090.yaml \ + --deploy-config vllm_omni/deploy/minicpmo_4_5_3gpu.yaml \ --trust-remote-code \ --host 0.0.0.0 --port 8099 ``` -## 2. Launch the Gradio demo +### Stage-based CLI (optional) + +Stage 0 (thinker + API) and stage 1 (talker) can run in separate processes: ```bash -bash examples/online_serving/minicpmo/run_gradio_demo.sh +# Stage 0 +CUDA_VISIBLE_DEVICES=0 vllm serve openbmb/MiniCPM-o-4_5 --omni \ + --trust-remote-code --port 8099 --stage-id 0 \ + --omni-master-address 127.0.0.1 --omni-master-port 26000 -# Or run the python entry point directly: -python examples/online_serving/minicpmo/gradio_demo.py \ +# Stage 1 (headless) +CUDA_VISIBLE_DEVICES=1 vllm serve openbmb/MiniCPM-o-4_5 --omni \ + --trust-remote-code --stage-id 1 --headless \ + --omni-master-address 127.0.0.1 --omni-master-port 26000 +``` + +### Per-stage overrides + +```bash +vllm serve openbmb/MiniCPM-o-4_5 --omni --trust-remote-code --port 8099 \ + --stage-overrides '{"0": {"gpu_memory_utilization": 0.65}}' +``` + +## Send multimodal requests + +```bash +cd examples/online_serving/minicpmo +``` + +### curl + +```bash +bash run_curl_multimodal_generation.sh text +bash run_curl_multimodal_generation.sh use_image +bash run_curl_multimodal_generation.sh use_audio '["text"]' # text-only +``` + +Text + speech smoke test (TTS needs top-level `chat_template_kwargs`): + +```bash +curl http://localhost:8099/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d '{ + "model": "openbmb/MiniCPM-o-4_5", + "messages": [{"role": "user", "content": "Say hello, then introduce vLLM in one sentence."}], + "modalities": ["text", "audio"], + "chat_template_kwargs": {"use_tts_template": true} + }' +``` + +### Python client + +```bash +python openai_chat_completion_client_for_multimodal_generation.py \ + --query-type use_image \ + --port 8099 \ + --host localhost + +# Text-only (faster; no <|tts_bos|>) +python openai_chat_completion_client_for_multimodal_generation.py \ + --query-type text \ + --modalities text \ + --prompt "Briefly introduce yourself." +``` + +Shared helpers also work if you pass MiniCPM defaults yourself: + +```bash +python ../openai_chat_completion_client_for_multimodal_generation.py \ + --model openbmb/MiniCPM-o-4_5 \ + --query-type text \ + --port 8099 +``` + +(Note: the shared client does **not** set `use_tts_template`; prefer the +MiniCPM-specific client above for speech.) + +### Gradio demo + +```bash +bash run_gradio_demo.sh + +# Or: +python gradio_demo.py \ --minicpmo45-api-base http://localhost:8099/v1 \ --minicpmo45-model openbmb/MiniCPM-o-4_5 \ --port 7862 ``` -Open `http://:7862` in a browser. +Open `http://:7862`. Uncheck **"Generate speech output (TTS)"** for +text-only responses. + +## Modality control + +| Modalities | Output | +|---|---| +| `["text"]` | Text only (no TTS bos) | +| `["text", "audio"]` / unset | Text + 24 kHz speech | + +Speech requires `chat_template_kwargs.use_tts_template=true` so the chat +template appends `<|tts_bos|>`. For **curl**, put that field at the request +root; nested `extra_body` is ignored. The OpenAI Python SDK may use +`extra_body` because it merges those fields into the root. ## Notes -- **TTS trigger**: the demo sets - `extra_body.chat_template_kwargs.use_tts_template=True`, which appends - `<|tts_bos|>` to the assistant prefix. -- Uncheck **"Generate speech output (TTS)"** to get text-only responses - (faster). -- The audio output is the raw WAV returned by the stage-1 talker + - Token2Wav; sample rate is 24 kHz. -- Video input is forwarded as a base64 `video_url` entry; the server needs - decord/torchvision to decode it. +- Stage 1 is capped at `max_num_seqs: 1` in the deploy YAML (talker shares + request-0 audio metadata). +- Output audio is base64 WAV in `message.audio.data` (24 kHz mono). +- Offline counterpart: + [`examples/offline_inference/minicpmo/`](../../offline_inference/minicpmo/) +- Recipe: + [`recipes/OpenBMB/MiniCPM-o-4_5.md`](../../../recipes/OpenBMB/MiniCPM-o-4_5.md) diff --git a/examples/online_serving/minicpmo/openai_chat_completion_client_for_multimodal_generation.py b/examples/online_serving/minicpmo/openai_chat_completion_client_for_multimodal_generation.py new file mode 100644 index 000000000..45afe7e5d --- /dev/null +++ b/examples/online_serving/minicpmo/openai_chat_completion_client_for_multimodal_generation.py @@ -0,0 +1,237 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""OpenAI chat-completions client for MiniCPM-o 4.5 online serving. + +Thin wrapper around the shared multimodal client helpers, with MiniCPM-specific +defaults (port 8099, system prompt, ``chat_template_kwargs.use_tts_template``). +""" + +from __future__ import annotations + +import base64 +import concurrent.futures +import os +import sys + +from openai import OpenAI + +from vllm_omni.utils.tracking_parser import TrackingArgumentParser + +# Reuse prompt builders / URL helpers from the shared online client. +_HERE = os.path.dirname(os.path.abspath(__file__)) +_ONLINE_ROOT = os.path.abspath(os.path.join(_HERE, "..")) +if _ONLINE_ROOT not in sys.path: + sys.path.insert(0, _ONLINE_ROOT) + +from openai_chat_completion_client_for_multimodal_generation import ( # noqa: E402 + _build_prompt_for_query_type, + _parse_csv_arg, + make_audio_output_filename, + query_map, +) + +SEED = 42 + +default_system = ( + "You are MiniCPM-o, a helpful multimodal assistant that can " + "understand images, audio and video, and respond in text and speech." +) + + +def get_system_prompt() -> dict: + return { + "role": "system", + "content": [{"type": "text", "text": default_system}], + } + + +def _wants_tts(modalities: list[str] | None) -> bool: + if modalities is None: + return True + return "audio" in modalities + + +def run_multimodal_generation(args, client: OpenAI) -> None: + model_name = args.model + video_path = getattr(args, "video_path", None) + image_path = getattr(args, "image_path", None) + audio_path = getattr(args, "audio_path", None) + custom_prompt = getattr(args, "prompt", None) + + if args.modalities is not None: + output_modalities = args.modalities.split(",") + else: + output_modalities = None + + use_tts = _wants_tts(output_modalities) + num_concurrent_requests = args.num_concurrent_requests + prompt_list = _parse_csv_arg(getattr(args, "prompts", None)) + + request_payloads = [] + for idx in range(num_concurrent_requests): + per_req_prompt = ( + prompt_list[idx] + if idx < len(prompt_list) + else (custom_prompt if idx == 0 or not prompt_list else prompt_list[-1]) + ) + prompt = _build_prompt_for_query_type( + query_type=args.query_type, + custom_prompt=per_req_prompt, + video_path=video_path, + image_path=image_path, + audio_path=audio_path, + ) + extra_body: dict = { + "chat_template_kwargs": {"use_tts_template": use_tts}, + } + if args.query_type == "use_audio_in_video": + # MiniCPM-o 4.5 does not use Qwen's use_audio_in_video path; keep + # the flag only if the shared query type is selected explicitly. + extra_body["mm_processor_kwargs"] = {"use_audio_in_video": True} + request_payloads.append({"prompt": prompt, "extra_body": extra_body}) + + with concurrent.futures.ThreadPoolExecutor(max_workers=num_concurrent_requests) as executor: + futures = [ + executor.submit( + client.chat.completions.create, + messages=[ + get_system_prompt(), + payload["prompt"], + ], + model=model_name, + modalities=output_modalities, + extra_body=payload["extra_body"], + stream=args.stream, + ) + for payload in request_payloads + ] + chat_completions = [future.result() for future in concurrent.futures.as_completed(futures)] + + assert len(chat_completions) == num_concurrent_requests + count = 0 + if not args.stream: + for chat_completion in chat_completions: + request_id = getattr(chat_completion, "id", None) + for choice in chat_completion.choices: + if choice.message.audio: + audio_data = base64.b64decode(choice.message.audio.data) + audio_file_path = make_audio_output_filename(request_id=request_id, index=count) + with open(audio_file_path, "wb") as f: + f.write(audio_data) + print(f"Audio saved to {audio_file_path}") + count += 1 + elif choice.message.content: + print("Chat completion output from text:", choice.message.content) + else: + printed_content = False + for chat_completion in chat_completions: + for chunk in chat_completion: + for choice in chunk.choices: + if hasattr(choice, "delta"): + content = getattr(choice.delta, "content", None) + else: + content = None + + if getattr(chunk, "modality", None) == "audio" and content: + audio_data = base64.b64decode(content) + request_id = getattr(chunk, "id", None) + audio_file_path = make_audio_output_filename(request_id=request_id, index=count) + with open(audio_file_path, "wb") as f: + f.write(audio_data) + print(f"\nAudio saved to {audio_file_path}") + count += 1 + elif getattr(chunk, "modality", None) == "text": + if not printed_content: + printed_content = True + print("\ncontent:", end="", flush=True) + print(content, end="", flush=True) + + +def parse_args(): + parser = TrackingArgumentParser(description="MiniCPM-o 4.5 online multimodal chat client") + parser.add_argument( + "--query-type", + "-q", + type=str, + default="text", + choices=query_map.keys(), + help="Query type.", + ) + parser.add_argument( + "--model", + "-m", + type=str, + default="openbmb/MiniCPM-o-4_5", + help="Model Name / Path", + ) + parser.add_argument( + "--video-path", + "-v", + type=str, + default=None, + help="Path to local video file or URL.", + ) + parser.add_argument( + "--image-path", + "-i", + type=str, + default=None, + help="Path to local image file or URL.", + ) + parser.add_argument( + "--audio-path", + "-a", + type=str, + default=None, + help="Path to local audio file or URL.", + ) + parser.add_argument( + "--prompt", + "-p", + type=str, + default=None, + help="Custom text prompt/question.", + ) + parser.add_argument( + "--modalities", + type=str, + default=None, + help="Output modalities, e.g. text or text,audio.", + ) + parser.add_argument( + "--stream", + action="store_true", + help="Stream the response.", + ) + parser.add_argument( + "--num-concurrent-requests", + type=int, + default=1, + help="Number of concurrent requests to send.", + ) + parser.add_argument( + "--port", + type=int, + default=8099, + help="Port of the vLLM Omni API server.", + ) + parser.add_argument( + "--host", + type=str, + default="localhost", + help="Host/IP of the vLLM Omni API server.", + ) + parser.add_argument( + "--prompts", + type=str, + default=None, + help="Comma-separated prompts for concurrent requests.", + ) + return parser.parse_args() + + +if __name__ == "__main__": + args = parse_args() + openai_api_base = f"http://{args.host}:{args.port}/v1" + client = OpenAI(api_key="EMPTY", base_url=openai_api_base) + run_multimodal_generation(args, client) diff --git a/examples/online_serving/minicpmo/run_curl_multimodal_generation.sh b/examples/online_serving/minicpmo/run_curl_multimodal_generation.sh new file mode 100755 index 000000000..426a2df00 --- /dev/null +++ b/examples/online_serving/minicpmo/run_curl_multimodal_generation.sh @@ -0,0 +1,154 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Default query type +QUERY_TYPE="${1:-text}" + +# Default modalities argument (JSON). Use null for server default (text+audio). +MODALITIES="${2:-null}" + +# Validate query type +if [[ ! "$QUERY_TYPE" =~ ^(text|use_audio|use_image|use_video)$ ]]; then + echo "Error: Invalid query type '$QUERY_TYPE'" + echo "Usage: $0 [text|use_audio|use_image|use_video] [modalities]" + echo " text: Text query" + echo " use_audio: Audio + text query" + echo " use_image: Image + text query" + echo " use_video: Video + text query" + echo " modalities: JSON list or null (default: null → text+audio)" + echo " examples: '[\"text\"]' '[\"text\",\"audio\"]'" + exit 1 +fi + +HOST="${HOST:-localhost}" +PORT="${PORT:-8099}" +MODEL="${MODEL:-openbmb/MiniCPM-o-4_5}" + +thinker_sampling_params='{ + "temperature": 0.0, + "top_p": 1.0, + "top_k": -1, + "max_tokens": 2048, + "seed": 42, + "detokenize": true, + "repetition_penalty": 1.1 +}' + +talker_sampling_params='{ + "temperature": 0.0, + "top_p": 1.0, + "top_k": -1, + "max_tokens": 1, + "seed": 42, + "detokenize": false +}' +# Above is optional; defaults live in vllm_omni/deploy/minicpmo_4_5.yaml. + +MARY_HAD_LAMB_AUDIO_URL="https://vllm-public-assets.s3.us-west-2.amazonaws.com/multimodal_asset/mary_had_lamb.ogg" +CHERRY_BLOSSOM_IMAGE_URL="https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/cherry_blossom.jpg" +SAMPLE_VIDEO_URL="https://huggingface.co/datasets/raushan-testing-hf/videos-test/resolve/main/sample_demo_1.mp4" + +case "$QUERY_TYPE" in + text) + user_content='[ + { + "type": "text", + "text": "Say hello, then introduce vLLM-Omni in one sentence." + } + ]' + ;; + use_audio) + user_content='[ + { + "type": "audio_url", + "audio_url": { + "url": "'"$MARY_HAD_LAMB_AUDIO_URL"'" + } + }, + { + "type": "text", + "text": "What is the content of this audio?" + } + ]' + ;; + use_image) + user_content='[ + { + "type": "image_url", + "image_url": { + "url": "'"$CHERRY_BLOSSOM_IMAGE_URL"'" + } + }, + { + "type": "text", + "text": "What is the content of this image?" + } + ]' + ;; + use_video) + user_content='[ + { + "type": "video_url", + "video_url": { + "url": "'"$SAMPLE_VIDEO_URL"'" + } + }, + { + "type": "text", + "text": "Why is this video funny?" + } + ]' + ;; +esac + +sampling_params_list='[ + '"$thinker_sampling_params"', + '"$talker_sampling_params"' +]' + +# TTS speech path needs use_tts_template at the request root (curl does not +# flatten nested extra_body). Skip it when the caller asked for text-only. +USE_TTS_TEMPLATE=true +if [[ "$MODALITIES" == '["text"]' || "$MODALITIES" == "['text']" ]]; then + USE_TTS_TEMPLATE=false +fi + +echo "Running query type: $QUERY_TYPE (host=$HOST port=$PORT model=$MODEL)" +echo "modalities=$MODALITIES use_tts_template=$USE_TTS_TEMPLATE" +echo "" + +request_body=$(cat < Online serving for omni multimodal chat (text / image / audio / video → text + 24 kHz speech) +> Online serving and offline inference for omni multimodal chat +> (text / image / audio / video → text + 24 kHz speech) ## Summary @@ -9,7 +10,7 @@ - Task: Omni multimodal chat — accepts text / image / audio / video input; emits text and 24 kHz mono speech in the same response - Mode: Online serving via the OpenAI-compatible `/v1/chat/completions` - API, plus a bundled Gradio demo (text + speech UI) + API (plus Gradio demo), and offline inference via `Omni.generate` - Maintainer: [`@tc-mb`](https://github.com/tc-mb) (MiniCPM-V / MiniCPM-o team) ## When to use this recipe @@ -37,6 +38,8 @@ shot. The recipe covers the shipped GPU layouts (single / 2 / 3 / 8 GPUs): [`vllm_omni/deploy/minicpmo_4_5_8x4090.yaml`](../../vllm_omni/deploy/minicpmo_4_5_8x4090.yaml) - Online example + Gradio demo: [`examples/online_serving/minicpmo/`](../../examples/online_serving/minicpmo/) +- Offline end-to-end example: + [`examples/offline_inference/minicpmo/`](../../examples/offline_inference/minicpmo/) - Pipeline / talker source: [`vllm_omni/model_executor/models/minicpmo_4_5/`](../../vllm_omni/model_executor/models/minicpmo_4_5/) - Stage-input processor (thinker → talker bridge): From 740bb31ff98781bf91bea59e20741507c1cc1da9 Mon Sep 17 00:00:00 2001 From: mjZhaoElaine <38148457+mjZhaoElaine@users.noreply.github.com> Date: Mon, 20 Jul 2026 22:49:08 -0700 Subject: [PATCH 034/240] [Bug] Fix Ming feature extractor/processor registration crash under transformers >= 5.x (#5243) Signed-off-by: Mengjie Zhao Co-authored-by: Claude Opus 4.8 (1M context) --- vllm_omni/transformers_utils/processors/ming.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/vllm_omni/transformers_utils/processors/ming.py b/vllm_omni/transformers_utils/processors/ming.py index b32f10d12..aa59e5324 100644 --- a/vllm_omni/transformers_utils/processors/ming.py +++ b/vllm_omni/transformers_utils/processors/ming.py @@ -24,6 +24,8 @@ from transformers.processing_utils import ProcessorMixin from transformers.tokenization_utils_base import PreTokenizedInput, TextInput from transformers.utils import logging +from vllm_omni.transformers_utils.configs.ming_flash_omni import BailingMM2Config + try: from transformers import AutoVideoProcessor except ImportError: @@ -479,5 +481,10 @@ class MingFlashOmniProcessor(ProcessorMixin): return list(dict.fromkeys(names)) -AutoFeatureExtractor.register("MingWhisperFeatureExtractor", MingWhisperFeatureExtractor) -AutoProcessor.register("MingFlashOmniProcessor", MingFlashOmniProcessor) +# transformers >= 5.x requires the config *class* (not a string) as the first +# arg to AutoFeatureExtractor/AutoProcessor.register — it does `key.__module__` +# internally. Passing "MingWhisperFeatureExtractor" raises: +# AttributeError: 'str' object has no attribute '__module__' +# Same API change as minicpmo_4_5_omni_llm.py / processing_gr00t_n1d7.py. +AutoFeatureExtractor.register(BailingMM2Config, MingWhisperFeatureExtractor, exist_ok=True) +AutoProcessor.register(BailingMM2Config, MingFlashOmniProcessor, exist_ok=True) From 5d44868e918ecf9d3a6f1158c45acfad5989e1a7 Mon Sep 17 00:00:00 2001 From: fatSheep <50252876+00fish0@users.noreply.github.com> Date: Tue, 21 Jul 2026 14:30:30 +0800 Subject: [PATCH 035/240] [Hardware][Ascend] Keep HiFT vocoder on NPU (#5242) Signed-off-by: fatSheep Co-authored-by: amy-why-3459 --- .../npu/test_step_audio2_token2wav.py | 196 ++++++++++++++++++ .../minicpmo_4_5/minicpmo_4_5_omni_tts.py | 2 +- .../minicpmo_4_5/minicpmo_4_5_token2wav.py | 4 +- .../step_audio2/step_audio2_token2wav.py | 5 +- .../npu/models/step_audio2_token2wav.py | 124 +++++++---- 5 files changed, 282 insertions(+), 49 deletions(-) create mode 100644 tests/platforms/npu/test_step_audio2_token2wav.py diff --git a/tests/platforms/npu/test_step_audio2_token2wav.py b/tests/platforms/npu/test_step_audio2_token2wav.py new file mode 100644 index 000000000..ae86f92d9 --- /dev/null +++ b/tests/platforms/npu/test_step_audio2_token2wav.py @@ -0,0 +1,196 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""CPU unit tests for the Step-Audio2 Ascend HiFT patch.""" + +from __future__ import annotations + +import importlib.util +import sys +import types +from pathlib import Path +from types import SimpleNamespace + +import numpy as np +import pytest +import torch +import torch.nn.functional as F + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +def _load_patch_module(monkeypatch: pytest.MonkeyPatch): + fake_logger = SimpleNamespace(info=lambda *_args, **_kwargs: None) + fake_vllm = types.ModuleType("vllm") + fake_vllm_logger = types.ModuleType("vllm.logger") + fake_vllm_logger.init_logger = lambda _name: fake_logger + monkeypatch.setitem(sys.modules, "vllm", fake_vllm) + monkeypatch.setitem(sys.modules, "vllm.logger", fake_vllm_logger) + + root = next(parent for parent in Path(__file__).resolve().parents if (parent / "vllm_omni").is_dir()) + path = root / "vllm_omni" / "platforms" / "npu" / "models" / "step_audio2_token2wav.py" + spec = importlib.util.spec_from_file_location("test_step_audio2_token2wav_npu_patch", path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _reference_f02sine(sine_gen, f0_values: torch.Tensor) -> torch.Tensor: + rad_values = (f0_values / sine_gen.sampling_rate) % 1 + rand_ini = torch.rand(f0_values.shape[0], f0_values.shape[2], device=f0_values.device) + rand_ini[:, 0] = 0 + rad_values[:, 0, :] = rad_values[:, 0, :] + rand_ini + rad_values = F.interpolate( + rad_values.transpose(1, 2), + scale_factor=1 / sine_gen.upsample_scale, + mode="linear", + ).transpose(1, 2) + phase = torch.cumsum(rad_values, dim=1) * 2 * np.pi + phase = F.interpolate( + phase.transpose(1, 2) * sine_gen.upsample_scale, + scale_factor=sine_gen.upsample_scale, + mode="linear", + ).transpose(1, 2) + return torch.sin(phase) + + +def test_even_scale_downsample_matches_linear_interpolate(monkeypatch: pytest.MonkeyPatch) -> None: + module = _load_patch_module(monkeypatch) + x = torch.randn(2, 3, 960) + + expected = F.interpolate(x, scale_factor=1 / 4, mode="linear") + actual = module._linear_downsample_even_scale(x, 4) + + torch.testing.assert_close(actual, expected, rtol=0, atol=0) + + +@pytest.mark.parametrize(("shape", "scale"), [((1, 2, 12), 3), ((1, 2, 10), 4)]) +def test_even_scale_downsample_rejects_unsupported_shapes( + monkeypatch: pytest.MonkeyPatch, + shape: tuple[int, ...], + scale: int, +) -> None: + module = _load_patch_module(monkeypatch) + with pytest.raises(ValueError): + module._linear_downsample_even_scale(torch.randn(shape), scale) + + +def test_hift_patch_is_exact_idempotent_and_stays_on_device(monkeypatch: pytest.MonkeyPatch) -> None: + module = _load_patch_module(monkeypatch) + + class FakeSineGen: + sampling_rate = 24000 + upsample_scale = 4 + flag_for_pulse = False + + def __init__(self): + self.original_calls = 0 + + def _f02sine(self, f0_values): + self.original_calls += 1 + return _reference_f02sine(self, f0_values) + + class FakeHiFT: + def __init__(self): + self.m_source = SimpleNamespace(l_sin_gen=FakeSineGen()) + + def to(self, *_args, **_kwargs): + raise AssertionError("HiFT must stay on its accelerator") + + hift = FakeHiFT() + module.patch_step_audio2_hift_for_npu(hift) + patched_method = hift.m_source.l_sin_gen._f02sine + module.patch_step_audio2_hift_for_npu(hift) + assert hift.m_source.l_sin_gen._f02sine is patched_method + + f0_values = torch.rand(1, 16, 3) + torch.manual_seed(7) + expected = _reference_f02sine(hift.m_source.l_sin_gen, f0_values) + torch.manual_seed(7) + actual = patched_method(f0_values) + + assert hift.m_source.l_sin_gen.original_calls == 0 + torch.testing.assert_close(actual, expected, rtol=0, atol=0) + + +@pytest.mark.parametrize( + ("flag_for_pulse", "upsample_scale", "input_length"), + [ + (True, 4, 16), + (False, 3, 12), + (False, 4.5, 18), + (False, 4, 10), + ], +) +def test_hift_patch_delegates_unsupported_cases_to_original_f02sine_on_cpu( + monkeypatch: pytest.MonkeyPatch, + flag_for_pulse: bool, + upsample_scale: float, + input_length: int, +) -> None: + module = _load_patch_module(monkeypatch) + calls: list[str] = [] + + class FakeSineGen: + def __init__(self): + self.flag_for_pulse = flag_for_pulse + self.upsample_scale = upsample_scale + + def _f02sine(self, f0_values): + calls.append(f0_values.device.type) + return f0_values + 1 + + hift = SimpleNamespace(m_source=SimpleNamespace(l_sin_gen=FakeSineGen())) + module.patch_step_audio2_hift_for_npu(hift) + + f0_values = torch.randn(1, input_length, 3) + output = hift.m_source.l_sin_gen._f02sine(f0_values) + + assert calls == ["cpu"] + assert output.device == f0_values.device + torch.testing.assert_close(output, f0_values + 1) + + +@pytest.mark.parametrize("upsample_scale", [0, -2]) +def test_hift_patch_rejects_non_positive_scale( + monkeypatch: pytest.MonkeyPatch, + upsample_scale: int, +) -> None: + module = _load_patch_module(monkeypatch) + + class FakeSineGen: + flag_for_pulse = False + + def __init__(self): + self.upsample_scale = upsample_scale + + def _f02sine(self, f0_values): + return f0_values + + hift = SimpleNamespace(m_source=SimpleNamespace(l_sin_gen=FakeSineGen())) + module.patch_step_audio2_hift_for_npu(hift) + + with pytest.raises(ValueError, match="upsample_scale must be positive"): + hift.m_source.l_sin_gen._f02sine(torch.randn(1, 16, 3)) + + +def test_hift_patch_rejects_causal_sinegen(monkeypatch: pytest.MonkeyPatch) -> None: + module = _load_patch_module(monkeypatch) + + class FakeCausalSineGen: + causal = True + + def _f02sine(self, f0_values): + return f0_values + + hift = SimpleNamespace(m_source=SimpleNamespace(l_sin_gen=FakeCausalSineGen())) + + with pytest.raises(ValueError, match="only supports non-causal SineGen2"): + module.patch_step_audio2_hift_for_npu(hift) + + +def test_hift_patch_reports_incompatible_layout(monkeypatch: pytest.MonkeyPatch) -> None: + module = _load_patch_module(monkeypatch) + + with pytest.raises(TypeError, match=r"m_source\.l_sin_gen\._f02sine"): + module.patch_step_audio2_hift_for_npu(SimpleNamespace()) diff --git a/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_tts.py b/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_tts.py index d14fe14cf..71681c222 100644 --- a/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_tts.py +++ b/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_tts.py @@ -514,7 +514,7 @@ class MiniCPMO45OmniTTSForConditionalGeneration(nn.Module, SupportsPP): # memory on the NPU sdpa path. Detach the Token2wav vocoder first # so the cast does not drag it onto the accelerator or downcast # its float32 flow/HiFT weights: it manages its own device - # placement (e.g. CPU HiFT on NPU) and may not be an nn.Module. + # placement and may not be an nn.Module. device = current_omni_platform.get_torch_device() audio_tok = getattr(self.tts_obj, "audio_tokenizer", None) if audio_tok is not None: diff --git a/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_token2wav.py b/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_token2wav.py index c069b8d09..da868cd58 100644 --- a/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_token2wav.py +++ b/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_token2wav.py @@ -6,8 +6,8 @@ ``from stepaudio2 import Token2wav`` entry point (``stepaudio2-minicpmo``). That package hard-codes ``.cuda()`` and duplicates the flow/HiFT stack that vLLM-Omni already vendors for Step-Audio2 (which also carries the Ascend/NPU -fixes: CPU HiFT, DiT mask expand, MATH SDPA, compile disable, PE buffer -extension). +fixes: HiFT linear downsample, DiT mask expand, MATH SDPA, compile disable, +PE buffer extension). This module exposes the same call surface MiniCPM expects (``__call__`` / ``set_stream_cache`` / ``stream``) while delegating the diff --git a/vllm_omni/model_executor/models/step_audio2/step_audio2_token2wav.py b/vllm_omni/model_executor/models/step_audio2/step_audio2_token2wav.py index 1ebad86ce..026276c52 100644 --- a/vllm_omni/model_executor/models/step_audio2/step_audio2_token2wav.py +++ b/vllm_omni/model_executor/models/step_audio2/step_audio2_token2wav.py @@ -100,8 +100,9 @@ class StepAudio2Token2WavCore(nn.Module): # and apply the Token2Wav NPU patch here (idempotent) before # ``_ensure_models_loaded`` runs. Covers every # construction site (Step-Audio2 wrapper + MiniCPM-o facade): patched - # ``_ensure_models_loaded`` offloads HiFT to CPU and patched ``forward`` - # expands CosyVoice DiT masks + forces MATH SDPA (avoids FA 161001). + # ``_ensure_models_loaded`` replaces HiFT's failing linear downsample, + # while patched ``forward`` expands CosyVoice DiT masks and forces + # MATH SDPA (avoids FA 161001). from vllm_omni.platforms import current_omni_platform if current_omni_platform.is_npu(): diff --git a/vllm_omni/platforms/npu/models/step_audio2_token2wav.py b/vllm_omni/platforms/npu/models/step_audio2_token2wav.py index 04768d04d..1641fee27 100644 --- a/vllm_omni/platforms/npu/models/step_audio2_token2wav.py +++ b/vllm_omni/platforms/npu/models/step_audio2_token2wav.py @@ -4,9 +4,8 @@ Ascend-specific workarounds that must not live in the shared GPU model file: -1. HiFT vocoder CPU offload — STFT/ISTFT and sine-source generation are - unstable on Ascend (aicore 507015). HiFT is tiny, so running it on CPU - is acceptable; inputs/outputs are moved transparently. +1. HiFT sine-source downsample — replace the failing 480x ``linear1d`` + downsample with its exact midpoint form while keeping HiFT on NPU. 2. CosyVoice2 DiT SDPA — force MATH backend (+ DiT attn mask expand) to avoid fused FA rejecting CosyVoice ``(B,1,1,S)`` masks (error 161001). """ @@ -17,7 +16,9 @@ from collections.abc import Iterator from contextlib import contextmanager, nullcontext from types import MethodType +import numpy as np import torch +import torch.nn.functional as F from vllm.logger import init_logger logger = init_logger(__name__) @@ -28,52 +29,87 @@ _original_forward = None _original_stream_chunk_for = None -def patch_hift_module_for_npu(hift: torch.nn.Module) -> None: - """Run the entire HiFT vocoder on CPU when on Ascend NPU. +def _linear_downsample_even_scale(x: torch.Tensor, scale: int) -> torch.Tensor: + """Match ``F.interpolate(..., mode="linear")`` for an even integer scale. - HiFT is tiny, so keeping it on NPU yields negligible speedup while - several of its operators are unstable on Ascend and trigger aicore - exceptions (runtime error 507015): the STFT/ISTFT as well as the - sine-source generation (``f0_upsamp`` + ``m_source`` cumsum/sin over - the full waveform length). Running the whole module on CPU sidesteps - all of them. Inputs are copied to CPU and outputs are moved back to - the original device transparently, so callers stay unchanged. + With ``align_corners=False``, every output location for an even integer + downsample lies exactly halfway between two source samples. Selecting and + averaging those samples avoids Ascend/pytorch#150's ``linear1d`` kernel. """ - if getattr(hift, "_npu_cpu_offload_patched", False): + if scale <= 0 or scale % 2: + raise ValueError(f"scale must be a positive even integer, got {scale}") + if x.shape[-1] % scale: + raise ValueError(f"input length {x.shape[-1]} must be divisible by scale {scale}") + + left = scale // 2 - 1 + right = scale // 2 + return (x[..., left::scale] + x[..., right::scale]) * 0.5 + + +def _run_original_f02sine_on_cpu(self, f0_values: torch.Tensor) -> torch.Tensor: + """Run the unmodified ``_f02sine`` without invoking NPU ``linear1d``.""" + output_device = f0_values.device + output = self._step_audio2_original_f02sine(f0_values.cpu()) + return output.to(output_device) + + +def _f02sine_with_npu_safe_downsample(self, f0_values: torch.Tensor) -> torch.Tensor: + """Use the exact NPU midpoint path, with a narrow CPU fallback.""" + if getattr(self, "flag_for_pulse", False): + return _run_original_f02sine_on_cpu(self, f0_values) + + upsample_scale = self.upsample_scale + if upsample_scale <= 0: + raise ValueError(f"upsample_scale must be positive, got {upsample_scale}") + + scale = int(upsample_scale) + midpoint_supported = scale == upsample_scale and scale % 2 == 0 and f0_values.shape[1] % scale == 0 + if not midpoint_supported: + return _run_original_f02sine_on_cpu(self, f0_values) + + rad_values = (f0_values / self.sampling_rate) % 1 + rand_ini = torch.rand(f0_values.shape[0], f0_values.shape[2], device=f0_values.device) + rand_ini[:, 0] = 0 + rad_values[:, 0, :] = rad_values[:, 0, :] + rand_ini + + rad_values = _linear_downsample_even_scale(rad_values.transpose(1, 2), scale).transpose(1, 2) + phase = torch.cumsum(rad_values, dim=1) * 2 * np.pi + phase = F.interpolate( + phase.transpose(1, 2) * self.upsample_scale, + scale_factor=self.upsample_scale, + mode="linear", + ).transpose(1, 2) + return torch.sin(phase) + + +def patch_step_audio2_hift_for_npu(hift: torch.nn.Module) -> None: + """Patch the non-causal Step-Audio2 HiFT implementation for Ascend. + + The ``flashcosyvoice.SineGen2`` instantiated by Step-Audio2 1.0.0 is + non-causal and reduces a full-rate phase tensor by ``1 / 480`` before + restoring it to the waveform rate. Ascend's ``upsample_linear1d`` kernel + can raise an AIVector UB-address exception (ACL 507015) for that reduction. + + The exact midpoint form keeps the common path on NPU. Unsupported or pulse + configurations delegate only ``_f02sine`` to CPU, preserving upstream + behavior without restoring the old whole-HiFT CPU offload. + """ + if getattr(hift, "_step_audio2_npu_downsample_patched", False): return - hift.to("cpu") - original_forward = hift.forward + try: + sine_gen = hift.m_source.l_sin_gen + original_f02sine = sine_gen._f02sine + except AttributeError as exc: + raise TypeError("expected a Step-Audio2 flashcosyvoice HiFT with m_source.l_sin_gen._f02sine") from exc - def _forward_on_cpu(_module, *args, **kwargs): - output_device: torch.device | None = None + if getattr(sine_gen, "causal", False): + raise ValueError("the Step-Audio2 NPU HiFT patch only supports non-causal SineGen2") - def to_cpu(value): - nonlocal output_device - if isinstance(value, torch.Tensor): - if output_device is None and value.device.type != "cpu": - output_device = value.device - return value.cpu() - return value - - cpu_args = tuple(to_cpu(a) for a in args) - cpu_kwargs = {k: to_cpu(v) for k, v in kwargs.items()} - output = original_forward(*cpu_args, **cpu_kwargs) - - if output_device is None: - return output - - def to_device(value): - if isinstance(value, torch.Tensor): - return value.to(output_device) - return value - - if isinstance(output, tuple): - return tuple(to_device(v) for v in output) - return to_device(output) - - hift.forward = MethodType(_forward_on_cpu, hift) - hift._npu_cpu_offload_patched = True + sine_gen._step_audio2_original_f02sine = original_f02sine + sine_gen._f02sine = MethodType(_f02sine_with_npu_safe_downsample, sine_gen) + hift._step_audio2_npu_downsample_patched = True + logger.info("Patched Step-Audio2 HiFT linear downsample for Ascend NPU") @contextmanager @@ -99,7 +135,7 @@ def _patched_ensure_models_loaded(self) -> None: _original_ensure_models_loaded(self) if was_loaded or self.device.type != "npu" or self._hift is None: return - patch_hift_module_for_npu(self._hift) + patch_step_audio2_hift_for_npu(self._hift) def _patched_forward(self, generated_speech_tokens, prompt_wav, return_bytes=True): From 0576264dd1a063ec0c8601b45e2926f039ef1752 Mon Sep 17 00:00:00 2001 From: Hongshaorou Date: Tue, 21 Jul 2026 15:44:05 +0800 Subject: [PATCH 036/240] [Bugfix] Fix MiniCPM-o 4.5 audio placeholder pool step (#5116) Signed-off-by: a-kaa Co-authored-by: amy-why-3459 --- .../minicpmo_4_5/test_audio_pool_step.py | 71 +++++++++++++++++++ .../minicpmo_4_5/minicpmo_4_5_omni_llm.py | 7 +- 2 files changed, 76 insertions(+), 2 deletions(-) create mode 100644 tests/model_executor/models/minicpmo_4_5/test_audio_pool_step.py diff --git a/tests/model_executor/models/minicpmo_4_5/test_audio_pool_step.py b/tests/model_executor/models/minicpmo_4_5/test_audio_pool_step.py new file mode 100644 index 000000000..9fe2f144a --- /dev/null +++ b/tests/model_executor/models/minicpmo_4_5/test_audio_pool_step.py @@ -0,0 +1,71 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Regression tests for MiniCPM-o 4.5 audio placeholder pooling.""" + +from __future__ import annotations + +import math +from types import SimpleNamespace + +import pytest + +from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni_llm import ( + MiniCPMO45OmniLLMProcessingInfo, + MiniCPMOConfig, +) + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +class _FakeProcessor: + """Minimal processor surface used by ``get_audio_placeholder``.""" + + def __init__(self) -> None: + # Reproduce the cached vLLM processor's legacy initial state. + self.pool_step = 2 + self.audio_processor = SimpleNamespace(hop_length=160) + self.image_processor = SimpleNamespace(mean=[0.0], std=[1.0]) + + def get_audio_placeholder( + self, + audio_lens: int, + chunk_input: bool = True, + chunk_length: int = 1, + ) -> str: + del chunk_input, chunk_length + feature_lens = math.ceil(audio_lens / self.audio_processor.hop_length) + cnn_feature_lens = (feature_lens - 1) // 2 + 1 + output_lens = (cnn_feature_lens - self.pool_step) // self.pool_step + 1 + return "" * output_lens + + +def _processing_info(pool_step: int, processor: _FakeProcessor) -> MiniCPMO45OmniLLMProcessingInfo: + info = object.__new__(MiniCPMO45OmniLLMProcessingInfo) + info.ctx = SimpleNamespace( + tokenizer=object(), + get_hf_config=lambda: SimpleNamespace(audio_pool_step=pool_step), + get_hf_processor=lambda **kwargs: processor, + ) + return info + + +@pytest.mark.parametrize( + ("configured_pool_step", "expected_placeholders"), + [(5, 50), (2, 125)], +) +def test_audio_placeholders_follow_checkpoint_pool_step( + configured_pool_step: int, + expected_placeholders: int, +) -> None: + """The prompt path must use the same pool step as the audio encoder.""" + processor = _FakeProcessor() + info = _processing_info(configured_pool_step, processor) + + placeholder = info.get_audio_placeholder(5 * 16_000) + + assert processor.pool_step == configured_pool_step + assert placeholder.count("") == expected_placeholders + + +def test_minicpmo_4_5_pool_step_defaults_to_checkpoint_value() -> None: + assert MiniCPMOConfig().audio_pool_step == 5 diff --git a/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py b/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py index 3019a5b3d..921095e75 100644 --- a/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py +++ b/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py @@ -328,7 +328,7 @@ class MiniCPMOConfig(Qwen2Config): tts_config=None, use_image_id=True, vision_batch_size=16, - audio_pool_step=2, + audio_pool_step=5, audio_chunk_length=1.0, stream_input=False, init_vision=True, @@ -2839,6 +2839,9 @@ class MiniCPMO45OmniLLMProcessingInfo(BaseProcessingInfo): ) -> str: hf_processor = self.get_hf_processor() + # Keep prompt placeholders aligned with the encoder's checkpoint config. + hf_processor.pool_step = self.get_default_audio_pool_step() + return hf_processor.get_audio_placeholder( audio_lens, chunk_input=chunk_input, @@ -2846,7 +2849,7 @@ class MiniCPMO45OmniLLMProcessingInfo(BaseProcessingInfo): ) def get_default_audio_pool_step(self) -> int: - return getattr(self.get_hf_config(), "audio_pool_step", 2) + return getattr(self.get_hf_config(), "audio_pool_step", 5) def get_default_audio_sampling_rate(self) -> int: return 16000 From 2f33df4678b2b13c70abd9ddffffc314908fb23a Mon Sep 17 00:00:00 2001 From: bastefaniak Date: Tue, 21 Jul 2026 09:56:31 +0200 Subject: [PATCH 037/240] [bugfix] Cosmos3 edge fix config use_k_norm_und_for_gen name (#5239) Signed-off-by: Bartosz Stefaniak --- .../models/cosmos3/test_cosmos3_transformer.py | 12 ++++++------ .../models/cosmos3/transformer_cosmos3.py | 2 +- .../models/cosmos3/transformer_cosmos3_edge.py | 16 ++++++++-------- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/tests/diffusion/models/cosmos3/test_cosmos3_transformer.py b/tests/diffusion/models/cosmos3/test_cosmos3_transformer.py index cb59dffe5..5dd8a875a 100644 --- a/tests/diffusion/models/cosmos3/test_cosmos3_transformer.py +++ b/tests/diffusion/models/cosmos3/test_cosmos3_transformer.py @@ -193,7 +193,7 @@ def test_edge_und_and_gen_use_relu2_and_no_und_qk_norm() -> None: SimpleNamespace( tf_model_config=_tiny_cosmos3_edge_config( num_hidden_layers=1, - use_k_norm_und_for_gen=False, + use_und_k_norm_for_gen=False, ), dtype=torch.float32, ) @@ -221,14 +221,14 @@ def test_edge_creates_gen_k_norm_when_enabled() -> None: tf_model_config=_tiny_cosmos3_edge_config( num_hidden_layers=1, qk_norm_for_diffusion=True, - use_k_norm_und_for_gen=True, + use_und_k_norm_for_gen=True, ), dtype=torch.float32, ) ) k_norm = model.language_model.layers[0].self_attn.k_norm_und_for_gen - assert model.use_k_norm_und_for_gen is True + assert model.use_und_k_norm_for_gen is True assert isinstance(k_norm, RMSNorm) assert isinstance(model.gen_layers[0].cross_attention.norm_q, RMSNorm) assert isinstance(model.gen_layers[0].cross_attention.norm_k, RMSNorm) @@ -244,7 +244,7 @@ def test_edge_supports_no_gen_qk_norm_variant() -> None: tf_model_config=_tiny_cosmos3_edge_config( num_hidden_layers=1, qk_norm_for_diffusion=False, - use_k_norm_und_for_gen=False, + use_und_k_norm_for_gen=False, ), dtype=torch.float32, ) @@ -266,7 +266,7 @@ def test_edge_validates_required_relu2_weights() -> None: nn.Module.__init__(model) model.num_hidden_layers = 1 model.qk_norm_for_diffusion = True - model.use_k_norm_und_for_gen = True + model.use_und_k_norm_for_gen = True complete = { "transformer.language_model.layers.0.mlp.up_proj.weight", @@ -288,7 +288,7 @@ def test_edge_validates_required_relu2_weights() -> None: with pytest.raises(ValueError, match=r"self_attn\.k_norm_und_for_gen"): model.validate_loaded_weights(missing_k_norm) - model.use_k_norm_und_for_gen = False + model.use_und_k_norm_for_gen = False model.validate_loaded_weights(missing_k_norm) diff --git a/vllm_omni/diffusion/models/cosmos3/transformer_cosmos3.py b/vllm_omni/diffusion/models/cosmos3/transformer_cosmos3.py index de9767d6f..88d280bd0 100644 --- a/vllm_omni/diffusion/models/cosmos3/transformer_cosmos3.py +++ b/vllm_omni/diffusion/models/cosmos3/transformer_cosmos3.py @@ -1123,7 +1123,7 @@ class Cosmos3VFMTransformer(nn.Module): ) self.patch_latent_dim = (self.latent_patch_size**2) * self.latent_channel_size - self.use_k_norm_und_for_gen = _tf_config_get(model_config, "use_k_norm_und_for_gen", None) + self.use_und_k_norm_for_gen = _tf_config_get(model_config, "use_und_k_norm_for_gen", None) dtype = od_config.dtype quant_config = getattr(od_config, "quantization_config", None) if od_config else None diff --git a/vllm_omni/diffusion/models/cosmos3/transformer_cosmos3_edge.py b/vllm_omni/diffusion/models/cosmos3/transformer_cosmos3_edge.py index 6a1b4313b..c9a7d856e 100644 --- a/vllm_omni/diffusion/models/cosmos3/transformer_cosmos3_edge.py +++ b/vllm_omni/diffusion/models/cosmos3/transformer_cosmos3_edge.py @@ -93,7 +93,7 @@ class Cosmos3EdgeCausalAttention(nn.Module): num_key_value_heads: int, head_dim: int, rms_norm_eps: float, - use_k_norm_und_for_gen: bool, + use_und_k_norm_for_gen: bool, quant_config: QuantizationConfig | None = None, prefix: str = "", ) -> None: @@ -144,7 +144,7 @@ class Cosmos3EdgeCausalAttention(nn.Module): prefix=f"{prefix}.to_out", ) - self.k_norm_und_for_gen = RMSNorm(self.head_dim, eps=rms_norm_eps) if use_k_norm_und_for_gen else None + self.k_norm_und_for_gen = RMSNorm(self.head_dim, eps=rms_norm_eps) if use_und_k_norm_for_gen else None self.attn = FrameworkAttention( num_heads=self.num_heads, head_size=self.head_dim, @@ -195,7 +195,7 @@ class Cosmos3EdgeUndDecoderLayer(nn.Module): num_key_value_heads: int, head_dim: int, rms_norm_eps: float, - use_k_norm_und_for_gen: bool, + use_und_k_norm_for_gen: bool, quant_config: QuantizationConfig | None = None, prefix: str = "", ) -> None: @@ -206,7 +206,7 @@ class Cosmos3EdgeUndDecoderLayer(nn.Module): num_key_value_heads=num_key_value_heads, head_dim=head_dim, rms_norm_eps=rms_norm_eps, - use_k_norm_und_for_gen=use_k_norm_und_for_gen, + use_und_k_norm_for_gen=use_und_k_norm_for_gen, quant_config=quant_config, prefix=f"{prefix}.self_attn", ) @@ -256,7 +256,7 @@ class Cosmos3EdgeLanguageModel(nn.Module): rms_norm_eps: float, rope_theta: float, mrope_section: list[int], - use_k_norm_und_for_gen: bool, + use_und_k_norm_for_gen: bool, quant_config: QuantizationConfig | None = None, prefix: str = "", ) -> None: @@ -276,7 +276,7 @@ class Cosmos3EdgeLanguageModel(nn.Module): num_key_value_heads=num_key_value_heads, head_dim=head_dim, rms_norm_eps=rms_norm_eps, - use_k_norm_und_for_gen=use_k_norm_und_for_gen, + use_und_k_norm_for_gen=use_und_k_norm_for_gen, quant_config=quant_config, prefix=f"{prefix}.layers.{i}", ) @@ -368,7 +368,7 @@ class Cosmos3EdgeVFMTransformer(Cosmos3VFMTransformer): return list(mrope_section) def _language_model_kwargs(self) -> dict[str, Any]: - return {"use_k_norm_und_for_gen": self.use_k_norm_und_for_gen} + return {"use_und_k_norm_for_gen": self.use_und_k_norm_for_gen} def validate_loaded_weights(self, loaded: set[str]) -> None: missing: list[str] = [] @@ -379,7 +379,7 @@ class Cosmos3EdgeVFMTransformer(Cosmos3VFMTransformer): f"gen_layers.{layer_idx}.mlp.up_proj.", f"gen_layers.{layer_idx}.mlp.down_proj.", ) - if self.use_k_norm_und_for_gen: + if self.use_und_k_norm_for_gen: required_markers = ( *required_markers, f"language_model.layers.{layer_idx}.self_attn.k_norm_und_for_gen.", From 5a6a93dc5f36ddaec0cf0cf2945a362b3a1bc580 Mon Sep 17 00:00:00 2001 From: wtz2333 <2955110911@qq.com> Date: Tue, 21 Jul 2026 17:27:29 +0800 Subject: [PATCH 038/240] [diffusion][model] Add LingBot Video dense and MoE support (#5035) Signed-off-by: hsliuustc0106 Signed-off-by: wtz2333 <2955110911@qq.com> Co-authored-by: hsliuustc0106 --- .buildkite/cuda/test-nightly.yml | 38 + benchmarks/README.md | 8 + benchmarks/lingbot_video/README.md | 62 + .../lingbot_video/dense_pipeline_parity.py | 217 ++++ .../lingbot_video/moe_transformer_parity.py | 399 +++++++ docs/models/supported_models.md | 1 + .../text_to_video/text_to_video_lingbot.py | 124 ++ recipes/README.md | 1 + recipes/Robbyant/LingBot-Video.md | 259 +++++ .../tests/test_lingbot_video_vllm_omni.json | 34 + .../models/lingbot_video/__init__.py | 2 + .../test_lingbot_video_moe_cuda.py | 183 +++ .../test_lingbot_video_transformer.py | 266 +++++ .../test_pipeline_lingbot_video.py | 219 ++++ .../e2e/online_serving/test_lingbot_video.py | 67 ++ .../test_lingbot_video_expansion.py | 83 ++ .../online_serving/test_lingbot_video_moe.py | 59 + tests/model_tests/diffusion/test_alignment.py | 1 + .../models/lingbot_video/__init__.py | 16 + .../lingbot_video_transformer.py | 1005 +++++++++++++++++ .../lingbot_video/pipeline_lingbot_video.py | 695 ++++++++++++ vllm_omni/diffusion/registry.py | 6 + vllm_omni/model_extras/lingbot_video.py | 16 + vllm_omni/model_extras/registry.py | 4 + 24 files changed, 3765 insertions(+) create mode 100644 benchmarks/lingbot_video/README.md create mode 100644 benchmarks/lingbot_video/dense_pipeline_parity.py create mode 100644 benchmarks/lingbot_video/moe_transformer_parity.py create mode 100644 examples/offline_inference/text_to_video/text_to_video_lingbot.py create mode 100644 recipes/Robbyant/LingBot-Video.md create mode 100644 tests/dfx/perf/tests/test_lingbot_video_vllm_omni.json create mode 100644 tests/diffusion/models/lingbot_video/__init__.py create mode 100644 tests/diffusion/models/lingbot_video/test_lingbot_video_moe_cuda.py create mode 100644 tests/diffusion/models/lingbot_video/test_lingbot_video_transformer.py create mode 100644 tests/diffusion/models/lingbot_video/test_pipeline_lingbot_video.py create mode 100644 tests/e2e/online_serving/test_lingbot_video.py create mode 100644 tests/e2e/online_serving/test_lingbot_video_expansion.py create mode 100644 tests/e2e/online_serving/test_lingbot_video_moe.py create mode 100644 vllm_omni/diffusion/models/lingbot_video/__init__.py create mode 100644 vllm_omni/diffusion/models/lingbot_video/lingbot_video_transformer.py create mode 100644 vllm_omni/diffusion/models/lingbot_video/pipeline_lingbot_video.py create mode 100644 vllm_omni/model_extras/lingbot_video.py diff --git a/.buildkite/cuda/test-nightly.yml b/.buildkite/cuda/test-nightly.yml index dd62ee68b..d0f866b5a 100644 --- a/.buildkite/cuda/test-nightly.yml +++ b/.buildkite/cuda/test-nightly.yml @@ -1322,6 +1322,44 @@ steps: path: /mnt/hf-cache type: DirectoryOrCreate + - label: ":full_moon: Diffusion X2V · LingBot Video MoE Function Test" + timeout_in_minutes: 90 + commands: + - pytest -s -v tests/e2e/online_serving/test_lingbot_video_moe.py -m "full_model and cuda" --run-level "full_model" + agents: + queue: "mithril-h100-pool" + plugins: + - kubernetes: + podSpec: + containers: + - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT + resources: + limits: + nvidia.com/gpu: 1 + volumeMounts: + - name: devshm + mountPath: /dev/shm + - name: hf-cache + mountPath: /root/.cache/huggingface + env: + - name: HF_HOME + value: /root/.cache/huggingface + - name: HF_TOKEN + valueFrom: + secretKeyRef: + name: hf-token-secret + key: token + nodeSelector: + node.kubernetes.io/instance-type: gpu-h100-sxm + volumes: + - name: devshm + emptyDir: + medium: Memory + - name: hf-cache + hostPath: + path: /mnt/hf-cache + type: DirectoryOrCreate + - label: ":full_moon: Diffusion X2V · Doc Test" timeout_in_minutes: 60 commands: diff --git a/benchmarks/README.md b/benchmarks/README.md index 37a6264a9..0a8fb8058 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -28,6 +28,14 @@ Benchmarks for GLM-Image performance across HuggingFace baseline, vLLM-Omni offl - **Tasks**: text-to-image and image-to-image - **Key metrics**: request/image throughput, latency percentiles, optional per-stage pipeline timings +### [LingBot-Video](lingbot_video/README.md) — Dense and MoE Parity + +Manual cross-runtime validation for the dense LingBot-Video pipeline and the +LingBot-Video MoE transformer. + +- **Dense pipeline**: decoded-video MAE, MSE, PSNR, latency, and optional steady-state timings +- **MoE transformer**: bitwise router, sparse-block, shared-expert, and full-transformer parity + ### [Distributed](distributed/omni_connectors/README.md) — RDMA Connector Testing RDMA environment setup and transfer tests for `MooncakeTransferEngineConnector`, including pytest-based single-node checks and manual cross-node benchmarks. diff --git a/benchmarks/lingbot_video/README.md b/benchmarks/lingbot_video/README.md new file mode 100644 index 000000000..687973659 --- /dev/null +++ b/benchmarks/lingbot_video/README.md @@ -0,0 +1,62 @@ +# LingBot-Video Benchmarks + +These manual benchmarks compare the upstream LingBot-Video implementation with +vLLM-Omni. Run all commands from the repository root. + +| Benchmark | Script | Purpose | +|-----------|--------|---------| +| Dense pipeline parity | `dense_pipeline_parity.py` | Compare generated videos and report MAE, MSE, PSNR, latency, and optional steady-state timings | +| MoE numerical parity | `moe_transformer_parity.py` | Check bitwise parity for the sparse MoE block and the full MoE transformer | + +Both benchmarks require a local LingBot-Video checkout and local model files. +They do not run in CI. + +## Dense pipeline parity + +The dense benchmark launches the upstream Diffusers inference script and the +vLLM-Omni offline example with identical prompts and sampling parameters. It +then compares the decoded MP4 frames. + +```bash +CUDA_VISIBLE_DEVICES=0 \ +python benchmarks/lingbot_video/dense_pipeline_parity.py \ + --model /path/to/lingbot-video-dense-1.3b \ + --official-repo /path/to/lingbot-video \ + --output-dir /tmp/lingbot_dense_parity +``` + +Use `--runs N` to additionally measure repeated in-process vLLM-Omni +requests. The video comparison reports MAE, MSE, and PSNR; it is an end-to-end +pipeline diagnostic rather than a bitwise transformer check. + +## MoE numerical parity + +The lightweight block comparison uses deterministic weights and inputs to +cover correction-bias routing, group-limited top-k, routed experts, FP32 +scatter-weighted restore, padding masks, and the shared expert. + +```bash +CUDA_VISIBLE_DEVICES=0 \ +python benchmarks/lingbot_video/moe_transformer_parity.py \ + --scope block \ + --official-repo /path/to/lingbot-video \ + --output-json /tmp/lingbot_moe_block_parity.json +``` + +The full comparison loads the upstream and vLLM-Omni 30B transformers +sequentially and compares their output tensors. It requires a CUDA device with +`torch._grouped_mm` and enough memory for one 30B BF16 transformer. + +```bash +CUDA_VISIBLE_DEVICES=0 \ +python benchmarks/lingbot_video/moe_transformer_parity.py \ + --scope transformer \ + --official-repo /path/to/lingbot-video \ + --model /path/to/lingbot-video-moe-30b-a3b \ + --output-json /tmp/lingbot_moe_transformer_parity.json +``` + +For the correctness oracle, the upstream transformer uses +`diffusers:_native_math` and vLLM-Omni uses +`TORCH_SDPA + SDPBackend.MATH`. The command exits with a nonzero status unless +the selected comparisons are bitwise equal. diff --git a/benchmarks/lingbot_video/dense_pipeline_parity.py b/benchmarks/lingbot_video/dense_pipeline_parity.py new file mode 100644 index 000000000..0c0c1d7a0 --- /dev/null +++ b/benchmarks/lingbot_video/dense_pipeline_parity.py @@ -0,0 +1,217 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from __future__ import annotations + +import argparse +import json +import re +import statistics +import subprocess +import sys +import time +from pathlib import Path +from typing import Any + +import imageio.v3 as iio +import numpy as np +import torch + +from vllm_omni.diffusion.data import DiffusionParallelConfig +from vllm_omni.entrypoints.omni import Omni +from vllm_omni.inputs.data import OmniDiffusionSamplingParams +from vllm_omni.platforms import current_omni_platform + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="LingBot dense parity and timing harness.") + parser.add_argument("--model", default="/home/models/lingbot-video-dense-1.3b") + parser.add_argument("--official-repo", default="/tmp/lingbot-video") + parser.add_argument("--output-dir", default="/tmp/lingbot_dense_parity") + parser.add_argument("--prompt", default="a robotic arm picks up a red block") + parser.add_argument("--height", type=int, default=192) + parser.add_argument("--width", type=int, default=320) + parser.add_argument("--num-frames", type=int, default=9) + parser.add_argument("--steps", type=int, default=2) + parser.add_argument("--guidance-scale", type=float, default=3.0) + parser.add_argument("--shift", type=float, default=3.0) + parser.add_argument("--seed", type=int, default=42) + parser.add_argument("--fps", type=int, default=24) + parser.add_argument("--runs", type=int, default=0, help="Optional in-process steady-state native runs.") + return parser.parse_args() + + +def _run(cmd: list[str], cwd: Path | None = None) -> tuple[float, str]: + start = time.perf_counter() + proc = subprocess.run(cmd, cwd=cwd, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + elapsed = time.perf_counter() - start + if proc.returncode != 0: + print(proc.stdout) + raise subprocess.CalledProcessError(proc.returncode, proc.args, output=proc.stdout) + return elapsed, proc.stdout + + +def _read_video(path: Path) -> np.ndarray: + frames = iio.imread(path, plugin="pyav") + if frames.dtype != np.float32: + frames = frames.astype(np.float32) / 255.0 + return frames + + +def _compare(a: np.ndarray, b: np.ndarray) -> dict[str, Any]: + if a.shape != b.shape: + return {"shape_a": list(a.shape), "shape_b": list(b.shape), "shape_match": False} + diff = a - b + mse = float(np.mean(diff * diff)) + mae = float(np.mean(np.abs(diff))) + psnr = float("inf") if mse == 0 else float(20.0 * np.log10(1.0 / np.sqrt(mse))) + return { + "shape": list(a.shape), + "shape_match": True, + "mae": mae, + "mse": mse, + "psnr": psnr, + } + + +def _extract_total_generation_seconds(log: str) -> float | None: + match = re.search(r"Total generation time:\s*([0-9.]+)\s*seconds", log) + return float(match.group(1)) if match else None + + +def _peak_memory_mb(output: Any) -> float: + result = output[0] if isinstance(output, list) and output else output + return float(getattr(result, "peak_memory_mb", 0.0) or 0.0) + + +def _benchmark_native_requests(args: argparse.Namespace) -> dict[str, Any]: + load_start = time.perf_counter() + omni = Omni( + model=args.model, + model_class_name="LingBotVideoPipeline", + flow_shift=args.shift, + parallel_config=DiffusionParallelConfig(), + ) + load_seconds = time.perf_counter() - load_start + prompt = {"prompt": args.prompt} + latencies: list[float] = [] + peak_mb = 0.0 + + for _ in range(args.runs): + generator = torch.Generator(device=current_omni_platform.device_type).manual_seed(args.seed) + sampling_params = OmniDiffusionSamplingParams( + height=args.height, + width=args.width, + num_frames=args.num_frames, + num_inference_steps=args.steps, + guidance_scale=args.guidance_scale, + generator=generator, + output_type="pt", + extra_args={"shift": args.shift}, + ) + start = time.perf_counter() + output = omni.generate(prompt, sampling_params) + latencies.append(time.perf_counter() - start) + peak_mb = max(peak_mb, _peak_memory_mb(output)) + del output + omni.close() + + return { + "load_seconds": load_seconds, + "run_seconds": latencies, + "mean_seconds": statistics.mean(latencies) if latencies else None, + "median_seconds": statistics.median(latencies) if latencies else None, + "min_seconds": min(latencies) if latencies else None, + "max_seconds": max(latencies) if latencies else None, + "peak_mb": peak_mb, + } + + +def main() -> None: + args = parse_args() + output_dir = Path(args.output_dir) + output_dir.mkdir(parents=True, exist_ok=True) + official_out = output_dir / "official_diffusers.mp4" + native_out = output_dir / "vllm_omni_native.mp4" + + official_cmd = [ + sys.executable, + str(Path(args.official_repo) / "scripts" / "inference.py"), + "--backend", + "diffusers", + "--model_dir", + args.model, + "--mode", + "t2v", + "--prompt", + args.prompt, + "--output", + str(official_out), + "--height", + str(args.height), + "--width", + str(args.width), + "--num_frames", + str(args.num_frames), + "--steps", + str(args.steps), + "--guidance_scale", + str(args.guidance_scale), + "--shift", + str(args.shift), + "--seed", + str(args.seed), + "--fps", + str(args.fps), + "--quiet_progress", + ] + native_cmd = [ + sys.executable, + "examples/offline_inference/text_to_video/text_to_video_lingbot.py", + "--model", + args.model, + "--prompt", + args.prompt, + "--output", + str(native_out), + "--height", + str(args.height), + "--width", + str(args.width), + "--num-frames", + str(args.num_frames), + "--num-inference-steps", + str(args.steps), + "--guidance-scale", + str(args.guidance_scale), + "--flow-shift", + str(args.shift), + "--seed", + str(args.seed), + "--fps", + str(args.fps), + ] + + official_time, official_log = _run(official_cmd, cwd=Path(args.official_repo)) + native_time, native_log = _run(native_cmd, cwd=Path.cwd()) + + result = { + "official_seconds": official_time, + "native_process_seconds": native_time, + "native_request_seconds": _extract_total_generation_seconds(native_log), + "official_output": str(official_out), + "native_output": str(native_out), + "accuracy": _compare(_read_video(official_out), _read_video(native_out)), + "official_log_tail": official_log.splitlines()[-20:], + "native_log_tail": native_log.splitlines()[-20:], + } + if args.runs > 0: + result["native_steady_state"] = _benchmark_native_requests(args) + + result_path = output_dir / "comparison.json" + result_path.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n") + print(json.dumps(result, indent=2, sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/lingbot_video/moe_transformer_parity.py b/benchmarks/lingbot_video/moe_transformer_parity.py new file mode 100644 index 000000000..cdad31f19 --- /dev/null +++ b/benchmarks/lingbot_video/moe_transformer_parity.py @@ -0,0 +1,399 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +"""Reproduce LingBot-Video MoE block and transformer numerical parity.""" + +from __future__ import annotations + +import argparse +import gc +import importlib +import json +import os +import sys +import time +from pathlib import Path +from typing import Any + +import torch + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--scope", + choices=("block", "transformer", "all"), + default="block", + help="Run the lightweight sparse-block check, the real transformer check, or both.", + ) + parser.add_argument( + "--official-repo", + type=Path, + required=True, + help="Local checkout of https://github.com/Robbyant/lingbot-video.", + ) + parser.add_argument( + "--model", + default=None, + help="Local MoE checkpoint path or a checkpoint already present in the Hugging Face cache.", + ) + parser.add_argument("--transformer-subfolder", default="transformer") + parser.add_argument("--output-json", type=Path, default=None) + parser.add_argument("--device", default="cuda") + parser.add_argument("--seed", type=int, default=42) + parser.add_argument("--num-frames", type=int, default=1) + parser.add_argument("--height", type=int, default=8) + parser.add_argument("--width", type=int, default=8) + parser.add_argument("--text-length", type=int, default=16) + return parser.parse_args() + + +def _load_official_module(official_repo: Path): + transformer_file = official_repo / "lingbot_video" / "transformer_lingbot_video.py" + if not transformer_file.is_file(): + raise FileNotFoundError(f"Upstream transformer not found: {transformer_file}") + sys.path.insert(0, str(official_repo)) + module = importlib.import_module("lingbot_video.transformer_lingbot_video") + module_path = Path(module.__file__).resolve() + if official_repo.resolve() not in module_path.parents: + raise RuntimeError(f"Imported LingBot module from {module_path}, not {official_repo.resolve()}") + return module + + +def _load_native_module(): + return importlib.import_module("vllm_omni.diffusion.models.lingbot_video.lingbot_video_transformer") + + +def _tensor_metrics(actual: torch.Tensor, expected: torch.Tensor) -> dict[str, Any]: + actual_float = actual.float() + expected_float = expected.float() + diff = (actual_float - expected_float).abs() + denominator = actual_float.norm() * expected_float.norm() + cosine = ( + float(torch.dot(actual_float.flatten(), expected_float.flatten()) / denominator) + if float(denominator) > 0 + else 1.0 + ) + return { + "shape": list(actual.shape), + "equal": bool(torch.equal(actual, expected)), + "max_abs": float(diff.max()) if diff.numel() else 0.0, + "mean_abs": float(diff.mean()) if diff.numel() else 0.0, + "rmse": float(torch.sqrt(torch.mean(diff.square()))) if diff.numel() else 0.0, + "cosine": cosine, + "finite": bool(torch.isfinite(actual).all() and torch.isfinite(expected).all()), + } + + +def _initialize_block(module: torch.nn.Module, seed: int) -> None: + generator = torch.Generator(device="cpu").manual_seed(seed) + with torch.no_grad(): + for parameter in module.parameters(): + values = torch.randn(parameter.shape, generator=generator, dtype=torch.float32) + parameter.copy_((values * 0.02).to(parameter.dtype)) + module.router.e_score_correction_bias.copy_(torch.tensor([0.9, 0.8, 1.0, 0.0, 0.7, 0.6, 0.5, 0.4])) + + +def _apply_padding( + scores: torch.Tensor, + padding_mask: torch.Tensor, + route_scale: float, +) -> torch.Tensor: + scores = scores * padding_mask.unsqueeze(-1).to(scores.dtype) + scores = scores / (scores.sum(dim=-1, keepdim=True) + 1e-9) + return scores * route_scale + + +def _run_block_parity( + official_module, + native_module, + *, + device: torch.device, + seed: int, +) -> dict[str, Any]: + if not hasattr(torch, "_grouped_mm"): + raise RuntimeError("The block parity check requires torch._grouped_mm.") + + common = { + "hidden_size": 16, + "num_experts": 8, + "top_k": 2, + "moe_intermediate_size": 8, + "score_func": "sigmoid", + "norm_topk_prob": True, + "n_group": 4, + "topk_group": 1, + "routed_scaling_factor": 1.5, + "n_shared_experts": 1, + } + official = official_module.LingBotVideoSparseMoeBlock( + intermediate_size=32, + **common, + ) + native = native_module.LingBotVideoSparseMoeBlock(**common) + _initialize_block(official, seed) + native.load_state_dict(official.state_dict(), strict=True) + official = official.to(device=device, dtype=torch.bfloat16).eval() + native = native.to(device=device, dtype=torch.bfloat16).eval() + + generator = torch.Generator(device=device).manual_seed(seed) + hidden_states = torch.randn( + 2, + 5, + common["hidden_size"], + generator=generator, + device=device, + dtype=torch.bfloat16, + ) + padding_mask = torch.tensor( + [1, 1, 1, 1, 1, 1, 1, 1, 0, 0], + device=device, + dtype=torch.float32, + ) + tokens = hidden_states.reshape(-1, common["hidden_size"]) + + with torch.inference_mode(): + official_router = official.router(tokens) + official_indices, official_scores = official_router[:2] + native_indices, native_scores = native.router(tokens) + + official_scores = _apply_padding( + official_scores, + padding_mask, + official.router.route_scale, + ) + native_scores = _apply_padding( + native_scores, + padding_mask, + native.router.route_scale, + ) + official_routed = official._run_selected_experts( + tokens, + official_scores, + official_indices, + ) + native_routed = native._run_selected_experts( + tokens, + native_scores, + native_indices, + ) + official_shared = official.shared_experts(hidden_states) + native_shared = native.shared_experts(hidden_states) + official_output = official(hidden_states, padding_mask=padding_mask) + native_output = native(hidden_states, padding_mask=padding_mask) + + result = { + "router_indices_equal": bool(torch.equal(native_indices, official_indices)), + "router_scores": _tensor_metrics(native_scores, official_scores), + "routed_output": _tensor_metrics(native_routed, official_routed), + "shared_output": _tensor_metrics(native_shared, official_shared), + "final_output": _tensor_metrics(native_output, official_output), + } + result["exact"] = bool( + result["router_indices_equal"] + and result["router_scores"]["equal"] + and result["routed_output"]["equal"] + and result["shared_output"]["equal"] + and result["final_output"]["equal"] + ) + return result + + +def _release_cuda() -> None: + gc.collect() + torch.accelerator.empty_cache() + torch.accelerator.synchronize() + + +def _make_transformer_inputs( + config, + args: argparse.Namespace, +) -> dict[str, torch.Tensor]: + generator = torch.Generator(device="cpu").manual_seed(args.seed) + hidden_states = torch.randn( + 1, + int(config.in_channels), + args.num_frames, + args.height, + args.width, + generator=generator, + dtype=torch.float32, + ).to(torch.bfloat16) + encoder_hidden_states = torch.randn( + 1, + args.text_length, + int(config.text_dim), + generator=generator, + dtype=torch.float32, + ).to(torch.bfloat16) + return { + "hidden_states": hidden_states, + "timestep": torch.tensor([500.0], dtype=torch.float32), + "encoder_hidden_states": encoder_hidden_states, + "encoder_attention_mask": torch.ones(1, args.text_length, dtype=torch.long), + } + + +def _load_transformer( + transformer_cls, + args: argparse.Namespace, + device: torch.device, +): + start = time.perf_counter() + model = transformer_cls.from_pretrained( + args.model, + subfolder=args.transformer_subfolder, + torch_dtype=torch.bfloat16, + local_files_only=True, + low_cpu_mem_usage=True, + ) + model = model.to(device=device, dtype=torch.bfloat16).eval() + torch.accelerator.synchronize() + return model, time.perf_counter() - start + + +def _forward_transformer( + model, + inputs: dict[str, torch.Tensor], + device: torch.device, + attention_context, +) -> tuple[torch.Tensor, float, float]: + torch.accelerator.reset_peak_memory_stats(device) + gpu_inputs = {name: value.to(device) for name, value in inputs.items()} + start = time.perf_counter() + with torch.inference_mode(), attention_context: + output = model(**gpu_inputs, return_dict=False)[0] + torch.accelerator.synchronize() + elapsed = time.perf_counter() - start + peak_gib = torch.accelerator.max_memory_reserved(device) / (1024**3) + return output.cpu(), elapsed, peak_gib + + +def _run_transformer_parity( + official_module, + native_module, + *, + args: argparse.Namespace, + device: torch.device, +) -> dict[str, Any]: + if args.model is None: + raise ValueError("--model is required for transformer parity.") + + from diffusers.models.attention_dispatch import attention_backend + from torch.nn.attention import SDPBackend, sdpa_kernel + + from vllm_omni.diffusion.config import set_current_diffusion_config + from vllm_omni.diffusion.data import AttentionConfig, AttentionSpec, OmniDiffusionConfig + + official, official_load = _load_transformer( + official_module.LingBotVideoTransformer3DModel, + args, + device, + ) + inputs = _make_transformer_inputs(official.config, args) + official_output, official_forward, official_peak = _forward_transformer( + official, + inputs, + device, + attention_backend("_native_math"), + ) + del official + _release_cuda() + + native_config = OmniDiffusionConfig( + diffusion_attention_config=AttentionConfig( + default=AttentionSpec(backend="TORCH_SDPA"), + ), + ) + with set_current_diffusion_config(native_config): + native, native_load = _load_transformer( + native_module.LingBotVideoTransformer3DModel, + args, + device, + ) + native_backend_prefs = {block.attn.attn.backend_pref for block in native.blocks} + if native_backend_prefs != {"TORCH_SDPA"}: + raise RuntimeError( + f"Failed to force the native transformer to TORCH_SDPA: resolved preferences were {native_backend_prefs}." + ) + native_output, native_forward, native_peak = _forward_transformer( + native, + inputs, + device, + sdpa_kernel(SDPBackend.MATH), + ) + del native + _release_cuda() + + metrics = _tensor_metrics(native_output, official_output) + return { + "backends": { + "official": "diffusers:_native_math", + "native": "TORCH_SDPA+SDPBackend.MATH", + }, + "official": { + "load_seconds": official_load, + "forward_seconds": official_forward, + "peak_reserved_gib": official_peak, + }, + "native": { + "load_seconds": native_load, + "forward_seconds": native_forward, + "peak_reserved_gib": native_peak, + }, + "output": metrics, + "exact": metrics["equal"], + } + + +def main() -> int: + args = parse_args() + device = torch.device(args.device) + if device.type != "cuda" or not torch.cuda.is_available(): + raise RuntimeError("LingBot MoE parity requires an available CUDA device.") + + os.environ["DIFFUSERS_ATTN_BACKEND"] = "_native_math" + os.environ["DIFFUSION_ATTENTION_BACKEND"] = "TORCH_SDPA" + os.environ["LINGBOT_MOE_EXPERT_BACKEND"] = "grouped_mm" + os.environ["LINGBOT_MOE_PAD_BACKEND"] = "loop" + os.environ["LINGBOT_MOE_REORDER_BACKEND"] = "sort" + os.environ["LINGBOT_MOE_RESTORE_BACKEND"] = "scatter" + + official_module = _load_official_module(args.official_repo) + native_module = _load_native_module() + + result: dict[str, Any] = { + "settings": { + "scope": args.scope, + "seed": args.seed, + "device": str(device), + "official_repo": str(args.official_repo.resolve()), + "model": args.model, + } + } + if args.scope in {"block", "all"}: + result["block"] = _run_block_parity( + official_module, + native_module, + device=device, + seed=args.seed, + ) + if args.scope in {"transformer", "all"}: + result["transformer"] = _run_transformer_parity( + official_module, + native_module, + args=args, + device=device, + ) + + result["exact"] = all(section["exact"] for name, section in result.items() if name in {"block", "transformer"}) + payload = json.dumps(result, indent=2, sort_keys=True) + print(payload) + if args.output_json is not None: + args.output_json.parent.mkdir(parents=True, exist_ok=True) + args.output_json.write_text(payload + "\n", encoding="utf-8") + return 0 if result["exact"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/docs/models/supported_models.md b/docs/models/supported_models.md index c1dac6ca7..dda7c5f2d 100644 --- a/docs/models/supported_models.md +++ b/docs/models/supported_models.md @@ -47,6 +47,7 @@ th { | `LTX2ImageToVideoTwoStagesPipeline` | LTX-2-I2V | `rootonchair/LTX-2-19b-distilled` | ✅︎ | ✅︎ | | | | `LTX23Pipeline` | LTX-2.3-T2V | `diffusers/LTX-2.3-Diffusers` | ✅︎ | ✅︎ | | | | `LTX23ImageToVideoPipeline` | LTX-2.3-I2V | `diffusers/LTX-2.3-Diffusers` | ✅︎ | ✅︎ | | | +| `LingBotVideoPipeline` | LingBot-Video dense and MoE T2V | `robbyant/lingbot-video-dense-1.3b`, `robbyant/lingbot-video-moe-30b-a3b` | ✅︎ | | | | | `DreamZeroPipeline` | DreamZero-DROID | `GEAR-Dreams/DreamZero-DROID` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | | `HeliosPipeline`, `HeliosPyramidPipeline` | Helios | `BestWishYsh/Helios-Base`, `BestWishYsh/Helios-Mid`, `BestWishYsh/Helios-Distilled` | ✅︎ | ✅︎ | ✅︎ | | | `MagiHumanPipeline` | MagiHuman | `SII-GAIR/daVinci-MagiHuman-Base-1080p` | ✅︎ | ✅︎ | | | diff --git a/examples/offline_inference/text_to_video/text_to_video_lingbot.py b/examples/offline_inference/text_to_video/text_to_video_lingbot.py new file mode 100644 index 000000000..a8f8a2015 --- /dev/null +++ b/examples/offline_inference/text_to_video/text_to_video_lingbot.py @@ -0,0 +1,124 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from __future__ import annotations + +import argparse +import time +from pathlib import Path +from typing import Any + +import numpy as np +import torch + +from vllm_omni.diffusion.data import DiffusionParallelConfig +from vllm_omni.entrypoints.omni import Omni +from vllm_omni.inputs.data import OmniDiffusionSamplingParams +from vllm_omni.outputs import OmniRequestOutput +from vllm_omni.platforms import current_omni_platform + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Generate text-to-video with a LingBot-Video checkpoint.") + parser.add_argument("--model", default="/home/models/lingbot-video-dense-1.3b") + parser.add_argument("--prompt", default="a robotic arm picks up a red block") + parser.add_argument("--negative-prompt", default="") + parser.add_argument("--output", default="lingbot_video_output.mp4") + parser.add_argument("--height", type=int, default=192) + parser.add_argument("--width", type=int, default=320) + parser.add_argument("--num-frames", type=int, default=9) + parser.add_argument("--num-inference-steps", type=int, default=2) + parser.add_argument("--guidance-scale", type=float, default=3.0) + parser.add_argument("--flow-shift", type=float, default=3.0) + parser.add_argument("--seed", type=int, default=42) + parser.add_argument("--fps", type=int, default=24) + parser.add_argument("--output-type", choices=["pt", "np"], default="pt") + parser.add_argument("--batch-cfg", action="store_true") + return parser.parse_args() + + +def _peak_memory_mb(output: Any) -> float: + result = output[0] if isinstance(output, list) and output else output + return float(getattr(result, "peak_memory_mb", 0.0) or 0.0) + + +def _extract_video(output: Any) -> Any: + if isinstance(output, list): + output = output[0] if output else None + if isinstance(output, OmniRequestOutput): + if output.final_output_type != "image": + raise ValueError(f"Unexpected output type {output.final_output_type!r}.") + if not output.images: + raise ValueError("No video frames found in OmniRequestOutput.") + return output.images[0] if len(output.images) == 1 else output.images + return output + + +def _to_frame_array(video: Any) -> np.ndarray: + if isinstance(video, torch.Tensor): + tensor = video.detach().cpu() + if tensor.dim() == 5: + tensor = tensor[0] if tensor.shape[1] not in (3, 4) else tensor[0].permute(1, 2, 3, 0) + elif tensor.dim() == 4 and tensor.shape[0] in (3, 4): + tensor = tensor.permute(1, 2, 3, 0) + if tensor.is_floating_point(): + if float(tensor.min()) < 0.0: + tensor = tensor.clamp(-1, 1) * 0.5 + 0.5 + else: + tensor = tensor.clamp(0, 1) + return tensor.float().numpy() + if isinstance(video, np.ndarray): + array = video[0] if video.ndim == 5 else video + if np.issubdtype(array.dtype, np.integer): + return array.astype(np.float32) / 255.0 + return array.astype(np.float32, copy=False) + if isinstance(video, list): + return np.stack([_to_frame_array(frame) for frame in video], axis=0) + return np.asarray(video, dtype=np.float32) + + +def main() -> None: + args = parse_args() + generator = torch.Generator(device=current_omni_platform.device_type).manual_seed(args.seed) + omni = Omni( + model=args.model, + model_class_name="LingBotVideoPipeline", + flow_shift=args.flow_shift, + parallel_config=DiffusionParallelConfig(), + ) + + prompt: dict[str, str] = {"prompt": args.prompt} + if args.negative_prompt: + prompt["negative_prompt"] = args.negative_prompt + + sampling_params = OmniDiffusionSamplingParams( + height=args.height, + width=args.width, + num_frames=args.num_frames, + num_inference_steps=args.num_inference_steps, + guidance_scale=args.guidance_scale, + generator=generator, + output_type=args.output_type, + extra_args={"batch_cfg": args.batch_cfg, "shift": args.flow_shift}, + ) + + start = time.perf_counter() + output = omni.generate(prompt, sampling_params) + elapsed = time.perf_counter() - start + print(f"Total generation time: {elapsed:.4f} seconds ({elapsed * 1000:.2f} ms)") + peak_mb = _peak_memory_mb(output) + if peak_mb: + print(f"Worker peak GPU memory (reserved): {peak_mb:.2f} MiB ({peak_mb / 1024:.2f} GiB)") + + video = _to_frame_array(_extract_video(output)) + output_path = Path(args.output) + output_path.parent.mkdir(parents=True, exist_ok=True) + from diffusers.utils import export_to_video + + export_to_video(list(video), str(output_path), fps=args.fps) + print(f"Saved generated video to {output_path}") + omni.close() + + +if __name__ == "__main__": + main() diff --git a/recipes/README.md b/recipes/README.md index d335183ca..be4c7cc08 100644 --- a/recipes/README.md +++ b/recipes/README.md @@ -49,6 +49,7 @@ recipes/ | [`LTX/LTX-2.3.md`](./LTX/LTX-2.3.md) | Text-to-video with audio generation (22B) | 1x GPU (96GB VRAM) | | [`MammothModa2/MammothModa2-Preview.md`](./MammothModa2/MammothModa2-Preview.md) | Text-to-image with the shared offline image example (AR → DiT) | 1x L40S 48GB / 1x ≥40GB GPU | | [`mistralai/Voxtral-TTS.md`](./mistralai/Voxtral-TTS.md) | Online serving for TTS | 1x RTX 4090 24GB | +| [`Robbyant/LingBot-Video.md`](./Robbyant/LingBot-Video.md) | Native dense and MoE text-to-video serving | Dense: 1x L20X; MoE: 1x L20X (~67.7 GiB peak) | | [`cosmos3/Cosmos3-Nano.md`](./cosmos3/Cosmos3-Nano.md) | Text-to-image, text-to-video, image-to-video, video-to-video generation, text to video with sound, action policy | 1x H200 141GB / B300 | | [`cosmos3/Cosmos3-Super.md`](./cosmos3/Cosmos3-Super.md) | 64B T2I / T2V / I2V / V2V generation (+ optional audio) / Action policy | 8x H200/H100/A100 / 2x H200 / B300 | | [`OpenBMB/MiniCPM-o-4_5.md`](./OpenBMB/MiniCPM-o-4_5.md) | Online serving for omni multimodal chat (text / image / audio / video → text + 24 kHz speech) | 2x A100/H100 80GB / 3x mid-tier GPU / 8x RTX 4090 24GB | diff --git a/recipes/Robbyant/LingBot-Video.md b/recipes/Robbyant/LingBot-Video.md new file mode 100644 index 000000000..289d36990 --- /dev/null +++ b/recipes/Robbyant/LingBot-Video.md @@ -0,0 +1,259 @@ +# LingBot-Video + +> Native dense and MoE text-to-video serving for LingBot-Video + +## Summary + +- Vendor: Robbyant +- Models: `robbyant/lingbot-video-dense-1.3b` and `robbyant/lingbot-video-moe-30b-a3b` +- Task: Text-to-video generation +- Mode: Offline generation and online serving with the OpenAI-compatible `/v1/videos` API +- Maintainer: Community + +## When to use this recipe + +Use this recipe when you want to run a dense or MoE LingBot-Video checkpoint +with vLLM-Omni's native pipeline. The runtime path does not import the upstream +`lingbot_video` Python package; it loads checkpoint components directly with +the in-tree `LingBotVideoPipeline`, dense or routed-MoE DiT blocks, shared +FlowUniPC scheduler, Qwen3-VL text encoder, and Wan VAE. + +This first MoE integration is intentionally text-to-video and single-GPU only. +It targets the base BF16 checkpoint without its optional refiner or additional +parallel, cache, quantized, or expert-kernel backends. + +## References + +- Dense checkpoint: +- MoE checkpoint: +- Upstream project: +- Related offline example: [`examples/offline_inference/text_to_video/text_to_video_lingbot.py`](../../examples/offline_inference/text_to_video/text_to_video_lingbot.py) +- Related online video API docs: [`docs/serving/videos_api.md`](../../docs/serving/videos_api.md) + +## Hardware Support + +This recipe documents the CUDA single-GPU dense and BF16 MoE checkpoint paths. +Multi-GPU parallelism, Cache-DiT, quantization, and CPU offload are not +validated for LingBot-Video in this PR. + +## GPU + +### 1 x NVIDIA L20X + +Both checkpoints have been smoke-tested on one NVIDIA L20X at `192x320`, +9 frames, and 2 steps. The MoE smoke reserved approximately `67.70 GiB` of GPU +memory, so use a GPU with at least about 70 GiB of available memory for this +small validation shape. Larger resolutions, frame counts, or concurrent +requests require additional headroom. + +The MoE path is validated with BF16 expert weights. + +#### Dense offline T2V + +```bash +CUDA_VISIBLE_DEVICES=0 \ +python examples/offline_inference/text_to_video/text_to_video_lingbot.py \ + --model robbyant/lingbot-video-dense-1.3b \ + --prompt "a robotic arm picks up a red block" \ + --output lingbot_t2v.mp4 \ + --height 192 \ + --width 320 \ + --num-frames 9 \ + --num-inference-steps 2 \ + --guidance-scale 3.0 \ + --flow-shift 3.0 \ + --seed 42 \ + --fps 24 +``` + +#### MoE offline T2V + +```bash +CUDA_VISIBLE_DEVICES=0 \ +python examples/offline_inference/text_to_video/text_to_video_lingbot.py \ + --model robbyant/lingbot-video-moe-30b-a3b \ + --prompt "a robotic arm picks up a red block" \ + --output lingbot_moe_t2v.mp4 \ + --height 192 \ + --width 320 \ + --num-frames 9 \ + --num-inference-steps 2 \ + --guidance-scale 3.0 \ + --flow-shift 3.0 \ + --seed 42 \ + --fps 24 +``` + +#### Online serving + +```bash +CUDA_VISIBLE_DEVICES=0 \ +vllm serve robbyant/lingbot-video-dense-1.3b \ + --omni \ + --model-class-name LingBotVideoPipeline \ + --default-sampling-params \ + '{"0":{"num_frames":81,"num_inference_steps":40,"guidance_scale":6.0}}' \ + --port 8091 +``` + +For the MoE checkpoint, use the same single-GPU pipeline: + +```bash +CUDA_VISIBLE_DEVICES=0 \ +vllm serve robbyant/lingbot-video-moe-30b-a3b \ + --omni \ + --model-class-name LingBotVideoPipeline \ + --default-sampling-params \ + '{"0":{"num_frames":81,"num_inference_steps":40,"guidance_scale":6.0}}' \ + --port 8091 +``` + +These stage defaults match the LingBot reference pipeline. Request-level +values continue to override them, so the smaller smoke request below remains +unchanged. + +When serving MoE, replace the request's `model` form value below with +`robbyant/lingbot-video-moe-30b-a3b`. + +After the server is ready, submit a text-to-video job: + +```bash +create_response=$(curl -s http://localhost:8091/v1/videos \ + -F "model=robbyant/lingbot-video-dense-1.3b" \ + -F "prompt=a robotic arm picks up a red block" \ + -F "width=320" \ + -F "height=192" \ + -F "num_frames=9" \ + -F "fps=24" \ + -F "num_inference_steps=2" \ + -F "guidance_scale=3.0" \ + -F "flow_shift=3.0" \ + -F "seed=42") + +video_id=$(echo "${create_response}" | jq -r '.id') +while true; do + status=$(curl -s "http://localhost:8091/v1/videos/${video_id}" | jq -r '.status') + if [ "${status}" = "completed" ]; then + break + fi + if [ "${status}" = "failed" ]; then + curl -s "http://localhost:8091/v1/videos/${video_id}" | jq . + exit 1 + fi + sleep 2 +done + +curl -L "http://localhost:8091/v1/videos/${video_id}/content" -o lingbot_t2v.mp4 +``` + +## Key Parameters + +| Parameter | Suggested smoke value | Notes | +|-----------|-----------------------|-------| +| `height` | `192` | Must be a multiple of 16 | +| `width` | `320` | Must be a multiple of 16 | +| `num_frames` | `9` | Must be `1` or `4n + 1`; this PR validates T2V with video outputs | +| `num_inference_steps` | `2` | Use more steps for quality sweeps | +| `guidance_scale` | `3.0` | CFG is active when this is greater than `1.0` | +| `flow_shift` | `3.0` | Scheduler flow-shift; aliases the pipeline's internal `shift` | +| `negative_prompt` | model default | Optional text describing artifacts to avoid | +| `fps` | `24` | Output MP4 frame rate | + +## Validation + +Local dense smoke run: + +- Shape: 9 frames at `192x320` +- Steps: 2 +- Request generation time: `0.2923s` +- Peak reserved GPU memory: `14548 MiB` + +Local dense parity harness against the upstream repository: + +- Shape: `[9, 192, 320, 3]` +- MAE: `0.0065238` +- MSE: `0.00006650` +- PSNR: `41.77 dB` +- Native request time: `0.2875s` + +Local MoE validation on one NVIDIA L20X used checkpoint revision +`f2e538f64afe00cc4ae674db2aeb52e2945edfd5`: + +- Loaded all `977` transformer state keys and `30,084,506,176` parameters. +- Router weights and correction biases stayed in FP32; routed and shared + expert weights loaded in BF16. +- The complete 48-layer transformer forward returned finite + `[1, 16, 1, 8, 8]` BF16 output. +- With both implementations using matched math SDPA backends, the complete + native transformer matched the upstream implementation bitwise + (`max_abs=0`, `mean_abs=0`). +- An isolated sparse MoE block exercising the router, grouped experts, scatter + restore, and shared expert also matched the upstream implementation bitwise. +- The native pipeline generated a 9-frame `192x320` MP4 in 2 steps with + `69326 MiB` (`67.70 GiB`) peak reserved GPU memory. +- The online `/v1/videos` smoke created, polled, and downloaded the generated + MP4 successfully (`1 passed`). + +Do not treat these as production benchmarks; they are functional smoke plus +controlled numerical-parity evidence for small validation inputs. + +## Reproducing MoE numerical parity + +The in-tree parity harness compares the native MoE implementation with a local +checkout of the upstream LingBot-Video implementation. It uses local files +only and does not download checkpoints. + +Set paths for the upstream checkout and the cached MoE checkpoint: + +```bash +export LINGBOT_VIDEO_REPO=/path/to/lingbot-video +export LINGBOT_VIDEO_MOE_MODEL=/path/to/lingbot-video-moe-30b-a3b +``` + +Run the lightweight sparse-block comparison: + +```bash +CUDA_VISIBLE_DEVICES=0 \ +python benchmarks/lingbot_video/moe_transformer_parity.py \ + --scope block \ + --official-repo "${LINGBOT_VIDEO_REPO}" \ + --output-json /tmp/lingbot_moe_block_parity.json +``` + +This path covers bias-corrected router selection, group-limited top-k, routed +experts, FP32 scatter-weighted restore, padding masks, and the shared expert. +It exits with a nonzero status unless the upstream and native block outputs are +bitwise equal. + +The real-checkpoint transformer comparison requires one GPU with enough memory +for the 30B checkpoint. Models are loaded sequentially, so two copies are not +resident on the GPU at the same time: + +```bash +CUDA_VISIBLE_DEVICES=0 \ +python benchmarks/lingbot_video/moe_transformer_parity.py \ + --scope transformer \ + --official-repo "${LINGBOT_VIDEO_REPO}" \ + --model "${LINGBOT_VIDEO_MOE_MODEL}" \ + --output-json /tmp/lingbot_moe_transformer_parity.json +``` + +Transformer parity fixes the official implementation to +`diffusers:_native_math` and the native implementation to +`TORCH_SDPA + SDPBackend.MATH`. The expected result is `exact=true`, +`output.equal=true`, and zero max/mean/RMSE error. Different fused attention +kernels can introduce BF16 rounding differences before MoE routing, so +automatic attention-backend comparisons are diagnostic only and are not the +bitwise correctness oracle. + +## Known Limitations + +- T2V base-transformer inference only. T2I, I2V, TI2V, and the checkpoint's + optional `refiner/` transformer are not supported by this PR. +- Only the BF16 MoE checkpoint is validated in this first integration. +- No HSDP, tensor, sequence, expert, or CFG parallelism is claimed. +- No Cache-DiT, TeaCache, CPU offload, VAE patch parallelism, or quantized + inference is claimed. +- Optional Triton, SGLang, FP8, and alternative fused-expert backends from the + upstream project are not included. +- Only one request per LingBot pipeline batch is currently supported. diff --git a/tests/dfx/perf/tests/test_lingbot_video_vllm_omni.json b/tests/dfx/perf/tests/test_lingbot_video_vllm_omni.json new file mode 100644 index 000000000..a6751bda1 --- /dev/null +++ b/tests/dfx/perf/tests/test_lingbot_video_vllm_omni.json @@ -0,0 +1,34 @@ +[ + { + "test_name": "test_lingbot_video_single_device", + "description": "Single-device LingBot dense T2V baseline at 320x192, 9 frames, 2 steps.", + "server_type": "vllm-omni", + "server_params": { + "model": "robbyant/lingbot-video-dense-1.3b", + "serve_args": { + "model-class-name": "LingBotVideoPipeline", + "enable-diffusion-pipeline-profiler": true + } + }, + "benchmark_params": [ + { + "name": "320x192_frames9_steps2", + "dataset": "random", + "task": "t2v", + "backend": "v1/videos", + "width": 320, + "height": 192, + "num-frames": 9, + "fps": 24, + "num-inference-steps": 2, + "num-prompts": 3, + "max-concurrency": 1, + "warmup-requests": 1, + "warmup-concurrency": 1, + "warmup-num-inference-steps": 2, + "enable-negative-prompt": true, + "skip-performance-assertion": true + } + ] + } +] diff --git a/tests/diffusion/models/lingbot_video/__init__.py b/tests/diffusion/models/lingbot_video/__init__.py new file mode 100644 index 000000000..208f01a7c --- /dev/null +++ b/tests/diffusion/models/lingbot_video/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project diff --git a/tests/diffusion/models/lingbot_video/test_lingbot_video_moe_cuda.py b/tests/diffusion/models/lingbot_video/test_lingbot_video_moe_cuda.py new file mode 100644 index 000000000..5ea19948b --- /dev/null +++ b/tests/diffusion/models/lingbot_video/test_lingbot_video_moe_cuda.py @@ -0,0 +1,183 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import pytest +import torch +import torch.nn.functional as F + +from tests.helpers.mark import hardware_test + +pytestmark = [pytest.mark.core_model, pytest.mark.diffusion] + + +def _upstream_sparse_moe_reference(block, hidden_states, padding_mask=None): + """Independent eager reference for the upstream LingBot sparse MoE path.""" + batch_size, _, hidden_size = hidden_states.shape + tokens = hidden_states.reshape(-1, hidden_size) + + logits = F.linear(tokens.float(), block.router.weight.float()) + if block.router.score_func == "softmax": + scores = F.softmax(logits, dim=-1) + else: + scores = logits.sigmoid() + scores_for_choice = scores + block.router.e_score_correction_bias.unsqueeze(0) + + if block.router.n_group is not None and block.router.n_group > 1: + experts_per_group = block.router.num_experts // block.router.n_group + grouped = scores_for_choice.view(-1, block.router.n_group, experts_per_group) + group_scores = grouped.topk(2, dim=-1)[0].sum(dim=-1) + group_idx = torch.topk( + group_scores, + k=block.router.topk_group, + dim=-1, + sorted=False, + )[1] + group_mask = torch.zeros_like(group_scores) + group_mask.scatter_(1, group_idx, 1) + score_mask = group_mask.unsqueeze(-1).expand_as(grouped).reshape_as(scores_for_choice) + scores_for_choice = scores_for_choice.masked_fill(~score_mask.bool(), float("-inf")) + + top_indices = torch.topk( + scores_for_choice, + k=block.router.top_k, + dim=-1, + sorted=False, + )[1] + top_scores = scores.gather(1, top_indices) + if block.router.top_k > 1 and block.router.norm_topk_prob: + top_scores = top_scores / (top_scores.sum(dim=-1, keepdim=True) + 1e-20) + top_scores = top_scores.to(tokens.dtype) * block.router.route_scale + + if padding_mask is not None: + mask = padding_mask.unsqueeze(-1).to(top_scores.dtype) + top_scores = top_scores * mask + top_scores = top_scores / (top_scores.sum(dim=-1, keepdim=True) + 1e-9) + top_scores = top_scores * block.router.route_scale + + routed = torch.zeros( + tokens.shape, + dtype=torch.float32, + device=tokens.device, + ) + for expert_idx in range(block.router.num_experts): + token_idx, route_idx = torch.where(top_indices == expert_idx) + if token_idx.numel() == 0: + continue + expert_tokens = tokens[token_idx] + h = F.silu(expert_tokens @ block.experts.w1[expert_idx].transpose(-2, -1)) + h = h * (expert_tokens @ block.experts.w3[expert_idx].transpose(-2, -1)) + expert_output = h @ block.experts.w2[expert_idx].transpose(-2, -1) + weighted = expert_output.float() * top_scores[token_idx, route_idx, None].float() + routed.index_add_(0, token_idx, weighted) + + output = routed.to(tokens.dtype).reshape(batch_size, -1, hidden_size) + if block.shared_experts is not None: + shared = F.linear(hidden_states, block.shared_experts.gate_proj.weight) + shared = F.silu(shared) * F.linear(hidden_states, block.shared_experts.up_proj.weight) + shared = F.linear(shared, block.shared_experts.down_proj.weight) + output = output + shared + return output, top_indices, top_scores + + +@hardware_test(res={"cuda": "L4"}, num_cards=1) +def test_grouped_mm_matches_per_expert_reference() -> None: + from vllm_omni.diffusion.models.lingbot_video import lingbot_video_transformer as module + + if not hasattr(torch, "_grouped_mm"): + pytest.skip("torch._grouped_mm is unavailable") + + torch.manual_seed(42) + block = module.LingBotVideoSparseMoeBlock( + hidden_size=16, + num_experts=4, + top_k=2, + moe_intermediate_size=8, + score_func="sigmoid", + norm_topk_prob=True, + n_group=None, + topk_group=None, + routed_scaling_factor=1.0, + n_shared_experts=None, + ).to(device="cuda", dtype=torch.bfloat16) + with torch.no_grad(): + block.experts.w1.normal_(mean=0.0, std=0.02) + block.experts.w2.normal_(mean=0.0, std=0.02) + block.experts.w3.normal_(mean=0.0, std=0.02) + tokens = torch.randn(23, 16, device="cuda", dtype=torch.bfloat16) + counts = torch.tensor([5, 0, 9, 9], device="cuda", dtype=torch.int64) + + expected = block._run_experts_for_loop(tokens, counts) + actual = block._run_grouped_experts(tokens, counts) + + torch.testing.assert_close(actual, expected) + + +@hardware_test(res={"cuda": "L4"}, num_cards=1) +def test_sparse_moe_block_matches_upstream_reference() -> None: + from vllm_omni.diffusion.models.lingbot_video import lingbot_video_transformer as module + + if not hasattr(torch, "_grouped_mm"): + pytest.skip("torch._grouped_mm is unavailable") + + torch.manual_seed(42) + block = module.LingBotVideoSparseMoeBlock( + hidden_size=16, + num_experts=8, + top_k=2, + moe_intermediate_size=8, + score_func="sigmoid", + norm_topk_prob=True, + n_group=4, + topk_group=1, + routed_scaling_factor=1.5, + n_shared_experts=1, + ) + with torch.no_grad(): + block.router.weight.zero_() + block.router.e_score_correction_bias.copy_(torch.tensor([0.9, 0.8, 1.0, 0.0, 0.7, 0.6, 0.5, 0.4])) + for parameter in ( + block.experts.w1, + block.experts.w2, + block.experts.w3, + block.shared_experts.gate_proj.weight, + block.shared_experts.up_proj.weight, + block.shared_experts.down_proj.weight, + ): + parameter.normal_(mean=0.0, std=0.02) + block = block.to(device="cuda", dtype=torch.bfloat16) + + hidden_states = torch.randn(2, 5, 16, device="cuda", dtype=torch.bfloat16) + padding_mask = torch.tensor( + [1, 1, 1, 1, 1, 1, 1, 1, 0, 0], + device="cuda", + dtype=torch.float32, + ) + + with torch.no_grad(): + expected, expected_indices, expected_scores = _upstream_sparse_moe_reference( + block, + hidden_states, + padding_mask, + ) + actual_indices, actual_scores = block.router(hidden_states.reshape(-1, 16)) + actual_scores = actual_scores * padding_mask.unsqueeze(-1).to(actual_scores.dtype) + actual_scores = actual_scores / (actual_scores.sum(dim=-1, keepdim=True) + 1e-9) + actual_scores = actual_scores * block.router.route_scale + actual = block(hidden_states, padding_mask=padding_mask) + + corrected_scores = ( + torch.full( + (8,), + 0.5, + device="cuda", + ) + + block.router.e_score_correction_bias + ) + unrestricted_indices = torch.topk(corrected_scores, k=2, sorted=False)[1] + + assert set(actual_indices[0].tolist()) == {0, 1} + assert set(unrestricted_indices.tolist()) == {0, 2} + assert torch.equal(actual_indices, expected_indices) + torch.testing.assert_close(actual_scores, expected_scores, rtol=0, atol=0) + torch.testing.assert_close(actual, expected, rtol=0, atol=0) + assert torch.isfinite(actual).all() diff --git a/tests/diffusion/models/lingbot_video/test_lingbot_video_transformer.py b/tests/diffusion/models/lingbot_video/test_lingbot_video_transformer.py new file mode 100644 index 000000000..e9fad3825 --- /dev/null +++ b/tests/diffusion/models/lingbot_video/test_lingbot_video_transformer.py @@ -0,0 +1,266 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import pytest +import torch + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion] + + +def _tiny_transformer(**overrides): + from vllm_omni.diffusion.models.lingbot_video import LingBotVideoTransformer3DModel + + config = { + "patch_size": (1, 1, 1), + "in_channels": 2, + "out_channels": 2, + "hidden_size": 16, + "num_attention_heads": 1, + "depth": 0, + "intermediate_size": 32, + "text_dim": 8, + "freq_dim": 8, + "axes_dims": (4, 4, 8), + "axes_lens": (32, 32, 32), + } + config.update(overrides) + return LingBotVideoTransformer3DModel(**config) + + +def test_joint_position_ids_video_then_text_order(): + from vllm_omni.diffusion.models.lingbot_video.lingbot_video_transformer import make_joint_position_ids + + positions = make_joint_position_ids(text_len=3, grid_t=1, grid_h=2, grid_w=2, device=torch.device("cpu")) + + assert positions.shape == (7, 3) + assert positions[:4, 0].tolist() == [4, 4, 4, 4] + assert positions[:4, 1:].tolist() == [[0, 0], [0, 1], [1, 0], [1, 1]] + assert positions[4:].tolist() == [[1, 0, 0], [2, 0, 0], [3, 0, 0]] + + +def test_tiny_transformer_depth_zero_forward_shape(): + model = _tiny_transformer() + hidden_states = torch.randn(1, 2, 1, 2, 2) + timestep = torch.tensor([300.0]) + encoder_hidden_states = torch.randn(1, 3, 8) + encoder_attention_mask = torch.ones(1, 3, dtype=torch.long) + + with torch.no_grad(): + out = model( + hidden_states, + timestep, + encoder_hidden_states, + encoder_attention_mask=encoder_attention_mask, + return_dict=False, + )[0] + + assert out.shape == hidden_states.shape + assert torch.isfinite(out).all() + + +def test_packed_attention_uses_sdpa_fallback_without_flash_varlen(monkeypatch): + from vllm_omni.diffusion.models.lingbot_video import lingbot_video_transformer as module + + monkeypatch.setattr(module, "flash_attn_varlen_func_v3", None) + attn = module.LingBotVideoAttention( + hidden_size=8, + num_heads=2, + norm_eps=1e-6, + qkv_bias=False, + out_bias=False, + ) + captured = {} + + def fake_sdpa_forward(query, key, value, attn_metadata): + captured["mask"] = attn_metadata.attn_mask + return torch.zeros_like(query) + + monkeypatch.setattr(attn.attn.sdpa_fallback, "forward", fake_sdpa_forward) + x = torch.randn(1, 5, 8) + rotary = torch.ones(1, 5, 2, dtype=torch.complex64) + packed_indices = { + "cu_seqlens_kv": torch.tensor([0, 2, 5], dtype=torch.int32), + "max_seqlen_in_batch_kv": 3, + "attention_mask": module._packed_block_attention_mask([2, 3], x.device), + } + + out = attn(x, rotary, packed_indices=packed_indices) + + assert out.shape == x.shape + mask = captured["mask"] + assert mask.shape == (1, 1, 5, 5) + assert mask[0, 0, :2, :2].all() + assert mask[0, 0, 2:, 2:].all() + assert not mask[0, 0, :2, 2:].any() + assert not mask[0, 0, 2:, :2].any() + + +def test_tiny_transformer_rejects_invalid_rope_dims(): + from vllm_omni.diffusion.models.lingbot_video import LingBotVideoTransformer3DModel + + with pytest.raises(AssertionError, match="head_dim"): + LingBotVideoTransformer3DModel( + hidden_size=16, + num_attention_heads=1, + axes_dims=(4, 4, 4), + depth=0, + ) + + +def test_transformer_to_keeps_sensitive_modules_in_fp32(): + model = _tiny_transformer() + + model.to(dtype=torch.bfloat16) + + assert model.patch_embedder.weight.dtype == torch.bfloat16 + assert model.time_embedder.linear_1.weight.dtype == torch.float32 + assert model.norm_out_modulation[1].weight.dtype == torch.float32 + + +def test_router_group_limited_topk_uses_bias_corrected_choice(): + from vllm_omni.diffusion.models.lingbot_video import lingbot_video_transformer as module + + router = module.LingBotVideoRouter( + hidden_size=2, + num_experts=4, + top_k=2, + score_func="sigmoid", + norm_topk_prob=True, + n_group=2, + topk_group=1, + route_scale=2.5, + ) + with torch.no_grad(): + router.weight.copy_( + torch.tensor( + [ + [8.0, 0.0], + [7.0, 0.0], + [-8.0, 0.0], + [-7.0, 0.0], + ] + ) + ) + router.e_score_correction_bias.copy_(torch.tensor([0.0, 0.0, 2.0, 2.0])) + + top_indices, top_scores = router(torch.tensor([[1.0, 0.0]])) + + assert set(top_indices[0].tolist()) == {2, 3} + assert torch.allclose(top_scores.sum(dim=-1), torch.tensor([2.5])) + low_score, high_score = sorted(top_scores[0].tolist()) + assert low_score < 0.8 + assert high_score > 1.7 + + +def test_sparse_moe_block_masks_padding_tokens(): + from vllm_omni.diffusion.models.lingbot_video import lingbot_video_transformer as module + + block = module.LingBotVideoSparseMoeBlock( + hidden_size=4, + num_experts=2, + top_k=1, + moe_intermediate_size=3, + score_func="sigmoid", + norm_topk_prob=True, + n_group=None, + topk_group=None, + routed_scaling_factor=1.0, + n_shared_experts=None, + ) + with torch.no_grad(): + block.router.weight.copy_( + torch.tensor( + [ + [8.0, 0.0, 0.0, 0.0], + [-8.0, 0.0, 0.0, 0.0], + ] + ) + ) + block.router.e_score_correction_bias.zero_() + block.experts.w1.fill_(0.5) + block.experts.w2.fill_(0.5) + block.experts.w3.fill_(0.5) + + hidden_states = torch.tensor([[[1.0, 0.0, 0.0, 0.0], [-1.0, 0.0, 0.0, 0.0]]]) + padding_mask = torch.tensor([1.0, 0.0]) + + out = block(hidden_states, padding_mask=padding_mask) + + assert out.shape == hidden_states.shape + assert torch.isfinite(out).all() + assert not torch.allclose(out[0, 0], torch.zeros_like(out[0, 0])) + assert torch.allclose(out[0, 1], torch.zeros_like(out[0, 1])) + + +@pytest.mark.parametrize( + "counts_list", + [ + [0, 0, 0, 0], + [1, 0, 7, 8, 9], + [16, 0, 1, 0], + [0, 0, 33, 0], + ], +) +def test_grouped_padding_matches_reference(counts_list): + from vllm_omni.diffusion.models.lingbot_video import lingbot_video_transformer as module + + align = 8 + counts = torch.tensor(counts_list, dtype=torch.int64) + num_tokens = int(counts.sum()) + tokens = torch.arange(num_tokens * 4, dtype=torch.float32).reshape(num_tokens, 4) + + actual = module.LingBotVideoSparseMoeBlock._pad_grouped_tokens( + tokens, + counts, + align, + ) + + num_experts = len(counts_list) + max_len = ((num_tokens + num_experts * align + align - 1) // align) * align + aligned_counts = [((max(count, align) + align - 1) // align) * align for count in counts_list] + expected_indices = [num_tokens] * max_len + source_start = 0 + write_start = 0 + for count, aligned_count in zip(counts_list, aligned_counts): + expected_indices[write_start : write_start + count] = range( + source_start, + source_start + count, + ) + source_start += count + write_start += aligned_count + + expected_indices_tensor = torch.tensor(expected_indices, dtype=torch.int64) + tokens_with_pad = torch.vstack((tokens, tokens.new_zeros((tokens.shape[-1],)))) + expected_aligned_counts = torch.tensor(aligned_counts, dtype=torch.int32) + + assert actual[0] == tokens_with_pad.shape + torch.testing.assert_close(actual[1], tokens_with_pad[expected_indices_tensor]) + torch.testing.assert_close(actual[2], expected_indices_tensor) + torch.testing.assert_close(actual[3], expected_aligned_counts) + + +def test_tiny_transformer_constructs_moe_and_dense_layers(): + from vllm_omni.diffusion.models.lingbot_video import lingbot_video_transformer as module + + model = _tiny_transformer( + depth=2, + num_experts=4, + num_experts_per_tok=2, + moe_intermediate_size=8, + decoder_sparse_step=1, + mlp_only_layers=(1,), + n_shared_experts=1, + n_group=2, + topk_group=1, + routed_scaling_factor=2.5, + ) + + assert isinstance(model.blocks[0].ffn, module.LingBotVideoSparseMoeBlock) + assert isinstance(model.blocks[1].ffn, module.LingBotVideoMLP) + assert "blocks.0.ffn.experts.w1" in model.state_dict() + assert "blocks.0.ffn.shared_experts.gate_proj.weight" in model.state_dict() + + model.to(dtype=torch.bfloat16) + + assert model.blocks[0].ffn.router.weight.dtype == torch.float32 + assert model.blocks[0].ffn.experts.w1.dtype == torch.bfloat16 diff --git a/tests/diffusion/models/lingbot_video/test_pipeline_lingbot_video.py b/tests/diffusion/models/lingbot_video/test_pipeline_lingbot_video.py new file mode 100644 index 000000000..1322a542f --- /dev/null +++ b/tests/diffusion/models/lingbot_video/test_pipeline_lingbot_video.py @@ -0,0 +1,219 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from types import SimpleNamespace + +import numpy as np +import pytest +import torch +from torch import nn + +from vllm_omni.diffusion.request import OmniDiffusionRequest +from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch +from vllm_omni.inputs.data import OmniDiffusionSamplingParams + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion] + + +def _make_pipeline(): + from vllm_omni.diffusion.models.lingbot_video.pipeline_lingbot_video import LingBotVideoPipeline + + pipeline = object.__new__(LingBotVideoPipeline) + nn.Module.__init__(pipeline) + pipeline.device = torch.device("cpu") + pipeline.default_negative_prompt = "default negative" + pipeline.od_config = SimpleNamespace(flow_shift=None) + return pipeline + + +def _make_request_batch(prompt, **sampling_overrides): + sampling = OmniDiffusionSamplingParams(**sampling_overrides) + request = OmniDiffusionRequest(prompt=prompt, sampling_params=sampling, request_id="lingbot-test") + return DiffusionRequestBatch([request]) + + +def test_lingbot_video_pipeline_import_and_registry(): + from vllm_omni.diffusion.models.lingbot_video import ( + LingBotVideoPipeline, + LingBotVideoTransformer3DModel, + get_lingbot_video_post_process_func, + ) + from vllm_omni.diffusion.registry import _DIFFUSION_MODELS, _DIFFUSION_POST_PROCESS_FUNCS + + assert LingBotVideoPipeline is not None + assert LingBotVideoTransformer3DModel is not None + assert get_lingbot_video_post_process_func is not None + assert _DIFFUSION_MODELS["LingBotVideoPipeline"] == ( + "lingbot_video", + "pipeline_lingbot_video", + "LingBotVideoPipeline", + ) + assert _DIFFUSION_POST_PROCESS_FUNCS["LingBotVideoPipeline"] == "get_lingbot_video_post_process_func" + + +def test_component_discovery_declarations(): + from vllm_omni.diffusion.models.lingbot_video import LingBotVideoPipeline + + assert LingBotVideoPipeline._dit_modules == ["transformer"] + assert LingBotVideoPipeline._encoder_modules == ["text_encoder"] + assert LingBotVideoPipeline._vae_modules == ["vae"] + assert LingBotVideoPipeline.supports_step_execution is False + + +def test_extra_body_params_include_video_flow_shift_alias(): + from vllm_omni.model_extras import get_extra_body_params + + params = get_extra_body_params("LingBotVideoPipeline") + assert "shift" in params + assert "flow_shift" in params + assert "batch_cfg" in params + + +def test_extract_prompt_string_and_mapping(): + from vllm_omni.diffusion.models.lingbot_video.pipeline_lingbot_video import _extract_prompt + + string_req = SimpleNamespace(prompt="a robot waves") + assert _extract_prompt(string_req) == ("a robot waves", None) + + mapping_req = SimpleNamespace(prompt={"prompt": "a robot walks", "negative_prompt": "low quality"}) + assert _extract_prompt(mapping_req) == ("a robot walks", "low quality") + + with pytest.raises(TypeError, match="Unsupported LingBot prompt type"): + _extract_prompt(SimpleNamespace(prompt=["not", "supported"])) + + +def test_check_inputs_accepts_t2v_shapes_and_rejects_invalid_values(): + from vllm_omni.diffusion.models.lingbot_video import LingBotVideoPipeline + + LingBotVideoPipeline.check_inputs(height=192, width=320, num_frames=1) + LingBotVideoPipeline.check_inputs(height=192, width=320, num_frames=9) + + with pytest.raises(ValueError, match="num_frames"): + LingBotVideoPipeline.check_inputs(height=192, width=320, num_frames=8) + with pytest.raises(ValueError, match="height"): + LingBotVideoPipeline.check_inputs(height=190, width=320, num_frames=9) + + +def test_postprocess_keeps_torch_by_default_and_converts_np_when_requested(): + from vllm_omni.diffusion.models.lingbot_video import get_lingbot_video_post_process_func + + frames = torch.ones(2, 4, 4, 3) + post = get_lingbot_video_post_process_func(SimpleNamespace()) + + assert post(frames, SimpleNamespace(output_type="pt")) is frames + array = post(frames, SimpleNamespace(output_type="np")) + assert isinstance(array, np.ndarray) + assert array.shape == (2, 4, 4, 3) + + +def test_forward_resolves_t2v_sampling_and_flow_shift_alias(): + pipeline = _make_pipeline() + calls = [] + + def fake_generate(**kwargs): + calls.append(kwargs) + return torch.zeros(9, 192, 320, 3) + + pipeline._generate = fake_generate + req = _make_request_batch( + {"prompt": "a robot arm", "negative_prompt": "bad hands"}, + height=192, + width=320, + num_frames=9, + num_inference_steps=2, + guidance_scale=3.0, + seed=42, + output_type="pt", + extra_args={"flow_shift": 4.0, "batch_cfg": True}, + ) + + out = pipeline.forward(req) + + assert torch.equal(out.output, torch.zeros(9, 192, 320, 3)) + assert len(calls) == 1 + call = calls[0] + assert call["prompt"] == "a robot arm" + assert call["negative_prompt"] == "bad hands" + assert call["height"] == 192 + assert call["width"] == 320 + assert call["num_frames"] == 9 + assert call["num_inference_steps"] == 2 + assert call["guidance_scale"] == 3.0 + assert call["shift"] == 4.0 + assert call["batch_cfg"] is True + assert call["output_type"] == "pt" + + +def test_forward_uses_lingbot_guidance_default_when_omitted(): + pipeline = _make_pipeline() + calls = [] + + def fake_generate(**kwargs): + calls.append(kwargs) + return torch.zeros(1) + + pipeline._generate = fake_generate + req = _make_request_batch( + "a robot arm", + height=192, + width=320, + num_frames=81, + num_inference_steps=2, + seed=42, + ) + + pipeline.forward(req) + + assert calls[0]["num_frames"] == 81 + assert calls[0]["guidance_scale"] == 6.0 + + +def test_forward_prefers_shift_over_flow_shift_and_defaults_negative_prompt(): + pipeline = _make_pipeline() + calls = [] + + def fake_generate(**kwargs): + calls.append(kwargs) + return torch.zeros(5, 192, 192, 3) + + pipeline._generate = fake_generate + req = _make_request_batch( + "a robot arm", + height=192, + width=192, + num_frames=5, + num_inference_steps=2, + guidance_scale=1.0, + seed=42, + extra_args={"shift": 5.0, "flow_shift": 4.0}, + ) + + pipeline.forward(req) + + assert calls[0]["shift"] == 5.0 + assert calls[0]["negative_prompt"] == "default negative" + + +def test_forward_rejects_multi_request_batches(): + pipeline = _make_pipeline() + first = OmniDiffusionRequest( + prompt="first", + sampling_params=OmniDiffusionSamplingParams(seed=1), + request_id="lingbot-first", + ) + second = OmniDiffusionRequest( + prompt="second", + sampling_params=OmniDiffusionSamplingParams(seed=2), + request_id="lingbot-second", + ) + + with pytest.raises(ValueError, match="only supports one request"): + pipeline.forward(DiffusionRequestBatch([first, second])) + + +def test_load_weights_rejects_external_weight_stream(): + pipeline = _make_pipeline() + + assert pipeline.load_weights([]) == set() + with pytest.raises(RuntimeError, match="components are loaded directly"): + pipeline.load_weights([("transformer.weight", torch.zeros(1))]) diff --git a/tests/e2e/online_serving/test_lingbot_video.py b/tests/e2e/online_serving/test_lingbot_video.py new file mode 100644 index 000000000..e7af994cb --- /dev/null +++ b/tests/e2e/online_serving/test_lingbot_video.py @@ -0,0 +1,67 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +""" +Online serving smoke for ``robbyant/lingbot-video-dense-1.3b``. + +The LingBot dense PR supports text-to-video only, so this module exercises the +native ``/v1/videos`` path with a small 9-frame request. +""" + +import os + +import pytest + +from tests.helpers.mark import hardware_marks +from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler + +os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" + +MODEL = "robbyant/lingbot-video-dense-1.3b" +PROMPT = "a robotic arm picks up a red block" +NEGATIVE_PROMPT = "low quality, blurry, watermark, text" +DEFAULT_SAMPLING_PARAMS = '{"0":{"num_frames":81,"num_inference_steps":40,"guidance_scale":6.0}}' + +SINGLE_CARD_FEATURE_MARKS = hardware_marks(res={"cuda": "H100"}) + + +def _get_diffusion_feature_cases(model: str): + return [ + pytest.param( + OmniServerParams( + model=model, + server_args=[ + "--model-class-name", + "LingBotVideoPipeline", + "--default-sampling-params", + DEFAULT_SAMPLING_PARAMS, + ], + ), + id="default", + marks=SINGLE_CARD_FEATURE_MARKS, + ), + ] + + +@pytest.mark.core_model +@pytest.mark.advanced_model +@pytest.mark.diffusion +@pytest.mark.parametrize("omni_server", _get_diffusion_feature_cases(MODEL), indirect=True) +def test_text_to_video_001(omni_server: OmniServer, openai_client: OpenAIClientHandler) -> None: + request_config = { + "model": omni_server.model, + "form_data": { + "model": omni_server.model, + "prompt": PROMPT, + "negative_prompt": NEGATIVE_PROMPT, + "height": 192, + "width": 320, + "num_frames": 9, + "fps": 24, + "num_inference_steps": 2, + "guidance_scale": 3.0, + "flow_shift": 3.0, + "seed": 42, + }, + } + openai_client.send_video_diffusion_request(request_config) diff --git a/tests/e2e/online_serving/test_lingbot_video_expansion.py b/tests/e2e/online_serving/test_lingbot_video_expansion.py new file mode 100644 index 000000000..cb5a314ff --- /dev/null +++ b/tests/e2e/online_serving/test_lingbot_video_expansion.py @@ -0,0 +1,83 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +""" +L4 expansion coverage for ``robbyant/lingbot-video-dense-1.3b``. + +This file remains dense-only. Basic single-GPU MoE serving is covered by +``test_lingbot_video_moe.py``; T2I, I2V, TI2V, and multi-GPU feature rows +belong in follow-up PRs. +""" + +import json +import os + +import pytest + +from tests.helpers.mark import hardware_marks +from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler + +os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" + +pytestmark = [pytest.mark.diffusion, pytest.mark.full_model] + +MODEL = "robbyant/lingbot-video-dense-1.3b" +PROMPT = "a robotic arm picks up a red block" +NEGATIVE_PROMPT = "low quality, blurry, watermark, text" + +SINGLE_CARD_FEATURE_MARKS = hardware_marks(res={"cuda": "H100"}) + + +def _get_diffusion_feature_cases(model: str): + return [ + pytest.param( + OmniServerParams( + model=model, + server_args=["--model-class-name", "LingBotVideoPipeline"], + ), + id="default", + marks=SINGLE_CARD_FEATURE_MARKS, + ), + ] + + +@pytest.mark.parametrize("omni_server", _get_diffusion_feature_cases(MODEL), indirect=True) +def test_cfg_off(omni_server: OmniServer, openai_client: OpenAIClientHandler) -> None: + request_config = { + "model": omni_server.model, + "form_data": { + "model": omni_server.model, + "prompt": PROMPT, + "height": 192, + "width": 192, + "num_frames": 9, + "fps": 24, + "num_inference_steps": 2, + "guidance_scale": 1.0, + "flow_shift": 3.0, + "seed": 42, + }, + } + openai_client.send_video_diffusion_request(request_config) + + +@pytest.mark.parametrize("omni_server", _get_diffusion_feature_cases(MODEL), indirect=True) +def test_batch_cfg_extra_params(omni_server: OmniServer, openai_client: OpenAIClientHandler) -> None: + request_config = { + "model": omni_server.model, + "form_data": { + "model": omni_server.model, + "prompt": PROMPT, + "negative_prompt": NEGATIVE_PROMPT, + "height": 192, + "width": 320, + "num_frames": 9, + "fps": 24, + "num_inference_steps": 2, + "guidance_scale": 3.0, + "flow_shift": 3.0, + "seed": 42, + "extra_params": json.dumps({"batch_cfg": True}, separators=(",", ":")), + }, + } + openai_client.send_video_diffusion_request(request_config) diff --git a/tests/e2e/online_serving/test_lingbot_video_moe.py b/tests/e2e/online_serving/test_lingbot_video_moe.py new file mode 100644 index 000000000..32a6cbf2f --- /dev/null +++ b/tests/e2e/online_serving/test_lingbot_video_moe.py @@ -0,0 +1,59 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +"""Single-GPU serving smoke for ``robbyant/lingbot-video-moe-30b-a3b``.""" + +import os + +import pytest + +from tests.helpers.mark import hardware_marks +from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler + +os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" + +MODEL = "robbyant/lingbot-video-moe-30b-a3b" +PROMPT = "a robotic arm picks up a red block" +NEGATIVE_PROMPT = "low quality, blurry, watermark, text" +SMOKE_DEFAULT_SAMPLING_PARAMS = '{"0":{"num_frames":81,"num_inference_steps":2,"guidance_scale":3.0}}' + +SINGLE_CARD_MARKS = hardware_marks(res={"cuda": "H100"}) + + +def _get_server_cases(model: str): + return [ + pytest.param( + OmniServerParams( + model=model, + server_args=[ + "--model-class-name", + "LingBotVideoPipeline", + "--default-sampling-params", + SMOKE_DEFAULT_SAMPLING_PARAMS, + ], + ), + id="default", + marks=SINGLE_CARD_MARKS, + ), + ] + + +@pytest.mark.full_model +@pytest.mark.diffusion +@pytest.mark.parametrize("omni_server", _get_server_cases(MODEL), indirect=True) +def test_text_to_video_moe(omni_server: OmniServer, openai_client: OpenAIClientHandler) -> None: + request_config = { + "model": omni_server.model, + "form_data": { + "model": omni_server.model, + "prompt": PROMPT, + "negative_prompt": NEGATIVE_PROMPT, + "height": 192, + "width": 320, + "num_frames": 9, + "fps": 24, + "flow_shift": 3.0, + "seed": 42, + }, + } + openai_client.send_video_diffusion_request(request_config) diff --git a/tests/model_tests/diffusion/test_alignment.py b/tests/model_tests/diffusion/test_alignment.py index c95644143..4b51220c7 100644 --- a/tests/model_tests/diffusion/test_alignment.py +++ b/tests/model_tests/diffusion/test_alignment.py @@ -67,6 +67,7 @@ EXCLUDED_MODELS = [ "AudioXPipeline", "HunyuanVideo15Pipeline", "HunyuanVideo15ImageToVideoPipeline", + "LingBotVideoPipeline", "MagiHumanPipeline", "OmniVoicePipeline", "OmniVoice", diff --git a/vllm_omni/diffusion/models/lingbot_video/__init__.py b/vllm_omni/diffusion/models/lingbot_video/__init__.py new file mode 100644 index 000000000..013f7b2c3 --- /dev/null +++ b/vllm_omni/diffusion/models/lingbot_video/__init__.py @@ -0,0 +1,16 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from vllm_omni.diffusion.models.lingbot_video.lingbot_video_transformer import ( + LingBotVideoTransformer3DModel, +) +from vllm_omni.diffusion.models.lingbot_video.pipeline_lingbot_video import ( + LingBotVideoPipeline, + get_lingbot_video_post_process_func, +) + +__all__ = [ + "LingBotVideoPipeline", + "LingBotVideoTransformer3DModel", + "get_lingbot_video_post_process_func", +] diff --git a/vllm_omni/diffusion/models/lingbot_video/lingbot_video_transformer.py b/vllm_omni/diffusion/models/lingbot_video/lingbot_video_transformer.py new file mode 100644 index 000000000..ad820eba3 --- /dev/null +++ b/vllm_omni/diffusion/models/lingbot_video/lingbot_video_transformer.py @@ -0,0 +1,1005 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# Adapted from LingBot-Video (https://github.com/Robbyant/lingbot-video). + +import math + +import torch +import torch.distributed as dist +import torch.nn as nn +import torch.nn.functional as F +from diffusers.configuration_utils import ConfigMixin, register_to_config +from diffusers.models.embeddings import TimestepEmbedding, Timesteps +from diffusers.models.modeling_outputs import Transformer2DModelOutput +from diffusers.models.modeling_utils import ModelMixin + +from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata +from vllm_omni.diffusion.attention.backends.utils.fa import ( + flash_attn_varlen_func as flash_attn_varlen_func_v3, +) +from vllm_omni.diffusion.attention.layer import Attention + +LINGBOT_VIDEO_FP32_MODULES = ( + "time_embedder", + "time_modulation", + "scale_shift_table", + "norm", + "norm1", + "norm2", + "norm_q", + "norm_k", + "norm_post_attn", + "norm_post_ffn", + "norm_out", + "norm_out_modulation", + "router", +) + + +def should_keep_in_fp32(name: str) -> bool: + return any(module_name in name.split(".") for module_name in LINGBOT_VIDEO_FP32_MODULES) + + +def _all_to_all_split_cat( + local_input: torch.Tensor, + scatter_dim: int, + gather_dim: int, + group: dist.ProcessGroup, +) -> torch.Tensor: + world_size = dist.get_world_size(group) + input_list = [tensor.contiguous() for tensor in torch.tensor_split(local_input, world_size, scatter_dim)] + output_list = [torch.empty_like(input_list[0]) for _ in range(world_size)] + dist.all_to_all(output_list, input_list, group=group) + return torch.cat(output_list, dim=gather_dim).contiguous() + + +class LingBotVideoRMSNorm(nn.Module): + """RMSNorm with fp32 accumulation.""" + + def __init__(self, dim: int, eps: float = 1e-6): + super().__init__() + self.weight = nn.Parameter(torch.ones(dim)) + self.variance_epsilon = eps + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + input_dtype = hidden_states.dtype + hidden_states = hidden_states.to(torch.float32) + variance = hidden_states.pow(2).mean(-1, keepdim=True) + hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon) + return (self.weight * hidden_states).to(input_dtype) + + +def apply_rotary_emb(x: torch.Tensor, freqs_cis: torch.Tensor) -> torch.Tensor: + """Apply complex RoPE to `(B, S, H, D)` attention tensors.""" + with torch.amp.autocast("cuda", enabled=False): + x_c = torch.view_as_complex(x.float().reshape(*x.shape[:-1], -1, 2)) + out = torch.view_as_real(x_c * freqs_cis.unsqueeze(2)).flatten(3) + return out.type_as(x) + + +class LingBotVideoRotaryEmbedding(nn.Module): + """Complex64 RoPE table indexed by position ids.""" + + def __init__(self, axes_dims: tuple[int, ...], axes_lens: tuple[int, ...], theta: float): + super().__init__() + self.axes_dims = tuple(axes_dims) + self.axes_lens = list(axes_lens) + self.theta = theta + self.freqs_cis = None + + @staticmethod + def precompute_freqs_cis(dim: tuple[int, ...], end: tuple[int, ...], theta: float): + freqs_cis = [] + for d, e in zip(dim, end): + freqs = 1.0 / (theta ** (torch.arange(0, d, 2, dtype=torch.float64, device="cpu") / d)) + timestep = torch.arange(e, device=freqs.device, dtype=torch.float64) + freqs = torch.outer(timestep, freqs).float() + freqs_cis.append(torch.polar(torch.ones_like(freqs), freqs).to(torch.complex64)) + return freqs_cis + + def forward(self, position_ids: torch.Tensor) -> torch.Tensor: + # position_ids: (S, 3) int -> (S, head_dim/2) complex64 + device = position_ids.device + max_vals = position_ids.max(dim=0).values.tolist() + needs_rebuild = self.freqs_cis is None or any( + max_val >= axis_len for max_val, axis_len in zip(max_vals, self.axes_lens) + ) + if needs_rebuild: + for i in range(len(self.axes_lens)): + if max_vals[i] >= self.axes_lens[i]: + self.axes_lens[i] = int(max_vals[i] * 1.5) + 1 + self.freqs_cis = self.precompute_freqs_cis(self.axes_dims, tuple(self.axes_lens), theta=self.theta) + self.freqs_cis = [freqs_cis.to(device) for freqs_cis in self.freqs_cis] + elif self.freqs_cis[0].device != device: + self.freqs_cis = [freqs_cis.to(device) for freqs_cis in self.freqs_cis] + + return torch.cat([self.freqs_cis[i][position_ids[:, i]] for i in range(len(self.axes_dims))], dim=-1) + + +def make_joint_position_ids(text_len: int, grid_t: int, grid_h: int, grid_w: int, device: torch.device) -> torch.Tensor: + """Return ``(t, h, w)`` positions with video rows before text rows. + + Text t-axis positions are 1..text_len, and video t-axis positions start at + text_len + 1. This matches the token order produced by ``_cat_interleave``. + """ + tt = torch.arange(grid_t, device=device, dtype=torch.int32) + (text_len + 1) + hh = torch.arange(grid_h, device=device, dtype=torch.int32) + ww = torch.arange(grid_w, device=device, dtype=torch.int32) + grid = torch.stack(torch.meshgrid(tt, hh, ww, indexing="ij"), dim=-1).flatten(0, 2) + text_t = torch.arange(text_len, device=device, dtype=torch.int32) + 1 + text_pos = torch.stack([text_t, torch.zeros_like(text_t), torch.zeros_like(text_t)], dim=-1) + return torch.cat([grid, text_pos], dim=0) # (Nx + L, 3) + + +def _cat_interleave( + a: torch.Tensor, + len_a: list[int], + b: torch.Tensor, + len_b: list[int], +) -> torch.Tensor: + a_split = torch.split(a, len_a, dim=1) + b_split = torch.split(b, len_b, dim=1) + blocks: list[torch.Tensor] = [] + for x_part, text_part in zip(a_split, b_split): + blocks.extend([x_part, text_part]) + return torch.cat(blocks, dim=1) + + +def _packed_block_attention_mask(sample_seq_lens: list[int], device: torch.device) -> torch.Tensor: + total_seq_len = sum(sample_seq_lens) + mask = torch.zeros( + 1, + 1, + total_seq_len, + total_seq_len, + dtype=torch.bool, + device=device, + ) + start = 0 + for seq_len in sample_seq_lens: + end = start + seq_len + mask[:, :, start:end, start:end] = True + start = end + return mask + + +class LingBotVideoTextEmbedder(nn.Module): + """Matches CondProjection: RMSNorm(text_dim, eps=1e-6 fixed) -> Linear-SiLU-Linear.""" + + def __init__(self, text_dim: int, hidden_size: int): + super().__init__() + self.norm = LingBotVideoRMSNorm(text_dim, eps=1e-6) + self.linear_1 = nn.Linear(text_dim, hidden_size, bias=True) + self.linear_2 = nn.Linear(hidden_size, hidden_size, bias=True) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x = self.norm(x) + return self.linear_2(F.silu(self.linear_1(x))) + + +class LingBotVideoAttention(nn.Module): + def __init__(self, hidden_size, num_heads, norm_eps, qkv_bias, out_bias): + super().__init__() + self.num_heads = num_heads + self.head_dim = hidden_size // num_heads + self.to_q = nn.Linear(hidden_size, hidden_size, bias=qkv_bias) + self.to_k = nn.Linear(hidden_size, hidden_size, bias=qkv_bias) + self.to_v = nn.Linear(hidden_size, hidden_size, bias=qkv_bias) + self.norm_q = LingBotVideoRMSNorm(self.head_dim, norm_eps) + self.norm_k = LingBotVideoRMSNorm(self.head_dim, norm_eps) + self.to_out = nn.Linear(hidden_size, hidden_size, bias=out_bias) + self.attn = Attention( + num_heads=num_heads, + head_size=self.head_dim, + softmax_scale=1.0 / math.sqrt(self.head_dim), + causal=False, + num_kv_heads=num_heads, + ) + + def forward( + self, + x, + rotary_emb, + attention_mask=None, + packed_indices: dict[str, torch.Tensor] | None = None, + parallel_config=None, + ): + B, S, _ = x.shape + q = self.to_q(x).unflatten(2, (self.num_heads, self.head_dim)) + k = self.to_k(x).unflatten(2, (self.num_heads, self.head_dim)) + v = self.to_v(x).unflatten(2, (self.num_heads, self.head_dim)) + q = apply_rotary_emb(self.norm_q(q), rotary_emb) + k = apply_rotary_emb(self.norm_k(k), rotary_emb) + if packed_indices is None: + out = self.attn(q, k, v, attn_metadata=AttentionMetadata(attn_mask=attention_mask)) + else: + packed_attention_mask = packed_indices.get("attention_mask") + if packed_attention_mask is not None and parallel_config is None: + out = self.attn.sdpa_fallback.forward( + q, + k, + v, + AttentionMetadata(attn_mask=packed_attention_mask), + ) + return self.to_out(out.flatten(2, 3).type_as(x)) + if flash_attn_varlen_func_v3 is None: + raise RuntimeError( + "A flash attention varlen function is required for packed context parallel attention." + ) + if parallel_config is None: + result = flash_attn_varlen_func_v3( + q=q.reshape(-1, self.num_heads, self.head_dim), + k=k.reshape(-1, self.num_heads, self.head_dim), + v=v.reshape(-1, self.num_heads, self.head_dim), + cu_seqlens_q=packed_indices["cu_seqlens_kv"], + cu_seqlens_k=packed_indices["cu_seqlens_kv"], + max_seqlen_q=packed_indices["max_seqlen_in_batch_kv"], + max_seqlen_k=packed_indices["max_seqlen_in_batch_kv"], + causal=False, + ) + out = result[0] if isinstance(result, tuple) else result + out = out.reshape(B, S, self.num_heads, self.head_dim) + else: + group = parallel_config.context_parallel_config._ulysses_mesh.get_group() + world_size = dist.get_world_size(group) + local_heads = self.num_heads // world_size + q_global = _all_to_all_split_cat( + q.reshape(B, S, self.num_heads * self.head_dim), + scatter_dim=2, + gather_dim=1, + group=group, + ).view(B, S * world_size, local_heads, self.head_dim) + k_global = _all_to_all_split_cat( + k.reshape(B, S, self.num_heads * self.head_dim), + scatter_dim=2, + gather_dim=1, + group=group, + ).view(B, S * world_size, local_heads, self.head_dim) + v_global = _all_to_all_split_cat( + v.reshape(B, S, self.num_heads * self.head_dim), + scatter_dim=2, + gather_dim=1, + group=group, + ).view(B, S * world_size, local_heads, self.head_dim) + q_flat = q_global.reshape(-1, local_heads, self.head_dim) + k_flat = k_global.reshape(-1, local_heads, self.head_dim) + v_flat = v_global.reshape(-1, local_heads, self.head_dim) + result = flash_attn_varlen_func_v3( + q=q_flat, + k=k_flat, + v=v_flat, + cu_seqlens_q=packed_indices["cu_seqlens_kv"], + cu_seqlens_k=packed_indices["cu_seqlens_kv"], + max_seqlen_q=packed_indices["max_seqlen_in_batch_kv"], + max_seqlen_k=packed_indices["max_seqlen_in_batch_kv"], + causal=False, + ) + out_global = result[0] if isinstance(result, tuple) else result + out_global = out_global.reshape(B, S * world_size, local_heads * self.head_dim) + out = _all_to_all_split_cat( + out_global, + scatter_dim=1, + gather_dim=2, + group=group, + ).view(B, S, self.num_heads, self.head_dim) + return self.to_out(out.flatten(2, 3).type_as(x)) + + +class LingBotVideoMLP(nn.Module): + def __init__(self, hidden_size, intermediate_size): + super().__init__() + self.gate_proj = nn.Linear(hidden_size, intermediate_size, bias=False) + self.up_proj = nn.Linear(hidden_size, intermediate_size, bias=False) + self.down_proj = nn.Linear(intermediate_size, hidden_size, bias=False) + + def forward(self, x): + return self.down_proj(F.silu(self.gate_proj(x)) * self.up_proj(x)) + + +class LingBotVideoRouter(nn.Module): + """Token-choice top-k router used by the LingBot MoE checkpoints. + + Selection uses the bias-corrected score, while the gating weights use the + original score. This asymmetry matches the reference LingBot inference path. + """ + + def __init__( + self, + hidden_size: int, + num_experts: int, + top_k: int, + score_func: str, + norm_topk_prob: bool, + n_group: int | None, + topk_group: int | None, + route_scale: float, + ): + super().__init__() + self.num_experts = num_experts + self.top_k = top_k + self.score_func = score_func + self.norm_topk_prob = norm_topk_prob + self.n_group = n_group + self.topk_group = topk_group + self.route_scale = route_scale + self.weight = nn.Parameter(torch.empty(num_experts, hidden_size)) + self.register_buffer( + "e_score_correction_bias", + torch.zeros(num_experts), + persistent=True, + ) + + def _group_limited_topk(self, scores_for_choice: torch.Tensor) -> torch.Tensor: + if self.n_group is None or self.topk_group is None: + raise ValueError("group-limited top-k requires n_group and topk_group.") + seq_len = scores_for_choice.shape[0] + experts_per_group = self.num_experts // self.n_group + grouped = scores_for_choice.view(seq_len, self.n_group, experts_per_group) + group_scores = grouped.topk(2, dim=-1)[0].sum(dim=-1) + group_idx = torch.topk(group_scores, k=self.topk_group, dim=-1, sorted=False)[1] + group_mask = torch.zeros_like(group_scores) + group_mask.scatter_(1, group_idx, 1) + score_mask = group_mask.unsqueeze(-1).expand(seq_len, self.n_group, experts_per_group).reshape(seq_len, -1) + masked = scores_for_choice.masked_fill(~score_mask.bool(), float("-inf")) + return torch.topk(masked, k=self.top_k, dim=-1, sorted=False)[1] + + def forward(self, tokens: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + with torch.amp.autocast(tokens.device.type, enabled=False): + logits = F.linear(tokens.float(), self.weight.float()) + if self.score_func == "softmax": + scores = F.softmax(logits, dim=-1) + elif self.score_func == "sigmoid": + scores = logits.sigmoid() + else: + raise ValueError(f"Unsupported LingBot router score_func: {self.score_func!r}.") + + scores_for_choice = scores + self.e_score_correction_bias.unsqueeze(0) + if self.n_group is not None and self.n_group > 1: + top_indices = self._group_limited_topk(scores_for_choice) + else: + top_indices = torch.topk(scores_for_choice, k=self.top_k, dim=-1, sorted=False)[1] + top_scores = scores.gather(1, top_indices) + if self.top_k > 1 and self.norm_topk_prob: + top_scores = top_scores / (top_scores.sum(dim=-1, keepdim=True) + 1e-20) + top_scores = top_scores * self.route_scale + return top_indices, top_scores.to(tokens.dtype) + + +class LingBotVideoGroupedExperts(nn.Module): + """Grouped expert weights. + + Weight layout matches the reference checkpoint: + w1 [E, I, H], w2 [E, H, I], w3 [E, I, H]. + """ + + def __init__(self, num_experts: int, hidden_size: int, intermediate_size: int): + super().__init__() + self.num_experts = num_experts + self.w1 = nn.Parameter(torch.empty(num_experts, intermediate_size, hidden_size)) + self.w2 = nn.Parameter(torch.empty(num_experts, hidden_size, intermediate_size)) + self.w3 = nn.Parameter(torch.empty(num_experts, intermediate_size, hidden_size)) + + +def _round_up_to_multiple(value: int, multiple: int) -> int: + return ((value + multiple - 1) // multiple) * multiple + + +class LingBotVideoSparseMoeBlock(nn.Module): + def __init__( + self, + hidden_size: int, + num_experts: int, + top_k: int, + moe_intermediate_size: int, + score_func: str, + norm_topk_prob: bool, + n_group: int | None, + topk_group: int | None, + routed_scaling_factor: float, + n_shared_experts: int | None, + ): + super().__init__() + self.hidden_size = hidden_size + self.num_experts = num_experts + self.router = LingBotVideoRouter( + hidden_size, + num_experts, + top_k, + score_func, + norm_topk_prob, + n_group, + topk_group, + routed_scaling_factor, + ) + self.experts = LingBotVideoGroupedExperts(num_experts, hidden_size, moe_intermediate_size) + self.shared_experts = None + if n_shared_experts is not None and n_shared_experts > 0: + self.shared_experts = LingBotVideoMLP( + hidden_size, + moe_intermediate_size * n_shared_experts, + ) + + @staticmethod + def _reorder_tokens( + tokens: torch.Tensor, + top_scores: torch.Tensor, + top_indices: torch.Tensor, + num_experts: int, + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, int, int]: + num_tokens = tokens.shape[0] + top_k = top_indices.shape[1] + flat_scores = top_scores.reshape(-1) + flat_indices = top_indices.reshape(-1) + active_positions = torch.where(flat_scores != 0)[0] + active_experts = flat_indices[active_positions] + + counts = torch.zeros(num_experts, device=tokens.device, dtype=torch.int64) + counts.scatter_add_( + 0, + active_experts, + torch.ones_like(active_experts, dtype=torch.int64), + ) + + sort_order = torch.argsort(active_experts, stable=True) + sorted_positions = active_positions[sort_order] + sorted_scores = flat_scores[sorted_positions] + original_token_idx = sorted_positions // top_k + permuted_tokens = tokens[original_token_idx] + return permuted_tokens, counts, sorted_positions, sorted_scores, num_tokens, top_k + + @staticmethod + def _pad_grouped_tokens( + tokens: torch.Tensor, + counts: torch.Tensor, + align: int = 8, + ) -> tuple[torch.Size, torch.Tensor, torch.Tensor, torch.Tensor]: + num_tokens = tokens.shape[0] + num_experts = int(counts.shape[0]) + max_len = _round_up_to_multiple(num_tokens + num_experts * align, align) + counts_i64 = counts.to(torch.int64) + total_per_expert = torch.clamp_min(counts_i64, align) + aligned_counts_i64 = (total_per_expert + align - 1) // align * align + write_offsets = torch.cumsum(aligned_counts_i64, dim=0) - aligned_counts_i64 + end_offsets = torch.cumsum(aligned_counts_i64, dim=0) + start_indices = torch.cumsum(counts_i64, dim=0) - counts_i64 + + slots = torch.arange(max_len, dtype=torch.int64, device=tokens.device) + expert_idx = torch.bucketize(slots, end_offsets, right=True) + valid_expert = expert_idx < num_experts + safe_expert_idx = expert_idx.clamp(max=num_experts - 1) + local_idx = slots - write_offsets[safe_expert_idx] + source_idx = start_indices[safe_expert_idx] + local_idx + valid = valid_expert & (local_idx < counts_i64[safe_expert_idx]) + fill = torch.full_like(source_idx, num_tokens) + permuted_indices = torch.where(valid, source_idx, fill) + + tokens_with_pad = torch.vstack((tokens, tokens.new_zeros((tokens.shape[-1],)))) + input_shape = tokens_with_pad.shape + return ( + input_shape, + tokens_with_pad[permuted_indices], + permuted_indices, + aligned_counts_i64.to(torch.int32), + ) + + @staticmethod + def _unpad_grouped_tokens( + output: torch.Tensor, + input_shape: torch.Size, + permuted_indices: torch.Tensor, + ) -> torch.Tensor: + unpermuted = output.new_empty(input_shape) + unpermuted[permuted_indices, :] = output + return unpermuted[:-1] + + def _run_experts_for_loop(self, tokens: torch.Tensor, counts: torch.Tensor) -> torch.Tensor: + count_list = counts.tolist() + splits = torch.split(tokens, count_list, dim=0) + outputs = [] + for expert_idx, expert_tokens in enumerate(splits): + if expert_tokens.numel() == 0: + continue + h = F.silu(expert_tokens @ self.experts.w1[expert_idx].transpose(-2, -1)) + h = h * (expert_tokens @ self.experts.w3[expert_idx].transpose(-2, -1)) + h = h @ self.experts.w2[expert_idx].transpose(-2, -1) + outputs.append(h) + if not outputs: + return tokens.new_zeros(tokens.shape) + return torch.cat(outputs, dim=0) + + def _run_grouped_experts(self, tokens: torch.Tensor, counts: torch.Tensor) -> torch.Tensor: + if not hasattr(torch, "_grouped_mm") or tokens.device.type != "cuda": + return self._run_experts_for_loop(tokens, counts) + input_shape, padded_tokens, permuted_indices, aligned_counts = self._pad_grouped_tokens(tokens, counts) + offsets = torch.cumsum(aligned_counts, dim=0, dtype=torch.int32) + h = F.silu( + torch._grouped_mm( + padded_tokens.bfloat16(), + self.experts.w1.bfloat16().transpose(-2, -1), + offs=offsets, + ) + ) + h = h * torch._grouped_mm( + padded_tokens.bfloat16(), + self.experts.w3.bfloat16().transpose(-2, -1), + offs=offsets, + ) + out = torch._grouped_mm( + h, + self.experts.w2.bfloat16().transpose(-2, -1), + offs=offsets, + ).type_as(padded_tokens) + return self._unpad_grouped_tokens(out, input_shape, permuted_indices) + + @staticmethod + def _restore_tokens( + expert_output: torch.Tensor, + sorted_positions: torch.Tensor, + sorted_scores: torch.Tensor, + num_tokens: int, + top_k: int, + ) -> torch.Tensor: + hidden_size = expert_output.shape[-1] + unsorted = torch.zeros( + (num_tokens * top_k, hidden_size), + dtype=expert_output.dtype, + device=expert_output.device, + ) + unsorted[sorted_positions] = expert_output + unsorted = unsorted.reshape(num_tokens, top_k, hidden_size) + + scores_unsorted = torch.zeros( + num_tokens * top_k, + dtype=sorted_scores.dtype, + device=sorted_scores.device, + ) + scores_unsorted[sorted_positions] = sorted_scores + scores_unsorted = scores_unsorted.reshape(num_tokens, top_k, 1) + return (unsorted.float() * scores_unsorted).sum(dim=1).to(expert_output.dtype) + + def _run_selected_experts( + self, + tokens: torch.Tensor, + top_scores: torch.Tensor, + top_indices: torch.Tensor, + ) -> torch.Tensor: + ( + permuted_tokens, + counts, + sorted_positions, + sorted_scores, + num_tokens, + top_k, + ) = self._reorder_tokens(tokens, top_scores, top_indices, self.router.num_experts) + expert_output = self._run_grouped_experts(permuted_tokens, counts) + return self._restore_tokens( + expert_output, + sorted_positions, + sorted_scores, + num_tokens, + top_k, + ) + + def forward( + self, + hidden_states: torch.Tensor, + padding_mask: torch.Tensor | None = None, + ) -> torch.Tensor: + batch_size = hidden_states.shape[0] + tokens = hidden_states.reshape(-1, self.hidden_size) + top_indices, top_scores = self.router(tokens) + if padding_mask is not None: + mask = padding_mask.unsqueeze(-1).to(top_scores.dtype) + top_scores = top_scores * mask + top_scores = top_scores / (top_scores.sum(dim=-1, keepdim=True) + 1e-9) + top_scores = top_scores * self.router.route_scale + + out = self._run_selected_experts(tokens, top_scores, top_indices) + out = out.reshape(batch_size, -1, self.hidden_size) + if self.shared_experts is not None: + out = out + self.shared_experts(hidden_states) + return out + + +class LingBotVideoBlock(nn.Module): + def __init__( + self, + hidden_size, + num_attention_heads, + intermediate_size, + norm_eps, + qkv_bias, + out_bias, + num_experts, + num_experts_per_tok, + moe_intermediate_size, + decoder_sparse_step, + mlp_only_layers, + n_shared_experts, + score_func, + norm_topk_prob, + n_group, + topk_group, + routed_scaling_factor, + layer_idx: int, + ): + super().__init__() + self.layer_idx = layer_idx + h = hidden_size + self.scale_shift_table = nn.Parameter(torch.zeros(1, 6 * h)) + self.norm1 = LingBotVideoRMSNorm(h, norm_eps) + self.attn = LingBotVideoAttention(h, num_attention_heads, norm_eps, qkv_bias, out_bias) + self.norm_post_attn = LingBotVideoRMSNorm(h, norm_eps) + self.norm2 = LingBotVideoRMSNorm(h, norm_eps) + use_sparse_moe = ( + num_experts > 0 and layer_idx not in mlp_only_layers and (layer_idx + 1) % decoder_sparse_step == 0 + ) + if use_sparse_moe: + self.ffn = LingBotVideoSparseMoeBlock( + hidden_size=h, + num_experts=num_experts, + top_k=num_experts_per_tok, + moe_intermediate_size=moe_intermediate_size, + score_func=score_func, + norm_topk_prob=norm_topk_prob, + n_group=n_group, + topk_group=topk_group, + routed_scaling_factor=routed_scaling_factor, + n_shared_experts=n_shared_experts, + ) + else: + self.ffn = LingBotVideoMLP(h, intermediate_size) + self.norm_post_ffn = LingBotVideoRMSNorm(h, norm_eps) + + def forward( + self, + x, + temb6, + rotary_emb, + attention_mask=None, + moe_padding_mask=None, + packed_indices: dict[str, torch.Tensor] | None = None, + parallel_config=None, + ): + expected_tokens = x.shape[0] * x.shape[1] + if temb6.ndim != 2 or temb6.shape[0] != expected_tokens: + raise ValueError( + "LingBotVideoBlock expects token-level temb6 with shape " + f"(B*S, 6D); got {tuple(temb6.shape)} for hidden states {tuple(x.shape)}." + ) + # AdaLN modulation and normalization stay in fp32 for the sensitive path. + mod = temb6.view(x.shape[0], x.shape[1], -1) + self.scale_shift_table.unsqueeze(0) + shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = mod.chunk(6, dim=-1) + gate_msa, gate_mlp = gate_msa.tanh(), gate_mlp.tanh() + scale_msa, scale_mlp = 1.0 + scale_msa, 1.0 + scale_mlp + + # AdaLN modulation and norms stay in fp32; cast to the transformer + # compute dtype only at Linear boundaries. + bulk_dtype = self.attn.to_q.weight.dtype + attn_in = (self.norm1(x) * scale_msa + shift_msa).to(bulk_dtype) + attn_out = self.attn( + attn_in, + rotary_emb, + attention_mask, + packed_indices=packed_indices, + parallel_config=parallel_config, + ) + x = x + (gate_msa * self.norm_post_attn(attn_out)).to(x.dtype) + + ffn_in = (self.norm2(x) * scale_mlp + shift_mlp).to(bulk_dtype) + if isinstance(self.ffn, LingBotVideoSparseMoeBlock): + ffn_out = self.ffn(ffn_in, padding_mask=moe_padding_mask) + else: + ffn_out = self.ffn(ffn_in) + ffn_normed = self.norm_post_ffn(ffn_out) + x = x + (gate_mlp * ffn_normed).to(x.dtype) + return x + + +class LingBotVideoTransformer3DModel(ModelMixin, ConfigMixin): + _supports_gradient_checkpointing = False + _repeated_blocks = ["LingBotVideoBlock"] + _layerwise_offload_blocks_attr = "blocks" + _no_split_modules = ["LingBotVideoBlock"] + _keep_in_fp32_modules = list(LINGBOT_VIDEO_FP32_MODULES) + + def to(self, *args, **kwargs): + device, dtype, non_blocking, _ = torch._C._nn._parse_to(*args, **kwargs) + if dtype is None or dtype == torch.float32: + return super().to(*args, **kwargs) + + dtype_is_floating = torch.is_floating_point(torch.empty((), dtype=dtype)) + if not dtype_is_floating: + return super().to(*args, **kwargs) + + if device is not None: + super().to(device=device, non_blocking=non_blocking) + + for name, param in self.named_parameters(): + if not torch.is_floating_point(param): + continue + target_dtype = torch.float32 if should_keep_in_fp32(name) else dtype + param.data = param.data.to(dtype=target_dtype, non_blocking=non_blocking) + if param.grad is not None: + param.grad.data = param.grad.data.to(dtype=target_dtype, non_blocking=non_blocking) + + for name, buffer in self.named_buffers(): + if not torch.is_floating_point(buffer): + continue + target_dtype = torch.float32 if should_keep_in_fp32(name) else dtype + buffer.data = buffer.data.to(dtype=target_dtype, non_blocking=non_blocking) + + return self + + @register_to_config + def __init__( + self, + patch_size: tuple[int, int, int] = (1, 2, 2), + in_channels: int = 16, + out_channels: int = 16, + hidden_size: int = 2048, + num_attention_heads: int = 16, + depth: int = 24, + intermediate_size: int = 6144, + text_dim: int = 2560, + freq_dim: int = 256, + norm_eps: float = 1e-6, + rope_theta: float = 256.0, + axes_dims: tuple[int, int, int] = (32, 48, 48), + axes_lens: tuple[int, int, int] = (8192, 1024, 1024), + qkv_bias: bool = False, + out_bias: bool = True, + patch_embed_bias: bool = True, + timestep_mlp_bias: bool = True, + num_experts: int = 0, + num_experts_per_tok: int = 8, + moe_intermediate_size: int = 512, + decoder_sparse_step: int = 1, + mlp_only_layers: tuple[int, ...] = (), + n_shared_experts: int | None = None, + score_func: str = "sigmoid", + norm_topk_prob: bool = True, + n_group: int | None = None, + topk_group: int | None = None, + routed_scaling_factor: float = 1.0, + ): + super().__init__() + head_dim = hidden_size // num_attention_heads + assert head_dim == sum(axes_dims), f"head_dim {head_dim} != sum(axes_dims) {sum(axes_dims)}" + mlp_only_layers = tuple(mlp_only_layers) + + self.patch_embedder = nn.Linear(in_channels * math.prod(patch_size), hidden_size, bias=patch_embed_bias) + self.time_proj = Timesteps(freq_dim, flip_sin_to_cos=True, downscale_freq_shift=0) + self.time_embedder = TimestepEmbedding(freq_dim, hidden_size, act_fn="silu", sample_proj_bias=timestep_mlp_bias) + self.time_modulation = nn.Sequential(nn.SiLU(), nn.Linear(hidden_size, 6 * hidden_size)) + self.text_embedder = LingBotVideoTextEmbedder(text_dim, hidden_size) + self.rope = LingBotVideoRotaryEmbedding(axes_dims, axes_lens, rope_theta) + self.blocks = nn.ModuleList( + [ + LingBotVideoBlock( + hidden_size=hidden_size, + num_attention_heads=num_attention_heads, + intermediate_size=intermediate_size, + norm_eps=norm_eps, + qkv_bias=qkv_bias, + out_bias=out_bias, + num_experts=num_experts, + num_experts_per_tok=num_experts_per_tok, + moe_intermediate_size=moe_intermediate_size, + decoder_sparse_step=decoder_sparse_step, + mlp_only_layers=mlp_only_layers, + n_shared_experts=n_shared_experts, + score_func=score_func, + norm_topk_prob=norm_topk_prob, + n_group=n_group, + topk_group=topk_group, + routed_scaling_factor=routed_scaling_factor, + layer_idx=i, + ) + for i in range(depth) + ] + ) + self.norm_out = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=norm_eps) + self.norm_out_modulation = nn.Sequential(nn.SiLU(), nn.Linear(hidden_size, 2 * hidden_size)) + self.proj_out = nn.Linear(hidden_size, math.prod(patch_size) * out_channels) + + def forward( + self, + hidden_states: torch.Tensor, # (B, C, T, H, W) + timestep: torch.Tensor, # (B,) in [0, 1000](= sigma*1000) + encoder_hidden_states: torch.Tensor, # (B, L, text_dim) + encoder_attention_mask: torch.Tensor | None = None, # (B, L) 1=valid + return_dict: bool = True, + ): + B, C, T, H, W = hidden_states.shape + pF, pH, pW = self.config.patch_size + gt, gh, gw = T // pF, H // pH, W // pW + n_video = gt * gh * gw + L = encoder_hidden_states.shape[1] + device = hidden_states.device + if encoder_attention_mask is not None: + text_lens = encoder_attention_mask.sum(dim=-1).long() + else: + text_lens = torch.full((B,), L, dtype=torch.long, device=device) + text_lens_list = [int(v) for v in text_lens.detach().cpu().tolist()] + packed_batch = B > 1 + + # patchify: token order (f h w), feature order (pf ph pw c) -- matches patchify_and_embed + patch_tokens = hidden_states.reshape(B, C, gt, pF, gh, pH, gw, pW) + patch_tokens = patch_tokens.permute(0, 2, 4, 6, 3, 5, 7, 1).reshape( + B, + n_video, + pF * pH * pW * C, + ) + if packed_batch: + x = torch.cat( + [self.patch_embedder(patch_tokens[i : i + 1]) for i in range(B)], + dim=1, + ) + else: + x = self.patch_embedder(patch_tokens) + + if packed_batch: + text_parts = [ + self.text_embedder(encoder_hidden_states[i : i + 1, : text_lens_list[i], :]) for i in range(B) + ] + text = torch.cat(text_parts, dim=1) + joint = _cat_interleave( + x, + [n_video] * B, + text, + text_lens_list, + ) + else: + text = self.text_embedder(encoder_hidden_states) + joint = torch.cat([x, text], dim=1) # [video; text] + joint_seq_len = joint.shape[1] + + # Per-sample RoPE: video t-axis start = real text length of this sample + 1 + rotary_parts = [self.rope(make_joint_position_ids(text_lens_list[i], gt, gh, gw, device)) for i in range(B)] + if packed_batch: + rotary = torch.cat(rotary_parts, dim=0).unsqueeze(0) + else: + rotary = torch.stack(rotary_parts, dim=0) # (B, S, head_dim/2) complex64 + + parallel_config = getattr(self, "_parallel_config", None) + use_packed_attention = parallel_config is not None + + attention_mask = None + moe_padding_mask = None + packed_indices = None + has_padding = encoder_attention_mask is not None and bool((text_lens < L).any()) + if packed_batch or use_packed_attention: + sample_seq_lens = [n_video + text_len for text_len in text_lens_list] + cu_seqlens = torch.zeros(B + 1, device=device, dtype=torch.int32) + cu_seqlens[1:] = torch.cumsum( + torch.tensor(sample_seq_lens, device=device, dtype=torch.int32), + dim=0, + ) + packed_indices = { + "cu_seqlens_kv": cu_seqlens, + "max_seqlen_in_batch_kv": max(sample_seq_lens), + } + if packed_batch and not use_packed_attention: + packed_indices["attention_mask"] = _packed_block_attention_mask(sample_seq_lens, device) + has_padding = False + if has_padding: + key_mask = torch.cat( + [torch.ones(B, n_video, dtype=torch.bool, device=device), encoder_attention_mask.bool()], + dim=1, + ) + attention_mask = key_mask[:, None, None, :] # (B,1,1,S) -> SDPA broadcast + moe_padding_mask = key_mask.reshape(-1).float() # (B*S,) + packed_cp = packed_indices is not None and parallel_config is not None + padding_size = 0 + if packed_cp: + cp_config = parallel_config.context_parallel_config + cp_world_size = int(getattr(cp_config, "ulysses_degree", getattr(cp_config, "_world_size", 1))) + padding_size = (cp_world_size - (joint_seq_len % cp_world_size)) % cp_world_size + if padding_size: + joint = torch.cat( + [ + joint, + torch.zeros( + joint.shape[0], + padding_size, + joint.shape[2], + device=joint.device, + dtype=joint.dtype, + ), + ], + dim=1, + ) + rotary = torch.cat( + [ + rotary, + torch.zeros( + rotary.shape[0], + padding_size, + rotary.shape[2], + device=rotary.device, + dtype=rotary.dtype, + ), + ], + dim=1, + ) + if packed_indices is None: + raise RuntimeError("packed_indices must be initialized for packed context parallel.") + packed_indices["cu_seqlens_kv"] = torch.cat( + [ + packed_indices["cu_seqlens_kv"], + packed_indices["cu_seqlens_kv"][-1:] + padding_size, + ], + dim=0, + ) + packed_indices["max_seqlen_in_batch_kv"] = max( + int(packed_indices["max_seqlen_in_batch_kv"]), + int(padding_size), + ) + joint_seq_len = joint.shape[1] + + timestep_for_embed = timestep.float() + timestep_proj = self.time_proj(timestep_for_embed) + t_emb = self.time_embedder(timestep_proj) # (B, D) + if packed_batch: + temb_input = torch.cat( + [t_emb[i : i + 1].unsqueeze(1).expand(1, n_video + text_lens_list[i], -1) for i in range(B)], + dim=1, + ) + if padding_size: + temb_input = torch.cat( + [ + temb_input, + torch.zeros( + temb_input.shape[0], + padding_size, + temb_input.shape[2], + device=temb_input.device, + dtype=temb_input.dtype, + ), + ], + dim=1, + ) + temb6 = self.time_modulation(temb_input.reshape(joint_seq_len, -1)) + temb6 = temb6.reshape(1, joint_seq_len, -1) + else: + temb_input = t_emb.unsqueeze(1).expand(B, joint_seq_len, -1) # (B, S, D) + temb6 = self.time_modulation(temb_input.reshape(B * joint_seq_len, -1)) + temb6 = temb6.reshape(B, joint_seq_len, -1) # (B, S, 6D) + + temb6 = temb6.reshape(temb6.shape[0] * temb6.shape[1], -1) + + for block in self.blocks: + joint = block( + joint, + temb6, + rotary, + attention_mask, + moe_padding_mask, + packed_indices=packed_indices, + parallel_config=parallel_config, + ) + final_mod = self.norm_out_modulation(temb_input.reshape(joint.shape[0] * joint.shape[1], -1)) + shift, scale = final_mod.reshape(joint.shape[0], joint.shape[1], -1).chunk(2, dim=-1) + final_hidden = self.norm_out(joint) * (1.0 + scale) + shift + projected = self.proj_out(final_hidden.to(self.proj_out.weight.dtype)) + if packed_cp: + if padding_size: + projected = projected[:, :-padding_size, :] + if packed_batch: + split_lengths: list[int] = [] + for text_len in text_lens_list: + split_lengths.extend([n_video, text_len]) + parts = torch.split(projected, split_lengths, dim=1) + x = torch.cat(parts[::2], dim=1).reshape(B, n_video, -1) + else: + x = projected[:, :n_video] + + # unpatchify (matches the rearrange in postprocess) + Cout = self.config.out_channels + x = x.reshape(B, gt, gh, gw, pF, pH, pW, Cout) + x = x.permute(0, 7, 1, 4, 2, 5, 3, 6).reshape(B, Cout, T, H, W) + + if not return_dict: + return (x,) + return Transformer2DModelOutput(sample=x) diff --git a/vllm_omni/diffusion/models/lingbot_video/pipeline_lingbot_video.py b/vllm_omni/diffusion/models/lingbot_video/pipeline_lingbot_video.py new file mode 100644 index 000000000..f54fa387c --- /dev/null +++ b/vllm_omni/diffusion/models/lingbot_video/pipeline_lingbot_video.py @@ -0,0 +1,695 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# Adapted from LingBot-Video (https://github.com/Robbyant/lingbot-video). + +from __future__ import annotations + +import os +from collections.abc import Iterable, Mapping +from contextlib import nullcontext +from typing import Any, ClassVar + +import numpy as np +import torch +import torch.distributed as dist +from diffusers import AutoencoderKLWan +from diffusers.utils.torch_utils import randn_tensor +from torch import nn +from transformers import Qwen3VLForConditionalGeneration, Qwen3VLProcessor + +from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig +from vllm_omni.diffusion.distributed.utils import get_local_device +from vllm_omni.diffusion.models.interface import SupportsComponentDiscovery +from vllm_omni.diffusion.models.lingbot_video.lingbot_video_transformer import LingBotVideoTransformer3DModel +from vllm_omni.diffusion.models.progress_bar import ProgressBarMixin +from vllm_omni.diffusion.models.schedulers import FlowUniPCMultistepScheduler +from vllm_omni.diffusion.request import OmniDiffusionRequest +from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch + +TOKEN_LENGTH = 37698 +HIDDEN_STATE_SKIP_LAYER = 0 +LOW_NOISE_TAIL_V1_DEFAULT_STEPS = 2 + +PROMPT_TEMPLATE = ( + "<|im_start|>system\nGiven a user input that may include a text prompt alone, " + "a text prompt with an image reference, or a text prompt with a video reference " + 'or a video reference alone, generate an "Enhanced prompt" that provides detailed ' + "visual descriptions suitable for video generation. Evaluate the level of detail " + "in the user's input: if it is simple, enrich it by adding specifics about colors, " + "shapes, sizes, textures, lighting, motion dynamics, camera movement, temporal " + "progression, and spatial relationships to create vivid, concrete, and temporally " + "coherent scenes to create vivid and concrete scenes. Please generate only the " + "enhanced description for the prompt below and avoid including any additional " + "commentary or evaluations:<|im_end|>\n<|im_start|>user\n{}<|im_end|>\n" + "<|im_start|>assistant\n" +) +DEFAULT_NEGATIVE_PROMPT = ( + '{"universal_negative": {"visual_quality": ["low quality", "worst quality", "blurry", ' + '"pixelated", "jpeg artifacts", "low resolution", "unstable color", "color flicker", ' + '"underexposed", "overexposed", "invisible subject", "subject hidden in darkness"], ' + '"artistic_style": ["painting", "illustration", "drawing", "cartoon", "3d render", ' + '"cgi", "sketch", "digital art"], "composition_and_content": ["text", "watermark", ' + '"signature", "logo", "subtitles", "pillarboxed", "side bars", ' + '"portrait image in landscape frame"], "temporal_and_motion_stability": ["flickering", ' + '"jittery", "motion blur", "temporal inconsistency", "warping", "morphing", ' + '"incoherent motion", "unnatural movement", "static object with sudden jump", ' + '"frame-to-frame inconsistency"], "material_and_structure": ["plastic-like glass", ' + '"unrealistic texture", "deformed bottle", "liquid freezing improperly", ' + '"distorted reflections"]}}' +) + + +def _dtype_from_name(value: Any, default: torch.dtype) -> torch.dtype: + if isinstance(value, torch.dtype): + return value + if value is None: + return default + normalized = str(value).lower() + if normalized in {"bf16", "bfloat16", "torch.bfloat16"}: + return torch.bfloat16 + if normalized in {"fp16", "float16", "torch.float16"}: + return torch.float16 + if normalized in {"fp32", "float32", "torch.float32"}: + return torch.float32 + raise ValueError(f"Unsupported LingBot dtype: {value!r}.") + + +def _extract_prompt(req: OmniDiffusionRequest) -> tuple[str, str | None]: + prompt_obj = req.prompt + if isinstance(prompt_obj, str): + return prompt_obj, None + if isinstance(prompt_obj, Mapping): + prompt = prompt_obj.get("prompt", "") + negative_prompt = prompt_obj.get("negative_prompt") + return str(prompt), None if negative_prompt is None else str(negative_prompt) + raise TypeError(f"Unsupported LingBot prompt type: {type(prompt_obj)!r}.") + + +def _module_dtype(module: torch.nn.Module) -> torch.dtype: + try: + return next(module.parameters()).dtype + except StopIteration: + return torch.float32 + + +def _module_device(module: torch.nn.Module) -> torch.device: + try: + return next(module.parameters()).device + except StopIteration: + return torch.device("cpu") + + +def _transformer_timestep(timestep: torch.Tensor, transformer_dtype: torch.dtype) -> torch.Tensor: + sigma = timestep.float() / 1000.0 + if transformer_dtype in {torch.bfloat16, torch.float16}: + sigma = sigma.to(transformer_dtype) + return (sigma * 1000.0).float() + + +def _transformer_autocast(device: torch.device, transformer_dtype: torch.dtype): + if device.type != "cuda" or transformer_dtype not in {torch.bfloat16, torch.float16}: + return nullcontext() + return torch.autocast(device_type="cuda", dtype=transformer_dtype) + + +def _group_global_rank(group: Any | None, group_rank: int) -> int: + if group is None: + return group_rank + get_global_rank = getattr(dist, "get_global_rank", None) + if get_global_rank is None: + return group_rank + return int(get_global_rank(group, group_rank)) + + +def _validate_refiner_sigmas(sigmas: np.ndarray, t_thresh: float | None = None) -> np.ndarray: + arr = np.asarray(list(sigmas), dtype=np.float64) + if arr.ndim != 1 or arr.size == 0: + raise ValueError("refiner sigma schedule must be a non-empty 1D list") + if not np.all(np.isfinite(arr)): + raise ValueError("refiner sigma schedule contains non-finite values") + if np.any(arr < 0.0) or np.any(arr > 1.0): + raise ValueError(f"refiner sigma schedule values must be in [0, 1], got {arr.tolist()}") + if arr.size > 1 and not np.all(np.diff(arr) < 0.0): + raise ValueError(f"refiner sigma schedule must be strictly descending, got {arr.tolist()}") + if t_thresh is not None and abs(float(arr[0]) - float(t_thresh)) > 1e-6: + raise ValueError(f"refiner sigma schedule must start at t_thresh={float(t_thresh)}, got {float(arr[0])}") + return arr + + +def _compute_refiner_sigmas( + *, + sigma_max: float, + sigma_min: float, + num_inference_steps: int, + shift: float, + t_thresh: float | None, + tail_steps: int = 0, +) -> np.ndarray | None: + if t_thresh is None: + return None + t_value = float(t_thresh) + if not (0.0 < t_value <= 1.0): + raise ValueError(f"refiner t_thresh must lie in (0, 1], got {t_value}") + steps = int(num_inference_steps) + if steps < 1: + raise ValueError(f"num_inference_steps must be >= 1, got {steps}") + tail = int(tail_steps or 0) + if tail < 0: + raise ValueError(f"refiner_sigma_tail_steps must be >= 0, got {tail}") + + base = np.linspace(float(sigma_max), float(sigma_min), steps + 1).copy()[:-1] + shift_value = float(shift) + shifted = shift_value * base / (1.0 + (shift_value - 1.0) * base) + eps = 1e-6 + sigmas = shifted[shifted <= t_value + eps] + if sigmas.size == 0 or abs(float(sigmas[0]) - t_value) > eps: + sigmas = np.concatenate([[t_value], sigmas]) + if tail > 0: + start = float(sigmas[-1]) + stop = min(float(sigma_min), start) + extra = np.linspace(start, stop, tail + 2, dtype=np.float64)[1:-1] + sigmas = np.concatenate([sigmas, extra]) + return _validate_refiner_sigmas(sigmas, t_value).astype(np.float32) + + +def _pad_prompt_embeds( + embeds: torch.Tensor, + mask: torch.Tensor, + target_length: int, +) -> tuple[torch.Tensor, torch.Tensor]: + if embeds.shape[0] != 1: + raise ValueError(f"batched CFG helper expects batch=1 inputs, got {embeds.shape[0]}") + if embeds.shape[1] > target_length: + raise ValueError(f"cannot pad length {embeds.shape[1]} down to {target_length}") + pad_len = target_length - embeds.shape[1] + if pad_len == 0: + return embeds, mask + embed_pad = torch.zeros(embeds.shape[0], pad_len, embeds.shape[2], dtype=embeds.dtype, device=embeds.device) + mask_pad = torch.zeros(mask.shape[0], pad_len, dtype=mask.dtype, device=mask.device) + return torch.cat([embeds, embed_pad], dim=1), torch.cat([mask, mask_pad], dim=1) + + +def _batch_cfg_prompt_inputs( + prompt_embeds: torch.Tensor, + prompt_mask: torch.Tensor, + negative_embeds: torch.Tensor, + negative_mask: torch.Tensor, + *, + null_cond_clone_zero: bool = False, +) -> tuple[torch.Tensor, torch.Tensor]: + if null_cond_clone_zero: + zero_negative = torch.zeros_like(prompt_embeds) + return ( + torch.cat([prompt_embeds, zero_negative], dim=0), + torch.cat([prompt_mask, prompt_mask.clone()], dim=0), + ) + + target_length = max(int(prompt_embeds.shape[1]), int(negative_embeds.shape[1])) + prompt_padded, prompt_mask_padded = _pad_prompt_embeds(prompt_embeds, prompt_mask, target_length) + negative_padded, negative_mask_padded = _pad_prompt_embeds(negative_embeds, negative_mask, target_length) + return ( + torch.cat([prompt_padded, negative_padded], dim=0), + torch.cat([prompt_mask_padded, negative_mask_padded], dim=0), + ) + + +def get_lingbot_video_post_process_func(od_config: OmniDiffusionConfig): + del od_config + + def post_process_func(frames: torch.Tensor, sampling_params=None): + output_type = getattr(sampling_params, "output_type", None) or "pt" + if output_type == "np" and isinstance(frames, torch.Tensor): + return frames.float().cpu().numpy() + return frames + + return post_process_func + + +class LingBotVideoPipeline(nn.Module, ProgressBarMixin, SupportsComponentDiscovery): + """Native vLLM-Omni entry for LingBot-Video checkpoints. + + The in-tree transformer supports both dense MLP blocks and routed MoE blocks. + Fused expert kernels and the optional ``refiner/`` transformer are not loaded + or executed. ``t_thresh`` only selects a low-noise sigma schedule for the + primary transformer; it does not enable automatic refiner orchestration. + """ + + supports_step_execution: ClassVar[bool] = False + _dit_modules: ClassVar[list[str]] = ["transformer"] + _encoder_modules: ClassVar[list[str]] = ["text_encoder"] + _vae_modules: ClassVar[list[str]] = ["vae"] + + def __init__(self, *, od_config: OmniDiffusionConfig, prefix: str = ""): + super().__init__() + del prefix + self.od_config = od_config + self.device = get_local_device() + self.vae_scale_factor_temporal = 4 + self.vae_scale_factor_spatial = 8 + self.token_length = TOKEN_LENGTH + self.hidden_state_skip_layer = HIDDEN_STATE_SKIP_LAYER + self.prompt_template = PROMPT_TEMPLATE + self._crop_start: int | None = None + + model = od_config.model + local_files_only = os.path.exists(model) + dtype = getattr(od_config, "dtype", torch.bfloat16) + model_config = getattr(od_config, "model_config", None) or {} + transformer_dtype = _dtype_from_name(model_config.get("transformer_dtype"), dtype) + text_encoder_dtype = _dtype_from_name(model_config.get("text_encoder_dtype"), dtype) + vae_dtype = _dtype_from_name(model_config.get("vae_dtype"), torch.float32) + + transformer_subfolder = str(model_config.get("transformer_subfolder", "transformer")) + text_encoder_subfolder = str(model_config.get("text_encoder_subfolder", "text_encoder")) + processor_subfolder = str(model_config.get("processor_subfolder", "processor")) + vae_subfolder = str(model_config.get("vae_subfolder", "vae")) + scheduler_subfolder = str(model_config.get("scheduler_subfolder", "scheduler")) + + self.transformer = LingBotVideoTransformer3DModel.from_pretrained( + model, + subfolder=transformer_subfolder, + torch_dtype=transformer_dtype, + local_files_only=local_files_only, + ).to(self.device) + text_encoder_kwargs: dict[str, Any] = { + "dtype": text_encoder_dtype, + "local_files_only": local_files_only, + } + self.text_encoder = Qwen3VLForConditionalGeneration.from_pretrained( + model, + subfolder=text_encoder_subfolder, + **text_encoder_kwargs, + ).to(self.device) + self.processor = Qwen3VLProcessor.from_pretrained( + model, + subfolder=processor_subfolder, + local_files_only=local_files_only, + ) + self.vae = AutoencoderKLWan.from_pretrained( + model, + subfolder=vae_subfolder, + torch_dtype=vae_dtype, + local_files_only=local_files_only, + ).to(self.device) + self.scheduler = FlowUniPCMultistepScheduler.from_pretrained( + model, + subfolder=scheduler_subfolder, + local_files_only=local_files_only, + ) + self.set_progress_bar_config(disable=bool(model_config.get("quiet_progress", True))) + self.default_negative_prompt = DEFAULT_NEGATIVE_PROMPT + + def to(self, *args, **kwargs): + device, dtype, non_blocking, _ = torch._C._nn._parse_to(*args, **kwargs) + super().to(*args, **kwargs) + if device is not None: + self.device = torch.device(device) + self.transformer.to(device=self.device, non_blocking=non_blocking) + self.text_encoder.to(device=self.device, non_blocking=non_blocking) + self.vae.to(device=self.device, non_blocking=non_blocking) + if dtype is not None: + self.transformer.to(dtype=dtype, non_blocking=non_blocking) + return self + + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: + consumed = list(weights) + if consumed: + raise RuntimeError( + f"{self.__class__.__name__}.load_weights received {len(consumed)} weight tensors; " + "LingBot components are loaded directly from subfolders during __init__." + ) + return {name for name, _ in self.named_parameters()} + + @staticmethod + def check_inputs(height: int, width: int, num_frames: int) -> None: + if num_frames != 1 and (num_frames - 1) % 4 != 0: + raise ValueError(f"`num_frames` must be 1 or 4n+1, got {num_frames}.") + if height % 16 != 0 or width % 16 != 0: + raise ValueError(f"`height` and `width` must be multiples of 16, got {height}x{width}.") + + @staticmethod + def apply_text_to_template(text: str, template: str = PROMPT_TEMPLATE) -> str: + return template.format(text) + + def _compute_crop_start(self) -> int: + if self._crop_start is None: + marker = "<|USER_INPUT_MARKER|>" + marked = self.prompt_template.format(marker) + marker_pos = marked.find(marker) + if marker_pos < 0: + self._crop_start = 0 + else: + prefix = self.processor( + text=marked[:marker_pos], + images=None, + videos=None, + return_tensors="pt", + ) + self._crop_start = int(prefix["input_ids"].shape[1]) + return self._crop_start + + def _build_prompt_inputs(self, prompt: str | list[str]): + prompts = [prompt] if isinstance(prompt, str) else list(prompt) + texts = [self.apply_text_to_template(text, self.prompt_template) for text in prompts] + return self.processor( + text=texts, + images=None, + videos=None, + do_resize=False, + truncation=True, + max_length=self.token_length, + padding="longest", + return_tensors="pt", + ) + + @torch.no_grad() + def encode_prompt( + self, + prompt: str | list[str], + *, + device: str | torch.device | None = None, + ) -> tuple[torch.Tensor, torch.Tensor]: + device = torch.device(device) if device is not None else self.device + inputs = self._build_prompt_inputs(prompt).to(device) + outputs = self.text_encoder( + **inputs, + output_hidden_states=self.hidden_state_skip_layer is not None, + ) + if self.hidden_state_skip_layer is not None: + prompt_embeds = outputs.hidden_states[-(self.hidden_state_skip_layer + 1)] + else: + prompt_embeds = outputs.last_hidden_state + + prompt_mask = inputs["attention_mask"] + crop_start = self._compute_crop_start() + if crop_start > 0: + prompt_embeds = prompt_embeds[:, crop_start:] + prompt_mask = prompt_mask[:, crop_start:] + + if prompt_embeds.shape[0] == 1: + true_len = int(prompt_mask[0].sum().item()) + prompt_embeds = prompt_embeds[:, :true_len] + prompt_mask = prompt_mask[:, :true_len] + return prompt_embeds, prompt_mask + + def prepare_latents( + self, + num_frames: int, + height: int, + width: int, + generator: torch.Generator | None, + latents: torch.Tensor | None, + device: torch.device, + ) -> torch.Tensor: + latent_frames = (num_frames - 1) // self.vae_scale_factor_temporal + 1 + latent_height = height // self.vae_scale_factor_spatial + latent_width = width // self.vae_scale_factor_spatial + shape = (1, self.transformer.config.in_channels, latent_frames, latent_height, latent_width) + if latents is None: + return randn_tensor(shape, generator=generator, device=device, dtype=torch.float32) + if tuple(latents.shape) != shape: + raise ValueError(f"`latents` shape must be {shape}, got {tuple(latents.shape)}.") + return latents.to(device=device, dtype=torch.float32) + + def _dit_latent_to_vae(self, latents: torch.Tensor) -> torch.Tensor: + mean = torch.tensor(self.vae.config.latents_mean, device=latents.device, dtype=torch.float32) + std_inv = 1.0 / torch.tensor(self.vae.config.latents_std, device=latents.device, dtype=torch.float32) + mean = mean.view(1, -1, 1, 1, 1) + std_inv = std_inv.view(1, -1, 1, 1, 1) + return latents.float() / std_inv + mean + + @torch.no_grad() + def _decode_latents(self, latents: torch.Tensor) -> torch.Tensor: + vae_device = _module_device(self.vae) + vae_dtype = _module_dtype(self.vae) + vae_latents = self._dit_latent_to_vae(latents).to(device=vae_device, dtype=torch.float32) + if vae_latents.ndim == 5: + vae_latents = vae_latents.contiguous(memory_format=torch.channels_last_3d) + autocast_dtype = ( + vae_dtype if vae_device.type == "cuda" and vae_dtype in {torch.bfloat16, torch.float16} else None + ) + with torch.autocast("cuda", dtype=autocast_dtype or torch.bfloat16, enabled=autocast_dtype is not None): + decoded = self.vae.decode(vae_latents) + frames = decoded[0] if isinstance(decoded, tuple) else decoded.sample + frames = frames.float().clamp_(-1, 1) + frames = (frames + 1.0) / 2.0 + frames = frames.permute(0, 2, 3, 4, 1).cpu() + return frames[0] + + @torch.no_grad() + def _generate( + self, + *, + prompt: str, + negative_prompt: str = DEFAULT_NEGATIVE_PROMPT, + height: int = 480, + width: int = 480, + num_frames: int = 81, + num_inference_steps: int = 40, + guidance_scale: float = 6.0, + shift: float = 3.0, + generator: torch.Generator | None = None, + latents: torch.Tensor | None = None, + prompt_embeds: torch.Tensor | None = None, + prompt_mask: torch.Tensor | None = None, + negative_prompt_embeds: torch.Tensor | None = None, + negative_prompt_mask: torch.Tensor | None = None, + output_type: str = "pt", + cfg_parallel_group: Any | None = None, + batch_cfg: bool = False, + null_cond_clone_zero: bool = False, + t_thresh: float | None = None, + refiner_sigma_tail_steps: int = LOW_NOISE_TAIL_V1_DEFAULT_STEPS, + offload_vae_during_denoise: bool = False, + **extra_args, + ) -> torch.Tensor: + del extra_args + self.check_inputs(height, width, num_frames) + device = self.device + do_cfg = guidance_scale > 1.0 + effective_batch_cfg = bool(batch_cfg) + cfg_parallel = cfg_parallel_group is not None + cfg_parallel_rank = 0 + if cfg_parallel: + if not dist.is_available() or not dist.is_initialized(): + raise ValueError("`cfg_parallel_group` requires an initialized process group.") + if effective_batch_cfg: + raise ValueError("`cfg_parallel_group` and `batch_cfg` are mutually exclusive.") + if not do_cfg: + raise ValueError("CFG parallel requires `guidance_scale > 1.0`.") + cfg_parallel_rank = dist.get_rank(cfg_parallel_group) + cfg_parallel_world_size = dist.get_world_size(cfg_parallel_group) + if cfg_parallel_world_size != 2: + raise ValueError(f"CFG parallel currently requires exactly 2 ranks, got {cfg_parallel_world_size}.") + + if prompt_embeds is not None: + if prompt_mask is None: + raise ValueError("`prompt_mask` is required when `prompt_embeds` is provided.") + prompt_embeds = prompt_embeds.to(device=device) + prompt_mask = prompt_mask.to(device=device) + if negative_prompt_embeds is not None: + if negative_prompt_mask is None: + raise ValueError("`negative_prompt_mask` is required when `negative_prompt_embeds` is provided.") + negative_prompt_embeds = negative_prompt_embeds.to(device=device) + negative_prompt_mask = negative_prompt_mask.to(device=device) + + negative_embeds = None + negative_mask = None + if cfg_parallel and cfg_parallel_rank == 1: + if negative_prompt_embeds is not None: + negative_embeds, negative_mask = negative_prompt_embeds, negative_prompt_mask + else: + negative_embeds, negative_mask = self.encode_prompt(negative_prompt, device=device) + prompt_embeds = prompt_mask = None + else: + if prompt_embeds is None: + prompt_embeds, prompt_mask = self.encode_prompt(prompt, device=device) + if do_cfg and not cfg_parallel: + if null_cond_clone_zero: + negative_embeds = torch.zeros_like(prompt_embeds) + negative_mask = prompt_mask.clone() + elif negative_prompt_embeds is not None: + negative_embeds, negative_mask = negative_prompt_embeds, negative_prompt_mask + else: + negative_embeds, negative_mask = self.encode_prompt(negative_prompt, device=device) + + latents = self.prepare_latents(num_frames, height, width, generator, latents, device) + sigmas = _compute_refiner_sigmas( + sigma_max=float(self.scheduler.sigma_max), + sigma_min=float(self.scheduler.sigma_min), + num_inference_steps=num_inference_steps, + shift=shift, + t_thresh=t_thresh, + tail_steps=refiner_sigma_tail_steps, + ) + if sigmas is None: + self.scheduler.set_timesteps(num_inference_steps, device=device, shift=shift) + else: + self.scheduler.set_timesteps(int(sigmas.shape[0]), device=device, sigmas=sigmas, shift=1.0) + + transformer_dtype = _module_dtype(self.transformer) + vae_restore_device: torch.device | None = None + vae_offloaded = False + if offload_vae_during_denoise and output_type != "latent": + vae_device = _module_device(self.vae) + if vae_device.type == "cuda": + self.vae.to("cpu") + torch.accelerator.empty_cache() + vae_restore_device = vae_device + vae_offloaded = True + + cfg_latent_src = _group_global_rank(cfg_parallel_group, 0) + cfg_uncond_src = _group_global_rank(cfg_parallel_group, 1) + for timestep in self.progress_bar(self.scheduler.timesteps): + if cfg_parallel: + dist.broadcast(latents, src=cfg_latent_src, group=cfg_parallel_group) + timestep_batch = _transformer_timestep(timestep, transformer_dtype).expand(1).to(device) + latent_model_input = latents + if cfg_parallel: + if cfg_parallel_rank == 0: + branch_embeds = prompt_embeds + branch_mask = prompt_mask + else: + branch_embeds = negative_embeds + branch_mask = negative_mask + if branch_embeds is None: + raise RuntimeError("CFG branch embeddings were not initialized.") + with _transformer_autocast(device, transformer_dtype): + branch_noise_pred = self.transformer( + latent_model_input, + timestep_batch, + branch_embeds.to(transformer_dtype), + encoder_attention_mask=branch_mask, + return_dict=False, + )[0].float() + if cfg_parallel_rank == 0: + noise_pred = branch_noise_pred + noise_pred_uncond = torch.empty_like(noise_pred) + else: + noise_pred_uncond = branch_noise_pred + dist.broadcast(noise_pred_uncond, src=cfg_uncond_src, group=cfg_parallel_group) + if cfg_parallel_rank != 0: + continue + noise_pred = noise_pred_uncond + guidance_scale * (noise_pred - noise_pred_uncond) + else: + if prompt_embeds is None: + raise RuntimeError("Prompt embeddings were not initialized.") + prompt_model_input = prompt_embeds.to(transformer_dtype) + if do_cfg and effective_batch_cfg: + if negative_embeds is None or negative_mask is None: + raise RuntimeError("Negative embeddings were not initialized for CFG.") + cfg_embeds, cfg_mask = _batch_cfg_prompt_inputs( + prompt_model_input, + prompt_mask, + negative_embeds.to(transformer_dtype), + negative_mask, + null_cond_clone_zero=False, + ) + cfg_latents = torch.cat([latent_model_input, latent_model_input], dim=0) + cfg_timesteps = torch.cat([timestep_batch, timestep_batch], dim=0) + with _transformer_autocast(device, transformer_dtype): + noise_batched = self.transformer( + cfg_latents, + cfg_timesteps, + cfg_embeds, + encoder_attention_mask=cfg_mask, + return_dict=False, + )[0].float() + noise_pred, noise_pred_uncond = noise_batched.chunk(2, dim=0) + noise_pred = noise_pred_uncond + guidance_scale * (noise_pred - noise_pred_uncond) + else: + with _transformer_autocast(device, transformer_dtype): + noise_pred = self.transformer( + latent_model_input, + timestep_batch, + prompt_model_input, + encoder_attention_mask=prompt_mask, + return_dict=False, + )[0].float() + + if do_cfg and not effective_batch_cfg: + if negative_embeds is None or negative_mask is None: + raise RuntimeError("Negative embeddings were not initialized for CFG.") + with _transformer_autocast(device, transformer_dtype): + noise_pred_uncond = self.transformer( + latent_model_input, + timestep_batch, + negative_embeds.to(transformer_dtype), + encoder_attention_mask=negative_mask, + return_dict=False, + )[0].float() + noise_pred = noise_pred_uncond + guidance_scale * (noise_pred - noise_pred_uncond) + + latents = self.scheduler.step(noise_pred, timestep, latents, return_dict=False, generator=generator)[0] + + if cfg_parallel: + dist.barrier(group=cfg_parallel_group) + if cfg_parallel_rank != 0: + return latents if output_type == "latent" else [] + + if output_type == "latent": + return latents + if output_type in {"pt", "np"}: + if vae_offloaded and vae_restore_device is not None: + self.vae.to(device=vae_restore_device) + torch.accelerator.empty_cache() + return self._decode_latents(latents) + raise ValueError(f"Unsupported output_type: {output_type}") + + @torch.inference_mode() + def forward(self, req: DiffusionRequestBatch) -> DiffusionOutput: + if req.num_reqs != 1: + raise ValueError(f"LingBotVideoPipeline only supports one request per batch, got {req.num_reqs}.") + request = req.requests[0] + prompt, prompt_negative = _extract_prompt(request) + sampling = request.sampling_params + extra_args = dict(sampling.extra_args or {}) + + generator = sampling.generator + if isinstance(generator, list): + generator = generator[0] if generator else None + if generator is None and sampling.seed is not None: + generator = torch.Generator(device=self.device).manual_seed(int(sampling.seed)) + + height = sampling.height if sampling.height is not None else extra_args.pop("height", 480) + width = sampling.width if sampling.width is not None else extra_args.pop("width", 480) + num_frames = sampling.num_frames or extra_args.pop("num_frames", 81) + num_inference_steps = ( + sampling.num_inference_steps + if sampling.num_inference_steps is not None + else extra_args.pop("num_inference_steps", 40) + ) + guidance_scale = ( + sampling.guidance_scale if sampling.guidance_scale_provided else extra_args.pop("guidance_scale", 6.0) + ) + shift = extra_args.pop( + "shift", + extra_args.pop("flow_shift", getattr(self.od_config, "flow_shift", None) or 3.0), + ) + negative_prompt = extra_args.pop("negative_prompt", prompt_negative or self.default_negative_prompt) + output_type = ( + sampling.output_type or getattr(self.od_config, "output_type", None) or extra_args.pop("output_type", "pt") + ) + if output_type not in {"pt", "np", "latent"}: + output_type = "pt" + sampling.output_type = output_type + + frames = self._generate( + prompt=prompt, + negative_prompt=negative_prompt, + height=height, + width=width, + num_frames=num_frames, + num_inference_steps=num_inference_steps, + guidance_scale=guidance_scale, + shift=shift, + generator=generator, + latents=sampling.latents, + output_type=output_type, + batch_cfg=bool(extra_args.pop("batch_cfg", False)), + null_cond_clone_zero=bool(extra_args.pop("null_cond_clone_zero", False)), + offload_vae_during_denoise=bool(extra_args.pop("offload_vae_during_denoise", False)), + t_thresh=extra_args.pop("t_thresh", None), + refiner_sigma_tail_steps=int(extra_args.pop("refiner_sigma_tail_steps", LOW_NOISE_TAIL_V1_DEFAULT_STEPS)), + ) + return DiffusionOutput(output=frames) diff --git a/vllm_omni/diffusion/registry.py b/vllm_omni/diffusion/registry.py index 22c36f32b..aea257dd6 100644 --- a/vllm_omni/diffusion/registry.py +++ b/vllm_omni/diffusion/registry.py @@ -256,6 +256,11 @@ _DIFFUSION_MODELS = { "pipeline_hunyuan_video_1_5_i2v", "HunyuanVideo15I2VPipeline", ), + "LingBotVideoPipeline": ( + "lingbot_video", + "pipeline_lingbot_video", + "LingBotVideoPipeline", + ), "MagiHumanPipeline": ( "magi_human", "pipeline_magi_human", @@ -536,6 +541,7 @@ _DIFFUSION_POST_PROCESS_FUNCS = { "Flux2Pipeline": "get_flux2_post_process_func", "HunyuanVideo15Pipeline": "get_hunyuan_video_15_post_process_func", "HunyuanVideo15ImageToVideoPipeline": "get_hunyuan_video_15_i2v_post_process_func", + "LingBotVideoPipeline": "get_lingbot_video_post_process_func", "MagiHumanPipeline": "get_magi_human_post_process_func", "OmniVoicePipeline": "get_omnivoice_post_process_func", "DreamIDOmniPipeline": "get_dreamid_omni_post_process_func", diff --git a/vllm_omni/model_extras/lingbot_video.py b/vllm_omni/model_extras/lingbot_video.py new file mode 100644 index 000000000..c7aae3763 --- /dev/null +++ b/vllm_omni/model_extras/lingbot_video.py @@ -0,0 +1,16 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +LINGBOT_VIDEO_EXTRA_BODY_PARAMS = frozenset( + { + "batch_cfg", + "flow_shift", + "negative_prompt", + "null_cond_clone_zero", + "offload_vae_during_denoise", + "output_type", + "refiner_sigma_tail_steps", + "shift", + "t_thresh", + } +) diff --git a/vllm_omni/model_extras/registry.py b/vllm_omni/model_extras/registry.py index 79f8b4360..ca13a35c5 100644 --- a/vllm_omni/model_extras/registry.py +++ b/vllm_omni/model_extras/registry.py @@ -34,6 +34,7 @@ from vllm_omni.model_extras.helios import ( HELIOS_EXTRA_BODY_PARAMS, HELIOS_EXTRA_OUTPUT_PARAMS, ) +from vllm_omni.model_extras.lingbot_video import LINGBOT_VIDEO_EXTRA_BODY_PARAMS from vllm_omni.model_extras.magi_human import ( MAGI_HUMAN_EXTRA_BODY_PARAMS, MAGI_HUMAN_EXTRA_OUTPUT_PARAMS, @@ -170,6 +171,9 @@ _EXTRA_SPECS: dict[str, dict[str, Any]] = { "extra_body_params": HELIOS_EXTRA_BODY_PARAMS, "extra_output_params": HELIOS_EXTRA_OUTPUT_PARAMS, }, + "LingBotVideoPipeline": { + "extra_body_params": LINGBOT_VIDEO_EXTRA_BODY_PARAMS, + }, "WanVACEPipeline": { "extra_body_params": VACE_EXTRA_BODY_PARAMS, "extra_output_params": VACE_EXTRA_OUTPUT_PARAMS, From 779466becb8f2c84efcfb1898358089db8fc767c Mon Sep 17 00:00:00 2001 From: SYLAR <125541396+lishunyang12@users.noreply.github.com> Date: Tue, 21 Jul 2026 18:11:16 +0800 Subject: [PATCH 039/240] [Bugfix][Quant] Skip quack FP8 GEMM when scales are unpopulated (#5262) Signed-off-by: lishunyang12 Co-authored-by: lishunyang12 --- vllm_omni/quantization/quack_fp8.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/vllm_omni/quantization/quack_fp8.py b/vllm_omni/quantization/quack_fp8.py index 865958386..e92515432 100644 --- a/vllm_omni/quantization/quack_fp8.py +++ b/vllm_omni/quantization/quack_fp8.py @@ -98,6 +98,29 @@ def quack_scaled_fp8_mm( return out +_valid_scale_ptrs: set[tuple[int, int]] = set() + + +def _scales_valid(scale_a: torch.Tensor, scale_b: torch.Tensor) -> bool: + """True when both per-tensor scales are finite and positive. + + vLLM initializes a per-tensor FP8 scale to ``finfo(float32).min`` and fills it at + weight-load time, so a call made before that (the dummy profiling forward) would + make ``alpha = scale_a * scale_b`` overflow to ``+inf`` and return an all-inf tile. + Cache only positive results: a buffer still holding the sentinel is re-checked and + picks up the fast path once the real scale is written. + """ + key = (scale_a.data_ptr(), scale_b.data_ptr()) + if key in _valid_scale_ptrs: + return True + ok = bool( + torch.isfinite(scale_a).all() and torch.isfinite(scale_b).all() and (scale_a > 0).all() and (scale_b > 0).all() + ) + if ok: + _valid_scale_ptrs.add(key) + return ok + + def install_quack_fp8_patch() -> None: if not quack_enabled(): return @@ -115,6 +138,10 @@ def install_quack_fp8_patch() -> None: return def apply_scaled_mm(self, *, A, B, out_dtype, As, Bs, bias, output_shape): # noqa: N803 + # An unpopulated scale makes alpha overflow to +inf; hand those calls to + # FlashInfer, whose bmm_fp8 tolerates them. + if As.numel() != 1 or Bs.numel() != 1 or not _scales_valid(As, Bs): + return original(self, A=A, B=B, out_dtype=out_dtype, As=As, Bs=Bs, bias=bias, output_shape=output_shape) try: out = quack_scaled_fp8_mm(A, B, As, Bs, out_dtype, bias) if out is not None: From 0034eaa9c3f39d49019f04a1582e1d664b239a2c Mon Sep 17 00:00:00 2001 From: Alicia <115451386+congw729@users.noreply.github.com> Date: Tue, 21 Jul 2026 20:26:22 +0800 Subject: [PATCH 040/240] Update perf baseline base on 7/5-7/11 7 days avg. (#5231) Signed-off-by: Alicia <115451386+congw729@users.noreply.github.com> --- .../dfx/perf/tests/test_bagel_vllm_omni.json | 32 ++++---- tests/dfx/perf/tests/test_higgs_audio_v3.json | 10 +-- .../tests/test_qwen3_omni_async_chunk.json | 80 +++++++++---------- .../tests/test_qwen3_omni_multi_replicas.json | 26 +++--- .../tests/test_qwen3_omni_no_async_chunk.json | 48 +++++------ .../perf/tests/test_qwen3_omni_vllm_text.json | 32 ++++---- .../test_qwen_image_edit_2511_vllm_omni.json | 20 ++--- .../tests/test_qwen_image_edit_vllm_omni.json | 22 ++--- .../test_qwen_image_layered_vllm_omni.json | 8 +- .../perf/tests/test_qwen_image_vllm_omni.json | 16 ++-- tests/dfx/perf/tests/test_tts.json | 60 +++++++------- tests/dfx/perf/tests/test_voxcpm2.json | 22 ++--- .../perf/tests/test_wan22_i2v_vllm_omni.json | 18 ++--- 13 files changed, 197 insertions(+), 197 deletions(-) diff --git a/tests/dfx/perf/tests/test_bagel_vllm_omni.json b/tests/dfx/perf/tests/test_bagel_vllm_omni.json index 303839387..e6777acfe 100644 --- a/tests/dfx/perf/tests/test_bagel_vllm_omni.json +++ b/tests/dfx/perf/tests/test_bagel_vllm_omni.json @@ -40,10 +40,10 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.3678, - "latency_mean": 2.691, - "peak_memory_mb_max": 33083.6, - "peak_memory_mb_mean": 33082.5 + "throughput_qps": 0.495, + "latency_mean": 2.0238, + "peak_memory_mb_max": 29393.4286, + "peak_memory_mb_mean": 29392.7143 } } ] @@ -90,10 +90,10 @@ "enable-negative-prompt": true, "num-input-images": 1, "baseline": { - "throughput_qps": 0.3124, - "latency_mean": 3.1686, - "peak_memory_mb_max": 33448.8, - "peak_memory_mb_mean": 33447.7 + "throughput_qps": 0.3451, + "latency_mean": 2.9031, + "peak_memory_mb_max": 30480.2857, + "peak_memory_mb_mean": 30479.2857 } } ] @@ -139,10 +139,10 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.3746, - "latency_mean": 2.6425, - "peak_memory_mb_max": 33077.0, - "peak_memory_mb_mean": 33077.0 + "throughput_qps": 0.5048, + "latency_mean": 1.987, + "peak_memory_mb_max": 29380.0, + "peak_memory_mb_mean": 29380.0 } } ] @@ -189,10 +189,10 @@ "enable-negative-prompt": true, "num-input-images": 1, "baseline": { - "throughput_qps": 0.1349, - "latency_mean": 7.339, - "peak_memory_mb_max": 33321.2, - "peak_memory_mb_mean": 33321.2 + "throughput_qps": 0.149, + "latency_mean": 6.7157, + "peak_memory_mb_max": 30574.0, + "peak_memory_mb_mean": 30574.0 } } ] diff --git a/tests/dfx/perf/tests/test_higgs_audio_v3.json b/tests/dfx/perf/tests/test_higgs_audio_v3.json index 9b7b2eb5c..04cc88ac2 100644 --- a/tests/dfx/perf/tests/test_higgs_audio_v3.json +++ b/tests/dfx/perf/tests/test_higgs_audio_v3.json @@ -47,10 +47,10 @@ "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { "median_audio_ttfp_ms": [ - 3000 + 223.3584 ], "median_audio_rtf": [ - 0.5 + 0.4393 ] } }, @@ -71,13 +71,13 @@ "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { "median_audio_ttfp_ms": [ - 5000 + 276.575 ], "median_audio_rtf": [ - 1.2 + 0.5073 ], "audio_throughput": [ - 30 + 27.3728 ] } } diff --git a/tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json b/tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json index 66ffa231f..013436d80 100644 --- a/tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json +++ b/tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json @@ -44,25 +44,25 @@ "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { "mean_ttft_ms": [ - 135.4379, - 187.0896, - 400, - 435.675, - 724.6836 + 96.4061, + 140.8147, + 271.9025, + 362.2916, + 507.776 ], "mean_e2el_ms": [ - 17619.9851, - 34811.8335, - 43656.9501, - 56657.1548, - 85097.6016 + 18507.4582, + 28365.4956, + 31907.9613, + 48161.5829, + 72630.8601 ], "mean_audio_rtf": [ - 0.1279, - 0.2238, - 0.2735, - 0.3712, - 0.5634 + 0.1175, + 0.18, + 0.2359, + 0.3278, + 0.6296 ] } }, @@ -91,13 +91,13 @@ "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { "mean_ttft_ms": [ - 340.329 + 354.0951 ], "mean_e2el_ms": [ - 4305.477 + 3952.5615 ], "mean_audio_rtf": [ - 0.1305 + 0.117 ] } }, @@ -128,13 +128,13 @@ "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { "mean_ttft_ms": [ - 220.0909 + 259.6317 ], "mean_e2el_ms": [ - 5475.9565 + 4253.3009 ], "mean_audio_rtf": [ - 0.1818 + 0.1448 ] } }, @@ -167,13 +167,13 @@ "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { "mean_ttft_ms": [ - 450.0155 + 457.4453 ], "mean_e2el_ms": [ - 8516.8316 + 5912.5623 ], "mean_audio_rtf": [ - 0.2717 + 0.1967 ] } }, @@ -206,18 +206,18 @@ "percentile-metrics": "ttft,tpot,itl,e2el", "baseline": { "mean_ttft_ms": [ - 78.4278, - 195.8037, - 333.7318, - 561.805, - 1038.043 + 90.8033, + 202.1779, + 326.5864, + 544.8914, + 961.3929 ], "mean_e2el_ms": [ - 4288.3029, - 7429.9429, - 9451.6184, - 12607.5292, - 18153.2547 + 4517.3411, + 7857.9711, + 9861.1687, + 13283.3363, + 19541.0736 ] } }, @@ -251,10 +251,10 @@ "percentile-metrics": "ttft,tpot,itl,e2el", "baseline": { "mean_ttft_ms": [ - 275.3875 + 318.048 ], "mean_e2el_ms": [ - 730.6995 + 807.6114 ] } }, @@ -290,10 +290,10 @@ "percentile-metrics": "ttft,tpot,itl,e2el", "baseline": { "mean_ttft_ms": [ - 166.2201 + 186.168 ], "mean_e2el_ms": [ - 673.5789 + 726.1137 ] } }, @@ -331,10 +331,10 @@ "percentile-metrics": "ttft,tpot,itl,e2el", "baseline": { "mean_ttft_ms": [ - 357.7781 + 375.5282 ], "mean_e2el_ms": [ - 953.3493 + 1006.1663 ] } } diff --git a/tests/dfx/perf/tests/test_qwen3_omni_multi_replicas.json b/tests/dfx/perf/tests/test_qwen3_omni_multi_replicas.json index ac6f679c1..4e54c6f16 100644 --- a/tests/dfx/perf/tests/test_qwen3_omni_multi_replicas.json +++ b/tests/dfx/perf/tests/test_qwen3_omni_multi_replicas.json @@ -35,10 +35,10 @@ "ignore_eos": true, "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "mean_ttft_ms": [203, 283, 470, 869], - "mean_tpot_ms": [28, 36, 41, 45], - "mean_audio_ttfp_ms": [861, 1200, 1528, 2100], - "mean_audio_rtf": [0.24, 0.30, 0.35, 0.39] + "mean_ttft_ms": [130.5751, 278.6469, 324.7249, 911.6085], + "mean_tpot_ms": [7.9854, 11.1209, 16.1507, 20.7485], + "mean_audio_ttfp_ms": [486.486, 954.1791, 1289.6514, 2196.8275], + "mean_audio_rtf": [0.1896, 0.3962, 0.4098, 0.7322] } }, { @@ -61,9 +61,9 @@ }, "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "mean_ttft_ms": [377], - "mean_audio_ttfp_ms": [684], - "mean_audio_rtf": [0.14] + "mean_ttft_ms": [340.5122], + "mean_audio_ttfp_ms": [463.0752], + "mean_audio_rtf": [0.1153] } }, { @@ -88,9 +88,9 @@ }, "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "mean_ttft_ms": [228], - "mean_audio_ttfp_ms": [686], - "mean_audio_rtf": [0.16] + "mean_ttft_ms": [236.926], + "mean_audio_ttfp_ms": [395.2182], + "mean_audio_rtf": [0.1259] } }, { @@ -117,9 +117,9 @@ }, "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "mean_ttft_ms": [522], - "mean_audio_ttfp_ms": [1119], - "mean_audio_rtf": [0.21] + "mean_ttft_ms": [470.6126], + "mean_audio_ttfp_ms": [665.5474], + "mean_audio_rtf": [0.166] } } ] diff --git a/tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json b/tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json index 78daf691b..71c90f0e7 100644 --- a/tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json +++ b/tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json @@ -44,25 +44,25 @@ "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { "mean_ttft_ms": [ - 109.1218, - 247.4411, - 354.9384, - 447.7885, - 694.7487 + 115.8348, + 173.1823, + 446.672, + 464.5985, + 3420.0634 ], "mean_e2el_ms": [ - 22413.667, - 30808.9715, - 35677.844, - 46345.8978, - 68350.4853 + 25070.9523, + 31992.0646, + 37619.7669, + 50393.8257, + 136619.2743 ], "mean_audio_rtf": [ - 0.1695, - 0.2033, - 0.2467, - 0.3234, - 0.4438 + 0.169, + 0.2111, + 0.2475, + 0.3551, + 0.9266 ] } }, @@ -91,13 +91,13 @@ "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { "mean_ttft_ms": [ - 323.3573 + 343.1377 ], "mean_e2el_ms": [ - 4623.4173 + 4811.1123 ], "mean_audio_rtf": [ - 0.1406 + 0.1463 ] } }, @@ -128,13 +128,13 @@ "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { "mean_ttft_ms": [ - 215.4502 + 245.5446 ], "mean_e2el_ms": [ - 4924.149 + 5019.1256 ], "mean_audio_rtf": [ - 0.1637 + 0.1713 ] } }, @@ -167,13 +167,13 @@ "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { "mean_ttft_ms": [ - 442.5815 + 506.1407 ], "mean_e2el_ms": [ - 6441.0986 + 6814.9656 ], "mean_audio_rtf": [ - 0.2091 + 0.2213 ] } } diff --git a/tests/dfx/perf/tests/test_qwen3_omni_vllm_text.json b/tests/dfx/perf/tests/test_qwen3_omni_vllm_text.json index b9d553098..4dedcd4b9 100644 --- a/tests/dfx/perf/tests/test_qwen3_omni_vllm_text.json +++ b/tests/dfx/perf/tests/test_qwen3_omni_vllm_text.json @@ -52,18 +52,18 @@ "percentile-metrics": "ttft,tpot,itl,e2el", "baseline": { "mean_ttft_ms": [ - 69.1276, - 255.1882, - 310.639, - 422.0337, - 532.0582 + 69.8081, + 173.0651, + 271.8108, + 413.4079, + 497.2546 ], "mean_e2el_ms": [ - 4382.7211, - 7510.3815, - 9440.4854, - 12553.0186, - 16932.6065 + 4720.8225, + 7937.2532, + 9962.3566, + 13724.5479, + 18324.091 ] } }, @@ -98,10 +98,10 @@ "percentile-metrics": "ttft,tpot,itl,e2el", "baseline": { "mean_ttft_ms": [ - 26.3381 + 25.9477 ], "mean_e2el_ms": [ - 485.2436 + 517.3155 ] } }, @@ -138,10 +138,10 @@ "percentile-metrics": "ttft,tpot,itl,e2el", "baseline": { "mean_ttft_ms": [ - 26.4517 + 26.7056 ], "mean_e2el_ms": [ - 517.2017 + 551.308 ] } }, @@ -180,10 +180,10 @@ "percentile-metrics": "ttft,tpot,itl,e2el", "baseline": { "mean_ttft_ms": [ - 27.6972 + 27.6906 ], "mean_e2el_ms": [ - 561.3684 + 596.5284 ] } } diff --git a/tests/dfx/perf/tests/test_qwen_image_edit_2511_vllm_omni.json b/tests/dfx/perf/tests/test_qwen_image_edit_2511_vllm_omni.json index 53e13daed..350cfb6bb 100644 --- a/tests/dfx/perf/tests/test_qwen_image_edit_2511_vllm_omni.json +++ b/tests/dfx/perf/tests/test_qwen_image_edit_2511_vllm_omni.json @@ -36,8 +36,8 @@ "baseline": { "throughput_qps": 0.0721, "latency_mean": 13.8398, - "peak_memory_mb_max": 54731.4, - "peak_memory_mb_mean": 54731.4 + "peak_memory_mb_max": 57610.0, + "peak_memory_mb_mean": 57610.0 } }, { @@ -54,8 +54,8 @@ "baseline": { "throughput_qps": 0.0186, "latency_mean": 53.6942, - "peak_memory_mb_max": 64402.4, - "peak_memory_mb_mean": 64402.4 + "peak_memory_mb_max": 67790.0, + "peak_memory_mb_mean": 67790.0 } } ] @@ -101,8 +101,8 @@ "baseline": { "throughput_qps": 0.0538, "latency_mean": 18.5591, - "peak_memory_mb_max": 53485.0, - "peak_memory_mb_mean": 53485.0 + "peak_memory_mb_max": 56449.4286, + "peak_memory_mb_mean": 56449.4286 } } ] @@ -158,8 +158,8 @@ "baseline": { "throughput_qps": 0.2776, "latency_mean": 3.5938, - "peak_memory_mb_max": 55214.0, - "peak_memory_mb_mean": 55214.0 + "peak_memory_mb_max": 58113.7143, + "peak_memory_mb_mean": 58075.1429 } }, { @@ -176,8 +176,8 @@ "baseline": { "throughput_qps": 0.1051, "latency_mean": 9.4918, - "peak_memory_mb_max": 64265.6, - "peak_memory_mb_mean": 64265.6 + "peak_memory_mb_max": 68107.4286, + "peak_memory_mb_mean": 68107.4286 } } ] diff --git a/tests/dfx/perf/tests/test_qwen_image_edit_vllm_omni.json b/tests/dfx/perf/tests/test_qwen_image_edit_vllm_omni.json index bfda30731..9c5ad947b 100644 --- a/tests/dfx/perf/tests/test_qwen_image_edit_vllm_omni.json +++ b/tests/dfx/perf/tests/test_qwen_image_edit_vllm_omni.json @@ -35,8 +35,8 @@ "baseline": { "throughput_qps": 0.1129, "latency_mean": 8.8352, - "peak_memory_mb_max": 55356.5, - "peak_memory_mb_mean": 55356.5 + "peak_memory_mb_max": 58572.0, + "peak_memory_mb_mean": 58572.0 } }, { @@ -50,10 +50,10 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.0240, + "throughput_qps": 0.024, "latency_mean": 41.5686, - "peak_memory_mb_max": 62762.7, - "peak_memory_mb_mean": 62762.7 + "peak_memory_mb_max": 66368.0, + "peak_memory_mb_mean": 66368.0 } } ] @@ -98,8 +98,8 @@ "baseline": { "throughput_qps": 0.0672, "latency_mean": 14.8517, - "peak_memory_mb_max": 53931.5, - "peak_memory_mb_mean": 53931.5 + "peak_memory_mb_max": 57076.0, + "peak_memory_mb_mean": 57076.0 } } ] @@ -154,8 +154,8 @@ "baseline": { "throughput_qps": 0.2879, "latency_mean": 3.4701, - "peak_memory_mb_max": 55525.6, - "peak_memory_mb_mean": 55525.6 + "peak_memory_mb_max": 58758.0, + "peak_memory_mb_mean": 58758.0 } }, { @@ -171,8 +171,8 @@ "baseline": { "throughput_qps": 0.1113, "latency_mean": 8.9656, - "peak_memory_mb_max": 63758.3, - "peak_memory_mb_mean": 63758.3 + "peak_memory_mb_max": 67424.0, + "peak_memory_mb_mean": 67424.0 } } ] diff --git a/tests/dfx/perf/tests/test_qwen_image_layered_vllm_omni.json b/tests/dfx/perf/tests/test_qwen_image_layered_vllm_omni.json index 3445dd94c..0553d891e 100644 --- a/tests/dfx/perf/tests/test_qwen_image_layered_vllm_omni.json +++ b/tests/dfx/perf/tests/test_qwen_image_layered_vllm_omni.json @@ -35,8 +35,8 @@ "baseline": { "throughput_qps": 0.0703, "latency_mean": 14.208, - "peak_memory_mb_max": 60543.5, - "peak_memory_mb_mean": 60543.5 + "peak_memory_mb_max": 63225.1429, + "peak_memory_mb_mean": 63225.1429 } }, { @@ -52,8 +52,8 @@ "baseline": { "throughput_qps": 0.0417, "latency_mean": 23.914, - "peak_memory_mb_max": 60543.5, - "peak_memory_mb_mean": 60543.5 + "peak_memory_mb_max": 63225.1429, + "peak_memory_mb_mean": 63225.1429 } } ] diff --git a/tests/dfx/perf/tests/test_qwen_image_vllm_omni.json b/tests/dfx/perf/tests/test_qwen_image_vllm_omni.json index b3daf7587..3fb882563 100644 --- a/tests/dfx/perf/tests/test_qwen_image_vllm_omni.json +++ b/tests/dfx/perf/tests/test_qwen_image_vllm_omni.json @@ -35,7 +35,7 @@ "baseline": { "throughput_qps": 0.4153, "latency_mean": 2.4065, - "peak_memory_mb_mean": 52289.9 + "peak_memory_mb_mean": 55121.4286 } }, { @@ -51,7 +51,7 @@ "baseline": { "throughput_qps": 0.0423, "latency_mean": 23.5991, - "peak_memory_mb_mean": 61360.5 + "peak_memory_mb_mean": 65118.0 } } ] @@ -93,7 +93,7 @@ "baseline": { "throughput_qps": 0.4175, "latency_mean": 2.3953, - "peak_memory_mb_mean": 56000 + "peak_memory_mb_mean": 55088.5714 } }, { @@ -109,7 +109,7 @@ "baseline": { "throughput_qps": 0.0423, "latency_mean": 23.5768, - "peak_memory_mb_mean": 65000 + "peak_memory_mb_mean": 66012.2857 } } ] @@ -154,7 +154,7 @@ "baseline": { "throughput_qps": 0.1214, "latency_mean": 8.2211, - "peak_memory_mb_mean": 51704.7 + "peak_memory_mb_mean": 54501.1429 } } ] @@ -209,7 +209,7 @@ "baseline": { "throughput_qps": 0.559, "latency_mean": 1.794, - "peak_memory_mb_mean": 52397.6775 + "peak_memory_mb_mean": 55290.5714 } }, { @@ -225,7 +225,7 @@ "baseline": { "throughput_qps": 0.1918, "latency_mean": 5.2022, - "peak_memory_mb_mean": 61468.8 + "peak_memory_mb_mean": 64838.8571 } } ] @@ -271,7 +271,7 @@ "baseline": { "throughput_qps": [0.4125, 0.6714, 0.6674, 0.8159], "latency_mean": [2.4215, 2.992, 5.9293, 9.656], - "peak_memory_mb_mean": [67000, 67000, 67000, 67000] + "peak_memory_mb_mean": [55147.4286, 55148.0, 55150.4643, 55152.3821] } } ] diff --git a/tests/dfx/perf/tests/test_tts.json b/tests/dfx/perf/tests/test_tts.json index 85236eab7..91037aab6 100644 --- a/tests/dfx/perf/tests/test_tts.json +++ b/tests/dfx/perf/tests/test_tts.json @@ -40,10 +40,10 @@ "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { "median_audio_ttfp_ms": [ - 139.9434 + 141.2791 ], "median_audio_rtf": [ - 0.1683 + 0.1591 ] } }, @@ -68,7 +68,7 @@ "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { "median_audio_ttfp_ms": [ - 227.5653, + 226.747, 647.1272, 4777.2714 ], @@ -78,9 +78,9 @@ 1.2282 ], "audio_throughput": [ - 23.501, - 40.634, - 43.0758 + 24.2296, + 40.3552, + 43.2572 ] } }, @@ -154,10 +154,10 @@ "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { "median_audio_ttfp_ms": [ - 49.7908 + 46.8434 ], "median_audio_rtf": [ - 0.1425 + 0.1367 ] } }, @@ -197,9 +197,9 @@ 1.1662 ], "audio_throughput": [ - 36.465, - 49.6268, - 48.3091 + 38.3858, + 55.3523, + 68.7032 ] } }, @@ -253,13 +253,13 @@ "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { "median_audio_ttfp_ms": [ - 56.6977 + 56.746 ], "median_audio_rtf": [ - 0.1555 + 0.1571 ], "audio_throughput": [ - 11.5292 + 11.1226 ] } }, @@ -284,10 +284,10 @@ "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { "median_audio_ttfp_ms": [ - 52.0447 + 46.3479 ], "median_audio_rtf": [ - 0.1452 + 0.1352 ] } }, @@ -326,9 +326,9 @@ 1.1703 ], "audio_throughput": [ - 40.2198, - 48.9046, - 48.0505 + 38.9378, + 55.1846, + 71.5708 ] } }, @@ -353,13 +353,13 @@ "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { "median_audio_ttfp_ms": [ - 56.3961 + 55.5029 ], "median_audio_rtf": [ - 0.1542 + 0.1508 ], "audio_throughput": [ - 9.6672 + 9.4885 ] } } @@ -403,10 +403,10 @@ "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { "median_audio_ttfp_ms": [ - 150 + 82.1886 ], "median_audio_rtf": [ - 0.15 + 0.1241 ] } }, @@ -430,13 +430,13 @@ "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { "median_audio_ttfp_ms": [ - 1500 + 160.9168 ], "median_audio_rtf": [ - 0.3 + 0.1907 ], "audio_throughput": [ - 30.0 + 39.2332 ] } }, @@ -460,13 +460,13 @@ "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { "median_audio_ttfp_ms": [ - 3000 + 100.1265 ], "median_audio_rtf": [ - 0.25 + 0.1437 ], "audio_throughput": [ - 4.0 + 11.7605 ] } } diff --git a/tests/dfx/perf/tests/test_voxcpm2.json b/tests/dfx/perf/tests/test_voxcpm2.json index ef72b5d88..12afdc132 100644 --- a/tests/dfx/perf/tests/test_voxcpm2.json +++ b/tests/dfx/perf/tests/test_voxcpm2.json @@ -38,10 +38,10 @@ "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { "median_audio_ttfp_ms": [ - 130.2115 + 137.645 ], "median_audio_rtf": [ - 0.0488 + 0.0497 ] } }, @@ -65,10 +65,10 @@ 475.3357 ], "median_audio_rtf": [ - 0.1671 + 0.1243 ], "audio_throughput": [ - 22.8717 + 36.6358 ] } }, @@ -115,10 +115,10 @@ "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { "median_audio_ttfp_ms": [ - 75.7427 + 79.6927 ], "median_audio_rtf": [ - 0.0419 + 0.0423 ] } }, @@ -142,13 +142,13 @@ "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { "median_audio_ttfp_ms": [ - 351.5855 + 262.6212 ], "median_audio_rtf": [ - 0.1207 + 0.0753 ], "audio_throughput": [ - 73.7632 + 103.5843 ] } }, @@ -175,10 +175,10 @@ 80.1095 ], "median_audio_rtf": [ - 0.0485 + 0.0479 ], "audio_throughput": [ - 18.3715 + 17.2918 ] } }, diff --git a/tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json b/tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json index 78f720a6a..53c7da9b7 100644 --- a/tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json +++ b/tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json @@ -42,9 +42,9 @@ } ], "baseline": { - "throughput_qps": 0.0465, - "latency_mean": 21.4354, - "peak_memory_mb_mean": 72557.58 + "throughput_qps": 0.0434, + "latency_mean": 23.0654, + "peak_memory_mb_mean": 76360.0 } } ] @@ -96,9 +96,9 @@ } ], "baseline": { - "throughput_qps": 0.0642, - "latency_mean": 15.5589, - "peak_memory_mb_mean": 45742.88 + "throughput_qps": 0.0606, + "latency_mean": 16.4928, + "peak_memory_mb_mean": 47164.8 } }, { @@ -121,9 +121,9 @@ } ], "baseline": { - "throughput_qps": 0.0128, - "latency_mean": 77.8565, - "peak_memory_mb_mean": 53947.84 + "throughput_qps": 0.0124, + "latency_mean": 80.7784, + "peak_memory_mb_mean": 55374.3429 } } ] From 7924beef84ff2846833f01b123e3c52f7ddaa1c6 Mon Sep 17 00:00:00 2001 From: SYLAR <125541396+lishunyang12@users.noreply.github.com> Date: Tue, 21 Jul 2026 20:28:31 +0800 Subject: [PATCH 041/240] [Refactor] Migrate replica-DP Wan2.2 example to --deploy-config (#5267) Signed-off-by: lishunyang12 Co-authored-by: lishunyang12 --- .../replica_data_parallel/README.md | 11 +++-- .../replica_data_parallel/run_server.sh | 2 +- .../replica_data_parallel/wan2_2_ti2v_dp.yaml | 42 ++++++------------- 3 files changed, 19 insertions(+), 36 deletions(-) diff --git a/examples/online_serving/replica_data_parallel/README.md b/examples/online_serving/replica_data_parallel/README.md index 4bc98d6f7..a230d71eb 100644 --- a/examples/online_serving/replica_data_parallel/README.md +++ b/examples/online_serving/replica_data_parallel/README.md @@ -12,7 +12,7 @@ This example uses `Wan-AI/Wan2.2-TI2V-5B-Diffusers` with one GPU per replica. | File | Purpose | |------|---------| -| `wan2_2_ti2v_dp.yaml` | Diffusion stage config; `runtime.num_replicas` + `runtime.devices` drive the fan-out. | +| `wan2_2_ti2v_dp.yaml` | Deploy config; `num_replicas` + `devices` drive the fan-out. | | `run_server.sh` | Substitutes `NUM_REPLICAS` / `DEVICES` into the config and serves. | | `bench_replica_dp.py` | Replica-agnostic load driver; reports throughput, latency, and a per-input isolation check. | @@ -82,10 +82,9 @@ output under `N>=2` concurrency must match its own single-replica baseline. - `runtime.num_replicas: N` fans out `N` replicas; `runtime.devices` assigns their GPUs. With `tensor_parallel_size = 1`, list one GPU per replica (`num_replicas * tensor_parallel_size` entries, pool mode). -- Replica fan-out here comes from the stage config, not a CLI flag. (The headless / +- Replica fan-out here comes from the deploy config, not a CLI flag. (The headless / multi-runtime launch path uses the process-local `--omni-dp-size-local`, which requires `--stage-id`.) -- This recipe serves via `--stage-configs-path`. Expressing replica-DP under the - newer `--deploy-config` schema is part of the ongoing serving-config alignment - (see the diffusion parallelism guide and the `#4707` discussion); `num_replicas` - is accepted by both paths. +- This recipe serves via `--deploy-config`. Replica fan-out comes from the deploy + config's `num_replicas` / `devices`; stage topology comes from the registered + `WanPipeline`. diff --git a/examples/online_serving/replica_data_parallel/run_server.sh b/examples/online_serving/replica_data_parallel/run_server.sh index 915732eee..6b6f641fd 100755 --- a/examples/online_serving/replica_data_parallel/run_server.sh +++ b/examples/online_serving/replica_data_parallel/run_server.sh @@ -30,4 +30,4 @@ echo "Serving ${MODEL}: num_replicas=${NUM_REPLICAS} devices=${DEVICES} (config vllm serve "$MODEL" --omni \ --host "$HOST" \ --port "$PORT" \ - --stage-configs-path "$CFG" + --deploy-config "$CFG" diff --git a/examples/online_serving/replica_data_parallel/wan2_2_ti2v_dp.yaml b/examples/online_serving/replica_data_parallel/wan2_2_ti2v_dp.yaml index b472d7e2d..52cf99778 100644 --- a/examples/online_serving/replica_data_parallel/wan2_2_ti2v_dp.yaml +++ b/examples/online_serving/replica_data_parallel/wan2_2_ti2v_dp.yaml @@ -8,36 +8,20 @@ # run_server.sh substitutes __NUM_REPLICAS__ / __DEVICES__ from env before serving. # For tensor_parallel_size = 1, `devices` lists one GPU per replica (pool mode): # num_replicas=4 -> devices "0,1,2,3". +# +# Stage topology (diffusion DiT, video output) comes from the registered +# WanPipeline, which auto-detects TI2V-5B mode from model_index.json; this file +# carries deployment knobs only. +trust_remote_code: true +distributed_executor_backend: mp -stage_args: +stages: - stage_id: 0 - stage_type: diffusion - runtime: - num_replicas: __NUM_REPLICAS__ - devices: "__DEVICES__" - max_batch_size: 1 - engine_args: - model_stage: dit - # `model_class_name` takes the diffusion registry KEY (WanPipeline), which - # resolves to the Wan22Pipeline class. Wan2.2 TI2V-5B is served by the base - # WanPipeline, which auto-detects TI2V-5B mode from model_index.json. - model_class_name: WanPipeline - max_num_seqs: 1 - enforce_eager: true - trust_remote_code: true - distributed_executor_backend: "mp" - vae_use_tiling: true - parallel_config: - tensor_parallel_size: 1 - - final_output: true - final_output_type: video - is_comprehension: false + num_replicas: __NUM_REPLICAS__ + devices: "__DEVICES__" + max_num_seqs: 1 + tensor_parallel_size: 1 + enforce_eager: true + vae_use_tiling: true default_sampling_params: seed: 42 - -runtime: - enabled: true - defaults: - window_size: -1 - max_inflight: 1 From 7b5b399210dec4297954c3396b0b86062f8f837f Mon Sep 17 00:00:00 2001 From: wangyu <53896905+yenuo26@users.noreply.github.com> Date: Tue, 21 Jul 2026 21:47:55 +0800 Subject: [PATCH 042/240] [CI] skip test_audio_in_video_default_loader_sampling_regression (#5248) (#5280) Signed-off-by: wangyu <410167048@qq.com> --- tests/e2e/online_serving/test_qwen3_omni_expansion.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/e2e/online_serving/test_qwen3_omni_expansion.py b/tests/e2e/online_serving/test_qwen3_omni_expansion.py index 3ded285ce..d9fa2e6e2 100644 --- a/tests/e2e/online_serving/test_qwen3_omni_expansion.py +++ b/tests/e2e/online_serving/test_qwen3_omni_expansion.py @@ -411,6 +411,7 @@ def test_audio_in_video_002(omni_server, openai_client) -> None: openai_client.send_omni_request(request_config, request_num=get_max_batch_size()) +@pytest.mark.skip(reason="https://github.com/vllm-project/vllm-omni/issues/5248") @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_audio_in_video_default_loader_sampling_regression(omni_server, openai_client) -> None: From 9ca98009b2cadd35affb7f6a7befdc5b535bfe31 Mon Sep 17 00:00:00 2001 From: wangyu <53896905+yenuo26@users.noreply.github.com> Date: Tue, 21 Jul 2026 23:29:06 +0800 Subject: [PATCH 043/240] [Bugfix][CI] Recognize local_model in check-test-ci-coverage hook (#5275) Signed-off-by: wangyu <410167048@qq.com> --- tools/pre_commit/check_test_marks.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tools/pre_commit/check_test_marks.py b/tools/pre_commit/check_test_marks.py index 9a727ab72..2afef8b6e 100644 --- a/tools/pre_commit/check_test_marks.py +++ b/tools/pre_commit/check_test_marks.py @@ -4,8 +4,9 @@ """Run a pre-commit hook that fails if test files are modified or added that (probably) never run in the CI. For now, this means that every tests file needs to have a CI level marker (e.g., core_model, advanced_model, full_model, -etc) and hardware mark / helper so that we ensure mutated tests will actually -be selected as long as there are pytest commands pointing at the right paths. +local_model, slow, etc) and hardware mark / helper so that we ensure mutated +tests will actually be selected as long as there are pytest commands pointing +at the right paths. """ import os @@ -13,7 +14,7 @@ import re import sys # CI level markers -LEVEL_MARKERS = ("core_model", "advanced_model", "full_model", "slow") +LEVEL_MARKERS = ("core_model", "advanced_model", "full_model", "local_model", "slow") # Hardware markers. These are platforms and resources that CI filters on. # NOTE: If new hardware marks are added to pyproject.toml etc, we need From d34b91722f372bc6c2888821849540c22b572edc Mon Sep 17 00:00:00 2001 From: akshatvishu Date: Tue, 21 Jul 2026 22:19:18 +0530 Subject: [PATCH 044/240] [BugFix][Diffusion] Preserve image payloads with reasoning metadata (#5238) Signed-off-by: akshatvishu --- .../test_diffusion_output_formatter.py | 35 +++++++++++++++++++ vllm_omni/diffusion/output_formatter.py | 7 ++-- 2 files changed, 38 insertions(+), 4 deletions(-) diff --git a/tests/diffusion/test_diffusion_output_formatter.py b/tests/diffusion/test_diffusion_output_formatter.py index 22b305eab..11354bed6 100644 --- a/tests/diffusion/test_diffusion_output_formatter.py +++ b/tests/diffusion/test_diffusion_output_formatter.py @@ -241,6 +241,41 @@ def test_formatter_preserves_text_envelope_metadata(monkeypatch: pytest.MonkeyPa } +@pytest.mark.parametrize( + "text_metadata", + [ + pytest.param({"think_text": "reasoning"}, id="sensenova-bagel-thinking"), + pytest.param({"ar_generated_text": "reasoning"}, id="hunyuan-image3-cot"), + ], +) +def test_formatter_preserves_image_with_reasoning_metadata( + monkeypatch: pytest.MonkeyPatch, + text_metadata: dict[str, str], +) -> None: + monkeypatch.setattr(output_formatter, "supports_audio_output", lambda _: False) + metadata = {"text": text_metadata} + postprocess_output = normalize_diffusion_postprocess_output( + { + "payload": {"image": "image-0"}, + "metadata": metadata, + } + ) + + [result] = format_diffusion_outputs( + request=_request("generate an image"), + od_config=_config(), + diffusion_output=DiffusionOutput(output=None), + output_data=None, + postprocess_output=postprocess_output, + timings=_timings(), + ) + + assert postprocess_output.primary_key == "image" + assert result.images == ["image-0"] + assert result.final_output_type == "image" + assert result.multimodal_output == {"metadata": metadata} + + def test_formatter_preserves_audio_output_with_model_sample_rate_fallback( monkeypatch: pytest.MonkeyPatch, ) -> None: diff --git a/vllm_omni/diffusion/output_formatter.py b/vllm_omni/diffusion/output_formatter.py index a36305fce..e6e2fb966 100644 --- a/vllm_omni/diffusion/output_formatter.py +++ b/vllm_omni/diffusion/output_formatter.py @@ -149,10 +149,9 @@ def format_diffusion_outputs( "postprocess_time_ms": timings.postprocess_time_s * 1000, } - # Detect text output: when the pipeline returns a string (e.g., - # SenseNova-U1 / BAGEL single-stage img2text / text2text), wrap it - # as a text-type response instead of an image. - is_text_output = postprocess_output.primary_key == "text" or "text" in postprocess_output.metadata + # Only the primary payload determines the response type. Some image models + # include reasoning text in metadata, but their final output is still an image. + is_text_output = postprocess_output.primary_key == "text" is_audio_output = supports_audio_output(od_config.model_class_name) audio_sample_rate = _metadata_audio_sample_rate(postprocess_output.metadata) From ee33954dff27da317be597449a6c1b5a5df4052b Mon Sep 17 00:00:00 2001 From: Jae Date: Tue, 21 Jul 2026 14:58:56 -0400 Subject: [PATCH 045/240] [Model] Support Boogu/Boogu-Image-0.1-Base and Boogu/Boogu-Image-0.1-Edit (#4995) Signed-off-by: zzehli Signed-off-by: jl Signed-off-by: Gao Han Co-authored-by: Cursor --- .buildkite/cuda/test-nightly.yml | 4 + docs/models/supported_models.md | 1 + recipes/Boogu/Boogu-Image.md | 180 +++ recipes/README.md | 1 + .../test_boogu_image_edit_vllm_omni.json | 35 + .../tests/test_boogu_image_vllm_omni.json | 35 + .../diffusion/models/boogu_image/__init__.py | 0 .../boogu_image/test_boogu_image_scheduler.py | 161 ++ .../test_boogu_image_transformer.py | 433 ++++++ .../boogu_image/test_pipeline_boogu_image.py | 900 +++++++++++ tests/e2e/online_serving/test_boogu_image.py | 94 ++ .../online_serving/test_boogu_image_edit.py | 156 ++ .../test_boogu_image_edit_expansion.py | 107 ++ .../test_boogu_image_expansion.py | 127 ++ .../openai_api/test_image_server.py | 2 + tests/model_tests/diffusion/test_alignment.py | 1 + vllm_omni/diffusion/model_metadata.py | 9 + .../diffusion/models/boogu_image/__init__.py | 12 + .../boogu_image/boogu_image_transformer.py | 1372 +++++++++++++++++ .../models/boogu_image/image_processor.py | 213 +++ .../boogu_image/pipeline_boogu_image.py | 793 ++++++++++ ...flow_match_euler_discrete_time_shifting.py | 318 ++++ vllm_omni/diffusion/registry.py | 7 + vllm_omni/diffusion/request.py | 7 + vllm_omni/entrypoints/openai/api_server.py | 4 + vllm_omni/inputs/data.py | 1 + 26 files changed, 4973 insertions(+) create mode 100644 recipes/Boogu/Boogu-Image.md create mode 100644 tests/dfx/perf/tests/test_boogu_image_edit_vllm_omni.json create mode 100644 tests/dfx/perf/tests/test_boogu_image_vllm_omni.json create mode 100644 tests/diffusion/models/boogu_image/__init__.py create mode 100644 tests/diffusion/models/boogu_image/test_boogu_image_scheduler.py create mode 100644 tests/diffusion/models/boogu_image/test_boogu_image_transformer.py create mode 100644 tests/diffusion/models/boogu_image/test_pipeline_boogu_image.py create mode 100644 tests/e2e/online_serving/test_boogu_image.py create mode 100644 tests/e2e/online_serving/test_boogu_image_edit.py create mode 100644 tests/e2e/online_serving/test_boogu_image_edit_expansion.py create mode 100644 tests/e2e/online_serving/test_boogu_image_expansion.py create mode 100644 vllm_omni/diffusion/models/boogu_image/__init__.py create mode 100644 vllm_omni/diffusion/models/boogu_image/boogu_image_transformer.py create mode 100644 vllm_omni/diffusion/models/boogu_image/image_processor.py create mode 100644 vllm_omni/diffusion/models/boogu_image/pipeline_boogu_image.py create mode 100644 vllm_omni/diffusion/models/boogu_image/scheduling_flow_match_euler_discrete_time_shifting.py diff --git a/.buildkite/cuda/test-nightly.yml b/.buildkite/cuda/test-nightly.yml index d0f866b5a..9568f60c4 100644 --- a/.buildkite/cuda/test-nightly.yml +++ b/.buildkite/cuda/test-nightly.yml @@ -649,6 +649,10 @@ steps: tests/e2e/online_serving/test_qwen_image_expansion.py tests/e2e/online_serving/test_qwen_image_edit_expansion.py tests/e2e/online_serving/test_qwen_image_layered_expansion.py + tests/e2e/online_serving/test_boogu_image.py + tests/e2e/online_serving/test_boogu_image_edit.py + tests/e2e/online_serving/test_boogu_image_expansion.py + tests/e2e/online_serving/test_boogu_image_edit_expansion.py tests/e2e/online_serving/test_bagel_expansion.py tests/e2e/online_serving/test_krea2_expansion.py tests/e2e/offline_inference/test_krea2_expansion.py diff --git a/docs/models/supported_models.md b/docs/models/supported_models.md index dda7c5f2d..ca3c38a91 100644 --- a/docs/models/supported_models.md +++ b/docs/models/supported_models.md @@ -54,6 +54,7 @@ th { | `OvisImagePipeline` | Ovis-Image | `OvisAI/Ovis-Image` | ✅︎ | ✅︎ | | ✅︎ | | `LongcatImagePipeline` | LongCat-Image | `meituan-longcat/LongCat-Image` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | | `LongCatImageEditPipeline` | LongCat-Image-Edit | `meituan-longcat/LongCat-Image-Edit` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | +| `BooguImagePipeline` | Boogu-Image | `Boogu/Boogu-Image-0.1-Base`, `Boogu/Boogu-Image-0.1-Edit` | ✅︎ | | | | | `StableDiffusionXLPipeline` | Stable-Diffusion-XL | `stabilityai/stable-diffusion-xl-base-1.0` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | | `StableDiffusion3Pipeline` | Stable-Diffusion-3 | `stabilityai/stable-diffusion-3.5-medium` | ✅︎ | ✅︎ | | ✅︎ | | `CosyVoice3Model` | CosyVoice3 | `FunAudioLLM/Fun-CosyVoice3-0.5B-2512` | ✅︎ | ✅︎ | | ✅︎ | diff --git a/recipes/Boogu/Boogu-Image.md b/recipes/Boogu/Boogu-Image.md new file mode 100644 index 000000000..b9bff789f --- /dev/null +++ b/recipes/Boogu/Boogu-Image.md @@ -0,0 +1,180 @@ +# Boogu-Image + +> Text-to-image and image-editing online serving (Boogu-Image-0.1-Base / -Edit) + +## Summary + +- Vendor: Boogu +- Model: `Boogu/Boogu-Image-0.1-Base` (text-to-image), `Boogu/Boogu-Image-0.1-Edit` (image editing) +- Task: Text-to-image generation and text-guided image editing (TI2I) +- Mode: Online serving with the OpenAI-compatible API +- Maintainer: Community + +## When to use this recipe + +Use this recipe when you want a known-good starting point for serving +`Boogu/Boogu-Image-0.1-Base` with vLLM-Omni's native pipeline (no +`--diffusion-load-format diffusers`, and the upstream `boogu` package is not +required). + +Boogu-Image-0.1 is an Apache-2.0 unified image generation and editing model +family. This recipe covers the Base text-to-image checkpoint, which pairs a +Qwen3-VL multimodal encoder with a Diffusion Transformer (DiT) and a flow-match +Euler scheduler with time-shift. It handles photorealistic generation and +Chinese/English text rendering. + +## References + +- Upstream model card: +- Project page: +- GitHub: +- Related example: [`examples/online_serving/text_to_image/`](../../examples/online_serving/text_to_image/README.md) + +## Hardware Support + +This recipe documents tested configurations for CUDA GPU serving. The native +pipeline runs single-GPU; multi-GPU parallelism, CPU offload, and cache +acceleration are not yet supported for this model (see Notes). + +## GPU + +### 1 x A100/H100 (Single GPU, 40GB+ VRAM) + +The model footprint is roughly 34.6 GiB on GPU, so a 40GB+ card is recommended. + +#### Command + +```bash +vllm serve Boogu/Boogu-Image-0.1-Base --omni --port 8091 +``` + +!!! note + If you hit Out-of-Memory (OOM) on a smaller card, enable VAE slicing and + tiling to reduce peak memory: `--vae-use-slicing --vae-use-tiling`. + +#### Verification + +After the server is ready, test with a simple request: + +```bash +curl -X POST http://localhost:8091/v1/images/generations \ + -H "Content-Type: application/json" \ + -d '{ + "model": "Boogu/Boogu-Image-0.1-Base", + "prompt": "A mountain lake at sunset, photorealistic, cinematic lighting", + "size": "1024x1024", + "num_inference_steps": 28, + "guidance_scale": 4.0, + "seed": 42 + }' | jq -r '.data[0].b64_json' | base64 -d > output.png +``` + +Or via the chat-completions endpoint (parameters go in `extra_body`): + +```bash +curl -s http://localhost:8091/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d '{ + "messages": [ + {"role": "user", "content": "A mountain lake at sunset, photorealistic, cinematic lighting"} + ], + "extra_body": { + "height": 1024, + "width": 1024, + "num_inference_steps": 28, + "guidance_scale": 4.0, + "seed": 42 + } + }' | jq -r '.choices[0].message.content[0].image_url.url' | cut -d',' -f2- | base64 -d > output.png +``` + +#### Notes + +- **Memory usage:** ~34.6 GiB on GPU. Use `--vae-use-slicing --vae-use-tiling` + to trim peak VRAM if needed. +- **Key flags:** + - `--omni` — enables vLLM-Omni diffusion serving. +- **Guidance:** Boogu-Image uses `guidance_scale` (mapped to the upstream + `text_guidance_scale`); the default is `4.0`. Classifier-free guidance is + active whenever `guidance_scale > 1.0`. +- **Recommended settings:** `num_inference_steps=28`-`50`, `guidance_scale=4.0`. + The model's maximum native resolution is 2K. +- **Known limitations (not yet supported):** CPU offload + (`--enable-cpu-offload` / `--enable-layerwise-offload`), Cache-DiT + (`--cache-backend cache_dit`), and multi-GPU parallelism (TP / SP / CFG / + HSDP) are planned follow-ups and are not validated for this model yet. + +## Image editing (Boogu-Image-0.1-Edit) + +The Edit checkpoint is served by the same native pipeline (`BooguImagePipeline`); +the image-editing (TI2I) path activates automatically when a request carries a +reference image. The Base text-to-image path is unaffected (no reference image +is sent). + +#### Command + +```bash +vllm serve Boogu/Boogu-Image-0.1-Edit --omni --port 8091 +``` + +#### Verification + +Edit an image with `/v1/images/edits` (the model-card example — change a photo +to a colored-pencil drawing). Diffusion parameters are plain multipart form +fields; add `guidance_scale_2` to enable image guidance (double-guidance path): + +```bash +curl -s http://localhost:8091/v1/images/edits \ + -F model="Boogu/Boogu-Image-0.1-Edit" \ + -F image="@input.png" \ + -F prompt="Change the style to a colored pencil drawing." \ + -F num_inference_steps=28 \ + -F guidance_scale=5.0 \ + -F guidance_scale_2=2.0 \ + -F seed=42 \ + | jq -r '.data[0].b64_json' | base64 -d > edited.png +``` + +Or via chat completions (attach the image as a data URL; parameters go in +`extra_body`): + +```bash +curl -s http://localhost:8091/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d '{ + "messages": [ + {"role": "user", "content": [ + {"type": "image_url", "image_url": {"url": "data:image/png;base64,"}}, + {"type": "text", "text": "Change the style to a colored pencil drawing."} + ]} + ], + "extra_body": { + "num_inference_steps": 28, + "guidance_scale": 5.0, + "guidance_scale_2": 1.0, + "seed": 42 + } + }' | jq -r '.choices[0].message.content[0].image_url.url' | cut -d',' -f2- | base64 -d > edited.png +``` + +#### Notes + +- **Single reference image:** only one input image is supported for now (the + upstream "Only support 1 reference image for now" limit). +- **Guidance semantics:** + - `guidance_scale` = text guidance (upstream `text_guidance_scale`, default + `4.0`); `> 1.0` enables text CFG. Editing typically uses `5.0`. + - `guidance_scale_2` = image guidance (upstream `image_guidance_scale`, + default `1.0` = off). Setting it `> 1.0` enables the double-guidance path + (3 model predictions per step), steering more strongly toward the reference + image. +- **Output resolution:** the output size follows the reference image (upstream + `align_res`, on by default for a single reference), so `height`/`width` are + derived from the input and requested sizes are not applied for edits. +- **Same limitations** as the Base checkpoint apply (no CPU offload, Cache-DiT, + or multi-GPU parallelism yet). +- **Offline editing:** the shared offline example + [`examples/offline_inference/image_to_image/image_edit.py`](../../examples/offline_inference/image_to_image/image_to_image.md) + supports Boogu-Image-Edit directly + (`--model Boogu/Boogu-Image-0.1-Edit --guidance-scale 5.0`, optional + `--guidance-scale-2`). diff --git a/recipes/README.md b/recipes/README.md index be4c7cc08..7129918e1 100644 --- a/recipes/README.md +++ b/recipes/README.md @@ -37,6 +37,7 @@ recipes/ | [`audiox/AudioX.md`](./audiox/AudioX.md) | Offline + online unified text/video→audio diffusion | 1x L4 24GB | | [`Baidu/ERNIE-Image.md`](./Baidu/ERNIE-Image.md) | Text-to-image online serving (ERNIE-Image 8B) | 1x or 2x RTX 4090 24GB | | [`Bagel/BAGEL-7B-MoT.md`](./Bagel/BAGEL-7B-MoT.md) | Text-to-image with shared online/offline examples | 1x A100 80GB / 2x CUDA GPUs | +| [`Boogu/Boogu-Image.md`](./Boogu/Boogu-Image.md) | Text-to-image online serving (Boogu-Image-0.1-Base) | 1x A100/H100 40GB+ | | [`BosonAI/Higgs-Audio-V3-TTS.md`](./BosonAI/Higgs-Audio-V3-TTS.md) | Online + offline multilingual TTS with voice cloning | 1x H100 80GB | | [`ByteDance/Lance.md`](./ByteDance/Lance.md) | Unified AR+diffusion: text/img/video gen + understanding (Lance 3B) | 1x B300 / A100 80GB | | [`fishaudio/Fish-Speech-S2-Pro.md`](./fishaudio/Fish-Speech-S2-Pro.md) | Online serving for TTS | 1x A800 80GB | diff --git a/tests/dfx/perf/tests/test_boogu_image_edit_vllm_omni.json b/tests/dfx/perf/tests/test_boogu_image_edit_vllm_omni.json new file mode 100644 index 000000000..028889af1 --- /dev/null +++ b/tests/dfx/perf/tests/test_boogu_image_edit_vllm_omni.json @@ -0,0 +1,35 @@ +[ + { + "test_name": "test_boogu_image_edit_single_device", + "description": "Single-device baseline (no parallelism), concurrency sweep 1/2/4 at 512x512, 28 steps. Text-only edit path (2 predictions/step); a double-guidance (guidance_scale_2) case is a follow-up since the random dataset cannot inject extra_body params. Baselines recorded on 1x A40 46GB (measured c1/c2/c4: qps 0.0337/0.0336/0.0336, latency_mean 29.64/56.50/101.48 s, peak mem 36754 MB) with ~10% margin.", + "server_type": "vllm-omni", + "server_params": { + "model": "Boogu/Boogu-Image-0.1-Edit", + "serve_args": { + "enable-diffusion-pipeline-profiler": true + } + }, + "benchmark_params": [ + { + "name": "512x512_steps28_i2i", + "dataset": "random", + "task": "i2i", + "width": 512, + "height": 512, + "num-inference-steps": 28, + "num-prompts": 10, + "max-concurrency": [1, 2, 4], + "warmup-requests": 4, + "warmup-concurrency": 4, + "warmup-num-inference-steps": 28, + "enable-negative-prompt": true, + "baseline": { + "throughput_qps": {"1": 0.0303, "2": 0.0302, "4": 0.0302}, + "latency_mean": {"1": 32.6, "2": 62.2, "4": 111.7}, + "peak_memory_mb_max": 40500, + "peak_memory_mb_mean": 40500 + } + } + ] + } +] diff --git a/tests/dfx/perf/tests/test_boogu_image_vllm_omni.json b/tests/dfx/perf/tests/test_boogu_image_vllm_omni.json new file mode 100644 index 000000000..4ee6ca577 --- /dev/null +++ b/tests/dfx/perf/tests/test_boogu_image_vllm_omni.json @@ -0,0 +1,35 @@ +[ + { + "test_name": "test_boogu_image_single_device", + "description": "Single-device baseline (no parallelism), concurrency sweep 1/2/4 at 512x512, 28 steps. Baselines recorded on 1x A40 46GB (measured c1/c2/c4: qps 0.0611/0.0609/0.0611, latency_mean 16.37/31.22/55.77 s, peak mem 36754 MB) with ~10% margin.", + "server_type": "vllm-omni", + "server_params": { + "model": "Boogu/Boogu-Image-0.1-Base", + "serve_args": { + "enable-diffusion-pipeline-profiler": true + } + }, + "benchmark_params": [ + { + "name": "512x512_steps28", + "dataset": "random", + "task": "t2i", + "width": 512, + "height": 512, + "num-inference-steps": 28, + "num-prompts": 10, + "max-concurrency": [1, 2, 4], + "warmup-requests": 4, + "warmup-concurrency": 4, + "warmup-num-inference-steps": 28, + "enable-negative-prompt": true, + "baseline": { + "throughput_qps": {"1": 0.0549, "2": 0.0547, "4": 0.0549}, + "latency_mean": {"1": 18.0, "2": 34.3, "4": 61.4}, + "peak_memory_mb_max": 40500, + "peak_memory_mb_mean": 40500 + } + } + ] + } +] diff --git a/tests/diffusion/models/boogu_image/__init__.py b/tests/diffusion/models/boogu_image/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/diffusion/models/boogu_image/test_boogu_image_scheduler.py b/tests/diffusion/models/boogu_image/test_boogu_image_scheduler.py new file mode 100644 index 000000000..a92d9e979 --- /dev/null +++ b/tests/diffusion/models/boogu_image/test_boogu_image_scheduler.py @@ -0,0 +1,161 @@ +# SPDX-License-Identifier: Apache-2.0 +"""L1 unit tests for the Boogu time-shifting flow-match Euler scheduler. + +Golden values were generated from the upstream boogu scheduler +(scripts/test_ported_scheduler_parity.py) with the released checkpoint's +scheduler_config.json, so CI does not need the `boogu` package installed. +""" + +import numpy as np +import pytest +import torch + +from vllm_omni.diffusion.models.boogu_image import FlowMatchEulerDiscreteScheduler + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + +# Ground truth from the released checkpoint's scheduler/scheduler_config.json. +CHECKPOINT_CONFIG = dict( + num_train_timesteps=1000, + do_shift=True, + dynamic_time_shift=False, + time_shift_version="v1", + seq_len=4096, + base_shift=0.5, + max_shift=1.15, +) + +NUM_TOKENS = 4096 * 4 # 1024x1024 resolution + +# Generated from the upstream boogu scheduler with CHECKPOINT_CONFIG. +GOLDEN_TIMESTEPS = { + 4: [0.00000000, 0.09546924, 0.24048907, 0.48715585], + 10: [ + 0.00000000, + 0.03398621, + 0.07335263, + 0.11948693, + 0.17429835, + 0.24048907, + 0.32201332, + 0.42489707, + 0.55880034, + 0.74024153, + ], +} + + +def make_scheduler(**overrides) -> FlowMatchEulerDiscreteScheduler: + return FlowMatchEulerDiscreteScheduler(**{**CHECKPOINT_CONFIG, **overrides}) + + +@pytest.mark.parametrize("n_steps", sorted(GOLDEN_TIMESTEPS)) +def test_golden_timesteps_checkpoint_config(n_steps): + scheduler = make_scheduler() + scheduler.set_timesteps(num_inference_steps=n_steps, num_tokens=NUM_TOKENS) + np.testing.assert_allclose( + scheduler.timesteps.numpy(), + np.asarray(GOLDEN_TIMESTEPS[n_steps], dtype=np.float32), + atol=1e-7, + ) + + +@pytest.mark.parametrize("n_steps", [4, 10, 30]) +def test_timestep_grid_structure(n_steps): + scheduler = make_scheduler() + scheduler.set_timesteps(num_inference_steps=n_steps, num_tokens=NUM_TOKENS) + + assert scheduler.timesteps.shape == (n_steps,) + assert scheduler._timesteps.shape == (n_steps + 1,) + # Ascending 0 -> 1 convention with the terminal 1.0 appended. + assert scheduler.timesteps[0].item() == 0.0 + assert torch.all(scheduler._timesteps[1:] > scheduler._timesteps[:-1]) + assert scheduler._timesteps[-1].item() == 1.0 + assert torch.equal(scheduler._timesteps[:-1], scheduler.timesteps) + + +def test_no_shift_is_uniform_grid(): + n_steps = 8 + scheduler = make_scheduler(do_shift=False) + scheduler.set_timesteps(num_inference_steps=n_steps, num_tokens=NUM_TOKENS) + expected = np.linspace(0, 1, n_steps + 1, dtype=np.float32)[:-1] + np.testing.assert_allclose(scheduler.timesteps.numpy(), expected, atol=1e-7) + + +def test_static_shift_ignores_num_tokens(): + # dynamic_time_shift=False: the shift comes from configured seq_len, so + # per-call num_tokens must not change the schedule. + scheduler_a = make_scheduler() + scheduler_a.set_timesteps(num_inference_steps=10, num_tokens=NUM_TOKENS) + scheduler_b = make_scheduler() + scheduler_b.set_timesteps(num_inference_steps=10, num_tokens=320 * 320) + assert torch.equal(scheduler_a.timesteps, scheduler_b.timesteps) + + +def test_step_arithmetic_and_index_advancement(): + n_steps = 4 + scheduler = make_scheduler() + scheduler.set_timesteps(num_inference_steps=n_steps, num_tokens=NUM_TOKENS) + + sample = torch.zeros(1, 4, 8, 8) + model_output = torch.ones(1, 4, 8, 8) + + assert scheduler.step_index is None + out = scheduler.step(model_output, scheduler.timesteps[0], sample) + assert scheduler.step_index == 1 + + # prev_sample = sample + (t_next - t) * model_output + expected_dt = scheduler._timesteps[1] - scheduler._timesteps[0] + assert torch.allclose(out.prev_sample, expected_dt * model_output) + + # Full loop reaches the end of the grid. + sample = out.prev_sample + for t in scheduler.timesteps[1:]: + sample = scheduler.step(model_output, t, sample, return_dict=False)[0] + assert scheduler.step_index == n_steps + # Integrating a constant velocity of 1 over t in [0, 1] gives exactly 1. + assert torch.allclose(sample, torch.ones_like(sample), atol=1e-6) + + +def test_step_rejects_integer_timestep(): + scheduler = make_scheduler() + scheduler.set_timesteps(num_inference_steps=4, num_tokens=NUM_TOKENS) + with pytest.raises(ValueError, match="integer indices"): + scheduler.step(torch.zeros(1), 0, torch.zeros(1)) + + +def test_set_begin_index(): + scheduler = make_scheduler() + scheduler.set_timesteps(num_inference_steps=4, num_tokens=NUM_TOKENS) + scheduler.set_begin_index(2) + assert scheduler.begin_index == 2 + + scheduler.step(torch.zeros(1), scheduler.timesteps[2], torch.zeros(1)) + assert scheduler.step_index == 3 + + +def test_len_and_config_roundtrip(): + scheduler = make_scheduler() + assert len(scheduler) == CHECKPOINT_CONFIG["num_train_timesteps"] + # ConfigMixin must retain the checkpoint config (needed for from_pretrained). + assert scheduler.config.do_shift is True + assert scheduler.config.dynamic_time_shift is False + assert scheduler.config.time_shift_version == "v1" + assert scheduler.config.seq_len == 4096 + + +def test_matches_upstream_boogu_scheduler(): + """Differential check against the upstream boogu package, when installed.""" + boogu_schedulers = pytest.importorskip("boogu.schedulers.scheduling_flow_match_euler_discrete_time_shifting") + UpstreamScheduler = boogu_schedulers.FlowMatchEulerDiscreteScheduler + + for dynamic in (False, True): + for version in ("v1", "v2"): + for n_steps in (4, 10, 30): + config = {**CHECKPOINT_CONFIG, "dynamic_time_shift": dynamic, "time_shift_version": version} + upstream = UpstreamScheduler(**config) + ported = make_scheduler(**config) + upstream.set_timesteps(num_inference_steps=n_steps, num_tokens=NUM_TOKENS) + ported.set_timesteps(num_inference_steps=n_steps, num_tokens=NUM_TOKENS) + assert torch.equal(upstream.timesteps, ported.timesteps), (dynamic, version, n_steps) + assert torch.equal(upstream._timesteps, ported._timesteps), (dynamic, version, n_steps) diff --git a/tests/diffusion/models/boogu_image/test_boogu_image_transformer.py b/tests/diffusion/models/boogu_image/test_boogu_image_transformer.py new file mode 100644 index 000000000..5551ba191 --- /dev/null +++ b/tests/diffusion/models/boogu_image/test_boogu_image_transformer.py @@ -0,0 +1,433 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import os +from types import SimpleNamespace + +import pytest +import torch + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion] + +# Small mock dimensions (head_dim = HIDDEN_SIZE / NUM_HEADS = 16 must equal +# sum(AXES_DIM_ROPE)). +HIDDEN_SIZE = 64 +NUM_HEADS = 4 +NUM_KV_HEADS = 2 +HEAD_DIM = HIDDEN_SIZE // NUM_HEADS +AXES_DIM_ROPE = (8, 4, 4) +AXES_LENS = (32, 16, 16) +MULTIPLE_OF = 32 +NORM_EPS = 1e-5 + + +@pytest.fixture(autouse=True) +def _init_distributed(): + """Initialize the minimal single-rank distributed environment required by + the vLLM parallel linear layers (tensor-parallel group must exist).""" + from vllm.distributed.parallel_state import ( + cleanup_dist_env_and_memory, + init_distributed_environment, + initialize_model_parallel, + ) + + os.environ.setdefault("MASTER_ADDR", "localhost") + os.environ.setdefault("MASTER_PORT", "29512") + init_distributed_environment( + world_size=1, + rank=0, + local_rank=0, + distributed_init_method="env://", + ) + initialize_model_parallel() + yield + cleanup_dist_env_and_memory() + + +@pytest.fixture(autouse=True) +def _force_default_gemm(monkeypatch): + """Force CPU-compatible GEMM dispatch for tests using CPU tensors. + + vLLM's dispatch_unquantized_gemm() selects the backend by platform, not by + tensor device; CPU test tensors can crash on non-default backends.""" + from vllm.model_executor.layers.utils import default_unquantized_gemm + + monkeypatch.setattr( + "vllm.model_executor.layers.linear.dispatch_unquantized_gemm", + lambda: default_unquantized_gemm, + ) + + +def _randomize_parameters(module: torch.nn.Module) -> None: + """Fill parameters with small random values. + + vLLM parallel linears allocate weights with `torch.empty` (real weights + arrive via `load_weights`), so uninitialized memory must be overwritten + before a forward pass.""" + with torch.no_grad(): + for param in module.parameters(): + param.uniform_(-0.02, 0.02) + + +def _identity_rotary_emb(batch_size: int, seq_len: int) -> torch.Tensor: + """Complex rotary frequencies encoding a zero rotation.""" + return torch.polar( + torch.ones(batch_size, seq_len, HEAD_DIM // 2), + torch.zeros(batch_size, seq_len, HEAD_DIM // 2), + ) + + +def _tiny_tf_model_config(**overrides): + config = { + "patch_size": 2, + "in_channels": 4, + "hidden_size": HIDDEN_SIZE, + "num_layers": 4, + "num_double_stream_layers": 2, + "num_refiner_layers": 2, + "num_attention_heads": NUM_HEADS, + "num_kv_heads": NUM_KV_HEADS, + "multiple_of": MULTIPLE_OF, + "norm_eps": NORM_EPS, + "axes_dim_rope": list(AXES_DIM_ROPE), + "axes_lens": list(AXES_LENS), + "instruction_feature_configs": { + "instruction_feat_dim": 32, + "reduce_type": "mean", + "num_instruction_feature_layers": 1, + }, + "prompt_tuning_configs": {"use_prompt_tuning": False}, + "timestep_scale": 1.0, + } + config.update(overrides) + return config + + +def _tiny_od_config(**overrides): + from vllm_omni.diffusion.data import TransformerConfig + + return SimpleNamespace( + tf_model_config=TransformerConfig.from_dict(_tiny_tf_model_config(**overrides)), + dtype=torch.float32, + ) + + +def test_boogu_image_transformer_import(): + from vllm_omni.diffusion.models.boogu_image import BooguImageTransformer2DModel + + assert BooguImageTransformer2DModel is not None + + +def test_single_stream_block_shape(): + from vllm_omni.diffusion.models.boogu_image.boogu_image_transformer import ( + BooguImageTransformerBlock, + ) + + block = BooguImageTransformerBlock( + dim=HIDDEN_SIZE, + num_attention_heads=NUM_HEADS, + num_kv_heads=NUM_KV_HEADS, + multiple_of=MULTIPLE_OF, + ffn_dim_multiplier=None, + norm_eps=NORM_EPS, + modulation=True, + ) + _randomize_parameters(block) + + batch_size, seq_len = 1, 16 + hidden_states = torch.randn(batch_size, seq_len, HIDDEN_SIZE) + attention_mask = torch.ones(batch_size, seq_len, dtype=torch.bool) + rotary_emb = _identity_rotary_emb(batch_size, seq_len) + temb = torch.randn(batch_size, min(HIDDEN_SIZE, 1024)) + + out = block(hidden_states, attention_mask, rotary_emb, temb) + assert out.shape == hidden_states.shape + assert torch.isfinite(out).all() + + with pytest.raises(ValueError, match="temb"): + block(hidden_states, attention_mask, rotary_emb, None) + + +def test_single_stream_block_no_modulation(): + from vllm_omni.diffusion.models.boogu_image.boogu_image_transformer import ( + BooguImageTransformerBlock, + ) + + block = BooguImageTransformerBlock( + dim=HIDDEN_SIZE, + num_attention_heads=NUM_HEADS, + num_kv_heads=NUM_KV_HEADS, + multiple_of=MULTIPLE_OF, + ffn_dim_multiplier=None, + norm_eps=NORM_EPS, + modulation=False, + ) + _randomize_parameters(block) + + batch_size, seq_len = 2, 8 + hidden_states = torch.randn(batch_size, seq_len, HIDDEN_SIZE) + out = block(hidden_states, None, _identity_rotary_emb(batch_size, seq_len)) + assert out.shape == hidden_states.shape + assert torch.isfinite(out).all() + + +def test_double_stream_block_shape(): + from vllm_omni.diffusion.models.boogu_image.boogu_image_transformer import ( + BooguImageDoubleStreamTransformerBlock, + ) + + block = BooguImageDoubleStreamTransformerBlock( + dim=HIDDEN_SIZE, + num_attention_heads=NUM_HEADS, + num_kv_heads=NUM_KV_HEADS, + multiple_of=MULTIPLE_OF, + ffn_dim_multiplier=None, + norm_eps=NORM_EPS, + modulation=True, + ) + _randomize_parameters(block) + + batch_size = 1 + img_len, instruct_len = 16, 8 + total_len = img_len + instruct_len + + img_hidden_states = torch.randn(batch_size, img_len, HIDDEN_SIZE) + instruct_hidden_states = torch.randn(batch_size, instruct_len, HIDDEN_SIZE) + img_attention_mask = torch.ones(batch_size, img_len, dtype=torch.bool) + joint_attention_mask = torch.ones(batch_size, total_len, dtype=torch.bool) + image_rotary_emb = _identity_rotary_emb(batch_size, img_len) + rotary_emb = _identity_rotary_emb(batch_size, total_len) + temb = torch.randn(batch_size, min(HIDDEN_SIZE, 1024)) + + img_out, instruct_out = block( + img_hidden_states, + instruct_hidden_states, + img_attention_mask, + joint_attention_mask, + image_rotary_emb, + rotary_emb, + temb=temb, + encoder_seq_lengths=[instruct_len] * batch_size, + seq_lengths=[total_len] * batch_size, + ) + + assert img_out.shape == img_hidden_states.shape + assert instruct_out.shape == instruct_hidden_states.shape + assert torch.isfinite(img_out).all() + assert torch.isfinite(instruct_out).all() + + +def test_transformer_instantiates(): + from vllm_omni.diffusion.models.boogu_image.boogu_image_transformer import ( + BooguImageTransformer2DModel, + ) + + model = BooguImageTransformer2DModel(od_config=_tiny_od_config()) + + assert BooguImageTransformer2DModel._repeated_blocks == [ + "BooguImageTransformerBlock", + "BooguImageContextRefinerTransformerBlock", + "BooguImageSingleStreamTransformerBlock", + ] + assert BooguImageTransformer2DModel._layerwise_offload_blocks_attrs == [ + "single_stream_layers", + "double_stream_layers", + ] + + assert len(model.noise_refiner) == 2 + assert len(model.ref_image_refiner) == 2 + assert len(model.context_refiner) == 2 + assert len(model.double_stream_layers) == 2 + assert len(model.single_stream_layers) == 2 # num_layers - num_double_stream_layers + assert model.image_index_embedding.shape == (5, HIDDEN_SIZE) + # patch_size^2 * in_channels -> hidden_size + assert model.x_embedder.in_features == 2 * 2 * 4 + assert model.x_embedder.out_features == HIDDEN_SIZE + + +def test_transformer_preprocesses_multiple_instruction_feature_layers(): + from vllm_omni.diffusion.models.boogu_image.boogu_image_transformer import ( + BooguImageTransformer2DModel, + ) + + instruction_feature_configs = { + "instruction_feat_dim": 32, + "reduce_type": "concat", + "num_instruction_feature_layers": 2, + } + model = BooguImageTransformer2DModel( + od_config=_tiny_od_config(instruction_feature_configs=instruction_feature_configs) + ) + hidden_states = [torch.randn(1, 8, 32), torch.randn(1, 8, 32)] + + processed = model.preprocess_instruction_hidden_states(hidden_states) + + assert model.preprocessed_instruction_feat_dim == 64 + assert processed.shape == (1, 8, 64) + assert torch.equal(processed, torch.cat(hidden_states, dim=-1)) + + +def test_transformer_validates_rope_dims(): + from vllm_omni.diffusion.models.boogu_image.boogu_image_transformer import ( + BooguImageTransformer2DModel, + ) + + with pytest.raises(ValueError, match="axes_dim_rope"): + BooguImageTransformer2DModel(od_config=_tiny_od_config(axes_dim_rope=[8, 8, 8])) + + +def test_transformer_rejects_prompt_tuning(): + from vllm_omni.diffusion.models.boogu_image.boogu_image_transformer import ( + BooguImageTransformer2DModel, + ) + + with pytest.raises(NotImplementedError, match="[Pp]rompt tuning"): + BooguImageTransformer2DModel(od_config=_tiny_od_config(prompt_tuning_configs={"use_prompt_tuning": True})) + + +def _native_to_checkpoint_name(name: str) -> str: + """Inverse of ``load_weights`` remapping: native param name -> diffusers name. + + - ``.to_out.`` -> ``.to_out.0.`` (diffusers ModuleList wrap). + - promoted joint-attention projections move back under ``.processor.``. + """ + if ".to_out." in name: + name = name.replace(".to_out.", ".to_out.0.") + for proj in ( + "img_to_q", + "img_to_k", + "img_to_v", + "instruct_to_q", + "instruct_to_k", + "instruct_to_v", + "instruct_out", + "img_out", + ): + token = f".img_instruct_attn.{proj}." + if token in name: + name = name.replace(token, f".img_instruct_attn.processor.{proj}.") + break + return name + + +def test_transformer_load_weights_round_trip(): + from vllm_omni.diffusion.models.boogu_image.boogu_image_transformer import ( + BooguImageTransformer2DModel, + ) + + model = BooguImageTransformer2DModel(od_config=_tiny_od_config()) + native_params = dict(model.named_parameters()) + + # Build synthetic diffusers-named weights (one per native parameter). + checkpoint_weights = {} + for native_name, param in native_params.items(): + checkpoint_weights[_native_to_checkpoint_name(native_name)] = torch.randn_like(param) + + # The remapping must be a bijection over the parameter set. + assert len(checkpoint_weights) == len(native_params) + + loaded = model.load_weights(list(checkpoint_weights.items())) + + # No missing / unexpected parameters. + assert loaded == set(native_params.keys()) + + # Values landed on the right parameters (TP=1: weight_loader copies verbatim). + reloaded = dict(model.named_parameters()) + for native_name in native_params: + expected = checkpoint_weights[_native_to_checkpoint_name(native_name)] + assert torch.allclose(reloaded[native_name], expected) + + +def test_transformer_load_weights_warns_for_unexpected_and_unloaded(caplog): + from vllm_omni.diffusion.models.boogu_image.boogu_image_transformer import ( + BooguImageTransformer2DModel, + ) + + model = BooguImageTransformer2DModel(od_config=_tiny_od_config()) + native_params = dict(model.named_parameters()) + loaded_name = next(iter(native_params)) + checkpoint_name = _native_to_checkpoint_name(loaded_name) + + loaded = model.load_weights( + [ + (checkpoint_name, torch.randn_like(native_params[loaded_name])), + ("unexpected.weight", torch.ones(1)), + ] + ) + + assert loaded == {loaded_name} + assert "Skipping unexpected checkpoint weight unexpected.weight" in caplog.text + assert "Model parameters not loaded from checkpoint" in caplog.text + assert next(name for name in native_params if name != loaded_name) in caplog.text + + +def test_transformer_forward_t2i_shape(): + from vllm_omni.diffusion.models.boogu_image.boogu_image_transformer import ( + BooguImageDoubleStreamRotaryPosEmbed, + BooguImageTransformer2DModel, + ) + + model = BooguImageTransformer2DModel(od_config=_tiny_od_config()) + _randomize_parameters(model) + model.eval() + + batch_size = 1 + in_channels = 4 + latent_h = latent_w = 8 # multiples of patch_size (2) + instruct_len = 8 + instruction_feat_dim = 32 # matches _tiny_tf_model_config + + latents = torch.randn(batch_size, in_channels, latent_h, latent_w) + timestep = torch.full((batch_size,), 0.5) + instruction_hidden_states = torch.randn(batch_size, instruct_len, instruction_feat_dim) + instruction_attention_mask = torch.ones(batch_size, instruct_len, dtype=torch.bool) + freqs_cis = BooguImageDoubleStreamRotaryPosEmbed.get_freqs_cis(model.axes_dim_rope, model.axes_lens, theta=10000) + + with torch.no_grad(): + out = model(latents, timestep, instruction_hidden_states, freqs_cis, instruction_attention_mask) + + assert out.shape == (batch_size, model.out_channels, latent_h, latent_w) + assert torch.isfinite(out).all() + + +def test_transformer_forward_ti2i_shape(): + """Editing path: a non-empty ``ref_image_hidden_states`` exercises the + reference-image patch embedder + refiner and must not change the output + shape (the output tracks the noise-latent dimensions).""" + from vllm_omni.diffusion.models.boogu_image.boogu_image_transformer import ( + BooguImageDoubleStreamRotaryPosEmbed, + BooguImageTransformer2DModel, + ) + + model = BooguImageTransformer2DModel(od_config=_tiny_od_config()) + _randomize_parameters(model) + model.eval() + + batch_size = 1 + in_channels = 4 + latent_h = latent_w = 8 + ref_h, ref_w = 6, 10 # a differently-sized reference latent + instruct_len = 8 + instruction_feat_dim = 32 + + latents = torch.randn(batch_size, in_channels, latent_h, latent_w) + timestep = torch.full((batch_size,), 0.5) + instruction_hidden_states = torch.randn(batch_size, instruct_len, instruction_feat_dim) + instruction_attention_mask = torch.ones(batch_size, instruct_len, dtype=torch.bool) + freqs_cis = BooguImageDoubleStreamRotaryPosEmbed.get_freqs_cis(model.axes_dim_rope, model.axes_lens, theta=10000) + + # One sample, one reference image (Boogu editing supports a single ref). + ref_image_hidden_states = [[torch.randn(in_channels, ref_h, ref_w)]] + + with torch.no_grad(): + out = model( + latents, + timestep, + instruction_hidden_states, + freqs_cis, + instruction_attention_mask, + ref_image_hidden_states=ref_image_hidden_states, + ) + + assert out.shape == (batch_size, model.out_channels, latent_h, latent_w) + assert torch.isfinite(out).all() diff --git a/tests/diffusion/models/boogu_image/test_pipeline_boogu_image.py b/tests/diffusion/models/boogu_image/test_pipeline_boogu_image.py new file mode 100644 index 000000000..d38be97fa --- /dev/null +++ b/tests/diffusion/models/boogu_image/test_pipeline_boogu_image.py @@ -0,0 +1,900 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +"""L1 unit tests for the native Boogu-Image pipeline. + +Two groups: + +1. Constructor tests: all ``from_pretrained`` calls are mocked (Ovis pattern) + so the pipeline ``__init__`` wiring is exercised without downloading + weights or building the real transformer. +2. Prompt-encoding tests: a pipeline shell (``object.__new__``) is wired with + small deterministic fakes so the ported chat templating, processor kwargs, + CFG handling, and reshape logic can be verified numerically on CPU. +""" + +from types import SimpleNamespace + +import pytest +import torch +from torch import nn + +from vllm_omni.diffusion.data import DiffusionParallelConfig, OmniDiffusionConfig, TransformerConfig + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion] + +_MODULE = "vllm_omni.diffusion.models.boogu_image.pipeline_boogu_image" + +_EMBED_DIM = 8 +_SEQ_LEN = 6 + + +# --------------------------------------------------------------------------- +# Constructor tests (mocked from_pretrained) +# --------------------------------------------------------------------------- + + +@pytest.fixture +def mock_dependencies(mocker, monkeypatch): + """Mock external components so ``__init__`` runs without real weights.""" + # The full VLM wrapper has an ``lm_head``; the pipeline must strip it and + # keep the inner ``.model`` as the encoder. + inner_encoder = mocker.MagicMock(name="inner_qwen3vl_model") + inner_encoder.dtype = torch.float32 + mllm_wrapper = mocker.MagicMock(name="qwen3vl_wrapper") + mllm_wrapper.model.to.return_value = inner_encoder + + mock_processor = mocker.MagicMock(name="processor") + + mock_vae = mocker.MagicMock(name="vae") + mock_vae.config.block_out_channels = [128, 256, 512, 512] # scale factor 8 + mock_vae.to.return_value = mock_vae + + mock_scheduler = mocker.MagicMock(name="scheduler") + + monkeypatch.setattr( + f"{_MODULE}.FlowMatchEulerDiscreteScheduler.from_pretrained", + lambda *a, **k: mock_scheduler, + ) + monkeypatch.setattr( + f"{_MODULE}.Qwen3VLForConditionalGeneration.from_pretrained", + lambda *a, **k: mllm_wrapper, + ) + monkeypatch.setattr( + f"{_MODULE}.Qwen3VLProcessor.from_pretrained", + lambda *a, **k: mock_processor, + ) + monkeypatch.setattr( + f"{_MODULE}.AutoencoderKL.from_pretrained", + lambda *a, **k: mock_vae, + ) + + mock_transformer_cls = mocker.MagicMock(name="transformer_cls") + mock_transformer_instance = mocker.MagicMock(name="transformer") + mock_transformer_cls.return_value = mock_transformer_instance + monkeypatch.setattr(f"{_MODULE}.BooguImageTransformer2DModel", mock_transformer_cls) + + # Treat the dummy model id as a local path: skips hub prefetch. + mocker.patch("os.path.exists", return_value=True) + + return { + "inner_encoder": inner_encoder, + "mllm_wrapper": mllm_wrapper, + "processor": mock_processor, + "vae": mock_vae, + "scheduler": mock_scheduler, + "transformer": mock_transformer_instance, + } + + +@pytest.fixture +def boogu_pipeline(mock_dependencies): + from vllm_omni.diffusion.models.boogu_image.pipeline_boogu_image import ( + BooguImagePipeline, + ) + + od_config = OmniDiffusionConfig( + model="dummy-boogu", + tf_model_config=TransformerConfig(params={}), + dtype=torch.float32, + num_gpus=1, + ) + return BooguImagePipeline(od_config=od_config) + + +def test_boogu_image_pipeline_import(): + from vllm_omni.diffusion.models.boogu_image import BooguImagePipeline + + assert BooguImagePipeline is not None + + +def test_component_discovery_declarations(): + from vllm_omni.diffusion.models.boogu_image import BooguImagePipeline + + # CPU offload / HSDP discovery must find ``mllm`` (there is no + # ``text_encoder`` attribute on this pipeline). + assert BooguImagePipeline._dit_modules == ["transformer"] + assert BooguImagePipeline._encoder_modules == ["mllm"] + assert BooguImagePipeline._vae_modules == ["vae"] + + +def test_constructor_wires_components(boogu_pipeline, mock_dependencies): + assert boogu_pipeline.scheduler is mock_dependencies["scheduler"] + assert boogu_pipeline.processor is mock_dependencies["processor"] + assert boogu_pipeline.vae is mock_dependencies["vae"] + assert boogu_pipeline.transformer is mock_dependencies["transformer"] + assert boogu_pipeline.vae_scale_factor == 8 + assert boogu_pipeline.default_sample_size == 128 + assert hasattr(boogu_pipeline, "load_weights") + + +def test_constructor_strips_mllm_lm_head(boogu_pipeline, mock_dependencies): + # Upstream encodes with the inner Qwen3VLModel, not the generation wrapper. + assert boogu_pipeline.mllm is mock_dependencies["inner_encoder"] + + +def test_constructor_weights_sources(boogu_pipeline): + (source,) = boogu_pipeline.weights_sources + assert source.model_or_path == "dummy-boogu" + assert source.subfolder == "transformer" + assert source.prefix == "transformer." + assert source.fall_back_to_pt is True + + +@pytest.mark.parametrize( + ("parallel_config", "cache_backend", "message"), + [ + (DiffusionParallelConfig(tensor_parallel_size=2), "none", "Tensor parallelism"), + (DiffusionParallelConfig(ulysses_degree=2), "none", "Sequence parallelism"), + (DiffusionParallelConfig(ring_degree=2), "none", "Sequence parallelism"), + (DiffusionParallelConfig(cfg_parallel_size=2), "none", "CFG parallelism"), + ( + DiffusionParallelConfig(use_hsdp=True, hsdp_shard_size=2), + "none", + "HSDP", + ), + (DiffusionParallelConfig(), "cache_dit", "Cache backend 'cache_dit'"), + (DiffusionParallelConfig(), "tea_cache", "Cache backend 'tea_cache'"), + ], +) +def test_constructor_rejects_unsupported_execution_modes( + mock_dependencies, + parallel_config, + cache_backend, + message, +): + from vllm_omni.diffusion.models.boogu_image.pipeline_boogu_image import ( + BooguImagePipeline, + ) + + od_config = OmniDiffusionConfig( + model="dummy-boogu", + tf_model_config=TransformerConfig(params={}), + dtype=torch.float32, + parallel_config=parallel_config, + cache_backend=cache_backend, + ) + + with pytest.raises(NotImplementedError, match=message): + BooguImagePipeline(od_config=od_config) + + # Validation happens before any checkpoint component is constructed. + mock_dependencies["mllm_wrapper"].model.to.assert_not_called() + + +# --------------------------------------------------------------------------- +# Prompt-encoding tests (deterministic fakes, no constructor) +# --------------------------------------------------------------------------- + + +class _RecordingProcessor: + """Fake Qwen3VLProcessor: deterministic token ids derived from the text.""" + + def __init__(self): + self.calls = [] + + def apply_chat_template(self, prompts, **kwargs): + self.calls.append({"prompts": prompts, "kwargs": kwargs}) + batch = len(prompts) + input_ids = torch.zeros(batch, _SEQ_LEN, dtype=torch.long) + for i, messages in enumerate(prompts): + system_text = messages[0]["content"][0]["text"] + user_text = messages[1]["content"][0]["text"] + input_ids[i, 0] = len(system_text) % 997 + input_ids[i, 1] = len(user_text) % 997 + input_ids[i, 2:] = torch.arange(2, _SEQ_LEN) + i + attention_mask = torch.ones(batch, _SEQ_LEN, dtype=torch.long) + attention_mask[:, -1] = 0 # fake right-padding + return {"input_ids": input_ids, "attention_mask": attention_mask} + + +class _FakeMLLM: + """Fake Qwen3VLModel: hidden states are a deterministic function of ids.""" + + dtype = torch.bfloat16 + + def __init__(self): + self.calls = [] + + def __call__(self, input_ids=None, attention_mask=None, output_hidden_states=False, **kwargs): + self.calls.append({"output_hidden_states": output_hidden_states, **kwargs}) + base = input_ids.to(torch.float32).unsqueeze(-1).repeat(1, 1, _EMBED_DIM) + hidden = base + torch.arange(_EMBED_DIM, dtype=torch.float32) + return SimpleNamespace(last_hidden_state=hidden, hidden_states=(hidden - 1, hidden)) + + +def _make_encode_pipeline(num_instruction_feature_layers: int = 1): + from vllm_omni.diffusion.models.boogu_image.pipeline_boogu_image import ( + SYSTEM_PROMPT_4_T2I_UNIFIED, + SYSTEM_PROMPT_4_TI2I_UNIFIED, + BooguImagePipeline, + ) + + pipeline = object.__new__(BooguImagePipeline) + nn.Module.__init__(pipeline) + pipeline._execution_device = torch.device("cpu") + pipeline.processor = _RecordingProcessor() + pipeline.mllm = _FakeMLLM() + pipeline.transformer = SimpleNamespace( + instruction_feature_configs={ + "instruction_feat_dim": _EMBED_DIM, + "num_instruction_feature_layers": num_instruction_feature_layers, + "reduce_type": "mean", + }, + dtype=torch.float32, + ) + pipeline.SYSTEM_PROMPT_4_T2I = SYSTEM_PROMPT_4_T2I_UNIFIED + pipeline.SYSTEM_PROMPT_DROP = SYSTEM_PROMPT_4_TI2I_UNIFIED + return pipeline + + +def test_apply_chat_template_system_prompt_selection(): + pipeline = _make_encode_pipeline() + + messages = pipeline._apply_chat_template("a cat on a mat") + assert messages[0]["role"] == "system" + assert messages[0]["content"][0]["text"] == pipeline.SYSTEM_PROMPT_4_T2I + assert messages[1]["role"] == "user" + assert messages[1]["content"][0]["text"] == "a cat on a mat" + + # Empty and whitespace-only instructions select the DROP prompt (this is + # the path the default negative prompt "" takes). + for empty in ("", " ", None): + messages = pipeline._apply_chat_template(empty) + assert messages[0]["content"][0]["text"] == pipeline.SYSTEM_PROMPT_DROP + + +def test_processor_called_with_upstream_kwargs(): + pipeline = _make_encode_pipeline() + pipeline.encode_prompt("a dog", do_classifier_free_guidance=False) + + (call,) = pipeline.processor.calls + kwargs = call["kwargs"] + assert kwargs["padding"] == "longest" + assert kwargs["padding_side"] == "right" + assert kwargs["truncation"] is False + assert kwargs["max_length"] == 1280 # upstream __call__ default + assert kwargs["tokenize"] is True + assert kwargs["return_dict"] is True + assert kwargs["return_tensors"] == "pt" + + +def test_encode_prompt_shapes_and_dtype(): + pipeline = _make_encode_pipeline() + embeds, mask, neg_embeds, neg_mask = pipeline.encode_prompt(["a dog", "a cat"]) + + assert embeds.shape == (2, _SEQ_LEN, _EMBED_DIM) + assert mask.shape == (2, _SEQ_LEN) + assert neg_embeds.shape == (2, _SEQ_LEN, _EMBED_DIM) + assert neg_mask.shape == (2, _SEQ_LEN) + # Cast to the MLLM dtype, mask passed through from the processor. + assert embeds.dtype == torch.bfloat16 + assert neg_embeds.dtype == torch.bfloat16 + assert torch.equal(mask[:, -1], torch.zeros(2, dtype=torch.long)) + + +def test_single_layer_encoding_requests_hidden_states_once(): + pipeline = _make_encode_pipeline() + + pipeline._get_instruction_feature_embeds("a dog") + + assert pipeline.mllm.calls == [{"output_hidden_states": True, "return_dict": True}] + + +def test_single_layer_encoding_propagates_mllm_failure(): + pipeline = _make_encode_pipeline() + failure = RuntimeError("mllm failed") + + class _FailingMLLM: + dtype = torch.bfloat16 + + def __init__(self): + self.calls = 0 + + def __call__(self, **kwargs): + self.calls += 1 + raise failure + + pipeline.mllm = _FailingMLLM() + + with pytest.raises(RuntimeError, match="mllm failed") as exc_info: + pipeline._get_instruction_feature_embeds("a dog") + + assert exc_info.value is failure + assert pipeline.mllm.calls == 1 + + +def test_encode_prompt_cfg_negative_default_is_empty_string(): + pipeline = _make_encode_pipeline() + pipeline.encode_prompt("a dog") + + assert len(pipeline.processor.calls) == 2 + negative_messages = pipeline.processor.calls[1]["prompts"] + assert len(negative_messages) == 1 + # The default "" negative prompt goes through the DROP system prompt. + assert negative_messages[0][0]["content"][0]["text"] == pipeline.SYSTEM_PROMPT_DROP + assert negative_messages[0][1]["content"][0]["text"] == "" + + +def test_encode_prompt_without_cfg_skips_negative(): + pipeline = _make_encode_pipeline() + embeds, mask, neg_embeds, neg_mask = pipeline.encode_prompt("a dog", do_classifier_free_guidance=False) + + assert len(pipeline.processor.calls) == 1 + assert embeds.shape == (1, _SEQ_LEN, _EMBED_DIM) + assert neg_embeds is None + assert neg_mask is None + + +def test_encode_prompt_explicit_negative_prompt(): + pipeline = _make_encode_pipeline() + pipeline.encode_prompt("a dog", negative_prompt="blurry, low quality") + + negative_messages = pipeline.processor.calls[1]["prompts"] + assert negative_messages[0][0]["content"][0]["text"] == pipeline.SYSTEM_PROMPT_4_T2I + assert negative_messages[0][1]["content"][0]["text"] == "blurry, low quality" + + +def test_encode_prompt_negative_batch_mismatch_raises(): + pipeline = _make_encode_pipeline() + with pytest.raises(ValueError, match="batch size"): + pipeline.encode_prompt(["a dog", "a cat"], negative_prompt=["only one"]) + + +def test_encode_prompt_num_images_per_prompt_repeats_embeds(): + pipeline = _make_encode_pipeline() + embeds, mask, neg_embeds, neg_mask = pipeline.encode_prompt("a dog", num_images_per_prompt=3) + + assert embeds.shape == (3, _SEQ_LEN, _EMBED_DIM) + assert mask.shape == (3, _SEQ_LEN) + assert torch.equal(embeds[0], embeds[1]) + assert torch.equal(embeds[0], embeds[2]) + assert torch.equal(mask[0], mask[1]) + assert neg_embeds.shape == (3, _SEQ_LEN, _EMBED_DIM) + + +def test_encode_prompt_precomputed_embeds_bypass_encoder(): + pipeline = _make_encode_pipeline() + precomputed = torch.randn(1, _SEQ_LEN, _EMBED_DIM) + precomputed_mask = torch.ones(1, _SEQ_LEN, dtype=torch.long) + neg_precomputed = torch.randn(1, _SEQ_LEN, _EMBED_DIM) + + embeds, mask, neg_embeds, _ = pipeline.encode_prompt( + "ignored", + prompt_embeds=precomputed, + prompt_attention_mask=precomputed_mask, + negative_prompt_embeds=neg_precomputed, + ) + + assert len(pipeline.processor.calls) == 0 + assert torch.equal(embeds, precomputed) + assert torch.equal(neg_embeds, neg_precomputed) + + +def test_reshape_embeds_and_mask_list_branch(): + # Multi-layer configs return a list of per-layer tensors; the reshape + # helper must handle both forms. + pipeline = _make_encode_pipeline() + layer = torch.arange(2 * _SEQ_LEN * _EMBED_DIM, dtype=torch.float32).view(2, _SEQ_LEN, _EMBED_DIM) + mask = torch.ones(2, _SEQ_LEN, dtype=torch.long) + + batch_size, seq_len, reshaped, reshaped_mask = pipeline._reshape_embeds_and_mask([layer, layer + 1], mask, 2) + + assert batch_size == 2 + assert seq_len == _SEQ_LEN + assert isinstance(reshaped, list) and len(reshaped) == 2 + assert reshaped[0].shape == (4, _SEQ_LEN, _EMBED_DIM) + # repeat(1, n, 1).view(b*n, ...) interleaves per-sample: rows [s0, s0, s1, s1] + assert torch.equal(reshaped[0][0], reshaped[0][1]) + assert torch.equal(reshaped[0][2], reshaped[0][3]) + + +class _FakeTransformer: + """Fake denoiser: velocity 0 (identity flow) with the config the loop reads.""" + + in_channels = 4 + axes_dim_rope = (8, 4, 4) + axes_lens = (32, 16, 16) + dtype = torch.float32 + instruction_feature_configs = { + "instruction_feat_dim": _EMBED_DIM, + "num_instruction_feature_layers": 1, + "reduce_type": "mean", + } + + def __call__(self, latents, timestep, instruction_embeds, freqs_cis, instruction_attention_mask, **kwargs): + return torch.zeros_like(latents) + + +class _FakeScheduler: + def set_timesteps(self, num_inference_steps, device=None, num_tokens=None): + self.timesteps = torch.linspace(0, 1, num_inference_steps + 1)[:-1] + + def step(self, model_output, t, latents, return_dict=False): + return (latents,) + + +class _FakeDecodeVAE: + dtype = torch.float32 + + def __init__(self): + self.config = SimpleNamespace(scaling_factor=1.0, shift_factor=0.0, block_out_channels=[128, 256, 512, 512]) + + def decode(self, latents, return_dict=False): + batch = latents.shape[0] + return (torch.zeros(batch, 3, 16, 16),) + + +def _make_forward_pipeline(): + pipeline = _make_encode_pipeline() + pipeline.transformer = _FakeTransformer() + pipeline.scheduler = _FakeScheduler() + pipeline.vae = _FakeDecodeVAE() + pipeline.vae_scale_factor = 8 + pipeline.default_sample_size = 128 + return pipeline + + +def _make_request_batch(prompt, **sampling_overrides): + from vllm_omni.inputs.data import OmniDiffusionSamplingParams + + sampling = OmniDiffusionSamplingParams(**sampling_overrides) + return SimpleNamespace(prompts=[prompt], sampling_params=sampling) + + +def test_forward_returns_diffusion_output(): + from vllm_omni.diffusion.data import DiffusionOutput + + pipeline = _make_forward_pipeline() + req = _make_request_batch("a cat", height=64, width=64, num_inference_steps=2) + + out = pipeline.forward(req) + + assert isinstance(out, DiffusionOutput) + assert isinstance(out.output, torch.Tensor) + assert out.output.shape[0] == 1 + assert torch.isfinite(out.output).all() + # CFG default is on (text guidance 4.0), so the encoder ran twice (pos + neg). + assert len(pipeline.processor.calls) == 2 + + +def test_forward_cfg_off_when_guidance_one(): + pipeline = _make_forward_pipeline() + req = _make_request_batch("a cat", height=64, width=64, num_inference_steps=2, guidance_scale=1.0) + # guidance_scale=1.0 is falsy-adjacent but explicitly disables CFG; the + # request layer would set guidance_scale_provided, so emulate that here. + req.sampling_params.guidance_scale_provided = True + + pipeline.forward(req) + + # Only the positive prompt is encoded when CFG is off. + assert len(pipeline.processor.calls) == 1 + + +# --------------------------------------------------------------------------- +# Editing / TI2I: pre-process function +# --------------------------------------------------------------------------- + + +def _make_edit_od_config(tmp_path, block_out_channels=(128, 256, 512, 512)): + import json + + vae_dir = tmp_path / "vae" + vae_dir.mkdir(parents=True, exist_ok=True) + (vae_dir / "config.json").write_text(json.dumps({"block_out_channels": list(block_out_channels)})) + return OmniDiffusionConfig( + model=str(tmp_path), + tf_model_config=TransformerConfig(params={}), + dtype=torch.float32, + num_gpus=1, + ) + + +@pytest.mark.parametrize( + "factory_name", + ["get_boogu_image_pre_process_func", "get_boogu_image_post_process_func"], +) +@pytest.mark.parametrize("config_contents", [None, "{not valid json"]) +def test_process_factory_reports_invalid_vae_config(tmp_path, factory_name, config_contents): + import importlib + + vae_dir = tmp_path / "vae" + vae_dir.mkdir() + if config_contents is not None: + (vae_dir / "config.json").write_text(config_contents) + + od_config = OmniDiffusionConfig( + model=str(tmp_path), + tf_model_config=TransformerConfig(params={}), + dtype=torch.float32, + num_gpus=1, + ) + factory = getattr(importlib.import_module(_MODULE), factory_name) + + with pytest.raises(RuntimeError, match=r"Failed to load Boogu VAE config from .*vae/config\.json"): + factory(od_config) + + +@pytest.mark.parametrize(("height", "width"), [(0, 16), (16, 0), (0, 0)]) +def test_image_processor_rejects_non_positive_dimensions(height, width): + from vllm_omni.diffusion.models.boogu_image.image_processor import BooguImageProcessor + + image_processor = BooguImageProcessor() + image = torch.zeros(1, 3, 16, 16) + + with pytest.raises(ValueError, match=rf"height={height}, width={width}"): + image_processor.get_new_height_width(image, height=height, width=width) + + +def _make_diffusion_request(prompt, **sampling_overrides): + from vllm_omni.diffusion.request import OmniDiffusionRequest + from vllm_omni.inputs.data import OmniDiffusionSamplingParams + + sampling = OmniDiffusionSamplingParams(**sampling_overrides) + return OmniDiffusionRequest(prompt=prompt, sampling_params=sampling, request_id="req-0") + + +def test_pre_process_no_image_is_noop(tmp_path): + import PIL.Image # noqa: F401 (import guard: PIL must be available) + + from vllm_omni.diffusion.models.boogu_image.pipeline_boogu_image import ( + get_boogu_image_pre_process_func, + ) + + pre = get_boogu_image_pre_process_func(_make_edit_od_config(tmp_path)) + + # Text-to-image request: no multimodal image -> returned unchanged. + req = _make_diffusion_request({"prompt": "a cat"}, height=123, width=456) + out = pre(req) + assert "additional_information" not in out.prompt + assert out.sampling_params.height == 123 + assert out.sampling_params.width == 456 + + # A plain-string prompt cannot carry an image either. + str_req = _make_diffusion_request("a cat") + assert pre(str_req).prompt == "a cat" + + +def test_pre_process_populates_reference_and_align_res(tmp_path): + import PIL.Image + + from vllm_omni.diffusion.models.boogu_image.pipeline_boogu_image import ( + get_boogu_image_pre_process_func, + ) + + pre = get_boogu_image_pre_process_func(_make_edit_od_config(tmp_path)) + + image = PIL.Image.new("RGB", (1000, 500)) # (width, height) + req = _make_diffusion_request({"prompt": "make it winter", "multi_modal_data": {"image": image}}) + out = pre(req) + + ai = out.prompt["additional_information"] + assert "prompt_image" in ai and "preprocessed_image" in ai + + # VLM copy is a PIL image, downscaled, never upscaled, aligned to 16. + prompt_image = ai["prompt_image"] + assert isinstance(prompt_image, PIL.Image.Image) + assert prompt_image.width % 16 == 0 and prompt_image.height % 16 == 0 + assert prompt_image.width <= 1000 and prompt_image.height <= 500 + assert max(prompt_image.width, prompt_image.height) <= 768 + + # VAE copy is a normalized [1, C, H, W] tensor aligned to 16. + vae = ai["preprocessed_image"] + assert isinstance(vae, torch.Tensor) and vae.ndim == 4 and vae.shape[0] == 1 + assert vae.shape[-1] % 16 == 0 and vae.shape[-2] % 16 == 0 + assert -1.0 <= float(vae.min()) and float(vae.max()) <= 1.0 + + # align_res: the request resolution follows the VAE-encoded reference dims. + assert out.sampling_params.height == vae.shape[-2] + assert out.sampling_params.width == vae.shape[-1] + + +def test_pre_process_rejects_multiple_images(tmp_path): + import PIL.Image + + from vllm_omni.diffusion.models.boogu_image.pipeline_boogu_image import ( + get_boogu_image_pre_process_func, + ) + + pre = get_boogu_image_pre_process_func(_make_edit_od_config(tmp_path)) + imgs = [PIL.Image.new("RGB", (64, 64)), PIL.Image.new("RGB", (64, 64))] + req = _make_diffusion_request({"prompt": "combine", "multi_modal_data": {"image": imgs}}) + + with pytest.raises(ValueError, match="single reference image"): + pre(req) + + +def test_pre_process_single_image_in_list_is_accepted(tmp_path): + import PIL.Image + + from vllm_omni.diffusion.models.boogu_image.pipeline_boogu_image import ( + get_boogu_image_pre_process_func, + ) + + pre = get_boogu_image_pre_process_func(_make_edit_od_config(tmp_path)) + req = _make_diffusion_request({"prompt": "edit", "multi_modal_data": {"image": [PIL.Image.new("RGB", (128, 128))]}}) + out = pre(req) + assert "preprocessed_image" in out.prompt["additional_information"] + + +# --------------------------------------------------------------------------- +# Editing / TI2I: chat template + image-aware encoding +# --------------------------------------------------------------------------- + + +def _make_edit_encode_pipeline(): + from vllm_omni.diffusion.models.boogu_image.pipeline_boogu_image import ( + SYSTEM_PROMPT_4_TI2I_UNIFIED, + ) + + pipeline = _make_encode_pipeline() + pipeline.SYSTEM_PROMPT_4_TI2I = SYSTEM_PROMPT_4_TI2I_UNIFIED + pipeline.SYSTEM_PROMPT_4_I2I = SYSTEM_PROMPT_4_TI2I_UNIFIED + return pipeline + + +def test_apply_chat_template_ti2i_places_image_before_text(): + import PIL.Image + + pipeline = _make_edit_encode_pipeline() + image = PIL.Image.new("RGB", (16, 16)) + + messages = pipeline._apply_chat_template("turn day into night", [image]) + assert messages[0]["content"][0]["text"] == pipeline.SYSTEM_PROMPT_4_TI2I + user_content = messages[1]["content"] + # Image content comes first, then the instruction text. + assert user_content[0]["type"] == "image" + assert user_content[0]["image"] is image + assert user_content[-1] == {"type": "text", "text": "turn day into night"} + + # Empty instruction with an image selects the I2I system prompt. + empty_messages = pipeline._apply_chat_template("", [image]) + assert empty_messages[0]["content"][0]["text"] == pipeline.SYSTEM_PROMPT_4_I2I + + +class _ImageAwareRecordingProcessor: + """Records whether reference images reached the processor.""" + + def __init__(self): + self.calls = [] + + def apply_chat_template(self, prompts, **kwargs): + has_image = [] + for messages in prompts: + user_content = messages[1]["content"] + has_image.append(any(c.get("type") == "image" for c in user_content)) + self.calls.append({"prompts": prompts, "kwargs": kwargs, "has_image": has_image}) + batch = len(prompts) + input_ids = torch.arange(batch * _SEQ_LEN, dtype=torch.long).view(batch, _SEQ_LEN) + attention_mask = torch.ones(batch, _SEQ_LEN, dtype=torch.long) + return {"input_ids": input_ids, "attention_mask": attention_mask} + + +def test_encode_prompt_attaches_images_to_positive_only(): + import PIL.Image + + pipeline = _make_edit_encode_pipeline() + pipeline.processor = _ImageAwareRecordingProcessor() + image = PIL.Image.new("RGB", (16, 16)) + + pipeline.encode_prompt( + "add a rainbow", + do_classifier_free_guidance=True, + input_images=[[image]], + ) + + # Positive call carries the image; the negative (CFG) call is image-free. + assert pipeline.processor.calls[0]["has_image"] == [True] + assert pipeline.processor.calls[1]["has_image"] == [False] + + +# --------------------------------------------------------------------------- +# Editing / TI2I: reference-latent VAE encode +# --------------------------------------------------------------------------- + + +class _FakeRefVAE: + """Fake VAE whose ``encode`` yields a known latent for shape/scaling checks.""" + + dtype = torch.float32 + + def __init__(self, scaling_factor=2.0, shift_factor=0.5): + self.config = SimpleNamespace(scaling_factor=scaling_factor, shift_factor=shift_factor) + self._latent = torch.ones(1, 4, 3, 5) + + def encode(self, img): + dist = SimpleNamespace(sample=lambda generator=None: self._latent.clone()) + return SimpleNamespace(latent_dist=dist) + + +def test_build_ref_latents_shape_and_scaling(): + pipeline = _make_encode_pipeline() + pipeline.vae = _FakeRefVAE(scaling_factor=2.0, shift_factor=0.5) + + preprocessed = torch.zeros(1, 3, 48, 80) # normalized image tensor + ref_latents = pipeline._build_ref_latents([preprocessed], num_images_per_prompt=1, device=torch.device("cpu")) + + assert len(ref_latents) == 1 + (sample_latents,) = ref_latents + assert isinstance(sample_latents, list) and len(sample_latents) == 1 + latent = sample_latents[0] + # squeeze(0) -> [C, H, W]; (1 - shift) * scaling = (1 - 0.5) * 2 = 1.0 + assert latent.shape == (4, 3, 5) + assert torch.allclose(latent, torch.ones(4, 3, 5)) + + +def test_build_ref_latents_expands_per_output_and_handles_none(): + pipeline = _make_encode_pipeline() + pipeline.vae = _FakeRefVAE() + + preprocessed = torch.zeros(1, 3, 48, 80) + ref_latents = pipeline._build_ref_latents([preprocessed, None], num_images_per_prompt=2, device=torch.device("cpu")) + + # Two samples x 2 outputs each = 4 entries; the None sample stays None. + assert len(ref_latents) == 4 + assert ref_latents[0] is ref_latents[1] # same sample repeated + assert ref_latents[2] is None and ref_latents[3] is None + + +# --------------------------------------------------------------------------- +# Editing / TI2I: forward CFG branch selection +# --------------------------------------------------------------------------- + + +class _RecordingRefTransformer(_FakeTransformer): + """Counts predictions per step and records the reference-latent argument.""" + + def __init__(self): + self.calls = [] + + def __call__(self, latents, timestep, instruction_embeds, freqs_cis, instruction_attention_mask, **kwargs): + self.calls.append(kwargs.get("ref_image_hidden_states")) + return torch.zeros_like(latents) + + +class _EditForwardVAE(_FakeDecodeVAE): + """Adds a fake ``encode`` so the editing forward path can build ref latents.""" + + def encode(self, img): + dist = SimpleNamespace(sample=lambda generator=None: torch.zeros(1, 4, 8, 8)) + return SimpleNamespace(latent_dist=dist) + + +def _make_edit_forward_pipeline(): + pipeline = _make_edit_encode_pipeline() + # Image-aware processor: reference images appear as ``{"type": "image"}`` + # content entries, which the default text-first fake cannot parse. + pipeline.processor = _ImageAwareRecordingProcessor() + pipeline.transformer = _RecordingRefTransformer() + pipeline.scheduler = _FakeScheduler() + pipeline.vae = _EditForwardVAE() + pipeline.vae_scale_factor = 8 + pipeline.default_sample_size = 128 + return pipeline + + +def _make_edit_request(**sampling_overrides): + import PIL.Image + + image = PIL.Image.new("RGB", (64, 64)) + prompt = { + "prompt": "make it winter", + "additional_information": { + "prompt_image": image, + "preprocessed_image": torch.zeros(1, 3, 64, 64), + }, + } + sampling = _sampling(**sampling_overrides) + return SimpleNamespace(prompts=[prompt], sampling_params=sampling) + + +def _sampling(**overrides): + from vllm_omni.inputs.data import OmniDiffusionSamplingParams + + return OmniDiffusionSamplingParams(**overrides) + + +def _count_per_step(calls, num_steps): + assert len(calls) % num_steps == 0 + return len(calls) // num_steps + + +def test_forward_ti2i_text_only_two_predictions_with_ref(): + pipeline = _make_edit_forward_pipeline() + num_steps = 2 + req = _make_edit_request(height=64, width=64, num_inference_steps=num_steps, guidance_scale=5.0) + req.sampling_params.guidance_scale_provided = True + + pipeline.forward(req) + + # Text-only ti2i: cond+ref and neg+ref -> 2 predictions/step, both carry ref. + assert _count_per_step(pipeline.transformer.calls, num_steps) == 2 + assert all(ref is not None for ref in pipeline.transformer.calls) + + +def test_forward_ti2i_double_guidance_three_predictions(): + pipeline = _make_edit_forward_pipeline() + num_steps = 2 + req = _make_edit_request( + height=64, width=64, num_inference_steps=num_steps, guidance_scale=5.0, guidance_scale_2=2.0 + ) + req.sampling_params.guidance_scale_provided = True + req.sampling_params.guidance_scale_2_provided = True + + pipeline.forward(req) + + # Double guidance: cond+ref, neg+ref, neg+no-ref -> 3 predictions/step. + assert _count_per_step(pipeline.transformer.calls, num_steps) == 3 + # Exactly one of the three per step drops the reference (neg+no-ref). + per_step = [pipeline.transformer.calls[i : i + 3] for i in range(0, len(pipeline.transformer.calls), 3)] + for step_calls in per_step: + assert sum(ref is None for ref in step_calls) == 1 + + +def test_forward_ti2i_image_only_two_predictions_drop_ref(): + pipeline = _make_edit_forward_pipeline() + num_steps = 2 + # text guidance 1.0 (off, provided) + image guidance 2.0 (provided). + req = _make_edit_request( + height=64, width=64, num_inference_steps=num_steps, guidance_scale=1.0, guidance_scale_2=2.0 + ) + req.sampling_params.guidance_scale_provided = True + req.sampling_params.guidance_scale_2_provided = True + + pipeline.forward(req) + + # Image-only ti2i: cond+ref and cond+no-ref -> 2 predictions/step. + assert _count_per_step(pipeline.transformer.calls, num_steps) == 2 + per_step = [pipeline.transformer.calls[i : i + 2] for i in range(0, len(pipeline.transformer.calls), 2)] + for step_calls in per_step: + assert sum(ref is None for ref in step_calls) == 1 + + +def test_forward_ti2i_no_guidance_single_prediction_with_ref(): + pipeline = _make_edit_forward_pipeline() + num_steps = 2 + # Both guidances off/unprovided -> image guidance forced to 1.0, no CFG. + req = _make_edit_request(height=64, width=64, num_inference_steps=num_steps, guidance_scale=1.0) + req.sampling_params.guidance_scale_provided = True + + pipeline.forward(req) + + assert _count_per_step(pipeline.transformer.calls, num_steps) == 1 + assert all(ref is not None for ref in pipeline.transformer.calls) + + +def test_forward_image_guidance_ignored_without_reference(): + # guidance_scale_2 is set but there is no reference image (t2i request); + # image guidance must be forced off so this stays plain t2i CFG. + pipeline = _make_edit_forward_pipeline() + num_steps = 2 + sampling = _sampling(height=64, width=64, num_inference_steps=num_steps, guidance_scale=5.0, guidance_scale_2=2.0) + sampling.guidance_scale_provided = True + sampling.guidance_scale_2_provided = True + req = SimpleNamespace(prompts=[{"prompt": "a cat"}], sampling_params=sampling) + + pipeline.forward(req) + + # t2i text CFG: cond + uncond -> 2 predictions/step, no ref anywhere. + assert _count_per_step(pipeline.transformer.calls, num_steps) == 2 + assert all(ref is None for ref in pipeline.transformer.calls) diff --git a/tests/e2e/online_serving/test_boogu_image.py b/tests/e2e/online_serving/test_boogu_image.py new file mode 100644 index 000000000..5a5494d7d --- /dev/null +++ b/tests/e2e/online_serving/test_boogu_image.py @@ -0,0 +1,94 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +""" +Online serving tests for ``Boogu/Boogu-Image-0.1-Base`` (text-to-image). + +- ``test_text_to_image_001``: single chat request, default server (``_get_default_case``). +- ``test_batch_001``: concurrent prompts via ``send_diffusion_request([cfg0, cfg1, ...])`` — one + dict per prompt; each entry carries its own ``messages`` / ``negative_prompt`` (see + ``TEST_PROMPTS``). + +Boogu-Image's pipeline reads ``sp.guidance_scale`` (not ``true_cfg_scale``), so the +``extra_body`` uses ``guidance_scale``. These are smoke tests (minimal steps); numeric +parity vs. the Diffusers baseline is covered separately (see the support plan, step 14). + +From ``tests/``:: + + pytest -s -v e2e/online_serving/test_boogu_image.py \ + -m "full_model and diffusion and H100" --run-level=full_model +""" + +import os + +import pytest + +from tests.helpers.mark import hardware_marks +from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler, dummy_messages_from_mix_data + +os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" + +pytestmark = [pytest.mark.diffusion, pytest.mark.full_model] + +MODEL = "Boogu/Boogu-Image-0.1-Base" +T2I_PROMPT = "A mountain lake at sunset, photorealistic, cinematic lighting" +NEGATIVE_PROMPT = "" + +SINGLE_CARD_FEATURE_MARKS = hardware_marks(res={"cuda": "H100"}) + +TEST_PROMPTS: list[dict[str, str]] = [ + {"prompt": "a cup of coffee on a table", "negative_prompt": ""}, + {"prompt": "a toy dinosaur on a sandy beach", "negative_prompt": ""}, + {"prompt": "a futuristic city skyline at sunset", "negative_prompt": ""}, +] + + +def _get_default_case(model: str): + """Return a single default ``OmniServerParams`` row (no extra ``server_args``).""" + return [ + pytest.param( + OmniServerParams(model=model), + id="default", + marks=SINGLE_CARD_FEATURE_MARKS, + ), + ] + + +@pytest.mark.parametrize("omni_server", _get_default_case(MODEL), indirect=True) +def test_text_to_image_001(omni_server: OmniServer, openai_client: OpenAIClientHandler) -> None: + """Default Boogu-Image T2I smoke (single ``default`` server config).""" + messages = dummy_messages_from_mix_data(content_text=T2I_PROMPT) + request_config = { + "model": omni_server.model, + "messages": messages, + "extra_body": { + "height": 512, + "width": 512, + "num_inference_steps": 2, + "negative_prompt": NEGATIVE_PROMPT, + "guidance_scale": 4.0, + "seed": 42, + }, + } + openai_client.send_diffusion_request(request_config) + + +@pytest.mark.parametrize("omni_server", _get_default_case(MODEL), indirect=True) +def test_batch_001(omni_server: OmniServer, openai_client: OpenAIClientHandler) -> None: + """Concurrent T2I: one ``request_config`` dict per prompt (``send_diffusion_request`` list mode).""" + request_config = [ + { + "model": omni_server.model, + "messages": dummy_messages_from_mix_data(content_text=prompt["prompt"]), + "extra_body": { + "height": 512, + "width": 512, + "num_inference_steps": 2, + "negative_prompt": prompt["negative_prompt"], + "guidance_scale": 4.0, + "seed": 42, + }, + } + for prompt in TEST_PROMPTS + ] + openai_client.send_diffusion_request(request_config) diff --git a/tests/e2e/online_serving/test_boogu_image_edit.py b/tests/e2e/online_serving/test_boogu_image_edit.py new file mode 100644 index 000000000..45f999d26 --- /dev/null +++ b/tests/e2e/online_serving/test_boogu_image_edit.py @@ -0,0 +1,156 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +""" +Online serving tests for Boogu-Image-0.1-Edit (image-to-image via chat completions). + +The Edit checkpoint shares the ``BooguImagePipeline`` class with the Base +text-to-image checkpoint; the editing (TI2I) path activates when the request +carries a reference image. Text guidance rides on ``guidance_scale`` (upstream +default 4.0), and image guidance rides on ``guidance_scale_2`` (upstream default +1.0 = off; a value > 1 enables the double-guidance path). Only a single +reference image is supported. + +- ``test_single_image_to_image_001``: one reference image, text guidance only. +- ``test_double_guidance_001``: one reference image, text + image guidance. +- ``test_images_edits_endpoint_001``: same text-guided edit via ``/v1/images/edits``. +- ``test_images_edits_endpoint_double_guidance_001``: double guidance via ``/v1/images/edits``. + +From ``tests/``:: + + pytest -s -v e2e/online_serving/test_boogu_image_edit.py \ + -m "full_model and diffusion and H100" --run-level=full_model +""" + +import base64 + +import pytest + +from tests.helpers.mark import hardware_marks +from tests.helpers.media import generate_synthetic_image +from tests.helpers.runtime import ( + OmniServer, + OmniServerParams, + OpenAIClientHandler, + dummy_messages_from_mix_data, +) + +pytestmark = [pytest.mark.diffusion, pytest.mark.full_model] + +EDIT_PROMPT = "Change the style to a colored pencil drawing." +SINGLE_CARD_FEATURE_MARKS = hardware_marks(res={"cuda": "H100"}) + + +def _get_diffusion_feature_cases(model: str): + """Return one ``default`` ``OmniServerParams`` row for ``model`` (no extra ``server_args``).""" + return [ + pytest.param( + OmniServerParams( + model=model, + ), + id="default", + marks=SINGLE_CARD_FEATURE_MARKS, + ), + ] + + +@pytest.mark.parametrize( + "omni_server", + _get_diffusion_feature_cases("Boogu/Boogu-Image-0.1-Edit"), + indirect=True, +) +def test_single_image_to_image_001(omni_server: OmniServer, openai_client: OpenAIClientHandler): + """Single-reference text-guided edit smoke for ``Boogu/Boogu-Image-0.1-Edit``.""" + image_data_url = f"data:image/jpeg;base64,{generate_synthetic_image(512, 512)['base64']}" + + messages = dummy_messages_from_mix_data(image_data_url=image_data_url, content_text=EDIT_PROMPT) + + request_config = { + "model": omni_server.model, + "messages": messages, + "extra_body": { + "num_inference_steps": 2, + # Boogu text guidance (upstream default 4.0); > 1 enables CFG. + "guidance_scale": 5.0, + "seed": 42, + }, + } + + openai_client.send_diffusion_request(request_config) + + +@pytest.mark.parametrize( + "omni_server", + _get_diffusion_feature_cases("Boogu/Boogu-Image-0.1-Edit"), + indirect=True, +) +def test_double_guidance_001(omni_server: OmniServer, openai_client: OpenAIClientHandler): + """Single-reference double-guidance edit (text + image guidance) smoke.""" + image_data_url = f"data:image/jpeg;base64,{generate_synthetic_image(512, 512)['base64']}" + + messages = dummy_messages_from_mix_data(image_data_url=image_data_url, content_text=EDIT_PROMPT) + + request_config = { + "model": omni_server.model, + "messages": messages, + "extra_body": { + "num_inference_steps": 2, + "guidance_scale": 5.0, + # Image guidance > 1 activates the 3-prediction double-guidance path. + "guidance_scale_2": 2.0, + "seed": 42, + }, + } + + openai_client.send_diffusion_request(request_config) + + +def _edits_request_config(model: str, *, guidance_scale_2: float | None = None) -> dict: + """Multipart ``/v1/images/edits`` config: one synthetic reference + smoke-depth params.""" + image_bytes = base64.b64decode(generate_synthetic_image(512, 512)["base64"]) + data = { + "model": model, + "prompt": EDIT_PROMPT, + "num_inference_steps": 2, + # Boogu text guidance (upstream default 4.0); > 1 enables CFG. + "guidance_scale": 5.0, + "seed": 42, + } + if guidance_scale_2 is not None: + data["guidance_scale_2"] = guidance_scale_2 + return { + "data": data, + "files": {"image": ("ref.jpg", image_bytes, "image/jpeg")}, + "timeout": 300, + } + + +def _assert_edits_response_has_image(responses) -> None: + assert responses and responses[0].success, "Expected a successful /v1/images/edits response" + body = responses[0].json_body + assert body and body.get("data"), "No image data returned by /v1/images/edits" + assert body["data"][0].get("b64_json"), "Missing b64_json in /v1/images/edits response" + + +@pytest.mark.parametrize( + "omni_server", + _get_diffusion_feature_cases("Boogu/Boogu-Image-0.1-Edit"), + indirect=True, +) +def test_images_edits_endpoint_001(omni_server: OmniServer, openai_client: OpenAIClientHandler): + """Single-reference text-guided edit via the OpenAI-compatible ``/v1/images/edits`` endpoint.""" + responses = openai_client.send_images_edits_http_request(_edits_request_config(omni_server.model)) + _assert_edits_response_has_image(responses) + + +@pytest.mark.parametrize( + "omni_server", + _get_diffusion_feature_cases("Boogu/Boogu-Image-0.1-Edit"), + indirect=True, +) +def test_images_edits_endpoint_double_guidance_001(omni_server: OmniServer, openai_client: OpenAIClientHandler): + """Double-guidance edit (text + image guidance) via ``/v1/images/edits``.""" + responses = openai_client.send_images_edits_http_request( + _edits_request_config(omni_server.model, guidance_scale_2=2.0) + ) + _assert_edits_response_has_image(responses) diff --git a/tests/e2e/online_serving/test_boogu_image_edit_expansion.py b/tests/e2e/online_serving/test_boogu_image_edit_expansion.py new file mode 100644 index 000000000..efc968eb6 --- /dev/null +++ b/tests/e2e/online_serving/test_boogu_image_edit_expansion.py @@ -0,0 +1,107 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +""" +L4 expansion coverage for ``Boogu/Boogu-Image-0.1-Edit`` (image editing / TI2I). + +The nightly smoke module (``test_boogu_image_edit.py``) already covers single-image +text-guided editing and the text+image double-guidance path. This expansion +module adds the two edit paths smoke does not exercise: + +- ``test_image_only_guidance`` — ``guidance_scale=1.0`` (text CFG off) with + ``guidance_scale_2=2.0`` (image CFG on): the image-only branch of upstream's + CFG-branch priority (double > text-only > image-only > t2i), which no existing + e2e test reaches. +- ``test_align_res_output_size`` — a non-square reference with no requested + ``height`` / ``width``: asserts the output resolution follows the reference + (upstream ``align_res``, on by default for a single reference). + +The Edit checkpoint shares ``BooguImagePipeline`` with the Base checkpoint; the +TI2I path activates when the request carries a reference image. Only a single +reference image is supported. Cases stay smoke-depth (``num_inference_steps=2``). +CPU offload, Cache-DiT, and multi-GPU parallelism are not yet supported for this +model and are intentionally omitted. + +From ``tests/``:: + + pytest -s -v e2e/online_serving/test_boogu_image_edit_expansion.py \ + -m "full_model and diffusion and H100" --run-level=full_model +""" + +import os + +import pytest + +from tests.helpers.mark import hardware_marks +from tests.helpers.media import generate_synthetic_image +from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler, dummy_messages_from_mix_data + +os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" + +pytestmark = [pytest.mark.diffusion, pytest.mark.full_model] + +MODEL = "Boogu/Boogu-Image-0.1-Edit" +EDIT_PROMPT = "Change the style to a colored pencil drawing." + +SINGLE_CARD_FEATURE_MARKS = hardware_marks(res={"cuda": "H100"}) + + +def _get_diffusion_feature_cases(model: str): + """Return the single-card server configuration supported for Boogu-Image-Edit today.""" + return [ + pytest.param( + OmniServerParams(model=model), + id="default", + marks=SINGLE_CARD_FEATURE_MARKS, + ), + ] + + +@pytest.mark.parametrize("omni_server", _get_diffusion_feature_cases(MODEL), indirect=True) +def test_image_only_guidance(omni_server: OmniServer, openai_client: OpenAIClientHandler) -> None: + """Image-only guidance: text CFG off (1.0), image CFG on (2.0) -> image-only CFG branch.""" + image_data_url = f"data:image/jpeg;base64,{generate_synthetic_image(512, 512)['base64']}" + messages = dummy_messages_from_mix_data(image_data_url=image_data_url, content_text=EDIT_PROMPT) + request_config = { + "model": omni_server.model, + "messages": messages, + "extra_body": { + "num_inference_steps": 2, + # Text guidance off, image guidance on -> upstream's image-only branch. + "guidance_scale": 1.0, + "guidance_scale_2": 2.0, + "seed": 42, + }, + } + openai_client.send_diffusion_request(request_config) + + +@pytest.mark.parametrize("omni_server", _get_diffusion_feature_cases(MODEL), indirect=True) +def test_align_res_output_size(omni_server: OmniServer, openai_client: OpenAIClientHandler) -> None: + """Output resolution follows the reference image (upstream ``align_res``). + + No ``height`` / ``width`` is requested. The reference is 512x384 — both + already multiples of 32 (>= the ``vae_scale_factor * 2`` alignment) and well + under the VAE ``max_pixels`` / ``max_side`` limits — so the align-res target + equals the reference dims regardless of the exact VAE scale factor. + """ + ref_width, ref_height = 512, 384 + image_data_url = f"data:image/jpeg;base64,{generate_synthetic_image(ref_width, ref_height)['base64']}" + messages = dummy_messages_from_mix_data(image_data_url=image_data_url, content_text=EDIT_PROMPT) + request_config = { + "model": omni_server.model, + "messages": messages, + "extra_body": { + "num_inference_steps": 2, + "guidance_scale": 5.0, + "seed": 42, + }, + } + responses = openai_client.send_diffusion_request(request_config) + + assert responses and responses[0].images, "No image returned for the edit request" + out_image = responses[0].images[0] + assert (out_image.width, out_image.height) == (ref_width, ref_height), ( + f"align_res should size the output to the reference dims {(ref_width, ref_height)}; " + f"got {(out_image.width, out_image.height)}" + ) diff --git a/tests/e2e/online_serving/test_boogu_image_expansion.py b/tests/e2e/online_serving/test_boogu_image_expansion.py new file mode 100644 index 000000000..ef6d3a62e --- /dev/null +++ b/tests/e2e/online_serving/test_boogu_image_expansion.py @@ -0,0 +1,127 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +""" +L4 expansion coverage for ``Boogu/Boogu-Image-0.1-Base`` (text-to-image). + +The nightly smoke module (``test_boogu_image.py``) already covers the default +512x512 single request and a concurrent batch. This expansion module adds the +request-level paths that smoke does not exercise, run against every currently +supported single-GPU server configuration: + +Server rows (single card): +- ``default`` — no extra ``server_args``. +- ``vae_slicing_tiling`` — ``--vae-use-slicing --vae-use-tiling`` (the recipe's + OOM mitigation; applied generically in ``registry.initialize_model``). + +Cases (one per test, each parametrized over both server rows): +- ``test_non_square_resolution`` — 768x1024, exercises the patchify / RoPE path + at a non-square aspect ratio. +- ``test_cfg_off`` — ``guidance_scale=1.0``, exercises the no-CFG denoise branch + (single prediction per step). +- ``test_high_resolution`` — 1024x1024, the recipe-recommended resolution. + +Boogu-Image reads ``sp.guidance_scale`` (not ``true_cfg_scale``). These stay +smoke-depth (``num_inference_steps=2``); numeric parity vs. Diffusers is covered +by the local parity harness (support plan, step 14). CPU offload, Cache-DiT, and +multi-GPU parallelism are not yet supported for this model and are intentionally +omitted (add rows here when those features land). + +From ``tests/``:: + + pytest -s -v e2e/online_serving/test_boogu_image_expansion.py \ + -m "full_model and diffusion and H100" --run-level=full_model +""" + +import os + +import pytest + +from tests.helpers.mark import hardware_marks +from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler, dummy_messages_from_mix_data + +os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" + +pytestmark = [pytest.mark.diffusion, pytest.mark.full_model] + +MODEL = "Boogu/Boogu-Image-0.1-Base" +T2I_PROMPT = "A mountain lake at sunset, photorealistic, cinematic lighting" +NEGATIVE_PROMPT = "" + +SINGLE_CARD_FEATURE_MARKS = hardware_marks(res={"cuda": "H100"}) + + +def _get_diffusion_feature_cases(model: str): + """Return the single-card server configurations supported for Boogu-Image today.""" + return [ + pytest.param( + OmniServerParams(model=model), + id="default", + marks=SINGLE_CARD_FEATURE_MARKS, + ), + pytest.param( + OmniServerParams( + model=model, + server_args=["--vae-use-slicing", "--vae-use-tiling"], + ), + id="vae_slicing_tiling", + marks=SINGLE_CARD_FEATURE_MARKS, + ), + ] + + +@pytest.mark.parametrize("omni_server", _get_diffusion_feature_cases(MODEL), indirect=True) +def test_non_square_resolution(omni_server: OmniServer, openai_client: OpenAIClientHandler) -> None: + """Non-square (768x1024) T2I exercises the patchify / RoPE path off the square grid.""" + messages = dummy_messages_from_mix_data(content_text=T2I_PROMPT) + request_config = { + "model": omni_server.model, + "messages": messages, + "extra_body": { + "height": 1024, + "width": 768, + "num_inference_steps": 2, + "negative_prompt": NEGATIVE_PROMPT, + "guidance_scale": 4.0, + "seed": 42, + }, + } + openai_client.send_diffusion_request(request_config) + + +@pytest.mark.parametrize("omni_server", _get_diffusion_feature_cases(MODEL), indirect=True) +def test_cfg_off(omni_server: OmniServer, openai_client: OpenAIClientHandler) -> None: + """``guidance_scale=1.0`` exercises the no-CFG denoise branch (one prediction per step).""" + messages = dummy_messages_from_mix_data(content_text=T2I_PROMPT) + request_config = { + "model": omni_server.model, + "messages": messages, + "extra_body": { + "height": 512, + "width": 512, + "num_inference_steps": 2, + "negative_prompt": NEGATIVE_PROMPT, + "guidance_scale": 1.0, + "seed": 42, + }, + } + openai_client.send_diffusion_request(request_config) + + +@pytest.mark.parametrize("omni_server", _get_diffusion_feature_cases(MODEL), indirect=True) +def test_high_resolution(omni_server: OmniServer, openai_client: OpenAIClientHandler) -> None: + """1024x1024 T2I at the recipe-recommended resolution.""" + messages = dummy_messages_from_mix_data(content_text=T2I_PROMPT) + request_config = { + "model": omni_server.model, + "messages": messages, + "extra_body": { + "height": 1024, + "width": 1024, + "num_inference_steps": 2, + "negative_prompt": NEGATIVE_PROMPT, + "guidance_scale": 4.0, + "seed": 42, + }, + } + openai_client.send_diffusion_request(request_config) diff --git a/tests/entrypoints/openai_api/test_image_server.py b/tests/entrypoints/openai_api/test_image_server.py index 5a7ef7bf0..34ae5ba20 100644 --- a/tests/entrypoints/openai_api/test_image_server.py +++ b/tests/entrypoints/openai_api/test_image_server.py @@ -1562,6 +1562,7 @@ def test_image_edit_parameter_pass(async_omni_test_client): "output_format": "jpeg", "num_inference_steps": 20, "guidance_scale": 8.0, + "guidance_scale_2": 2.0, "seed": 1234, "negative_prompt": "negative", "n": 2, @@ -1576,6 +1577,7 @@ def test_image_edit_parameter_pass(async_omni_test_client): assert captured_prompt["negative_prompt"] == "negative" assert captured_sampling_params.num_inference_steps == 20 assert captured_sampling_params.guidance_scale == 8.0 + assert captured_sampling_params.guidance_scale_2 == 2.0 assert captured_sampling_params.seed == 1234 assert captured_sampling_params.num_outputs_per_prompt == 2 assert captured_sampling_params.width == 16 diff --git a/tests/model_tests/diffusion/test_alignment.py b/tests/model_tests/diffusion/test_alignment.py index 4b51220c7..4bc740b01 100644 --- a/tests/model_tests/diffusion/test_alignment.py +++ b/tests/model_tests/diffusion/test_alignment.py @@ -45,6 +45,7 @@ EXCLUDED_MODELS = [ "WanI2VDMD2Pipeline", "LongCatImagePipeline", "BagelPipeline", + "BooguImagePipeline", "LancePipeline", "MingImagePipeline", "InternVLAA1Pipeline", diff --git a/vllm_omni/diffusion/model_metadata.py b/vllm_omni/diffusion/model_metadata.py index f33463384..7e02cf5d8 100644 --- a/vllm_omni/diffusion/model_metadata.py +++ b/vllm_omni/diffusion/model_metadata.py @@ -15,6 +15,8 @@ class DiffusionModelMetadata: QWEN_IMAGE_EDIT_PLUS_MAX_INPUT_IMAGES = 4 # Upstream HunyuanImage-3.0 "Multi-Image Fusion" caps reference images at 3. HUNYUAN_IMAGE3_MAX_INPUT_IMAGES = 3 +# Boogu-Image editing (TI2I) supports a single reference image for now. +BOOGU_IMAGE_MAX_INPUT_IMAGES = 1 _DIFFUSION_MODEL_METADATA: dict[str, DiffusionModelMetadata] = { @@ -26,6 +28,13 @@ _DIFFUSION_MODEL_METADATA: dict[str, DiffusionModelMetadata] = { supports_multimodal_inputs=True, max_multimodal_image_inputs=HUNYUAN_IMAGE3_MAX_INPUT_IMAGES, ), + # Shared by the Base (text-to-image) and Edit (TI2I) checkpoints, which use + # the same ``BooguImagePipeline`` class. Text-to-image requests simply carry + # no reference image. + "BooguImagePipeline": DiffusionModelMetadata( + supports_multimodal_inputs=True, + max_multimodal_image_inputs=BOOGU_IMAGE_MAX_INPUT_IMAGES, + ), } diff --git a/vllm_omni/diffusion/models/boogu_image/__init__.py b/vllm_omni/diffusion/models/boogu_image/__init__.py new file mode 100644 index 000000000..7ec9c167c --- /dev/null +++ b/vllm_omni/diffusion/models/boogu_image/__init__.py @@ -0,0 +1,12 @@ +from .boogu_image_transformer import BooguImageTransformer2DModel +from .image_processor import BooguImageProcessor +from .pipeline_boogu_image import BooguImagePipeline, get_boogu_image_pre_process_func +from .scheduling_flow_match_euler_discrete_time_shifting import FlowMatchEulerDiscreteScheduler + +__all__ = [ + "BooguImagePipeline", + "BooguImageProcessor", + "BooguImageTransformer2DModel", + "FlowMatchEulerDiscreteScheduler", + "get_boogu_image_pre_process_func", +] diff --git a/vllm_omni/diffusion/models/boogu_image/boogu_image_transformer.py b/vllm_omni/diffusion/models/boogu_image/boogu_image_transformer.py new file mode 100644 index 000000000..e7489c868 --- /dev/null +++ b/vllm_omni/diffusion/models/boogu_image/boogu_image_transformer.py @@ -0,0 +1,1372 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# +# Native vLLM-Omni port of the Boogu-Image transformer. +# +# Ported from the upstream `boogu` package +# (boogu/models/transformers/transformer_boogu.py and friends) with the +# following changes: +# - Diffusers mixins (ModelMixin/ConfigMixin/PeftAdapterMixin) removed; +# configuration is read from `od_config.tf_model_config`. +# - diffusers Attention + custom processors replaced by the vLLM-Omni +# `Attention` layer (backend/SP/KV-quant handled by the framework). +# - Q/K/V and FFN projections use vLLM parallel linears for TP support. +# - Training-time features (prompt tuning, gradient checkpointing) and +# inference caches (TeaCache/TaylorSeer) are not ported. + +import itertools +from collections.abc import Iterable + +import torch +import torch.nn as nn +import torch.nn.functional as F +from diffusers.models.embeddings import Timesteps, get_1d_rotary_pos_embed +from einops import rearrange, repeat +from vllm.logger import init_logger +from vllm.model_executor.layers.layernorm import RMSNorm +from vllm.model_executor.layers.linear import ( + ColumnParallelLinear, + ReplicatedLinear, + RowParallelLinear, +) +from vllm.model_executor.model_loader.weight_utils import default_weight_loader + +from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata +from vllm_omni.diffusion.attention.layer import Attention +from vllm_omni.diffusion.data import OmniDiffusionConfig + +logger = init_logger(__name__) + + +def apply_rotary_emb(x: torch.Tensor, freqs_cis: torch.Tensor) -> torch.Tensor: + """Apply rotary embeddings in complex form (upstream `use_real=False` path). + + Args: + x: Query or key tensor of shape [B, S, H, D]. + freqs_cis: Complex frequency tensor of shape [B, S, D // 2]. + """ + x_rotated = torch.view_as_complex(x.float().reshape(*x.shape[:-1], x.shape[-1] // 2, 2)) + freqs_cis = freqs_cis.unsqueeze(2) + x_out = torch.view_as_real(x_rotated * freqs_cis).flatten(3) + return x_out.type_as(x) + + +def swiglu(x: torch.Tensor, y: torch.Tensor) -> torch.Tensor: + return F.silu(x.float(), inplace=False).to(x.dtype) * y + + +class TimestepEmbedding(nn.Module): + def __init__(self, in_channels: int, time_embed_dim: int) -> None: + super().__init__() + self.linear_1 = nn.Linear(in_channels, time_embed_dim, bias=True) + self.act = nn.SiLU() + self.linear_2 = nn.Linear(time_embed_dim, time_embed_dim, bias=True) + + def forward(self, sample: torch.Tensor) -> torch.Tensor: + return self.linear_2(self.act(self.linear_1(sample))) + + +class LuminaRMSNormZero(nn.Module): + """AdaRMS modulation: projects `temb` into scale/gate terms.""" + + def __init__(self, embedding_dim: int, norm_eps: float) -> None: + super().__init__() + self.silu = nn.SiLU() + self.linear = nn.Linear(min(embedding_dim, 1024), 4 * embedding_dim, bias=True) + self.norm = RMSNorm(embedding_dim, eps=norm_eps) + + def forward( + self, x: torch.Tensor, emb: torch.Tensor + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + emb = self.linear(self.silu(emb)) + scale_msa, gate_msa, scale_mlp, gate_mlp = emb.chunk(4, dim=1) + x = self.norm(x) * (1 + scale_msa[:, None]) + return x, gate_msa, scale_mlp, gate_mlp + + +class LuminaLayerNormContinuous(nn.Module): + """Final AdaLN + optional output projection (upstream `norm_out`).""" + + def __init__( + self, + embedding_dim: int, + conditioning_embedding_dim: int, + elementwise_affine: bool = True, + eps: float = 1e-5, + bias: bool = True, + out_dim: int | None = None, + ) -> None: + super().__init__() + self.silu = nn.SiLU() + self.linear_1 = nn.Linear(conditioning_embedding_dim, embedding_dim, bias=bias) + self.norm = nn.LayerNorm(embedding_dim, eps, elementwise_affine, bias) + self.linear_2 = nn.Linear(embedding_dim, out_dim, bias=bias) if out_dim is not None else None + + def forward(self, x: torch.Tensor, conditioning_embedding: torch.Tensor) -> torch.Tensor: + scale = self.linear_1(self.silu(conditioning_embedding).to(x.dtype)) + x = self.norm(x) * (1 + scale)[:, None, :] + if self.linear_2 is not None: + x = self.linear_2(x) + return x + + +class LuminaFeedForward(nn.Module): + """SwiGLU feed-forward with tensor-parallel projections.""" + + def __init__( + self, + dim: int, + inner_dim: int, + multiple_of: int = 256, + ffn_dim_multiplier: float | None = None, + ) -> None: + super().__init__() + if ffn_dim_multiplier is not None: + inner_dim = int(ffn_dim_multiplier * inner_dim) + inner_dim = multiple_of * ((inner_dim + multiple_of - 1) // multiple_of) + + self.linear_1 = ColumnParallelLinear(dim, inner_dim, bias=False) # gate + self.linear_3 = ColumnParallelLinear(dim, inner_dim, bias=False) # input + self.linear_2 = RowParallelLinear(inner_dim, dim, bias=False) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + h1, _ = self.linear_1(x) + h2, _ = self.linear_3(x) + out, _ = self.linear_2(swiglu(h1, h2)) + return out + + +class Lumina2CombinedTimestepCaptionEmbedding(nn.Module): + def __init__( + self, + hidden_size: int = 4096, + instruction_feat_dim: int = 2048, + frequency_embedding_size: int = 256, + norm_eps: float = 1e-5, + timestep_scale: float = 1.0, + ) -> None: + super().__init__() + self.time_proj = Timesteps( + num_channels=frequency_embedding_size, + flip_sin_to_cos=True, + downscale_freq_shift=0.0, + scale=timestep_scale, + ) + self.timestep_embedder = TimestepEmbedding( + in_channels=frequency_embedding_size, time_embed_dim=min(hidden_size, 1024) + ) + self.caption_embedder = nn.Sequential( + RMSNorm(instruction_feat_dim, eps=norm_eps), + nn.Linear(instruction_feat_dim, hidden_size, bias=True), + ) + + def forward( + self, + timestep: torch.Tensor, + instruction_hidden_states: torch.Tensor, + dtype: torch.dtype, + ) -> tuple[torch.Tensor, torch.Tensor]: + timestep_proj = self.time_proj(timestep).to(dtype=dtype) + time_embed = self.timestep_embedder(timestep_proj) + caption_embed = self.caption_embedder(instruction_hidden_states) + return time_embed, caption_embed + + +class BooguImageDoubleStreamRotaryPosEmbed(nn.Module): + """3-axis RoPE producing per-segment frequency tensors. + + Returns separate frequency tensors for instruction tokens, reference-image + tokens, noise-image tokens, the full joint sequence, and the combined + (ref + noise) image sequence used by the double-stream blocks. + """ + + def __init__( + self, + theta: int, + axes_dim: tuple[int, int, int], + axes_lens: tuple[int, int, int] = (300, 512, 512), + patch_size: int = 2, + ) -> None: + super().__init__() + self.theta = theta + self.axes_dim = axes_dim + self.axes_lens = axes_lens + self.patch_size = patch_size + + @staticmethod + def get_freqs_cis( + axes_dim: tuple[int, int, int], axes_lens: tuple[int, int, int], theta: int + ) -> list[torch.Tensor]: + freqs_cis = [] + freqs_dtype = torch.float32 if torch.backends.mps.is_available() else torch.float64 + for d, e in zip(axes_dim, axes_lens): + emb = get_1d_rotary_pos_embed(d, e, theta=theta, freqs_dtype=freqs_dtype) + freqs_cis.append(emb) + return freqs_cis + + def _get_freqs_cis(self, freqs_cis: list[torch.Tensor], ids: torch.Tensor) -> torch.Tensor: + device = ids.device + if ids.device.type == "mps": + ids = ids.to("cpu") + + result = [] + for i in range(len(self.axes_dim)): + freqs = freqs_cis[i].to(ids.device) + index = ids[:, :, i : i + 1].repeat(1, 1, freqs.shape[-1]).to(torch.int64) + result.append(torch.gather(freqs.unsqueeze(0).repeat(index.shape[0], 1, 1), dim=1, index=index)) + return torch.cat(result, dim=-1).to(device) + + def forward( + self, + freqs_cis: list[torch.Tensor], + attention_mask: torch.Tensor, + l_effective_ref_img_len: list[list[int]], + l_effective_img_len: list[int], + ref_img_sizes: list[list[tuple[int, int]] | None], + img_sizes: list[tuple[int, int]], + device: torch.device, + ): + batch_size = len(attention_mask) + p = self.patch_size + + encoder_seq_len = attention_mask.shape[1] + l_effective_cap_len = attention_mask.sum(dim=1).tolist() + + seq_lengths = [ + cap_len + sum(ref_img_len) + img_len + for cap_len, ref_img_len, img_len in zip(l_effective_cap_len, l_effective_ref_img_len, l_effective_img_len) + ] + + max_seq_len = max(seq_lengths) + max_ref_img_len = max(sum(ref_img_len) for ref_img_len in l_effective_ref_img_len) + max_img_len = max(l_effective_img_len) + + position_ids = torch.zeros(batch_size, max_seq_len, 3, dtype=torch.int32, device=device) + + for i, (cap_seq_len, seq_len) in enumerate(zip(l_effective_cap_len, seq_lengths)): + position_ids[i, :cap_seq_len] = repeat( + torch.arange(cap_seq_len, dtype=torch.int32, device=device), "l -> l 3" + ) + + pe_shift = cap_seq_len + pe_shift_len = cap_seq_len + + if ref_img_sizes[i] is not None: + for ref_img_size, ref_img_len in zip(ref_img_sizes[i], l_effective_ref_img_len[i]): + H, W = ref_img_size + ref_H_tokens, ref_W_tokens = H // p, W // p + assert ref_H_tokens * ref_W_tokens == ref_img_len + + row_ids = repeat( + torch.arange(ref_H_tokens, dtype=torch.int32, device=device), + "h -> h w", + w=ref_W_tokens, + ).flatten() + col_ids = repeat( + torch.arange(ref_W_tokens, dtype=torch.int32, device=device), + "w -> h w", + h=ref_H_tokens, + ).flatten() + position_ids[i, pe_shift_len : pe_shift_len + ref_img_len, 0] = pe_shift + position_ids[i, pe_shift_len : pe_shift_len + ref_img_len, 1] = row_ids + position_ids[i, pe_shift_len : pe_shift_len + ref_img_len, 2] = col_ids + + pe_shift += max(ref_H_tokens, ref_W_tokens) + pe_shift_len += ref_img_len + + H, W = img_sizes[i] + H_tokens, W_tokens = H // p, W // p + assert H_tokens * W_tokens == l_effective_img_len[i] + + row_ids = repeat(torch.arange(H_tokens, dtype=torch.int32, device=device), "h -> h w", w=W_tokens).flatten() + col_ids = repeat(torch.arange(W_tokens, dtype=torch.int32, device=device), "w -> h w", h=H_tokens).flatten() + + assert pe_shift_len + l_effective_img_len[i] == seq_len + position_ids[i, pe_shift_len:seq_len, 0] = pe_shift + position_ids[i, pe_shift_len:seq_len, 1] = row_ids + position_ids[i, pe_shift_len:seq_len, 2] = col_ids + + freqs_cis = self._get_freqs_cis(freqs_cis, position_ids) + + cap_freqs_cis = torch.zeros( + batch_size, encoder_seq_len, freqs_cis.shape[-1], device=device, dtype=freqs_cis.dtype + ) + ref_img_freqs_cis = torch.zeros( + batch_size, max_ref_img_len, freqs_cis.shape[-1], device=device, dtype=freqs_cis.dtype + ) + img_freqs_cis = torch.zeros(batch_size, max_img_len, freqs_cis.shape[-1], device=device, dtype=freqs_cis.dtype) + + combined_img_seq_lengths = [ + sum(ref_img_len) + img_len for ref_img_len, img_len in zip(l_effective_ref_img_len, l_effective_img_len) + ] + max_combined_img_len = max(combined_img_seq_lengths) + + combined_img_freqs_cis = torch.zeros( + batch_size, max_combined_img_len, freqs_cis.shape[-1], device=device, dtype=freqs_cis.dtype + ) + + for i, (cap_seq_len, ref_img_len, img_len, seq_len) in enumerate( + zip(l_effective_cap_len, l_effective_ref_img_len, l_effective_img_len, seq_lengths) + ): + cap_freqs_cis[i, :cap_seq_len] = freqs_cis[i, :cap_seq_len] + ref_img_freqs_cis[i, : sum(ref_img_len)] = freqs_cis[i, cap_seq_len : cap_seq_len + sum(ref_img_len)] + img_freqs_cis[i, :img_len] = freqs_cis[ + i, cap_seq_len + sum(ref_img_len) : cap_seq_len + sum(ref_img_len) + img_len + ] + + combined_img_freqs_cis[i, : sum(ref_img_len)] = freqs_cis[i, cap_seq_len : cap_seq_len + sum(ref_img_len)] + combined_img_freqs_cis[i, sum(ref_img_len) : sum(ref_img_len) + img_len] = freqs_cis[ + i, cap_seq_len + sum(ref_img_len) : cap_seq_len + sum(ref_img_len) + img_len + ] + + return ( + cap_freqs_cis, + ref_img_freqs_cis, + img_freqs_cis, + freqs_cis, + l_effective_cap_len, + seq_lengths, + combined_img_freqs_cis, + combined_img_seq_lengths, + ) + + +def _concat_instruction_image_features( + img_tensors: list[torch.Tensor], + instruct_tensors: list[torch.Tensor], + encoder_seq_lengths: list[int], + seq_lengths: list[int], +) -> list[torch.Tensor]: + """Pack per-stream tensors into joint sequences (instruction first, then image).""" + assert len(img_tensors) == len(instruct_tensors) + + batch_size = img_tensors[0].shape[0] + max_seq_len = max(seq_lengths) + + concatenated_list = [] + for img_tensor, instruct_tensor in zip(img_tensors, instruct_tensors): + feature_dim = img_tensor.shape[-1] + concatenated = img_tensor.new_zeros(batch_size, max_seq_len, feature_dim) + for i, (encoder_seq_len, seq_len) in enumerate(zip(encoder_seq_lengths, seq_lengths)): + concatenated[i, :encoder_seq_len] = instruct_tensor[i, :encoder_seq_len] + concatenated[i, encoder_seq_len:seq_len] = img_tensor[i, : seq_len - encoder_seq_len] + concatenated_list.append(concatenated) + + return concatenated_list + + +def _split_instruction_image_features( + hidden_states: torch.Tensor, + encoder_seq_lengths: list[int], + seq_lengths: list[int], +) -> tuple[torch.Tensor, torch.Tensor]: + """Unpack a joint sequence back into (instruction, image) streams.""" + batch_size = hidden_states.shape[0] + feature_dim = hidden_states.shape[-1] + + max_instruct_len = max(encoder_seq_lengths) + max_img_len = max(seq_len - encoder_seq_len for seq_len, encoder_seq_len in zip(seq_lengths, encoder_seq_lengths)) + + instruct_hidden_states = hidden_states.new_zeros(batch_size, max_instruct_len, feature_dim) + img_hidden_states = hidden_states.new_zeros(batch_size, max_img_len, feature_dim) + + for i, (encoder_seq_len, seq_len) in enumerate(zip(encoder_seq_lengths, seq_lengths)): + img_len = seq_len - encoder_seq_len + instruct_hidden_states[i, :encoder_seq_len] = hidden_states[i, :encoder_seq_len] + img_hidden_states[i, :img_len] = hidden_states[i, encoder_seq_len:seq_len] + + return instruct_hidden_states, img_hidden_states + + +class BooguImageSelfAttention(nn.Module): + """GQA self-attention with QK RMSNorm and complex RoPE. + + Replaces the upstream diffusers `Attention` + Boogu attention processors; + the vLLM-Omni `Attention` layer picks the kernel backend and handles + SP/KV-cache concerns. + """ + + def __init__(self, dim: int, num_attention_heads: int, num_kv_heads: int) -> None: + super().__init__() + self.head_dim = dim // num_attention_heads + kv_dim = self.head_dim * num_kv_heads + + self.to_q = ColumnParallelLinear(dim, dim, bias=False) + self.to_k = ColumnParallelLinear(dim, kv_dim, bias=False) + self.to_v = ColumnParallelLinear(dim, kv_dim, bias=False) + self.norm_q = RMSNorm(self.head_dim, eps=1e-5) + self.norm_k = RMSNorm(self.head_dim, eps=1e-5) + self.to_out = RowParallelLinear(dim, dim, bias=False) + + self.num_local_heads = self.to_q.output_size_per_partition // self.head_dim + self.num_local_kv_heads = self.to_k.output_size_per_partition // self.head_dim + + self.attn = Attention( + num_heads=self.num_local_heads, + head_size=self.head_dim, + softmax_scale=self.head_dim**-0.5, + causal=False, + num_kv_heads=self.num_local_kv_heads, + ) + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: torch.Tensor | None, + rotary_emb: torch.Tensor | None, + ) -> torch.Tensor: + dtype = hidden_states.dtype + + query, _ = self.to_q(hidden_states) + key, _ = self.to_k(hidden_states) + value, _ = self.to_v(hidden_states) + + query = query.unflatten(-1, (self.num_local_heads, self.head_dim)) + key = key.unflatten(-1, (self.num_local_kv_heads, self.head_dim)) + value = value.unflatten(-1, (self.num_local_kv_heads, self.head_dim)) + + query = self.norm_q(query) + key = self.norm_k(key) + + if rotary_emb is not None: + query = apply_rotary_emb(query, rotary_emb) + key = apply_rotary_emb(key, rotary_emb) + query, key = query.to(dtype), key.to(dtype) + + attn_metadata = AttentionMetadata(attn_mask=attention_mask) if attention_mask is not None else None + attn_output = self.attn(query, key, value, attn_metadata) + attn_output = attn_output.flatten(2, 3).type_as(hidden_states) + + attn_output, _ = self.to_out(attn_output) + return attn_output + + +class BooguImageJointAttention(nn.Module): + """Joint instruction+image attention for the double-stream blocks. + + The upstream implementation keeps the per-stream Q/K/V and output + projections on the attention *processor* + (`BooguImageDoubleStreamSelfAttnProcessor*`); here they are promoted onto + this module. The final joint output projection corresponds to the upstream + `img_instruct_attn.to_out[0]`. + """ + + def __init__(self, dim: int, num_attention_heads: int, num_kv_heads: int) -> None: + super().__init__() + self.head_dim = dim // num_attention_heads + kv_dim = self.head_dim * num_kv_heads + + self.img_to_q = ColumnParallelLinear(dim, dim, bias=False) + self.img_to_k = ColumnParallelLinear(dim, kv_dim, bias=False) + self.img_to_v = ColumnParallelLinear(dim, kv_dim, bias=False) + + self.instruct_to_q = ColumnParallelLinear(dim, dim, bias=False) + self.instruct_to_k = ColumnParallelLinear(dim, kv_dim, bias=False) + self.instruct_to_v = ColumnParallelLinear(dim, kv_dim, bias=False) + + self.norm_q = RMSNorm(self.head_dim, eps=1e-5) + self.norm_k = RMSNorm(self.head_dim, eps=1e-5) + + # Per-stream output projections (attention output is head-sharded + # under TP, hence row-parallel). + self.instruct_out = RowParallelLinear(dim, dim, bias=False) + self.img_out = RowParallelLinear(dim, dim, bias=False) + # Final joint projection applied to the merged full-dim sequence. + self.to_out = ReplicatedLinear(dim, dim, bias=False) + + self.num_local_heads = self.img_to_q.output_size_per_partition // self.head_dim + self.num_local_kv_heads = self.img_to_k.output_size_per_partition // self.head_dim + + self.attn = Attention( + num_heads=self.num_local_heads, + head_size=self.head_dim, + softmax_scale=self.head_dim**-0.5, + causal=False, + num_kv_heads=self.num_local_kv_heads, + ) + + def forward( + self, + img_hidden_states: torch.Tensor, + instruct_hidden_states: torch.Tensor, + joint_attention_mask: torch.Tensor | None, + rotary_emb: torch.Tensor | None, + encoder_seq_lengths: list[int], + seq_lengths: list[int], + ) -> torch.Tensor: + dtype = img_hidden_states.dtype + batch_size = img_hidden_states.shape[0] + + img_query, _ = self.img_to_q(img_hidden_states) + img_key, _ = self.img_to_k(img_hidden_states) + img_value, _ = self.img_to_v(img_hidden_states) + + instruct_query, _ = self.instruct_to_q(instruct_hidden_states) + instruct_key, _ = self.instruct_to_k(instruct_hidden_states) + instruct_value, _ = self.instruct_to_v(instruct_hidden_states) + + query, key, value = _concat_instruction_image_features( + [img_query, img_key, img_value], + [instruct_query, instruct_key, instruct_value], + encoder_seq_lengths, + seq_lengths, + ) + + query = query.view(batch_size, -1, self.num_local_heads, self.head_dim) + key = key.view(batch_size, -1, self.num_local_kv_heads, self.head_dim) + value = value.view(batch_size, -1, self.num_local_kv_heads, self.head_dim) + + query = self.norm_q(query) + key = self.norm_k(key) + + if rotary_emb is not None: + query = apply_rotary_emb(query, rotary_emb) + key = apply_rotary_emb(key, rotary_emb) + query, key = query.to(dtype), key.to(dtype) + + attn_metadata = AttentionMetadata(attn_mask=joint_attention_mask) if joint_attention_mask is not None else None + attn_output = self.attn(query, key, value, attn_metadata) + attn_output = attn_output.flatten(2, 3).to(dtype) + + instruct_attn_out, img_attn_out = _split_instruction_image_features( + attn_output, encoder_seq_lengths, seq_lengths + ) + instruct_projected, _ = self.instruct_out(instruct_attn_out) + img_projected, _ = self.img_out(img_attn_out) + + merged = _concat_instruction_image_features( + [img_projected], [instruct_projected], encoder_seq_lengths, seq_lengths + )[0] + merged, _ = self.to_out(merged) + return merged + + +class BooguImageTransformerBlock(nn.Module): + """Single-stream block: GQA attention + SwiGLU FFN with optional AdaRMS modulation.""" + + def __init__( + self, + dim: int, + num_attention_heads: int, + num_kv_heads: int, + multiple_of: int, + ffn_dim_multiplier: float | None, + norm_eps: float, + modulation: bool = True, + ) -> None: + super().__init__() + self.head_dim = dim // num_attention_heads + self.modulation = modulation + + self.attn = BooguImageSelfAttention(dim, num_attention_heads, num_kv_heads) + self.feed_forward = LuminaFeedForward( + dim=dim, + inner_dim=4 * dim, + multiple_of=multiple_of, + ffn_dim_multiplier=ffn_dim_multiplier, + ) + + if modulation: + self.norm1 = LuminaRMSNormZero(embedding_dim=dim, norm_eps=norm_eps) + else: + self.norm1 = RMSNorm(dim, eps=norm_eps) + + self.ffn_norm1 = RMSNorm(dim, eps=norm_eps) + self.norm2 = RMSNorm(dim, eps=norm_eps) + self.ffn_norm2 = RMSNorm(dim, eps=norm_eps) + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: torch.Tensor | None, + image_rotary_emb: torch.Tensor | None, + temb: torch.Tensor | None = None, + ) -> torch.Tensor: + if self.modulation: + if temb is None: + raise ValueError("temb must be provided when modulation is enabled") + norm_hidden_states, gate_msa, scale_mlp, gate_mlp = self.norm1(hidden_states, temb) + attn_output = self.attn(norm_hidden_states, attention_mask, image_rotary_emb) + hidden_states = hidden_states + gate_msa.unsqueeze(1).tanh() * self.norm2(attn_output) + mlp_output = self.feed_forward(self.ffn_norm1(hidden_states) * (1 + scale_mlp.unsqueeze(1))) + hidden_states = hidden_states + gate_mlp.unsqueeze(1).tanh() * self.ffn_norm2(mlp_output) + else: + norm_hidden_states = self.norm1(hidden_states) + attn_output = self.attn(norm_hidden_states, attention_mask, image_rotary_emb) + hidden_states = hidden_states + self.norm2(attn_output) + mlp_output = self.feed_forward(self.ffn_norm1(hidden_states)) + hidden_states = hidden_states + self.ffn_norm2(mlp_output) + + return hidden_states + + +class BooguImageNoiseRefinerTransformerBlock(BooguImageTransformerBlock): + pass + + +class BooguImageRefImgRefinerTransformerBlock(BooguImageTransformerBlock): + pass + + +class BooguImageContextRefinerTransformerBlock(BooguImageTransformerBlock): + pass + + +class BooguImageSingleStreamTransformerBlock(BooguImageTransformerBlock): + pass + + +class BooguImageDoubleStreamTransformerBlock(nn.Module): + """Double-stream block: instruction and image tokens are processed in + parallel streams coupled by a joint attention, plus an extra image + self-attention.""" + + def __init__( + self, + dim: int, + num_attention_heads: int, + num_kv_heads: int, + multiple_of: int, + ffn_dim_multiplier: float | None, + norm_eps: float, + modulation: bool = True, + ) -> None: + super().__init__() + self.head_dim = dim // num_attention_heads + self.num_attention_heads = num_attention_heads + self.modulation = modulation + self.hidden_size = dim + + self.img_instruct_attn = BooguImageJointAttention(dim, num_attention_heads, num_kv_heads) + self.img_self_attn = BooguImageSelfAttention(dim, num_attention_heads, num_kv_heads) + + self.img_feed_forward = LuminaFeedForward( + dim=dim, + inner_dim=4 * dim, + multiple_of=multiple_of, + ffn_dim_multiplier=ffn_dim_multiplier, + ) + + if modulation: + # Image modulation terms: cross-attn, MLP, self-attn. + self.img_norm1 = LuminaRMSNormZero(embedding_dim=dim, norm_eps=norm_eps) + self.img_norm2 = LuminaRMSNormZero(embedding_dim=dim, norm_eps=norm_eps) + self.img_norm3 = LuminaRMSNormZero(embedding_dim=dim, norm_eps=norm_eps) + else: + self.img_norm1 = RMSNorm(dim, eps=norm_eps) + self.img_norm2 = RMSNorm(dim, eps=norm_eps) + self.img_norm3 = RMSNorm(dim, eps=norm_eps) + + self.img_ffn_norm1 = RMSNorm(dim, eps=norm_eps) + self.img_attn_norm = RMSNorm(dim, eps=norm_eps) + self.img_self_attn_norm = RMSNorm(dim, eps=norm_eps) + self.img_ffn_norm2 = RMSNorm(dim, eps=norm_eps) + + self.instruct_feed_forward = LuminaFeedForward( + dim=dim, + inner_dim=4 * dim, + multiple_of=multiple_of, + ffn_dim_multiplier=ffn_dim_multiplier, + ) + + if modulation: + # Instruction modulation terms: cross-attn, MLP. + self.instruct_norm1 = LuminaRMSNormZero(embedding_dim=dim, norm_eps=norm_eps) + self.instruct_norm2 = LuminaRMSNormZero(embedding_dim=dim, norm_eps=norm_eps) + else: + self.instruct_norm1 = RMSNorm(dim, eps=norm_eps) + self.instruct_norm2 = RMSNorm(dim, eps=norm_eps) + + self.instruct_ffn_norm1 = RMSNorm(dim, eps=norm_eps) + self.instruct_attn_norm = RMSNorm(dim, eps=norm_eps) + self.instruct_ffn_norm2 = RMSNorm(dim, eps=norm_eps) + + def forward( + self, + img_hidden_states: torch.Tensor, # [B, L_img, D] (ref_img + noise_img) + instruct_hidden_states: torch.Tensor, # [B, L_instruct, D] + img_attention_mask: torch.Tensor | None, # [B, L_img] + joint_attention_mask: torch.Tensor | None, # [B, L_total] + image_rotary_emb: torch.Tensor | None, # [B, L_img, head_dim // 2] + rotary_emb: torch.Tensor | None, # [B, L_total, head_dim // 2] + temb: torch.Tensor | None = None, + encoder_seq_lengths: list[int] | None = None, + seq_lengths: list[int] | None = None, + ) -> tuple[torch.Tensor, torch.Tensor]: + if self.modulation and temb is None: + raise ValueError("temb must be provided when modulation is enabled") + + batch_size = img_hidden_states.shape[0] + L_instruct = instruct_hidden_states.shape[1] + L_img = img_hidden_states.shape[1] + + if self.modulation: + img_norm1_out, img_gate_msa, img_scale_mlp, img_gate_mlp = self.img_norm1(img_hidden_states, temb) + img_norm2_out, img_shift_mlp, _, _ = self.img_norm2(img_hidden_states, temb) + img_norm3_out, img_gate_self, _, _ = self.img_norm3(img_hidden_states, temb) + + ( + instruct_norm1_out, + instruct_gate_msa, + instruct_scale_mlp, + instruct_gate_mlp, + ) = self.instruct_norm1(instruct_hidden_states, temb) + instruct_norm2_out, instruct_shift_mlp, _, _ = self.instruct_norm2(instruct_hidden_states, temb) + + joint_attn_out = self.img_instruct_attn( + img_norm1_out, + instruct_norm1_out, + joint_attention_mask, + rotary_emb, + encoder_seq_lengths, + seq_lengths, + ) + + # Split the joint output back into instruction/image segments. + instruct_attn_out = instruct_hidden_states.new_zeros(batch_size, L_instruct, self.hidden_size) + img_attn_out = img_hidden_states.new_zeros(batch_size, L_img, self.hidden_size) + for i, (encoder_seq_len, seq_len) in enumerate(zip(encoder_seq_lengths, seq_lengths)): + instruct_attn_out[i, :encoder_seq_len] = joint_attn_out[i, :encoder_seq_len] + img_attn_out[i, : seq_len - encoder_seq_len] = joint_attn_out[i, encoder_seq_len:seq_len] + + img_self_attn_out = self.img_self_attn(img_norm3_out, img_attention_mask, image_rotary_emb) + + img_hidden_states = img_hidden_states + img_gate_msa.unsqueeze(1).tanh() * self.img_attn_norm(img_attn_out) + img_hidden_states = img_hidden_states + img_gate_self.unsqueeze(1).tanh() * self.img_self_attn_norm( + img_self_attn_out + ) + + img_mlp_input = (1 + img_scale_mlp.unsqueeze(1)) * img_norm2_out + img_shift_mlp.unsqueeze(1) + img_mlp_out = self.img_feed_forward(self.img_ffn_norm1(img_mlp_input)) + img_hidden_states = img_hidden_states + img_gate_mlp.unsqueeze(1).tanh() * self.img_ffn_norm2(img_mlp_out) + + instruct_hidden_states = instruct_hidden_states + instruct_gate_msa.unsqueeze( + 1 + ).tanh() * self.instruct_attn_norm(instruct_attn_out) + + instruct_mlp_input = ( + 1 + instruct_scale_mlp.unsqueeze(1) + ) * instruct_norm2_out + instruct_shift_mlp.unsqueeze(1) + instruct_mlp_out = self.instruct_feed_forward(self.instruct_ffn_norm1(instruct_mlp_input)) + instruct_hidden_states = instruct_hidden_states + instruct_gate_mlp.unsqueeze( + 1 + ).tanh() * self.instruct_ffn_norm2(instruct_mlp_out) + + else: + img_norm1_out = self.img_norm1(img_hidden_states) + img_norm3_out = self.img_norm3(img_hidden_states) + instruct_norm1_out = self.instruct_norm1(instruct_hidden_states) + + joint_attn_out = self.img_instruct_attn( + img_norm1_out, + instruct_norm1_out, + joint_attention_mask, + rotary_emb, + encoder_seq_lengths, + seq_lengths, + ) + + instruct_attn_out = instruct_hidden_states.new_zeros(batch_size, L_instruct, self.hidden_size) + img_attn_out = img_hidden_states.new_zeros(batch_size, L_img, self.hidden_size) + for i, (encoder_seq_len, seq_len) in enumerate(zip(encoder_seq_lengths, seq_lengths)): + instruct_attn_out[i, :encoder_seq_len] = joint_attn_out[i, :encoder_seq_len] + img_attn_out[i, : seq_len - encoder_seq_len] = joint_attn_out[i, encoder_seq_len:seq_len] + + img_self_attn_out = self.img_self_attn(img_norm3_out, img_attention_mask, image_rotary_emb) + + img_hidden_states = img_hidden_states + self.img_attn_norm(img_attn_out) + img_hidden_states = img_hidden_states + self.img_self_attn_norm(img_self_attn_out) + img_norm2_out = self.img_norm2(img_hidden_states) + img_mlp_out = self.img_feed_forward(self.img_ffn_norm1(img_norm2_out)) + img_hidden_states = img_hidden_states + self.img_ffn_norm2(img_mlp_out) + + instruct_hidden_states = instruct_hidden_states + self.instruct_attn_norm(instruct_attn_out) + instruct_norm2_out = self.instruct_norm2(instruct_hidden_states) + instruct_mlp_out = self.instruct_feed_forward(self.instruct_ffn_norm1(instruct_norm2_out)) + instruct_hidden_states = instruct_hidden_states + self.instruct_ffn_norm2(instruct_mlp_out) + + return img_hidden_states, instruct_hidden_states + + +def _cal_preprocessed_instruction_feat_dim(instruction_feature_configs: dict) -> int: + num_instruction_feature_layers = max(instruction_feature_configs.get("num_instruction_feature_layers", 1), 1) + instruction_feat_dim = instruction_feature_configs.get("instruction_feat_dim", 4096) + reduce_type = instruction_feature_configs.get("reduce_type", "concat") + if "cat" in reduce_type.lower(): + return num_instruction_feature_layers * instruction_feat_dim + elif "mean" in reduce_type.lower(): + return instruction_feat_dim + else: + raise ValueError(f"Invalid reduce_type: {reduce_type}") + + +class BooguImageTransformer2DModel(nn.Module): + """Boogu-Image transformer with mixed stream topology. + + Early layers use double-stream (dual-stream) processing, then switch to + single-stream joint processing. Context/noise/reference-image refiner + blocks run before the main stack. + """ + + # Noise refiner, reference image refiner, and double stream layers are + # kept out of regional torch.compile for numerical stability (matches the + # upstream `_repeated_blocks`). + _repeated_blocks = [ + "BooguImageTransformerBlock", + "BooguImageContextRefinerTransformerBlock", + "BooguImageSingleStreamTransformerBlock", + ] + _layerwise_offload_blocks_attrs = ["single_stream_layers", "double_stream_layers"] + + def __init__(self, od_config: OmniDiffusionConfig) -> None: + super().__init__() + self.od_config = od_config + cfg = od_config.tf_model_config + + patch_size = cfg.get("patch_size", 2) + in_channels = cfg.get("in_channels", 16) + out_channels = cfg.get("out_channels", None) + hidden_size = cfg.get("hidden_size", 2304) + num_layers = cfg.get("num_layers", 26) + num_double_stream_layers = cfg.get("num_double_stream_layers", 2) + num_refiner_layers = cfg.get("num_refiner_layers", 2) + num_attention_heads = cfg.get("num_attention_heads", 24) + num_kv_heads = cfg.get("num_kv_heads", 8) + multiple_of = cfg.get("multiple_of", 256) + ffn_dim_multiplier = cfg.get("ffn_dim_multiplier", None) + norm_eps = cfg.get("norm_eps", 1e-5) + axes_dim_rope = tuple(cfg.get("axes_dim_rope", (40, 40, 40))) + axes_lens = tuple(cfg.get("axes_lens", (2048, 1664, 1664))) + instruction_feature_configs = cfg.get( + "instruction_feature_configs", + {"instruction_feat_dim": 1024, "reduce_type": "mean", "num_instruction_feature_layers": 1}, + ) + prompt_tuning_configs = cfg.get("prompt_tuning_configs", {"use_prompt_tuning": False}) + timestep_scale = cfg.get("timestep_scale", 1.0) + + if (hidden_size // num_attention_heads) != sum(axes_dim_rope): + raise ValueError( + f"hidden_size // num_attention_heads ({hidden_size // num_attention_heads}) " + f"must equal sum(axes_dim_rope) ({sum(axes_dim_rope)})" + ) + if num_double_stream_layers > num_layers: + raise ValueError( + f"num_double_stream_layers ({num_double_stream_layers}) cannot be greater than " + f"num_layers ({num_layers})" + ) + if prompt_tuning_configs.get("use_prompt_tuning", False): + raise NotImplementedError( + "Prompt tuning is a training-time feature and is not supported by the " + "native vLLM-Omni BooguImageTransformer2DModel." + ) + + self.patch_size = patch_size + self.in_channels = in_channels + self.out_channels = out_channels or in_channels + self.hidden_size = hidden_size + self.num_attention_heads = num_attention_heads + self.num_double_stream_layers = num_double_stream_layers + self.num_single_stream_layers = num_layers - num_double_stream_layers + self.axes_dim_rope = axes_dim_rope + self.axes_lens = axes_lens + self.instruction_feature_configs = instruction_feature_configs + self.preprocessed_instruction_feat_dim = _cal_preprocessed_instruction_feat_dim(instruction_feature_configs) + + self.rope_embedder = BooguImageDoubleStreamRotaryPosEmbed( + theta=10000, + axes_dim=axes_dim_rope, + axes_lens=axes_lens, + patch_size=patch_size, + ) + + self.x_embedder = nn.Linear( + in_features=patch_size * patch_size * in_channels, + out_features=hidden_size, + ) + self.ref_image_patch_embedder = nn.Linear( + in_features=patch_size * patch_size * in_channels, + out_features=hidden_size, + ) + + self.time_caption_embed = Lumina2CombinedTimestepCaptionEmbedding( + hidden_size=hidden_size, + instruction_feat_dim=self.preprocessed_instruction_feat_dim, + norm_eps=norm_eps, + timestep_scale=timestep_scale, + ) + + self.noise_refiner = nn.ModuleList( + [ + BooguImageNoiseRefinerTransformerBlock( + hidden_size, + num_attention_heads, + num_kv_heads, + multiple_of, + ffn_dim_multiplier, + norm_eps, + modulation=True, + ) + for _ in range(num_refiner_layers) + ] + ) + + self.ref_image_refiner = nn.ModuleList( + [ + BooguImageRefImgRefinerTransformerBlock( + hidden_size, + num_attention_heads, + num_kv_heads, + multiple_of, + ffn_dim_multiplier, + norm_eps, + modulation=True, + ) + for _ in range(num_refiner_layers) + ] + ) + + self.context_refiner = nn.ModuleList( + [ + BooguImageContextRefinerTransformerBlock( + hidden_size, + num_attention_heads, + num_kv_heads, + multiple_of, + ffn_dim_multiplier, + norm_eps, + modulation=False, + ) + for _ in range(num_refiner_layers) + ] + ) + + # Mixed architecture: dual-stream first, then single-stream. + self.double_stream_layers = nn.ModuleList( + [ + BooguImageDoubleStreamTransformerBlock( + hidden_size, + num_attention_heads, + num_kv_heads, + multiple_of, + ffn_dim_multiplier, + norm_eps, + modulation=True, + ) + for _ in range(num_double_stream_layers) + ] + ) + + self.single_stream_layers = nn.ModuleList( + [ + BooguImageSingleStreamTransformerBlock( + hidden_size, + num_attention_heads, + num_kv_heads, + multiple_of, + ffn_dim_multiplier, + norm_eps, + modulation=True, + ) + for _ in range(self.num_single_stream_layers) + ] + ) + + self.norm_out = LuminaLayerNormContinuous( + embedding_dim=hidden_size, + conditioning_embedding_dim=min(hidden_size, 1024), + elementwise_affine=False, + eps=1e-6, + bias=True, + out_dim=patch_size * patch_size * self.out_channels, + ) + + # Distinguish multiple reference images (max 5). + self.image_index_embedding = nn.Parameter(torch.randn(5, hidden_size)) + + def preprocess_instruction_hidden_states(self, raw_instruction_hidden_states): + """Reduce the raw MLLM hidden states to the transformer feature dim. + + Mirrors upstream ``preprocess_instruction_hidden_states``: a single + tensor passes through unchanged; a list of per-layer states is combined + by ``concat`` or ``mean`` according to ``instruction_feature_configs``. + """ + cfg = self.instruction_feature_configs + num_instruction_feature_layers = max(cfg.get("num_instruction_feature_layers", 1), 1) + reduce_type = cfg.get("reduce_type", "concat") + + if isinstance(raw_instruction_hidden_states, torch.Tensor): + instruction_hidden_states = raw_instruction_hidden_states + elif isinstance(raw_instruction_hidden_states, (list, tuple)): + assert len(raw_instruction_hidden_states) == num_instruction_feature_layers + if "cat" in reduce_type.lower(): + instruction_hidden_states = torch.cat(raw_instruction_hidden_states, dim=-1) + elif "mean" in reduce_type.lower(): + instruction_hidden_states = torch.mean(torch.stack(raw_instruction_hidden_states), dim=0) + else: + raise ValueError(f"Invalid reduce_type: {reduce_type}") + else: + raise ValueError( + "Invalid type of raw_instruction_hidden_states, expected torch.Tensor or list, " + f"but got {type(raw_instruction_hidden_states)}" + ) + + assert self.preprocessed_instruction_feat_dim == instruction_hidden_states.shape[-1] + return instruction_hidden_states + + def flat_and_pad_to_seq(self, hidden_states, ref_image_hidden_states): + """Flatten patch tokens and pad to batched sequences. + + Ported from upstream; for text-to-image ``ref_image_hidden_states`` is + ``None`` and the reference-image branch collapses to zero-length. + """ + batch_size = len(hidden_states) + p = self.patch_size + device = hidden_states[0].device + + img_sizes = [(img.size(1), img.size(2)) for img in hidden_states] + l_effective_img_len = [(H // p) * (W // p) for (H, W) in img_sizes] + + if ref_image_hidden_states is not None: + ref_img_sizes = [ + [(img.size(1), img.size(2)) for img in imgs] if imgs is not None else None + for imgs in ref_image_hidden_states + ] + l_effective_ref_img_len = [ + [(ref_img_size[0] // p) * (ref_img_size[1] // p) for ref_img_size in _ref_img_sizes] + if _ref_img_sizes is not None + else [0] + for _ref_img_sizes in ref_img_sizes + ] + else: + ref_img_sizes = [None for _ in range(batch_size)] + l_effective_ref_img_len = [[0] for _ in range(batch_size)] + + max_ref_img_len = max(sum(ref_img_len) for ref_img_len in l_effective_ref_img_len) + max_img_len = max(l_effective_img_len) + + # Reference-image patch embeddings. + flat_ref_img_hidden_states = [] + for i in range(batch_size): + if ref_img_sizes[i] is not None: + imgs = [] + for ref_img in ref_image_hidden_states[i]: + C, H, W = ref_img.size() + ref_img = rearrange(ref_img, "c (h p1) (w p2) -> (h w) (p1 p2 c)", p1=p, p2=p) + imgs.append(ref_img) + flat_ref_img_hidden_states.append(torch.cat(imgs, dim=0)) + else: + flat_ref_img_hidden_states.append(None) + + # Noise-image patch embeddings. + flat_hidden_states = [] + for i in range(batch_size): + img = hidden_states[i] + C, H, W = img.size() + img = rearrange(img, "c (h p1) (w p2) -> (h w) (p1 p2 c)", p1=p, p2=p) + flat_hidden_states.append(img) + + padded_ref_img_hidden_states = torch.zeros( + batch_size, + max_ref_img_len, + flat_hidden_states[0].shape[-1], + device=device, + dtype=flat_hidden_states[0].dtype, + ) + padded_ref_img_mask = torch.zeros(batch_size, max_ref_img_len, dtype=torch.bool, device=device) + for i in range(batch_size): + if ref_img_sizes[i] is not None: + padded_ref_img_hidden_states[i, : sum(l_effective_ref_img_len[i])] = flat_ref_img_hidden_states[i] + padded_ref_img_mask[i, : sum(l_effective_ref_img_len[i])] = True + + padded_hidden_states = torch.zeros( + batch_size, + max_img_len, + flat_hidden_states[0].shape[-1], + device=device, + dtype=flat_hidden_states[0].dtype, + ) + padded_img_mask = torch.zeros(batch_size, max_img_len, dtype=torch.bool, device=device) + for i in range(batch_size): + padded_hidden_states[i, : l_effective_img_len[i]] = flat_hidden_states[i] + padded_img_mask[i, : l_effective_img_len[i]] = True + + return ( + padded_hidden_states, + padded_ref_img_hidden_states, + padded_img_mask, + padded_ref_img_mask, + l_effective_ref_img_len, + l_effective_img_len, + ref_img_sizes, + img_sizes, + ) + + def img_patch_embed_and_refine( + self, + hidden_states, + ref_image_hidden_states, + padded_img_mask, + padded_ref_img_mask, + noise_rotary_emb, + ref_img_rotary_emb, + l_effective_ref_img_len, + l_effective_img_len, + temb, + ): + """Embed image patches and run the refiner blocks. + + The reference-image refiner is skipped when there are no reference-image + tokens (text-to-image), which is numerically identical to upstream (the + combined sequence only reads ``[:sum(ref_img_len)]`` = empty) while + avoiding a degenerate zero-length attention. + """ + batch_size = len(hidden_states) + max_combined_img_len = max( + img_len + sum(ref_img_len) for img_len, ref_img_len in zip(l_effective_img_len, l_effective_ref_img_len) + ) + + hidden_states = self.x_embedder(hidden_states) + ref_image_hidden_states = self.ref_image_patch_embedder(ref_image_hidden_states) + + for i in range(batch_size): + shift = 0 + for j, ref_img_len in enumerate(l_effective_ref_img_len[i]): + ref_image_hidden_states[i, shift : shift + ref_img_len, :] = ( + ref_image_hidden_states[i, shift : shift + ref_img_len, :] + self.image_index_embedding[j] + ) + shift += ref_img_len + + for layer in self.noise_refiner: + hidden_states = layer(hidden_states, padded_img_mask, noise_rotary_emb, temb) + + flat_l_effective_ref_img_len = list(itertools.chain(*l_effective_ref_img_len)) + num_ref_images = len(flat_l_effective_ref_img_len) + max_ref_img_len = max(flat_l_effective_ref_img_len) + + if max_ref_img_len > 0: + batch_ref_img_mask = ref_image_hidden_states.new_zeros(num_ref_images, max_ref_img_len, dtype=torch.bool) + batch_ref_image_hidden_states = ref_image_hidden_states.new_zeros( + num_ref_images, max_ref_img_len, self.hidden_size + ) + batch_ref_img_rotary_emb = hidden_states.new_zeros( + num_ref_images, max_ref_img_len, ref_img_rotary_emb.shape[-1], dtype=ref_img_rotary_emb.dtype + ) + batch_temb = temb.new_zeros(num_ref_images, *temb.shape[1:], dtype=temb.dtype) + + # Flatten reference images into a temporary batch. + idx = 0 + for i in range(batch_size): + shift = 0 + for ref_img_len in l_effective_ref_img_len[i]: + batch_ref_img_mask[idx, :ref_img_len] = True + batch_ref_image_hidden_states[idx, :ref_img_len] = ref_image_hidden_states[ + i, shift : shift + ref_img_len + ] + batch_ref_img_rotary_emb[idx, :ref_img_len] = ref_img_rotary_emb[i, shift : shift + ref_img_len] + batch_temb[idx] = temb[i] + shift += ref_img_len + idx += 1 + + for layer in self.ref_image_refiner: + batch_ref_image_hidden_states = layer( + batch_ref_image_hidden_states, batch_ref_img_mask, batch_ref_img_rotary_emb, batch_temb + ) + + # Restore reference-image sequence layout. + idx = 0 + for i in range(batch_size): + shift = 0 + for ref_img_len in l_effective_ref_img_len[i]: + ref_image_hidden_states[i, shift : shift + ref_img_len] = batch_ref_image_hidden_states[ + idx, :ref_img_len + ] + shift += ref_img_len + idx += 1 + + combined_img_hidden_states = hidden_states.new_zeros(batch_size, max_combined_img_len, self.hidden_size) + for i, (ref_img_len, img_len) in enumerate(zip(l_effective_ref_img_len, l_effective_img_len)): + combined_img_hidden_states[i, : sum(ref_img_len)] = ref_image_hidden_states[i, : sum(ref_img_len)] + combined_img_hidden_states[i, sum(ref_img_len) : sum(ref_img_len) + img_len] = hidden_states[i, :img_len] + + return combined_img_hidden_states + + def forward( + self, + hidden_states: torch.Tensor | list[torch.Tensor], + timestep: torch.Tensor, + instruction_hidden_states: torch.Tensor, + freqs_cis: torch.Tensor, + instruction_attention_mask: torch.Tensor, + ref_image_hidden_states: list[list[torch.Tensor]] | None = None, + ) -> torch.Tensor: + """Denoise one step: refiner -> double-stream -> fuse -> single-stream -> unpatchify. + + Ported from upstream ``BooguImageTransformer2DModel.forward`` with the + TeaCache/TaylorSeer/PEFT/gradient-checkpointing branches removed. Returns + the velocity prediction as a ``[B, C, H_lat, W_lat]`` tensor. + """ + instruction_hidden_states = self.preprocess_instruction_hidden_states(instruction_hidden_states) + + batch_size = len(hidden_states) + is_hidden_states_tensor = isinstance(hidden_states, torch.Tensor) + if is_hidden_states_tensor: + assert hidden_states.ndim == 4 + hidden_states = [_hidden_states for _hidden_states in hidden_states] + + device = hidden_states[0].device + + # Timestep and instruction embedding. + temb, instruction_hidden_states = self.time_caption_embed( + timestep, instruction_hidden_states, hidden_states[0].dtype + ) + + # Flatten and pad token sequences. + ( + hidden_states, + ref_image_hidden_states, + img_mask, + ref_img_mask, + l_effective_ref_img_len, + l_effective_img_len, + ref_img_sizes, + img_sizes, + ) = self.flat_and_pad_to_seq(hidden_states, ref_image_hidden_states) + + # Build rotary embeddings and sequence lengths. + ( + context_rotary_emb, + ref_img_rotary_emb, + noise_rotary_emb, + rotary_emb, + encoder_seq_lengths, + seq_lengths, + combined_img_rotary_emb, + combined_img_seq_lengths, + ) = self.rope_embedder( + freqs_cis, + instruction_attention_mask, + l_effective_ref_img_len, + l_effective_img_len, + ref_img_sizes, + img_sizes, + device, + ) + + # Context refinement. + for layer in self.context_refiner: + instruction_hidden_states = layer(instruction_hidden_states, instruction_attention_mask, context_rotary_emb) + + # Image patch embedding and refinement. + combined_img_hidden_states = self.img_patch_embed_and_refine( + hidden_states, + ref_image_hidden_states, + img_mask, + ref_img_mask, + noise_rotary_emb, + ref_img_rotary_emb, + l_effective_ref_img_len, + l_effective_img_len, + temb, + ) + + instruct_hidden_states = instruction_hidden_states + img_hidden_states = combined_img_hidden_states + + # Joint mask for [instruct + image]. + max_seq_len = max(seq_lengths) + joint_attention_mask = hidden_states.new_zeros(batch_size, max_seq_len, dtype=torch.bool) + for i, seq_len in enumerate(seq_lengths): + joint_attention_mask[i, :seq_len] = True + + # Dual-stream (double-stream) stage. + if self.num_double_stream_layers > 0: + max_img_len = max(combined_img_seq_lengths) + img_attention_mask = hidden_states.new_zeros(batch_size, max_img_len, dtype=torch.bool) + for i, img_seq_len in enumerate(combined_img_seq_lengths): + img_attention_mask[i, :img_seq_len] = True + + for layer in self.double_stream_layers: + img_hidden_states, instruct_hidden_states = layer( + img_hidden_states, + instruct_hidden_states, + img_attention_mask, + joint_attention_mask, + combined_img_rotary_emb, + rotary_emb, + temb, + encoder_seq_lengths, + seq_lengths, + ) + + # Fuse streams to joint sequence. + joint_hidden_states = hidden_states.new_zeros(batch_size, max(seq_lengths), self.hidden_size) + for i, (encoder_seq_len, seq_len) in enumerate(zip(encoder_seq_lengths, seq_lengths)): + joint_hidden_states[i, :encoder_seq_len] = instruct_hidden_states[i, :encoder_seq_len] + joint_hidden_states[i, encoder_seq_len:seq_len] = img_hidden_states[i, : seq_len - encoder_seq_len] + + # Single-stream stage. + hidden_states = joint_hidden_states + for layer in self.single_stream_layers: + hidden_states = layer(hidden_states, joint_attention_mask, rotary_emb, temb) + + # Output projection. + hidden_states = self.norm_out(hidden_states, temb) + + # Reshape back to image format. + p = self.patch_size + output = [] + for i, (img_size, img_len, seq_len) in enumerate(zip(img_sizes, l_effective_img_len, seq_lengths)): + height, width = img_size + img_tokens = hidden_states[i][seq_len - img_len : seq_len] + img_output = rearrange( + img_tokens, + "(h w) (p1 p2 c) -> c (h p1) (w p2)", + h=height // p, + w=width // p, + p1=p, + p2=p, + ) + output.append(img_output) + + if is_hidden_states_tensor: + output = torch.stack(output, dim=0) + + return output + + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: + """Load diffusers-named checkpoint weights into the native module. + + Two name promotions relative to upstream (see step 8/10 findings): + + - ``*.img_instruct_attn.processor.{img,instruct}_{to_q,to_k,to_v}`` / + ``{instruct,img}_out`` -> drop ``.processor`` (upstream keeps the + joint-attention projections on the attention processor; the native + module hosts them directly). + - ``*.to_out.0.weight`` -> ``*.to_out.weight`` (diffusers wraps the + output projection in a ``ModuleList``; the native module uses a plain + linear). + """ + params_dict = dict(self.named_parameters()) + loaded_params: set[str] = set() + + for name, loaded_weight in weights: + original_name = name + if ".img_instruct_attn.processor." in name: + name = name.replace(".img_instruct_attn.processor.", ".img_instruct_attn.") + if ".to_out.0." in name: + name = name.replace(".to_out.0.", ".to_out.") + + if name not in params_dict: + logger.warning("Skipping unexpected checkpoint weight %s", original_name) + continue + + param = params_dict[name] + weight_loader = getattr(param, "weight_loader", default_weight_loader) + weight_loader(param, loaded_weight) + loaded_params.add(name) + + unloaded_params = sorted(params_dict.keys() - loaded_params) + if unloaded_params: + logger.warning("Model parameters not loaded from checkpoint: %s", unloaded_params) + + return loaded_params diff --git a/vllm_omni/diffusion/models/boogu_image/image_processor.py b/vllm_omni/diffusion/models/boogu_image/image_processor.py new file mode 100644 index 000000000..3edf67bcc --- /dev/null +++ b/vllm_omni/diffusion/models/boogu_image/image_processor.py @@ -0,0 +1,213 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +"""Native port of the upstream ``BooguImageProcessor``. + +Ported from the external ``boogu`` package +(``boogu/pipelines/image_processor.py``) so the native vLLM-Omni pipeline can +reproduce Boogu-Image's reference-image and VLM preprocessing without requiring +``pip install boogu-image``. + +Only the two Boogu-specific methods are ported: ``get_new_height_width`` (the +``max_pixels`` / ``max_side_length`` aware downscale-only resize) and +``preprocess`` (which routes through it). Everything else is inherited from the +stock diffusers ``VaeImageProcessor``. +""" + +import warnings + +import numpy as np +import PIL.Image +import torch +from diffusers.image_processor import ( + PipelineImageInput, + VaeImageProcessor, + is_valid_image_imagelist, +) + + +class BooguImageProcessor(VaeImageProcessor): + """VaeImageProcessor variant with Boogu-Image pixel/side-length constraints. + + Resizing never upscales (the ratio is clamped to ``<= 1``) and always aligns + the target height/width to multiples of ``vae_scale_factor``. + """ + + def __init__( + self, + do_resize: bool = True, + vae_scale_factor: int = 16, + resample: str = "lanczos", + max_pixels: int | None = None, + max_side_length: int | None = None, + do_normalize: bool = True, + do_binarize: bool = False, + do_convert_rgb: bool = False, + do_convert_grayscale: bool = False, + ): + super().__init__( + do_resize=do_resize, + vae_scale_factor=vae_scale_factor, + resample=resample, + do_normalize=do_normalize, + do_binarize=do_binarize, + do_convert_rgb=do_convert_rgb, + do_convert_grayscale=do_convert_grayscale, + ) + self.max_pixels = max_pixels + self.max_side_length = max_side_length + + def get_new_height_width( + self, + image: PIL.Image.Image | np.ndarray | torch.Tensor, + height: int | None = None, + width: int | None = None, + max_pixels: int | None = None, + max_side_length: int | None = None, + ) -> tuple[int, int]: + """Return target ``(height, width)`` after downscale + alignment. + + Faithful port of upstream ``BooguImageProcessor.get_new_height_width``. + """ + if height is None: + if isinstance(image, PIL.Image.Image): + height = image.height + elif isinstance(image, torch.Tensor): + height = image.shape[2] + else: + height = image.shape[1] + + if width is None: + if isinstance(image, PIL.Image.Image): + width = image.width + elif isinstance(image, torch.Tensor): + width = image.shape[3] + else: + width = image.shape[2] + + if max_side_length is None: + max_side_length = self.max_side_length + + if max_pixels is None: + max_pixels = self.max_pixels + + if height <= 0 or width <= 0: + raise ValueError(f"Image height and width must be positive, got height={height}, width={width}") + + max_side_length_ratio = 1.0 + if max_side_length is not None: + if height > width: + max_side_length_ratio = max_side_length / height + else: + max_side_length_ratio = max_side_length / width + + cur_pixels = height * width + max_pixels_ratio = (max_pixels / cur_pixels) ** 0.5 if max_pixels is not None else 1.0 + # Clamp ratio to <=1 to avoid upscaling input images in preprocessing. + ratio = min(max_pixels_ratio, max_side_length_ratio, 1.0) + + new_height, new_width = ( + int(height * ratio) // self.config.vae_scale_factor * self.config.vae_scale_factor, + int(width * ratio) // self.config.vae_scale_factor * self.config.vae_scale_factor, + ) + return new_height, new_width + + def preprocess( + self, + image: PipelineImageInput, + height: int | None = None, + width: int | None = None, + max_pixels: int | None = None, + max_side_length: int | None = None, + resize_mode: str = "default", + crops_coords: tuple[int, int, int, int] | None = None, + ) -> torch.Tensor: + """Preprocess an image into a normalized ``[B, C, H, W]`` tensor. + + Faithful port of upstream ``BooguImageProcessor.preprocess`` (PixArt-style + downscale). Only the PIL branch is exercised by the native pipeline, but + the numpy/tensor branches are kept for parity. + """ + supported_formats = (PIL.Image.Image, np.ndarray, torch.Tensor) + + if self.config.do_convert_grayscale and isinstance(image, (torch.Tensor, np.ndarray)) and image.ndim == 3: + if isinstance(image, torch.Tensor): + image = image.unsqueeze(1) + else: + if image.shape[-1] == 1: + image = np.expand_dims(image, axis=0) + else: + image = np.expand_dims(image, axis=-1) + + if isinstance(image, list) and isinstance(image[0], np.ndarray) and image[0].ndim == 4: + warnings.warn( + "Passing `image` as a list of 4d np.ndarray is deprecated." + "Please concatenate the list along the batch dimension and pass it as a single 4d np.ndarray", + FutureWarning, + ) + image = np.concatenate(image, axis=0) + if isinstance(image, list) and isinstance(image[0], torch.Tensor) and image[0].ndim == 4: + warnings.warn( + "Passing `image` as a list of 4d torch.Tensor is deprecated." + "Please concatenate the list along the batch dimension and pass it as a single 4d torch.Tensor", + FutureWarning, + ) + image = torch.cat(image, axis=0) + + if not is_valid_image_imagelist(image): + supported_formats_str = ", ".join(str(x) for x in supported_formats) + raise ValueError(f"Input is in incorrect format. Currently, we only support {supported_formats_str}") + + if not isinstance(image, list): + image = [image] + + if isinstance(image[0], PIL.Image.Image): + if crops_coords is not None: + image = [i.crop(crops_coords) for i in image] + if self.config.do_resize: + height, width = self.get_new_height_width(image[0], height, width, max_pixels, max_side_length) + image = [self.resize(i, height, width, resize_mode=resize_mode) for i in image] + if self.config.do_convert_rgb: + image = [self.convert_to_rgb(i) for i in image] + elif self.config.do_convert_grayscale: + image = [self.convert_to_grayscale(i) for i in image] + image = self.pil_to_numpy(image) + image = self.numpy_to_pt(image) + + elif isinstance(image[0], np.ndarray): + image = np.concatenate(image, axis=0) if image[0].ndim == 4 else np.stack(image, axis=0) + image = self.numpy_to_pt(image) + height, width = self.get_new_height_width(image, height, width, max_pixels, max_side_length) + if self.config.do_resize: + image = self.resize(image, height, width) + + elif isinstance(image[0], torch.Tensor): + image = torch.cat(image, axis=0) if image[0].ndim == 4 else torch.stack(image, axis=0) + + if self.config.do_convert_grayscale and image.ndim == 3: + image = image.unsqueeze(1) + + channel = image.shape[1] + if channel == self.config.vae_latent_channels: + return image + + height, width = self.get_new_height_width(image, height, width, max_pixels, max_side_length) + if self.config.do_resize: + image = self.resize(image, height, width) + + do_normalize = self.config.do_normalize + if do_normalize and image.min() < 0: + warnings.warn( + "Passing `image` as torch tensor with value range in [-1,1] is deprecated. The expected value range " + f"for image tensor is [0,1] when passing as pytorch tensor or numpy Array. You passed `image` with " + f"value range [{image.min()},{image.max()}]", + FutureWarning, + ) + do_normalize = False + if do_normalize: + image = self.normalize(image) + + if self.config.do_binarize: + image = self.binarize(image) + + return image diff --git a/vllm_omni/diffusion/models/boogu_image/pipeline_boogu_image.py b/vllm_omni/diffusion/models/boogu_image/pipeline_boogu_image.py new file mode 100644 index 000000000..7ef6877c8 --- /dev/null +++ b/vllm_omni/diffusion/models/boogu_image/pipeline_boogu_image.py @@ -0,0 +1,793 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +"""Native vLLM-Omni pipeline for Boogu-Image-0.1. + +Ported from the upstream ``boogu`` package +(``boogu/pipelines/boogu/pipeline_boogu.py``) with the following changes: + +- Diffusers ``DiffusionPipeline``/``register_modules`` machinery replaced by a + plain ``nn.Module`` constructed from ``OmniDiffusionConfig`` (components are + loaded from the checkpoint subfolders; transformer weights arrive later via + ``weights_sources`` + ``load_weights``). +- Upstream ``encode_instruction`` is exposed as ``encode_prompt`` (the + vLLM-Omni convention, also hooked by the prompt-embed cache); only the + text-to-image path is ported. Instruction rewriting, prompt tuning, + reference-image (TI2I/I2I) encoding, double guidance, and vision-token + stripping are not ported. +- ``forward()`` (denoise loop + VAE decode) is intentionally not implemented + yet; it lands together with the post-process registration. +""" + +import json +import os +from collections.abc import Iterable +from typing import ClassVar, cast + +import PIL.Image +import torch +import torch.nn.functional as F +from diffusers.image_processor import VaeImageProcessor +from diffusers.models.autoencoders.autoencoder_kl import AutoencoderKL +from diffusers.utils.torch_utils import randn_tensor +from torch import nn +from transformers import Qwen3VLForConditionalGeneration, Qwen3VLProcessor +from vllm.logger import init_logger +from vllm.model_executor.models.utils import AutoWeightsLoader + +from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig +from vllm_omni.diffusion.distributed.utils import get_local_device +from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader +from vllm_omni.diffusion.model_loader.hub_prefetch import from_pretrained_with_prefetch, prefetch_subfolders +from vllm_omni.diffusion.models.boogu_image.boogu_image_transformer import ( + BooguImageDoubleStreamRotaryPosEmbed, + BooguImageTransformer2DModel, +) +from vllm_omni.diffusion.models.boogu_image.image_processor import BooguImageProcessor +from vllm_omni.diffusion.models.boogu_image.scheduling_flow_match_euler_discrete_time_shifting import ( + FlowMatchEulerDiscreteScheduler, +) +from vllm_omni.diffusion.models.interface import SupportImageInput, SupportsComponentDiscovery +from vllm_omni.diffusion.models.progress_bar import ProgressBarMixin +from vllm_omni.diffusion.request import OmniDiffusionRequest +from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch +from vllm_omni.model_executor.model_loader.weight_utils import download_weights_from_hf_specific + +logger = init_logger(__name__) + +# Reference-image preprocessing limits (upstream ``BooguImagePipeline.__call__`` +# defaults). The VLM copy is aggressively downscaled for the Qwen3VL encoder; +# the VAE copy keeps near-native resolution for the reference latents. +_MAX_VLM_INPUT_PIL_PIXELS = 384 * 384 +_MAX_VLM_INPUT_PIL_SIDE_LENGTH = 384 * 2 +_MAX_INPUT_IMAGE_PIXELS = 2048 * 2048 +_MAX_INPUT_IMAGE_SIDE_LENGTH = 2048 * 2 + + +def _load_vae_scale_factor(model_path: str) -> int: + vae_config_path = os.path.join(model_path, "vae/config.json") + try: + with open(vae_config_path) as f: + vae_config = json.load(f) + except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc: + raise RuntimeError(f"Failed to load Boogu VAE config from {vae_config_path}: {exc}") from exc + + if "block_out_channels" not in vae_config: + return 8 + return 2 ** (len(vae_config["block_out_channels"]) - 1) + + +def get_boogu_image_post_process_func(od_config: OmniDiffusionConfig): + """Build the post-process callable that converts decoded tensors to images. + + Upstream ``BooguImageProcessor`` only customizes *pre*-processing; the + ``postprocess`` path is inherited from the stock diffusers + ``VaeImageProcessor``, so we reuse it directly here. + """ + model_name = od_config.model + if os.path.exists(model_name): + model_path = model_name + else: + model_path = download_weights_from_hf_specific(model_name, None, ["*"]) + + vae_scale_factor = _load_vae_scale_factor(model_path) + + image_processor = VaeImageProcessor(vae_scale_factor=vae_scale_factor * 2) + + def post_process_func(images: torch.Tensor): + return image_processor.postprocess(images) + + return post_process_func + + +def get_boogu_image_pre_process_func(od_config: OmniDiffusionConfig): + """Build the pre-process callable for Boogu-Image reference (edit) input. + + Text-to-image requests carry no image and are passed through unchanged (the + Base checkpoint shares this pipeline class). Edit (TI2I) requests carry a + single reference PIL image on ``prompt["multi_modal_data"]["image"]``; it is + resized twice — once for the Qwen3VL encoder (``prompt_image``) and once for + the VAE reference latents (``preprocessed_image``) — and stashed in + ``additional_information`` for ``forward`` to consume. Mirrors upstream + ``preprocess_vlm_input_pil_images`` + ``prepare_image``. + + For a single reference image, upstream ``align_res`` (default ``True``) + derives the output resolution from the VAE-encoded reference dimensions, so + the request height/width are overwritten accordingly. + """ + model_name = od_config.model + if os.path.exists(model_name): + model_path = model_name + else: + model_path = download_weights_from_hf_specific(model_name, None, ["*"]) + + vae_scale_factor = _load_vae_scale_factor(model_path) + + # Upstream builds ``BooguImageProcessor(vae_scale_factor=vae_scale_factor*2)`` + # so all resize targets align to multiples of ``vae_scale_factor * 2``. + image_processor = BooguImageProcessor(vae_scale_factor=vae_scale_factor * 2, do_resize=True) + + def pre_process_func(request: OmniDiffusionRequest): + prompt = request.prompt + if isinstance(prompt, str): + # Plain-text prompt cannot carry an image -> text-to-image, no-op. + return request + + multi_modal_data = prompt.get("multi_modal_data") or {} + raw_image = multi_modal_data.get("image") + if not raw_image: + # No reference image -> text-to-image, no-op (Base checkpoint). + return request + + if isinstance(raw_image, list): + if len(raw_image) > 1: + raise ValueError(f"Boogu-Image editing supports a single reference image; received {len(raw_image)}.") + raw_image = raw_image[0] + + if isinstance(raw_image, str): + image = PIL.Image.open(raw_image) + else: + image = cast(PIL.Image.Image, raw_image) + image = image.convert("RGB") + + if "additional_information" not in prompt: + prompt["additional_information"] = {} + + # VLM-resized copy (PIL) for the Qwen3VL instruction encoder. + vlm_height, vlm_width = image_processor.get_new_height_width( + image, None, None, _MAX_VLM_INPUT_PIL_PIXELS, _MAX_VLM_INPUT_PIL_SIDE_LENGTH + ) + prompt_image = image_processor.resize(image, vlm_height, vlm_width) + + # VAE-ready copy (normalized [1, C, H, W] tensor) for reference latents. + preprocessed_image = image_processor.preprocess( + image, max_pixels=_MAX_INPUT_IMAGE_PIXELS, max_side_length=_MAX_INPUT_IMAGE_SIDE_LENGTH + ) + + # align_res: single-image output resolution follows the reference dims. + request.sampling_params.height = int(preprocessed_image.shape[-2]) + request.sampling_params.width = int(preprocessed_image.shape[-1]) + + prompt["additional_information"]["preprocessed_image"] = preprocessed_image + prompt["additional_information"]["prompt_image"] = prompt_image + request.prompt = prompt + return request + + return pre_process_func + + +# System prompts matching upstream dataset logic (ported verbatim from +# ``BooguImagePipeline.__init__``). +SYSTEM_PROMPT_4_TI2I_UNIFIED = ( + "Describe the key features of the input image (color, shape, size, texture, objects, background), " + "then explain how the user's text instruction should alter or modify the image. Generate a new image " + "that meets the user's requirements while maintaining consistency with the original input where appropriate." +) +SYSTEM_PROMPT_4_T2I_UNIFIED = ( + "You are a helpful assistant that generates high-quality images based on user instructions. " + "The instructions are as follows." +) + + +class BooguImagePipeline(nn.Module, ProgressBarMixin, SupportsComponentDiscovery, SupportImageInput): + """Boogu-Image text-to-image and image-editing (TI2I) pipeline. + + Native vLLM-Omni implementation. A request with a reference image (edit / + TI2I) is served by the same class as text-to-image; the reference latents + and Qwen3VL image tokens are threaded through ``forward`` and the ported + transformer's reference-image refiner path. + """ + + supports_request_batch = False + + support_image_input: ClassVar[bool] = True + color_format: ClassVar[str] = "RGB" + + _dit_modules: ClassVar[list[str]] = ["transformer"] + _encoder_modules: ClassVar[list[str]] = ["mllm"] + _vae_modules: ClassVar[list[str]] = ["vae"] + + def __init__( + self, + *, + od_config: OmniDiffusionConfig, + prefix: str = "", + ) -> None: + super().__init__() + self.od_config = od_config + self._raise_unsupported_features() + self.weights_sources = [ + DiffusersPipelineLoader.ComponentSource( + model_or_path=od_config.model, + subfolder="transformer", + revision=None, + prefix="transformer.", + fall_back_to_pt=True, + ) + ] + + self._execution_device = get_local_device() + model = od_config.model + local_files_only = os.path.exists(model) + + # See ``hub_prefetch.py`` for the transformers v5 multi-worker subfolder + # race; prefetch the whole component set before any from_pretrained. + boogu_subfolders = ["scheduler", "vae", "mllm", "processor"] + prefetch_subfolders(model, boogu_subfolders, local_files_only=local_files_only) + + self.scheduler = FlowMatchEulerDiscreteScheduler.from_pretrained( + model, subfolder="scheduler", local_files_only=local_files_only + ) + + mllm = from_pretrained_with_prefetch( + Qwen3VLForConditionalGeneration.from_pretrained, + model, + subfolder="mllm", + prefetch_list=boogu_subfolders, + local_files_only=local_files_only, + torch_dtype=od_config.dtype, + ) + # Upstream reuses the full VLM as an optional instruction rewriter and + # encodes with its inner model (no ``lm_head``); the rewriter is not + # ported, so keep only the inner ``Qwen3VLModel`` as the encoder. + if hasattr(mllm, "lm_head"): + mllm = mllm.model + self.mllm = mllm.to(self._execution_device) + + self.processor = Qwen3VLProcessor.from_pretrained( + model, subfolder="processor", local_files_only=local_files_only + ) + + self.vae = from_pretrained_with_prefetch( + AutoencoderKL.from_pretrained, + model, + subfolder="vae", + prefetch_list=boogu_subfolders, + local_files_only=local_files_only, + ).to(self._execution_device) + + self.transformer = BooguImageTransformer2DModel(od_config=od_config) + + self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) if getattr(self, "vae", None) else 8 + self.default_sample_size = 128 + + self.SYSTEM_PROMPT_4_T2I = SYSTEM_PROMPT_4_T2I_UNIFIED + # Upstream uses the TI2I prompt for empty instructions (the default + # negative prompt "" hits this path). + self.SYSTEM_PROMPT_DROP = SYSTEM_PROMPT_4_TI2I_UNIFIED + # Edit (TI2I / I2I) system prompts (image present in the chat template). + self.SYSTEM_PROMPT_4_TI2I = SYSTEM_PROMPT_4_TI2I_UNIFIED + self.SYSTEM_PROMPT_4_I2I = SYSTEM_PROMPT_4_TI2I_UNIFIED + + def _raise_unsupported_features(self) -> None: + """Reject execution modes that do not have Boogu-specific support.""" + parallel_config = self.od_config.parallel_config + if parallel_config.tensor_parallel_size > 1: + raise NotImplementedError("Tensor parallelism is not supported by BooguImagePipeline.") + if (parallel_config.sequence_parallel_size or 1) > 1: + raise NotImplementedError("Sequence parallelism is not supported by BooguImagePipeline.") + if parallel_config.cfg_parallel_size > 1: + raise NotImplementedError("CFG parallelism is not supported by BooguImagePipeline.") + if parallel_config.use_hsdp: + raise NotImplementedError("HSDP is not supported by BooguImagePipeline.") + if self.od_config.cache_backend not in (None, "", "none"): + raise NotImplementedError( + f"Cache backend '{self.od_config.cache_backend}' is not supported by BooguImagePipeline." + ) + + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: + loader = AutoWeightsLoader(self) + return loader.load_weights(weights) + + # ------------------------------------------------------------------ + # Prompt encoding (upstream ``encode_instruction``, t2i path) + # ------------------------------------------------------------------ + + def _apply_chat_template( + self, + instruction: str, + input_pil_images: list[PIL.Image.Image] | None = None, + ) -> list[dict]: + """Build the chat messages for one instruction (text-to-image or edit). + + Mirrors upstream ``_apply_chat_template`` (``system_prompt_follows_task_type`` + is always ``False`` here): the system prompt is picked by whether images + are present and whether the instruction is empty, and reference images + are placed *before* the instruction text in the user turn. + """ + user_text_content = [{"type": "text", "text": instruction}] + + has_images = input_pil_images is not None and len(input_pil_images) > 0 + instruction_empty = instruction is None or len(instruction.strip()) == 0 + + if not has_images: + system_prompt = self.SYSTEM_PROMPT_DROP if instruction_empty else self.SYSTEM_PROMPT_4_T2I + else: + system_prompt = self.SYSTEM_PROMPT_4_I2I if instruction_empty else self.SYSTEM_PROMPT_4_TI2I + + system_role = {"role": "system", "content": [{"type": "text", "text": system_prompt}]} + if not has_images: + return [system_role, {"role": "user", "content": user_text_content}] + + images_content = [{"type": "image", "image": pil_img} for pil_img in input_pil_images] + return [system_role, {"role": "user", "content": images_content + user_text_content}] + + def _get_instruction_feature_embeds( + self, + instruction: str | list[str], + input_pil_images: list[list[PIL.Image.Image] | None] | None = None, + device: torch.device | None = None, + max_sequence_length: int = 256, + truncate_instruction_sequence: bool = False, + ) -> tuple[torch.Tensor, torch.Tensor]: + """Encode instructions (and optional reference images) with Qwen3VL. + + ``input_pil_images`` is a per-sample list (outer length == batch size); + each entry is the sample's already-VLM-resized reference images or + ``None``. Returns the last hidden state (or the last-N layers as a list + when the transformer config asks for more than one) and the attention + mask. + """ + device = device or self._execution_device + instruction = [instruction] if isinstance(instruction, str) else instruction + + if input_pil_images is None: + per_sample_images: list[list[PIL.Image.Image] | None] = [None] * len(instruction) + else: + assert len(input_pil_images) == len(instruction), ( + "`input_pil_images` outer length must match the instruction batch size." + ) + per_sample_images = input_pil_images + + prompts = [self._apply_chat_template(text, per_sample_images[i]) for i, text in enumerate(instruction)] + + vlm_inputs = self.processor.apply_chat_template( + prompts, + padding="longest", + max_length=max_sequence_length, + truncation=truncate_instruction_sequence, + padding_side="right", + return_tensors="pt", + tokenize=True, + return_dict=True, + ) + for k in vlm_inputs.keys(): + if isinstance(vlm_inputs[k], torch.Tensor): + vlm_inputs[k] = vlm_inputs[k].to(device) + + final_instruction_mask = vlm_inputs["attention_mask"] + + num_instruction_feature_layers = self.transformer.instruction_feature_configs.get( + "num_instruction_feature_layers", 1 + ) + + with torch.no_grad(): + text_encoder_outputs = self.mllm(**vlm_inputs, output_hidden_states=True, return_dict=True) + if num_instruction_feature_layers > 1: + instruction_feats = list(text_encoder_outputs.hidden_states)[-num_instruction_feature_layers:] + else: + instruction_feats = text_encoder_outputs.hidden_states[-1] + + dtype = self.mllm.dtype if self.mllm is not None else self.transformer.dtype + + if isinstance(instruction_feats, (list, tuple)): + final_instruction_feats = [feat.to(dtype=dtype, device=device) for feat in instruction_feats] + else: + final_instruction_feats = instruction_feats.to(dtype=dtype, device=device) + final_instruction_mask = final_instruction_mask.to(device=device) + + return final_instruction_feats, final_instruction_mask + + def _reshape_embeds_and_mask(self, embeds, mask, num_images_per_prompt: int): + """Duplicate embeddings/mask for each generation per prompt (mps-friendly).""" + if isinstance(embeds, (list, tuple)): + batch_size, seq_len, _ = embeds[0].shape + reshaped_embeds = [] + for embed in embeds: + embed = embed.repeat(1, num_images_per_prompt, 1) + reshaped_embeds.append(embed.view(batch_size * num_images_per_prompt, seq_len, -1)) + else: + batch_size, seq_len, _ = embeds.shape + embeds = embeds.repeat(1, num_images_per_prompt, 1) + reshaped_embeds = embeds.view(batch_size * num_images_per_prompt, seq_len, -1) + + mask = mask.repeat(num_images_per_prompt, 1) + reshaped_mask = mask.view(batch_size * num_images_per_prompt, -1) + + return batch_size, seq_len, reshaped_embeds, reshaped_mask + + def encode_prompt( + self, + prompt: str | list[str], + do_classifier_free_guidance: bool = True, + negative_prompt: str | list[str] | None = None, + num_images_per_prompt: int = 1, + device: torch.device | None = None, + prompt_embeds: torch.Tensor | None = None, + negative_prompt_embeds: torch.Tensor | None = None, + prompt_attention_mask: torch.Tensor | None = None, + negative_prompt_attention_mask: torch.Tensor | None = None, + max_sequence_length: int = 1280, + truncate_instruction_sequence: bool = False, + input_images: list[list[PIL.Image.Image] | None] | None = None, + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor | None, torch.Tensor | None]: + """Encode prompt (and negative prompt for CFG) into Qwen3VL hidden states. + + Port of upstream ``encode_instruction`` for text-to-image and the + text-guided image-editing (TI2I) path. Reference images are attached to + the *positive* instruction only (upstream default + ``use_input_images_4_neg_instruct=False``). Instruction rewriting, + prompt tuning, and double-guidance empty instructions are not ported. + The default ``max_sequence_length`` matches the upstream ``__call__`` + default (1280), not the upstream ``encode_instruction`` default (256). + + Args: + input_images: Per-sample list (outer length == batch size) of + already-VLM-resized reference images, or ``None`` for pure + text-to-image. + + Returns: + ``(prompt_embeds, prompt_attention_mask, negative_prompt_embeds, + negative_prompt_attention_mask)`` where each embeds tensor has shape + ``[batch_size * num_images_per_prompt, seq_len, dim]``. The negative + pair is ``None`` when ``do_classifier_free_guidance`` is off and no + precomputed negative embeddings were passed. + """ + device = device or self._execution_device + + prompt = [prompt] if isinstance(prompt, str) else prompt + + if prompt_embeds is None: + prompt_embeds, prompt_attention_mask = self._get_instruction_feature_embeds( + instruction=prompt, + input_pil_images=input_images, + device=device, + max_sequence_length=max_sequence_length, + truncate_instruction_sequence=truncate_instruction_sequence, + ) + + batch_size, _, prompt_embeds, prompt_attention_mask = self._reshape_embeds_and_mask( + prompt_embeds, prompt_attention_mask, num_images_per_prompt + ) + + if do_classifier_free_guidance and negative_prompt_embeds is None: + negative_prompt = negative_prompt if negative_prompt is not None else "" + negative_prompt = batch_size * [negative_prompt] if isinstance(negative_prompt, str) else negative_prompt + + if batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt` has batch size {len(negative_prompt)}, but `prompt` has" + f" batch size {batch_size}. Please make sure that passed `negative_prompt`" + " matches the batch size of `prompt`." + ) + + negative_prompt_embeds, negative_prompt_attention_mask = self._get_instruction_feature_embeds( + instruction=negative_prompt, + device=device, + max_sequence_length=max_sequence_length, + truncate_instruction_sequence=truncate_instruction_sequence, + ) + + _, _, negative_prompt_embeds, negative_prompt_attention_mask = self._reshape_embeds_and_mask( + negative_prompt_embeds, negative_prompt_attention_mask, num_images_per_prompt + ) + + return ( + prompt_embeds, + prompt_attention_mask, + negative_prompt_embeds, + negative_prompt_attention_mask, + ) + + # ------------------------------------------------------------------ + # Denoise loop + VAE decode (upstream ``__call__`` / ``processing``, t2i) + # ------------------------------------------------------------------ + + def prepare_latents(self, batch_size, num_channels_latents, height, width, dtype, device, generator, latents=None): + """Sample initial noise latents (upstream ``prepare_latents``).""" + height = int(height) // self.vae_scale_factor + width = int(width) // self.vae_scale_factor + shape = (batch_size, num_channels_latents, height, width) + if latents is None: + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + else: + latents = latents.to(device) + return latents + + def _resolve_output_size(self, height, width): + """t2i branch of upstream ``_resolve_output_and_original_size``. + + Clamps the working resolution to ``max_input_image_pixels`` (2048**2), + rounding down to a multiple of ``vae_scale_factor * 2``; the requested + size is remembered so the decoded image can be resized back. + """ + img_scale_num = self.vae_scale_factor * 2 + ori_height, ori_width = height, width + max_pixels = 2048 * 2048 + cur_pixels = height * width + ratio = min((max_pixels / cur_pixels) ** 0.5, 1.0) + height = int(height * ratio) // img_scale_num * img_scale_num + width = int(width * ratio) // img_scale_num * img_scale_num + return height, width, ori_height, ori_width + + def predict( + self, t, latents, instruction_embeds, freqs_cis, instruction_attention_mask, ref_image_hidden_states=None + ): + """One transformer velocity prediction (upstream ``predict``). + + ``ref_image_hidden_states`` is ``None`` for text-to-image, or the + per-sample reference latents (``list[list[Tensor[C, H, W]]]``) for the + image-editing path. + """ + timestep = t.expand(latents.shape[0]).to(latents.dtype) + return self.transformer( + latents, + timestep, + instruction_embeds, + freqs_cis, + instruction_attention_mask, + ref_image_hidden_states=ref_image_hidden_states, + ) + + def _encode_vae_image(self, img: torch.Tensor, generator=None) -> torch.Tensor: + """Encode an image tensor into the VAE latent space (upstream ``encode_vae``). + + Upstream leaves ``latent_dist.sample()`` unseeded; the native path + threads the request generator through so a fixed seed gives a + reproducible reference latent. + """ + z0 = self.vae.encode(img.to(dtype=self.vae.dtype)).latent_dist.sample(generator=generator) + if self.vae.config.shift_factor is not None: + z0 = z0 - self.vae.config.shift_factor + if self.vae.config.scaling_factor is not None: + z0 = z0 * self.vae.config.scaling_factor + return z0.to(dtype=self.vae.dtype) + + def _build_ref_latents( + self, + preprocessed_images: list[torch.Tensor | None], + num_images_per_prompt: int, + device: torch.device, + generator=None, + ) -> list[list[torch.Tensor] | None]: + """VAE-encode per-sample reference images into the transformer's format. + + Mirrors upstream ``prepare_image``: returns a list of length + ``batch_size * num_images_per_prompt`` where each entry is either + ``None`` (no reference / text-to-image) or a list of ``[C, H, W]`` + reference latents (one per reference image). Boogu editing uses a single + reference image, so each non-empty entry is a one-element list. + """ + # ``latent_dist.sample`` accepts only a single generator; a per-output + # generator list (num_outputs_per_prompt > 1) falls back to unseeded. + vae_generator = generator if isinstance(generator, torch.Generator) else None + + ref_latents: list[list[torch.Tensor] | None] = [] + for image in preprocessed_images: + if image is None: + sample_latents: list[torch.Tensor] | None = None + else: + latent = self._encode_vae_image(image.to(device=device), generator=vae_generator).squeeze(0) + sample_latents = [latent] + for _ in range(num_images_per_prompt): + ref_latents.append(sample_latents) + return ref_latents + + @staticmethod + def _extract_reference_images( + prompts: list, + ) -> tuple[list[PIL.Image.Image | None], list[torch.Tensor | None]]: + """Pull per-sample reference images out of ``additional_information``. + + Returns ``(prompt_images, preprocessed_images)`` where entries are + ``None`` for pure text-to-image samples. Populated by + :func:`get_boogu_image_pre_process_func`. + """ + prompt_images: list[PIL.Image.Image | None] = [] + preprocessed_images: list[torch.Tensor | None] = [] + for p in prompts: + if isinstance(p, str): + prompt_images.append(None) + preprocessed_images.append(None) + continue + ai = p.get("additional_information") or {} + prompt_images.append(ai.get("prompt_image")) + preprocessed_images.append(ai.get("preprocessed_image")) + return prompt_images, preprocessed_images + + def forward(self, req: DiffusionRequestBatch) -> DiffusionOutput: + # Prompt / negative-prompt extraction (mirrors the Ovis pattern; the + # online API sometimes passes ``{"negative_prompt": None}``). + prompt = [p if isinstance(p, str) else (p.get("prompt") or "") for p in req.prompts] + if all(isinstance(p, str) or p.get("negative_prompt") is None for p in req.prompts): + negative_prompt = None + else: + negative_prompt = ["" if isinstance(p, str) else (p.get("negative_prompt") or "") for p in req.prompts] + + # Reference (edit / TI2I) images, if any. + prompt_images, preprocessed_images = self._extract_reference_images(req.prompts) + has_reference = any(img is not None for img in preprocessed_images) + task_type = "ti2i" if has_reference else "t2i" + + sp = req.sampling_params + device = self._execution_device + + height = sp.height or self.default_sample_size * self.vae_scale_factor + width = sp.width or self.default_sample_size * self.vae_scale_factor + num_inference_steps = sp.num_inference_steps or 50 + # Upstream default text guidance is 4.0; the engine coerces an unset + # guidance_scale to 1.0, so only honor a caller-provided value. + text_guidance_scale = sp.guidance_scale if sp.guidance_scale_provided else 4.0 + # Image guidance rides on ``guidance_scale_2`` (upstream default 1.0 = + # off); only a caller-provided value enables the double-guidance path. + image_guidance_scale = sp.guidance_scale_2 if sp.guidance_scale_2_provided else 1.0 + if not has_reference: + image_guidance_scale = 1.0 + num_images_per_prompt = sp.num_outputs_per_prompt if sp.num_outputs_per_prompt > 0 else 1 + generator = sp.generator + max_sequence_length = sp.max_sequence_length or 1280 + output_type = sp.output_type or "pil" + cfg_range = (0.0, 1.0) + + # Negative instruction embeddings are needed whenever text guidance is + # active (t2i text CFG, ti2i text-only, and ti2i double guidance). + do_classifier_free_guidance = text_guidance_scale > 1.0 + + batch_size = len(prompt) + + # Per-sample VLM reference images for the positive instruction only + # (upstream default ``use_input_images_4_neg_instruct=False``). + input_images = None + if has_reference: + input_images = [[img] if img is not None else None for img in prompt_images] + + # 1. Encode prompts. + ( + instruction_embeds, + instruction_attention_mask, + negative_instruction_embeds, + negative_instruction_attention_mask, + ) = self.encode_prompt( + prompt, + do_classifier_free_guidance=do_classifier_free_guidance, + negative_prompt=negative_prompt, + num_images_per_prompt=num_images_per_prompt, + device=device, + max_sequence_length=max_sequence_length, + input_images=input_images, + ) + + # 2. Resolve working / output resolution. + height, width, ori_height, ori_width = self._resolve_output_size(height, width) + + # 3. Reference latents (edit path) and initial noise latents. + dtype = self.vae.dtype + latent_channels = self.transformer.in_channels + ref_latents = None + if has_reference: + ref_latents = self._build_ref_latents(preprocessed_images, num_images_per_prompt, device, generator) + + latents = self.prepare_latents( + batch_size * num_images_per_prompt, + latent_channels, + height, + width, + instruction_embeds.dtype, + device, + generator, + ) + + freqs_cis = BooguImageDoubleStreamRotaryPosEmbed.get_freqs_cis( + self.transformer.axes_dim_rope, + self.transformer.axes_lens, + theta=10000, + ) + + # 4. Timesteps (the ported scheduler consumes ``num_tokens``). + num_tokens = latents.shape[-2] * latents.shape[-1] + self.scheduler.set_timesteps(num_inference_steps, device=device, num_tokens=num_tokens) + timesteps = self.scheduler.timesteps + num_timesteps = len(timesteps) + + # 5. Denoise loop with (sequential) classifier-free guidance. Reproduces + # the branch priority of upstream ``processing`` (double > text-only > + # image-only > t2i text). Reference latents are kept in the conditional + # (and, for text-only ti2i, the unconditional) predictions. + with self.progress_bar(total=num_timesteps) as progress_bar: + for i, t in enumerate(timesteps): + in_cfg_range = cfg_range[0] <= i / num_timesteps <= cfg_range[1] + text_gs = text_guidance_scale if in_cfg_range else 1.0 + image_gs = image_guidance_scale if in_cfg_range else 1.0 + + model_pred = self.predict( + t, latents, instruction_embeds, freqs_cis, instruction_attention_mask, ref_latents + ) + + if task_type == "ti2i" and text_gs > 1.0 and image_gs > 1.0: + # Double guidance: 3 predictions (cond+ref, neg+ref, neg+no-ref). + model_pred_drop_text = self.predict( + t, + latents, + negative_instruction_embeds, + freqs_cis, + negative_instruction_attention_mask, + ref_latents, + ) + model_pred_drop_all = self.predict( + t, + latents, + negative_instruction_embeds, + freqs_cis, + negative_instruction_attention_mask, + None, + ) + delta_text = model_pred - model_pred_drop_text + delta_image = model_pred_drop_text - model_pred_drop_all + model_pred = model_pred + (text_gs - 1) * delta_text + (image_gs - 1) * delta_image + elif task_type == "ti2i" and text_gs > 1.0: + # Text-only ti2i guidance: reference kept in the uncond pred. + model_pred_drop_text = self.predict( + t, + latents, + negative_instruction_embeds, + freqs_cis, + negative_instruction_attention_mask, + ref_latents, + ) + model_pred = model_pred + (text_gs - 1) * (model_pred - model_pred_drop_text) + elif task_type == "ti2i" and image_gs > 1.0: + # Image-only ti2i guidance: drop the reference in the uncond pred. + model_pred_drop_image = self.predict( + t, latents, instruction_embeds, freqs_cis, instruction_attention_mask, None + ) + model_pred = model_pred + (image_gs - 1) * (model_pred - model_pred_drop_image) + elif text_gs > 1.0: + # Text-to-image classifier-free guidance. + model_pred_drop_all = self.predict( + t, + latents, + negative_instruction_embeds, + freqs_cis, + negative_instruction_attention_mask, + None, + ) + model_pred = model_pred + (text_gs - 1) * (model_pred - model_pred_drop_all) + + latents = self.scheduler.step(model_pred, t, latents, return_dict=False)[0] + latents = latents.to(dtype=instruction_embeds.dtype) + progress_bar.update() + + # 6. Decode. + if output_type == "latent": + image = latents + else: + latents = latents.to(dtype=dtype) + if self.vae.config.scaling_factor is not None: + latents = latents / self.vae.config.scaling_factor + if self.vae.config.shift_factor is not None: + latents = latents + self.vae.config.shift_factor + image = self.vae.decode(latents, return_dict=False)[0] + if (ori_height, ori_width) != (height, width): + image = F.interpolate(image, size=(ori_height, ori_width), mode="bilinear") + + return DiffusionOutput(output=image) diff --git a/vllm_omni/diffusion/models/boogu_image/scheduling_flow_match_euler_discrete_time_shifting.py b/vllm_omni/diffusion/models/boogu_image/scheduling_flow_match_euler_discrete_time_shifting.py new file mode 100644 index 000000000..656594221 --- /dev/null +++ b/vllm_omni/diffusion/models/boogu_image/scheduling_flow_match_euler_discrete_time_shifting.py @@ -0,0 +1,318 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# Ported for vLLM-Omni native Boogu-Image support from +# boogu/schedulers/scheduling_flow_match_euler_discrete_time_shifting.py +# (https://github.com/boogu-project/Boogu-Image). The scheduling logic is +# kept verbatim; only debug comments were removed and typing modernized. +# +# --- Upstream header preserved below --- +# +# Copyright (C) 2026 Boogu Team. +# +# This file is adapted by Boogu Team from prior open-source scheduler work. +# Boogu-specific modifications include static/dynamic time-shift handling used +# by the released Boogu pipeline. +# +# Original work: +# Copyright 2024 Stability AI, Katherine Crowson and The HuggingFace Team. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import math +from dataclasses import dataclass + +import numpy as np +import torch +from diffusers.configuration_utils import ConfigMixin, register_to_config +from diffusers.schedulers.scheduling_utils import SchedulerMixin +from diffusers.utils import BaseOutput, logging + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +@dataclass +class FlowMatchEulerDiscreteSchedulerOutput(BaseOutput): + """ + Output class for the scheduler's `step` function output. + + Args: + prev_sample (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)` for images): + Computed sample `(x_{t-1})` of previous timestep. `prev_sample` should be used as next model input in the + denoising loop. + """ + + prev_sample: torch.FloatTensor + + +class FlowMatchEulerDiscreteScheduler(SchedulerMixin, ConfigMixin): + """ + Euler scheduler with Boogu's training-consistent time shifting. + + Timesteps run ascending 0 -> 1 (not descending sigmas as in stock + diffusers), `set_timesteps` accepts a `num_tokens` argument for the + dynamic-shift variants, and `step()` integrates with `t_next - t` + directly. + + This model inherits from [`SchedulerMixin`] and [`ConfigMixin`]. Check the superclass documentation for the generic + methods the library implements for all schedulers such as loading and saving. + + Args: + num_train_timesteps (`int`, defaults to 1000): + The number of diffusion steps to train the model. + do_shift (`bool`, defaults to `True`): + Whether to apply the training-consistent time shift in `set_timesteps`. + dynamic_time_shift (`bool`, defaults to `True`): + If `True`, the shift depends on the per-sample `num_tokens`; if `False`, it uses the configured `seq_len`. + time_shift_version (`str`, defaults to `"v2"`): + `"v1"` (logistic transform with a linear mu mapping) or `"v2"` (rational transform). + seq_len (`int`, *optional*): + Token count used to compute the static shift when `dynamic_time_shift=False` (mirrors training). + base_shift (`float`, defaults to 0.5): + v1 linear mapping lower bound (matches training defaults). + max_shift (`float`, defaults to 1.15): + v1 linear mapping upper bound (matches training defaults). + """ + + _compatibles = [] + order = 1 + + @register_to_config + def __init__( + self, + num_train_timesteps: int = 1000, + do_shift: bool = True, + dynamic_time_shift: bool = True, + time_shift_version: str = "v2", + # seq_len is used to mirror training-side static time shift (when dynamic_time_shift=False) + # In training, seq_len is the token count used to compute shift. + seq_len: int | None = None, + # v1 linear mapping range (matches training defaults) + base_shift: float = 0.5, + max_shift: float = 1.15, + time_shift_v2_half_scaling_factor: float = 60.0, + ): + timesteps = torch.linspace(0, 1, num_train_timesteps + 1, dtype=torch.float32)[:-1] + + self.timesteps = timesteps + + self._step_index = None + self._begin_index = None + self.time_shift_v2_scaling_factor = time_shift_v2_half_scaling_factor * 2 + + @property + def step_index(self): + """ + The index counter for current timestep. It will increase 1 after each scheduler step. + """ + return self._step_index + + @property + def begin_index(self): + """ + The index for the first timestep. It should be set from pipeline with `set_begin_index` method. + """ + return self._begin_index + + # Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler.set_begin_index + def set_begin_index(self, begin_index: int = 0): + """ + Sets the begin index for the scheduler. This function should be run from pipeline before the inference. + + Args: + begin_index (`int`): + The begin index for the scheduler. + """ + self._begin_index = begin_index + + def index_for_timestep(self, timestep, schedule_timesteps=None): + if schedule_timesteps is None: + schedule_timesteps = self._timesteps + + indices = (schedule_timesteps == timestep).nonzero() + + # The sigma index that is taken for the **very** first `step` + # is always the second index (or the last index if there is only 1) + # This way we can ensure we don't accidentally skip a sigma in + # case we start in the middle of the denoising schedule (e.g. for image-to-image) + pos = 1 if len(indices) > 1 else 0 + + return indices[pos].item() + + # --- Helpers to mirror training-side shift logic --- + @staticmethod + def _get_lin_function(x1: float = 256, y1: float = 0.5, x2: float = 4096, y2: float = 1.15): + m = (y2 - y1) / (x2 - x1) + b = y1 - m * x1 + return lambda x: m * x + b + + @staticmethod + def _time_shift_v1(t_np: np.ndarray, mu: float, sigma: float = 1.0) -> np.ndarray: + # Matches training: t <- 1 - t; logistic transform; then t <- 1 - t + eps = 1e-8 + t1 = 1.0 - t_np + t1 = np.clip(t1, eps, 1.0 - eps) + num = math.exp(mu) + denom = num + np.power(1.0 / t1 - 1.0, sigma) + y = num / denom + out = 1.0 - y + return out.astype(np.float32) + + @staticmethod + def _time_shift_v2(t_np: np.ndarray, m: float) -> np.ndarray: + # Matches training: t' = t / (m - m t + t) + return (t_np / (m - m * t_np + t_np)).astype(np.float32) + + def set_timesteps( + self, + num_inference_steps: int | None = None, + device: str | torch.device | None = None, + timesteps: list[float] | None = None, + num_tokens: int | None = None, + ): + """ + Sets the discrete timesteps used for the diffusion chain (to be run before inference). + + Args: + num_inference_steps (`int`): + The number of diffusion steps used when generating samples with a pre-trained model. + device (`str` or `torch.device`, *optional*): + The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. + timesteps (`list[float]`, *optional*): + Custom timesteps to use. If provided, `num_inference_steps` is ignored. + num_tokens (`int`, *optional*): + Per-sample token count, used by the dynamic time-shift variants. + """ + + if timesteps is None: + self.num_inference_steps = num_inference_steps + t_arr = np.linspace(0, 1, num_inference_steps + 1, dtype=np.float32)[:-1] + + # Apply training-consistent time shift only when requested + if self.config.do_shift: + # dynamic or static + if self.config.dynamic_time_shift: + # dynamic: depend on per-sample token count + if self.config.time_shift_version == "v1": + # In training dynamic v1: mu is computed from tokens' linear map where + # tokens are approximately (H_lat//2)*(W_lat//2). We approximate this with num_tokens//4. + if num_tokens is not None and num_tokens > 0: + tokens_reduced = max(1, int(num_tokens) // 4) + lin = self._get_lin_function(y1=self.config.base_shift, y2=self.config.max_shift) + mu = lin(tokens_reduced) ## 4096 for 1024x1024 resolution + + t_arr = self._time_shift_v1(t_arr, mu, sigma=1.0) + # else: no-op if we lack num_tokens + elif self.config.time_shift_version == "v2": + # MUST remain identical to current behavior when v2 + dynamic=True + # m = sqrt(num_tokens) / 40; t' = t / (m - m t + t) + # When input resolution is 320 * 320, m = 1, when input resolution is 512 * 512, + # m = 1.6, when input resolution is 1024 * 1024, m = 3.2 + if num_tokens is not None and num_tokens > 0: + m = float(np.sqrt(num_tokens)) / self.time_shift_v2_scaling_factor + t_arr = self._time_shift_v2(t_arr, m) + # else: no-op + else: + # static: depend on seq_len configured at scheduler init + if self.config.time_shift_version == "v1": + if self.config.seq_len is not None and self.config.seq_len > 0: + lin = self._get_lin_function(y1=self.config.base_shift, y2=self.config.max_shift) + mu = lin(int(self.config.seq_len)) + t_arr = self._time_shift_v1(t_arr, mu, sigma=1.0) + elif self.config.time_shift_version == "v2": + if self.config.seq_len is not None and self.config.seq_len > 0: + # training static v2 uses m = sqrt(seq_len) / 40 + m = float(np.sqrt(self.config.seq_len)) / self.time_shift_v2_scaling_factor + t_arr = self._time_shift_v2(t_arr, m) + + timesteps = t_arr + + timesteps = torch.from_numpy(timesteps).to(dtype=torch.float32, device=device) + _timesteps = torch.cat([timesteps, torch.ones(1, device=timesteps.device)]) + + self.timesteps = timesteps + self._timesteps = _timesteps + self._step_index = None + self._begin_index = None + + def _init_step_index(self, timestep): + if self.begin_index is None: + if isinstance(timestep, torch.Tensor): + timestep = timestep.to(self.timesteps.device) + self._step_index = self.index_for_timestep(timestep) + else: + self._step_index = self._begin_index + + def step( + self, + model_output: torch.FloatTensor, + timestep: float | torch.FloatTensor, + sample: torch.FloatTensor, + generator: torch.Generator | None = None, # noqa: ARG002 - kept for API compatibility + return_dict: bool = True, + ) -> FlowMatchEulerDiscreteSchedulerOutput | tuple: + """ + Predict the sample from the previous timestep by reversing the SDE. This function propagates the diffusion + process from the learned model outputs (most often the predicted noise). + + Args: + model_output (`torch.FloatTensor`): + The direct output from learned diffusion model. + timestep (`float`): + The current discrete timestep in the diffusion chain. + sample (`torch.FloatTensor`): + A current instance of a sample created by the diffusion process. + generator (`torch.Generator`, *optional*): + A random number generator. + return_dict (`bool`): + Whether or not to return a [`FlowMatchEulerDiscreteSchedulerOutput`] or tuple. + + Returns: + [`FlowMatchEulerDiscreteSchedulerOutput`] or `tuple`: + If return_dict is `True`, [`FlowMatchEulerDiscreteSchedulerOutput`] is returned, otherwise a tuple is + returned where the first element is the sample tensor. + """ + + if isinstance(timestep, int) or isinstance(timestep, torch.IntTensor) or isinstance(timestep, torch.LongTensor): + raise ValueError( + ( + "Passing integer indices (e.g. from `enumerate(timesteps)`) as timesteps to" + " `EulerDiscreteScheduler.step()` is not supported. Make sure to pass" + " one of the `scheduler.timesteps` as a timestep." + ), + ) + + if self.step_index is None: + self._init_step_index(timestep) + # Upcast to avoid precision issues when computing prev_sample + sample = sample.to(torch.float32) + t = self._timesteps[self.step_index] + t_next = self._timesteps[self.step_index + 1] + + prev_sample = sample + (t_next - t) * model_output + + # Cast sample back to model compatible dtype + prev_sample = prev_sample.to(model_output.dtype) + + # upon completion increase step index by one + self._step_index += 1 + + if not return_dict: + return (prev_sample,) + + return FlowMatchEulerDiscreteSchedulerOutput(prev_sample=prev_sample) + + def __len__(self): + return self.config.num_train_timesteps diff --git a/vllm_omni/diffusion/registry.py b/vllm_omni/diffusion/registry.py index aea257dd6..41412424d 100644 --- a/vllm_omni/diffusion/registry.py +++ b/vllm_omni/diffusion/registry.py @@ -141,6 +141,11 @@ _DIFFUSION_MODELS = { "pipeline_bagel", "BagelPipeline", ), + "BooguImagePipeline": ( + "boogu_image", + "pipeline_boogu_image", + "BooguImagePipeline", + ), "LancePipeline": ( "lance", "pipeline_lance", @@ -503,6 +508,7 @@ _DIFFUSION_POST_PROCESS_FUNCS = { "GlmImagePipeline": "get_glm_image_post_process_func", "ZImagePipeline": "get_post_process_func", "OvisImagePipeline": "get_ovis_image_post_process_func", + "BooguImagePipeline": "get_boogu_image_post_process_func", "WanPipeline": "get_wan22_post_process_func", "WanVACEPipeline": "get_wan22_vace_post_process_func", "LTX2Pipeline": "get_ltx2_post_process_func", @@ -566,6 +572,7 @@ _DIFFUSION_PRE_PROCESS_FUNCS = { # `pre_process_func` function must be placed in {mod_folder}/{mod_relname}.py, # where mod_folder and mod_relname are defined and mapped using `_DIFFUSION_MODELS` via the `arch` key "GlmImagePipeline": "get_glm_image_pre_process_func", + "BooguImagePipeline": "get_boogu_image_pre_process_func", "QwenImageEditPipeline": "get_qwen_image_edit_pre_process_func", "QwenImageEditPlusPipeline": "get_qwen_image_edit_plus_pre_process_func", "LongCatImageEditPipeline": "get_longcat_image_edit_pre_process_func", diff --git a/vllm_omni/diffusion/request.py b/vllm_omni/diffusion/request.py index 42181419c..3ad3f1541 100644 --- a/vllm_omni/diffusion/request.py +++ b/vllm_omni/diffusion/request.py @@ -56,6 +56,13 @@ class OmniDiffusionRequest: ): self.sampling_params.do_classifier_free_guidance = True + # Detect whether the caller explicitly provided guidance_scale_2 + # BEFORE auto-filling it. Pipelines with a model-specific second + # guidance default (e.g. Boogu image_guidance_scale=1.0) must be able + # to tell an explicit value apart from the guidance_scale fallback. + if self.sampling_params.guidance_scale_2 is not None: + self.sampling_params.guidance_scale_2_provided = True + # Auto-fill guidance_scale_2 from the (now-resolved) guidance_scale # so downstream code always has a valid value. if self.sampling_params.guidance_scale_2 is None: diff --git a/vllm_omni/entrypoints/openai/api_server.py b/vllm_omni/entrypoints/openai/api_server.py index a9251a8d8..c9ddab2e9 100755 --- a/vllm_omni/entrypoints/openai/api_server.py +++ b/vllm_omni/entrypoints/openai/api_server.py @@ -1955,6 +1955,7 @@ async def edit_images( negative_prompt: str | None = Form(None), num_inference_steps: int | None = Form(None), guidance_scale: float | None = Form(None), + guidance_scale_2: float | None = Form(None), strength: float | None = Form(None), true_cfg_scale: float | None = Form(None), seed: int | None = Form(None), @@ -2120,6 +2121,7 @@ async def edit_images( # 3.4 Add optional parameters ONLY if provided _update_if_not_none(gen_params, "num_inference_steps", num_inference_steps) _update_if_not_none(gen_params, "guidance_scale", guidance_scale) + _update_if_not_none(gen_params, "guidance_scale_2", guidance_scale_2) _update_if_not_none(gen_params, "strength", strength) _update_if_not_none(gen_params, "true_cfg_scale", true_cfg_scale) # If seed is not provided, generate a random one to ensure @@ -2185,6 +2187,8 @@ async def edit_images( extra_body["num_inference_steps"] = num_inference_steps if guidance_scale is not None: extra_body["guidance_scale"] = guidance_scale + if guidance_scale_2 is not None: + extra_body["guidance_scale_2"] = guidance_scale_2 if strength is not None: extra_body["strength"] = strength if true_cfg_scale is not None: diff --git a/vllm_omni/inputs/data.py b/vllm_omni/inputs/data.py index 73ac80642..f576b9724 100644 --- a/vllm_omni/inputs/data.py +++ b/vllm_omni/inputs/data.py @@ -248,6 +248,7 @@ class OmniDiffusionSamplingParams: guidance_scale: float = 0.0 guidance_scale_provided: bool = False guidance_scale_2: float | None = None + guidance_scale_2_provided: bool = False guidance_rescale: float = 0.0 strength: float | None = None # I2I: Z-Image specific now, uses to control denoising start timestep decode_timestep: float | list[float] | None = None From ea34bde619a07ebcbc007c0e33688c325ef62ac4 Mon Sep 17 00:00:00 2001 From: rein yang <73573651+R2-Y@users.noreply.github.com> Date: Wed, 22 Jul 2026 09:23:03 +0800 Subject: [PATCH 046/240] [Bugfix] MiniCPMo-4.5 bugfix (#5233) Signed-off-by: R2-Y Co-authored-by: Guanzhe (Leo) Huang <116847811+leohuang257@users.noreply.github.com> Co-authored-by: Cursor Co-authored-by: amy-why-3459 --- .../test_long_audio_processing.py | 154 ++++++++++++++++++ .../minicpmo_4_5/minicpmo_4_5_omni_llm.py | 34 +++- 2 files changed, 186 insertions(+), 2 deletions(-) create mode 100644 tests/model_executor/models/minicpmo_4_5/test_long_audio_processing.py diff --git a/tests/model_executor/models/minicpmo_4_5/test_long_audio_processing.py b/tests/model_executor/models/minicpmo_4_5/test_long_audio_processing.py new file mode 100644 index 000000000..ba4015b98 --- /dev/null +++ b/tests/model_executor/models/minicpmo_4_5/test_long_audio_processing.py @@ -0,0 +1,154 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Regression tests for MiniCPM-o 4.5 >30s audio preprocessing. + +Covers: + - ``_minicpmo_field_config``: <=30s audio keeps the batched layout + - >30s audio switches to ``flat`` with one slice per source audio + - mixed long + short audios group chunks by source audio + - no audio inputs -> batched (default path unchanged) + - ``process_audios``: >30s audio unpads per chunk without ``TypeError`` + - multiple >30s audios unpad in chunk order + - <=30s audio unpadding byte-identical to the pre-fix behavior +""" + +from __future__ import annotations + +from types import SimpleNamespace + +import numpy as np +import pytest +import torch +from vllm.multimodal.inputs import ( + MultiModalBatchedField, + MultiModalFlatField, +) + +from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni_llm import ( + MiniCPMO45OmniLLMMultiModalProcessor, + MiniCPMOMultiModalDataParser, + _minicpmo_field_config, +) + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + +_SAMPLE_RATE = 16_000 +_FEAT_DIM = 128 +# Whisper pads every 30s chunk to 3000 mel frames. +_CHUNK_FRAMES = 3000 + + +class TestMiniCPMOFieldConfig: + def test_short_audios_stay_batched(self) -> None: + # two <=30s audios: one feature entry + one 1-element lens per audio + hf_inputs = { + "audio_features": [torch.zeros(_FEAT_DIM, _CHUNK_FRAMES)] * 2, + "audio_feature_lens": [torch.tensor([980]), torch.tensor([1500])], + } + + config = _minicpmo_field_config(hf_inputs) + + assert isinstance(config["audio_features"].field, MultiModalBatchedField) + + def test_long_audio_groups_chunks_per_audio(self) -> None: + # one 45s audio -> 2 chunks but a single lens tensor + hf_inputs = { + "audio_features": [torch.zeros(_FEAT_DIM, _CHUNK_FRAMES)] * 2, + "audio_feature_lens": [torch.tensor([_CHUNK_FRAMES, 1500])], + } + + config = _minicpmo_field_config(hf_inputs) + + field = config["audio_features"].field + assert isinstance(field, MultiModalFlatField) + assert field.slices == [slice(0, 2)] + + def test_mixed_long_and_short_audios(self) -> None: + # 45s audio (2 chunks) + 10s audio (1 chunk) + hf_inputs = { + "audio_features": [torch.zeros(_FEAT_DIM, _CHUNK_FRAMES)] * 3, + "audio_feature_lens": [ + torch.tensor([_CHUNK_FRAMES, 1500]), + torch.tensor([980]), + ], + } + + config = _minicpmo_field_config(hf_inputs) + + field = config["audio_features"].field + assert isinstance(field, MultiModalFlatField) + assert field.slices == [slice(0, 2), slice(2, 3)] + + def test_no_audio_inputs_stay_batched(self) -> None: + config = _minicpmo_field_config({}) + + assert isinstance(config["audio_features"].field, MultiModalBatchedField) + + +def _make_processor( + fake_hf_outputs: dict[str, list[torch.Tensor]], +) -> MiniCPMO45OmniLLMMultiModalProcessor: + """Build a processor without a checkpoint: only what ``process_audios`` + touches, with the HF processor call returning canned Whisper-style output.""" + processor = object.__new__(MiniCPMO45OmniLLMMultiModalProcessor) + processor.info = SimpleNamespace(audio_pattern="()") + processor.data_parser = MiniCPMOMultiModalDataParser(target_sr=_SAMPLE_RATE) + + def fake_base_call_hf_processor(prompts, mm_data, mm_kwargs, tok_kwargs, *, out_keys): + return {key: fake_hf_outputs[key] for key in out_keys} + + processor._base_call_hf_processor = fake_base_call_hf_processor + return processor + + +def _audio_seconds(seconds: float) -> np.ndarray: + return np.zeros(int(seconds * _SAMPLE_RATE), dtype=np.float32) + + +class TestProcessAudiosUnpadding: + def test_long_audio_unpads_per_chunk(self) -> None: + # 45s audio: 2 padded chunks, one lens tensor -> pre-fix TypeError + chunk_lens = [_CHUNK_FRAMES, 1500] + fake_outputs = { + "audio_features": [torch.randn(_FEAT_DIM, _CHUNK_FRAMES) for _ in chunk_lens], + "audio_feature_lens": [torch.tensor(chunk_lens)], + } + processor = _make_processor(fake_outputs) + + result = processor.process_audios({"audios": [_audio_seconds(45)]}, {}, {}) + + features = result["audio_features"] + assert [feat.shape[-1] for feat in features] == chunk_lens + for feat, padded, length in zip(features, fake_outputs["audio_features"], chunk_lens): + assert torch.equal(feat, padded[:, :length]) + + def test_multiple_long_audios(self) -> None: + # two >30s audios: 4 chunks total, 2-element lens tensor each + fake_outputs = { + "audio_features": [torch.randn(_FEAT_DIM, _CHUNK_FRAMES) for _ in range(4)], + "audio_feature_lens": [ + torch.tensor([_CHUNK_FRAMES, 1200]), + torch.tensor([_CHUNK_FRAMES, 800]), + ], + } + processor = _make_processor(fake_outputs) + + result = processor.process_audios({"audios": [_audio_seconds(42), _audio_seconds(38)]}, {}, {}) + + widths = [feat.shape[-1] for feat in result["audio_features"]] + assert widths == [_CHUNK_FRAMES, 1200, _CHUNK_FRAMES, 800] + + def test_short_audio_unchanged(self) -> None: + # <=30s audio: single-chunk unpadding identical to pre-fix behavior + fake_outputs = { + "audio_features": [torch.randn(_FEAT_DIM, _CHUNK_FRAMES)], + "audio_feature_lens": [torch.tensor([980])], + } + processor = _make_processor(fake_outputs) + + result = processor.process_audios({"audios": [_audio_seconds(9.8)]}, {}, {}) + + features = result["audio_features"] + assert len(features) == 1 + assert features[0].shape == (_FEAT_DIM, 980) + assert torch.equal(features[0], fake_outputs["audio_features"][0][:, :980]) diff --git a/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py b/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py index 921095e75..5d05e4756 100644 --- a/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py +++ b/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py @@ -3175,9 +3175,33 @@ class MiniCPMOAudioEmbeddingItems(DictEmbeddingItems): def _minicpmo_field_config(hf_inputs: Mapping[str, torch.Tensor]): + audio_features = hf_inputs.get("audio_features") + audio_feature_lens = hf_inputs.get("audio_feature_lens") + audio_features_cfg = MultiModalFieldConfig.batched("audio") + + if audio_features is not None and audio_feature_lens is not None: + num_features = len(audio_features) + num_audios = len(audio_feature_lens) + + if num_features > num_audios: + chunks_per_audio = [lens.numel() if isinstance(lens, torch.Tensor) else 1 for lens in audio_feature_lens] + if sum(chunks_per_audio) != num_features: + chunks_per_audio = [ + max(int((lens != 0).sum()), 1) if isinstance(lens, torch.Tensor) else 1 + for lens in audio_feature_lens + ] + + slice_idxs = [0] + for num_chunks in chunks_per_audio: + slice_idxs.append(slice_idxs[-1] + num_chunks) + audio_features_cfg = MultiModalFieldConfig.flat( + "audio", + [slice(slice_idxs[i], slice_idxs[i + 1]) for i in range(len(chunks_per_audio))], + ) + return dict( **_minicpmv_field_config(hf_inputs), - audio_features=MultiModalFieldConfig.batched("audio"), + audio_features=audio_features_cfg, audio_feature_lens=MultiModalFieldConfig.batched("audio"), audio_embeds=MultiModalFieldConfig.batched("audio"), ) @@ -3369,11 +3393,17 @@ class MiniCPMO45OmniLLMMultiModalProcessor(BaseMultiModalProcessor[MiniCPMO45Omn out_keys={"audio_features", "audio_feature_lens"}, ) + flat_feature_lens: list[int] = [] + for lens in audio_inputs["audio_feature_lens"]: + if isinstance(lens, torch.Tensor): + flat_feature_lens.extend(lens.flatten().tolist()) + else: + flat_feature_lens.append(int(lens)) unpadded_audio_features = [ feat[:, :feature_len] for feat, feature_len in zip( audio_inputs["audio_features"], - audio_inputs["audio_feature_lens"], + flat_feature_lens, ) ] audio_inputs["audio_features"] = unpadded_audio_features From 2cf5f57b3e0a5bc820f9f382ea88db68006915e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zeyu=20Huang=20=7C=20=E9=BB=83=E6=BE=A4=E5=AE=87?= <11222265+fhfuih@users.noreply.github.com> Date: Wed, 22 Jul 2026 09:30:59 +0800 Subject: [PATCH 047/240] [Refactor]: clean up diffusion executor lifecycle (#5215) Signed-off-by: Huang, Zeyu <11222265+fhfuih@users.noreply.github.com> --- .../test_multiproc_engine_concurrency.py | 37 ++++----- vllm_omni/diffusion/executor/abstract.py | 8 +- .../diffusion/executor/multiproc_executor.py | 81 ++++++++++--------- vllm_omni/diffusion/stage_diffusion_proc.py | 18 ++--- 4 files changed, 79 insertions(+), 65 deletions(-) diff --git a/tests/diffusion/test_multiproc_engine_concurrency.py b/tests/diffusion/test_multiproc_engine_concurrency.py index 070d899e6..9b8e08a74 100644 --- a/tests/diffusion/test_multiproc_engine_concurrency.py +++ b/tests/diffusion/test_multiproc_engine_concurrency.py @@ -57,10 +57,8 @@ def _make_executor(num_gpus: int = 1): Returns ``(executor, request_queue, result_queue)``. """ od_cfg = SimpleNamespace(num_gpus=num_gpus, streaming_output=False) - monkeypatch = pytest.MonkeyPatch() - monkeypatch.setattr(MultiprocDiffusionExecutor, "_init_executor", lambda self: None) - executor = MultiprocDiffusionExecutor(od_cfg) - monkeypatch.undo() + executor = object.__new__(MultiprocDiffusionExecutor) + executor.od_config = od_cfg req_q: queue.Queue = queue.Queue() res_q: queue.Queue = queue.Queue() @@ -73,7 +71,7 @@ def _make_executor(num_gpus: int = 1): executor._result_mq = mock_rmq executor._closed = False executor._processes = [] - executor.is_failed = False + executor._is_failed = False executor._failure_callbacks = [] return executor, req_q, res_q @@ -622,7 +620,7 @@ class TestMultiprocExecutorRaisesEngineDeadError: executor._broadcast_mq = MagicMock() executor._result_mq = MagicMock() executor._result_mq.dequeue = MagicMock(side_effect=TimeoutError) - executor.is_failed = True + executor._is_failed = True with pytest.raises(EngineDeadError): executor.collective_rpc( @@ -643,7 +641,7 @@ class TestMultiprocExecutorRaisesEngineDeadError: nonlocal call_count call_count += 1 if call_count == 1: - executor.is_failed = True + executor._is_failed = True raise TimeoutError return orig_dequeue(timeout=timeout) @@ -920,7 +918,7 @@ def _make_short_lived_process() -> mp.Process: class TestMultiprocExecutorWorkerMonitor: - """Integration tests for ``start_worker_monitor``. + """Integration tests for ``_start_worker_monitor``. Uses real short-lived subprocesses so that OS-level sentinel fd readiness is exercised end-to-end. @@ -928,17 +926,17 @@ class TestMultiprocExecutorWorkerMonitor: def test_worker_monitor_sets_is_failed_and_calls_callbacks_on_death(self): """When a worker process dies, the monitor thread must: - 1. Set ``is_failed = True`` + 1. Set ``_is_failed = True`` 2. Call ``shutdown()`` (which sets ``_closed = True``) 3. Invoke all registered failure callbacks """ executor = object.__new__(MultiprocDiffusionExecutor) executor._closed = False - executor.is_failed = False + executor._is_failed = False executor._failure_callbacks = [] executor._broadcast_mq = None executor._result_mq = None - executor.resources = None + executor._shutdown_cleaner = None # Use a no-op so shutdown() doesn't crash on None resources. executor._finalizer = lambda: None @@ -948,35 +946,38 @@ class TestMultiprocExecutorWorkerMonitor: callback_called = threading.Event() executor.register_failure_callback(callback_called.set) - executor.start_worker_monitor() + executor._start_worker_monitor() # Wait for the process to exit and the monitor to react. proc.join(5) - assert _poll_flag(lambda: executor.is_failed), "is_failed was not set" + assert _poll_flag(lambda: executor._is_failed), "_is_failed was not set" assert executor._closed, "shutdown() was not called" assert callback_called.wait(timeout=2), "failure callback was not invoked" + assert executor.is_dead def test_worker_monitor_noop_when_already_closed(self): """If ``_closed`` is already True when the process dies (orderly - shutdown), the monitor must *not* set ``is_failed``.""" + shutdown), the monitor must *not* set ``_is_failed``.""" executor = object.__new__(MultiprocDiffusionExecutor) executor._closed = True # already shut down - executor.is_failed = False + executor._is_failed = False executor._failure_callbacks = [] executor._broadcast_mq = None executor._result_mq = None - executor.resources = None + executor._shutdown_cleaner = None executor._finalizer = lambda: None proc = _make_short_lived_process() executor._processes = [proc] - executor.start_worker_monitor() + executor._start_worker_monitor() proc.join(5) # Give the monitor thread a chance to run (it should early-return). time.sleep(0.3) - assert not executor.is_failed, "is_failed should remain False on orderly shutdown" + assert not executor._is_failed, "_is_failed should remain False on orderly shutdown" + # Orderly close still reports dead via the public accessor. + assert executor.is_dead class TestStageDiffusionClientProcMonitor: diff --git a/vllm_omni/diffusion/executor/abstract.py b/vllm_omni/diffusion/executor/abstract.py index 8bfb5923a..17e5c0504 100644 --- a/vllm_omni/diffusion/executor/abstract.py +++ b/vllm_omni/diffusion/executor/abstract.py @@ -59,7 +59,7 @@ class DiffusionExecutor(ABC): raise ValueError(f"Unknown distributed executor backend: {distributed_executor_backend}") return executor_class - def __init__(self, od_config: OmniDiffusionConfig): + def __init__(self, od_config: OmniDiffusionConfig) -> None: self.od_config = od_config self._init_executor() @@ -68,6 +68,12 @@ class DiffusionExecutor(ABC): """Initialize the executor (e.g., launch workers, setup IPC).""" pass + @property + @abstractmethod + def is_dead(self) -> bool: + """Whether the executor is shut down or has failed fatally.""" + pass + @abstractmethod def execute_request(self, scheduler_output: DiffusionSchedulerOutput) -> BaseRunnerOutput: """Execute request-mode work from a scheduler output.""" diff --git a/vllm_omni/diffusion/executor/multiproc_executor.py b/vllm_omni/diffusion/executor/multiproc_executor.py index 2e491ff69..d0ed76bb9 100644 --- a/vllm_omni/diffusion/executor/multiproc_executor.py +++ b/vllm_omni/diffusion/executor/multiproc_executor.py @@ -7,10 +7,11 @@ import time import weakref from collections.abc import Callable from dataclasses import dataclass -from typing import TYPE_CHECKING, Any +from multiprocessing.synchronize import Event +from typing import TYPE_CHECKING, Any, cast import zmq -from vllm.distributed.device_communicators.shm_broadcast import MessageQueue +from vllm.distributed.device_communicators.shm_broadcast import Handle, MessageQueue from vllm.logger import init_logger from vllm.v1.engine.exceptions import EngineDeadError @@ -29,29 +30,21 @@ _DEQUEUE_TIMEOUT_S = 5.0 @dataclass -class BackgroundResources: - """ - Used as a finalizer for clean shutdown. - """ +class _ExecutorShutdownCleaner: + """Finalizer that shuts down executor worker processes.""" broadcast_mq: MessageQueue | None = None - result_mq: MessageQueue | None = None num_workers: int = 0 processes: list[mp.Process] | None = None - def __call__(self): + def __call__(self) -> None: """Clean up background resources.""" - if hasattr(self, "wake_events") and self.wake_events: - for ev in self.wake_events: - ev.set() - if self.broadcast_mq is not None: try: for _ in range(self.num_workers): self.broadcast_mq.enqueue(SHUTDOWN_MESSAGE, timeout=1.0) self.broadcast_mq = None - self.result_mq = None except Exception as exc: logger.warning("Failed to send shutdown signal: %s", exc) @@ -72,10 +65,11 @@ class MultiprocDiffusionExecutor(DiffusionExecutor): def _init_executor(self) -> None: self._processes: list[mp.Process] = [] self._closed = False - self.is_failed = False + self._is_failed = False self._failure_callbacks: list[Callable[[], None]] = [] + self._result_mq: MessageQueue | None = None - num_workers = self.od_config.num_gpus + num_workers = cast(int, self.od_config.num_gpus) self.wake_events = [mp.Event() for _ in range(num_workers)] self._broadcast_mq = self._init_broadcast_queue(num_workers) @@ -83,18 +77,23 @@ class MultiprocDiffusionExecutor(DiffusionExecutor): # Launch workers processes, result_handle = self._launch_workers(broadcast_handle, self.wake_events) - self._result_mq = self._init_result_queue(result_handle) self._processes = processes - self.resources = BackgroundResources( + shutdown_cleaner = _ExecutorShutdownCleaner( broadcast_mq=self._broadcast_mq, - result_mq=self._result_mq, num_workers=num_workers, processes=self._processes, ) - self._finalizer = weakref.finalize(self, self.resources) + self._shutdown_cleaner: _ExecutorShutdownCleaner | None = shutdown_cleaner + self._finalizer = weakref.finalize(self, shutdown_cleaner) - self.start_worker_monitor() + try: + self._result_mq = self._init_result_queue(result_handle) + except Exception: + self.shutdown() + raise + + self._start_worker_monitor() def _init_broadcast_queue(self, num_workers: int) -> MessageQueue: return MessageQueue( @@ -103,20 +102,21 @@ class MultiprocDiffusionExecutor(DiffusionExecutor): local_reader_ranks=list(range(num_workers)), ) - def _init_result_queue(self, result_handle) -> MessageQueue | None: + def _init_result_queue(self, result_handle: Handle | None) -> MessageQueue: if result_handle is None: - logger.error("Failed to get result queue handle from workers") - return None + raise RuntimeError("Failed to get result queue handle from workers") return MessageQueue.create_from_handle(result_handle, 0) def _ensure_open(self) -> None: if self._closed: raise RuntimeError("DiffusionExecutor is closed.") if self._result_mq is None: - raise RuntimeError("Result queue not initialized") + raise RuntimeError("Result queue is closed") + if self._broadcast_mq is None: + raise RuntimeError("Broadcast queue is closed") def _dequeue_one_with_failure_polling(self, deadline: float | None, method: str) -> Any: - """Block until one result message, polling ``is_failed`` between chunk timeouts.""" + """Block until one result message, polling ``_is_failed`` between chunk timeouts.""" while True: if deadline is None: chunk_timeout = _DEQUEUE_TIMEOUT_S @@ -126,9 +126,9 @@ class MultiprocDiffusionExecutor(DiffusionExecutor): raise TimeoutError(f"RPC call to {method} timed out.") chunk_timeout = min(_DEQUEUE_TIMEOUT_S, remaining) try: - return self._result_mq.dequeue(timeout=chunk_timeout) + return self._result_mq.dequeue(timeout=chunk_timeout) # pyright: ignore[reportOptionalMemberAccess] MQ is not None before shutdown except (TimeoutError, zmq.error.Again): - if self.is_failed: + if self._is_failed: raise EngineDeadError() continue @@ -183,11 +183,15 @@ class MultiprocDiffusionExecutor(DiffusionExecutor): MultiprocDiffusionExecutor._raise_for_rpc_error_dict(response) return response - def _launch_workers(self, broadcast_handle, wake_events): + def _launch_workers( + self, + broadcast_handle: Handle, + wake_events: list[Event], + ) -> tuple[list[mp.Process], Handle | None]: od_config = self.od_config logger.info("Starting server...") - num_gpus = od_config.num_gpus + num_gpus = cast(int, od_config.num_gpus) mp.set_start_method("spawn", force=True) processes = [] @@ -221,7 +225,6 @@ class MultiprocDiffusionExecutor(DiffusionExecutor): processes.append(process) # Wait for all workers to be ready - scheduler_infos = [] result_handle = None for writer in scheduler_pipe_writers: writer.close() @@ -241,14 +244,18 @@ class MultiprocDiffusionExecutor(DiffusionExecutor): if i == 0: result_handle = data.get("result_handle") - scheduler_infos.append(data) reader.close() logger.debug("All workers are ready") return processes, result_handle - def start_worker_monitor(self) -> None: + @property + def is_dead(self) -> bool: + """Whether the executor is shut down or a worker has failed fatally.""" + return self._closed or self._is_failed + + def _start_worker_monitor(self) -> None: # Monitors worker process liveness. If any die unexpectedly, # logs an error, shuts down the executor and invokes the failure # callback to inform the engine. @@ -287,7 +294,7 @@ class MultiprocDiffusionExecutor(DiffusionExecutor): "Diffusion worker(s) died unexpectedly: %s", details, ) - self.is_failed = True + self._is_failed = True self.shutdown() @@ -416,7 +423,7 @@ class MultiprocDiffusionExecutor(DiffusionExecutor): try: # Broadcast RPC request to all workers via unified message queue - self._broadcast_mq.enqueue(rpc_request) + self._broadcast_mq.enqueue(rpc_request) # pyright: ignore[reportOptionalMemberAccess] MQ is not None before shutdown # Only rank 0 has a result_mq, so we always expect exactly 1 response num_responses = 1 @@ -440,12 +447,12 @@ class MultiprocDiffusionExecutor(DiffusionExecutor): raise def check_health(self) -> None: - if self.is_failed: + if self._is_failed: raise EngineDeadError() self._ensure_open() for p in self._processes: if not p.is_alive(): - self.is_failed = True + self._is_failed = True raise EngineDeadError(f"Worker process {p.name} is dead") def shutdown(self) -> None: @@ -455,5 +462,5 @@ class MultiprocDiffusionExecutor(DiffusionExecutor): finally: self._broadcast_mq = None self._result_mq = None - self.resources = None + self._shutdown_cleaner = None self._processes = [] diff --git a/vllm_omni/diffusion/stage_diffusion_proc.py b/vllm_omni/diffusion/stage_diffusion_proc.py index e74a34b59..0e00f8eea 100644 --- a/vllm_omni/diffusion/stage_diffusion_proc.py +++ b/vllm_omni/diffusion/stage_diffusion_proc.py @@ -40,6 +40,7 @@ from vllm_omni.outputs import OmniRequestOutput if TYPE_CHECKING: from vllm_omni.diffusion.data import OmniDiffusionConfig + from vllm_omni.diffusion.executor.abstract import DiffusionExecutor logger = init_logger(__name__) @@ -92,20 +93,19 @@ class StageDiffusionProc: def _is_executor_dead(self) -> bool: """True iff the multiproc executor has been closed or marked failed. - Detects the "workers died but the diffusion proc is still pulling - requests" case: ``MultiprocDiffusionExecutor`` sets ``_closed = True`` - and ``is_failed = True`` from its worker-monitor thread the moment any - worker process exits; every subsequent ``execute_request`` / - ``collective_rpc`` then raises ``RuntimeError("DiffusionExecutor is - closed.")`` inside the engine. Callers in ``run_loop`` use this to - decide whether a per-request failure is recoverable or fatal. + requests" case: ``MultiprocDiffusionExecutor`` sets ``is_dead`` from its + worker-monitor thread the moment any worker process exits; every + subsequent ``execute_request`` / ``collective_rpc`` then raises + ``RuntimeError("DiffusionExecutor is closed.")`` inside the engine. + Callers in ``run_loop`` use this to decide whether a per-request + failure is recoverable or fatal. """ if self._engine is None: return False - executor = getattr(self._engine, "executor", None) + executor: DiffusionExecutor | None = getattr(self._engine, "executor", None) if executor is None: return False - return bool(getattr(executor, "_closed", False) or getattr(executor, "is_failed", False)) + return executor.is_dead def _signal_fatal_engine_failure(self, reason: str) -> None: """Idempotently signal ``run_loop`` to tear down on a fatal engine error.""" From dcf8b07d39d553e34338c997e5497a0a503cfdc4 Mon Sep 17 00:00:00 2001 From: "Bingyu (Spencer) Liu" Date: Wed, 22 Jul 2026 09:44:15 +0800 Subject: [PATCH 048/240] [Core] Cleanup Omni connector runtime and transfer backends (#5096) Signed-off-by: spencerr221 --- docs/configuration/stage_configs.md | 2 +- .../design/feature/disaggregated_inference.md | 2 - .../shared_memory_connector.md | 29 +- docs/design/feature/ray_based_execution.md | 2 +- .../offline_inference/covo_audio/end2end.py | 1 - .../offline_inference/mimo_audio/end2end.py | 1 - .../offline_inference/step_audio2/end2end.py | 1 - .../text_to_speech/ming_tts/runner.py | 1 - tests/config/test_omni_config.py | 2 - .../omni_connectors/test_adapter_and_flow.py | 5 +- .../omni_connectors/test_basic_connectors.py | 57 ++- .../test_omni_connector_configs.py | 1 - .../omni_connectors/test_shm_connector.py | 29 +- tests/e2e/accuracy/test_hunyuan_image3.py | 1 - tests/engine/test_async_omni_engine_abort.py | 3 - tests/entrypoints/test_pd_disaggregation.py | 4 +- tests/helpers/runtime.py | 2 - tests/worker/test_omni_connector_mixin.py | 29 +- vllm_omni/config/omni_config.py | 1 - vllm_omni/deploy/aura_omni.yaml | 1 - vllm_omni/deploy/bagel.yaml | 2 - .../deploy/dynin_omni_multiconnector.yaml | 2 - vllm_omni/deploy/fish_qwen3_omni.yaml | 1 - vllm_omni/deploy/fish_qwen3_omni_2gpu.yaml | 1 - ..._qwen3_omni_high_concurrency_dual_gpu.yaml | 1 - ...wen3_omni_high_concurrency_single_gpu.yaml | 1 - vllm_omni/deploy/higgs_audio_v2.yaml | 1 - vllm_omni/deploy/higgs_multimodal_qwen3.yaml | 1 - ...iggs_multimodal_qwen3_high_throughput.yaml | 1 - .../higgs_multimodal_qwen3_low_latency.yaml | 1 - vllm_omni/deploy/hunyuan_image_3_moe.yaml | 2 - vllm_omni/deploy/indextts2.yaml | 1 - vllm_omni/deploy/indextts2_low_latency.yaml | 1 - vllm_omni/deploy/mimo_audio.yaml | 1 - vllm_omni/deploy/ming_flash_omni_image.yaml | 2 - vllm_omni/deploy/moss_sound_effect.yaml | 1 - vllm_omni/deploy/moss_tts.yaml | 1 - vllm_omni/deploy/moss_tts_80gb.yaml | 1 - vllm_omni/deploy/moss_tts_local.yaml | 1 - vllm_omni/deploy/moss_tts_realtime.yaml | 1 - vllm_omni/deploy/moss_ttsd.yaml | 1 - vllm_omni/deploy/moss_voice_generator.yaml | 1 - vllm_omni/deploy/qwen3_tts.yaml | 1 - .../deploy/qwen3_tts_high_concurrency.yaml | 1 - vllm_omni/deploy/voxtral_tts.yaml | 1 - .../mooncake_transfer_engine_connector.py | 399 ++++++++++-------- .../connectors/shm_connector.py | 82 ++-- .../omni_connectors/utils/initialization.py | 22 +- vllm_omni/engine/arg_utils.py | 1 - .../xpu/stage_configs/voxtral_tts.yaml | 1 - vllm_omni/worker/gpu_ar_model_runner.py | 1 - .../worker/gpu_generation_model_runner.py | 1 - .../omni_connector_model_runner_mixin.py | 87 +--- 53 files changed, 349 insertions(+), 448 deletions(-) diff --git a/docs/configuration/stage_configs.md b/docs/configuration/stage_configs.md index 1aa4ca1b3..e49487b72 100644 --- a/docs/configuration/stage_configs.md +++ b/docs/configuration/stage_configs.md @@ -65,7 +65,7 @@ connectors: | Connector class | Use case | `extra` keys | |-----------------|----------|--------------| -| `SharedMemoryConnector` | Same-host KV transfer between stages (default for bundled YAMLs). | `shm_threshold_bytes` (int, default `65536`). | +| `SharedMemoryConnector` | Same-host KV transfer between stages (default for bundled YAMLs). | None. All payloads use shared memory. | | `MooncakeStoreConnector` | Cross-host KV transfer over TCP. Required for multi-node deployments. | `host`, `metadata_server`, `master`, `segment` (int bytes), `localbuf` (int bytes), `proto` (`"tcp"` / `"rdma"`). | A stage references a connector by name in its `input_connectors` / `output_connectors`: diff --git a/docs/design/feature/disaggregated_inference.md b/docs/design/feature/disaggregated_inference.md index 15d9e09df..c31086c0d 100644 --- a/docs/design/feature/disaggregated_inference.md +++ b/docs/design/feature/disaggregated_inference.md @@ -66,8 +66,6 @@ runtime: connectors: connector_of_shared_memory: name: SharedMemoryConnector - extra: - shm_threshold_bytes: 65536 ``` Wire stages to connectors: diff --git a/docs/design/feature/omni_connectors/shared_memory_connector.md b/docs/design/feature/omni_connectors/shared_memory_connector.md index 5b35014f2..62d77cd6d 100644 --- a/docs/design/feature/omni_connectors/shared_memory_connector.md +++ b/docs/design/feature/omni_connectors/shared_memory_connector.md @@ -8,9 +8,7 @@ auto-configured when no explicit connector is specified for an edge. ## How It Works All payloads are serialized and stored in shared memory (`/dev/shm`); the SHM -segment name is returned in metadata. The configuration exposes a -`shm_threshold_bytes` field for a future inline-vs-SHM split, but the current -implementation always uses shared memory regardless of payload size. +segment name is returned in metadata. ## Configuration @@ -19,8 +17,6 @@ runtime: connectors: connector_of_shared_memory: name: SharedMemoryConnector - extra: - shm_threshold_bytes: 65536 ``` ## Notes @@ -166,27 +162,10 @@ This path is mainly for older code paths and is not the preferred mode for the c ### 6. Key Implementation Characteristics -#### 6.1 Threshold Exists, but the Current Code Always Uses SHM +#### 6.1 All Payloads Use Shared Memory -The class keeps a `shm_threshold_bytes` field and still exposes metrics for inline writes. However, the current implementation uses: - -```python -if True: - ... -``` - -inside `put()`, which means the current code path always writes to shared memory. - -So the design still suggests a future split between: - -- small payloads inline -- large payloads in shared memory - -but the current behavior is effectively: - -- all payloads go through shared memory - -This should be documented because it affects real runtime behavior. +`put()` writes every serialized payload to shared memory. The connector has no +inline-payload path or size threshold. #### 6.2 Cleanup Is Currently Passive diff --git a/docs/design/feature/ray_based_execution.md b/docs/design/feature/ray_based_execution.md index ae10d661f..4f4caa642 100644 --- a/docs/design/feature/ray_based_execution.md +++ b/docs/design/feature/ray_based_execution.md @@ -47,7 +47,7 @@ When running on Ray, the system automatically adapts its communication strategy: * **Cross-Node**: Recommended to use `MooncakeTransferEngineConnector` (RDMA, fastest) or `MooncakeStoreConnector` (TCP fallback). * **Same-Node**: Can still use `SharedMemoryConnector` for efficiency, or Ray's native object store (plasma). -* **SHM threshold default differs**: when `worker_backend="ray"`, the SharedMemoryConnector default threshold is set to `sys.maxsize`, which forces payloads to go inline (no SHM). Override `shm_threshold_bytes` in the connector config if you want SHM for Ray runs. +* **SHM payload routing**: `SharedMemoryConnector` always stores payloads in `/dev/shm`. ### 2.4 Internal Helpers diff --git a/examples/offline_inference/covo_audio/end2end.py b/examples/offline_inference/covo_audio/end2end.py index ae7e5c6bb..ce728373d 100644 --- a/examples/offline_inference/covo_audio/end2end.py +++ b/examples/offline_inference/covo_audio/end2end.py @@ -64,7 +64,6 @@ def main(args): stage_init_timeout=args.stage_init_timeout, batch_timeout=args.batch_timeout, init_timeout=args.init_timeout, - shm_threshold_bytes=args.shm_threshold_bytes, ) # Stage 0: fused_thinker_talker diff --git a/examples/offline_inference/mimo_audio/end2end.py b/examples/offline_inference/mimo_audio/end2end.py index 139351aab..4ba8f483a 100644 --- a/examples/offline_inference/mimo_audio/end2end.py +++ b/examples/offline_inference/mimo_audio/end2end.py @@ -188,7 +188,6 @@ def main(args): init_sleep_seconds=args.init_sleep_seconds, batch_timeout=args.batch_timeout, init_timeout=args.init_timeout, - shm_threshold_bytes=args.shm_threshold_bytes, ) thinker_sampling_params = SamplingParams( diff --git a/examples/offline_inference/step_audio2/end2end.py b/examples/offline_inference/step_audio2/end2end.py index 8cc7e2a26..e0d1836a8 100644 --- a/examples/offline_inference/step_audio2/end2end.py +++ b/examples/offline_inference/step_audio2/end2end.py @@ -258,7 +258,6 @@ def main(args): "init_sleep_seconds": args.init_sleep_seconds, "batch_timeout": args.batch_timeout, "init_timeout": args.init_timeout, - "shm_threshold_bytes": args.shm_threshold_bytes, "worker_backend": args.worker_backend, "ray_address": args.ray_address, "trust_remote_code": True, diff --git a/examples/offline_inference/text_to_speech/ming_tts/runner.py b/examples/offline_inference/text_to_speech/ming_tts/runner.py index 35852ce04..6f9d75740 100644 --- a/examples/offline_inference/text_to_speech/ming_tts/runner.py +++ b/examples/offline_inference/text_to_speech/ming_tts/runner.py @@ -140,7 +140,6 @@ def build_engine_kwargs(args, stats_log_file: str | None) -> dict: "stage_init_timeout": args.stage_init_timeout, "init_timeout": args.init_timeout, "batch_timeout": args.batch_timeout, - "shm_threshold_bytes": args.shm_threshold_bytes, "worker_backend": args.worker_backend, } if stats_log_file is not None: diff --git a/tests/config/test_omni_config.py b/tests/config/test_omni_config.py index 23e53ea3d..1da71b21d 100644 --- a/tests/config/test_omni_config.py +++ b/tests/config/test_omni_config.py @@ -302,7 +302,6 @@ def test_from_pipeline_config_maps_orchestrator_cli_overrides(): "omni_dp_size_local": 2, "omni_lb_policy": "round_robin", "omni_heartbeat_timeout": 9.5, - "shm_threshold_bytes": 4096, "batch_timeout": 3, }, ) @@ -317,7 +316,6 @@ def test_from_pipeline_config_maps_orchestrator_cli_overrides(): assert orchestrator_config.omni_dp_size_local == 2 assert orchestrator_config.omni_lb_policy == "round_robin" assert orchestrator_config.omni_heartbeat_timeout == 9.5 - assert orchestrator_config.shm_threshold_bytes == 4096 assert orchestrator_config.batch_timeout == 3 diff --git a/tests/distributed/omni_connectors/test_adapter_and_flow.py b/tests/distributed/omni_connectors/test_adapter_and_flow.py index 548d16eb6..92dc1d7c0 100644 --- a/tests/distributed/omni_connectors/test_adapter_and_flow.py +++ b/tests/distributed/omni_connectors/test_adapter_and_flow.py @@ -142,11 +142,10 @@ def test_recv_no_connector(): def test_shm_connector_flow(mocker: MockerFixture): """ Verify the full flow: Send -> Adapter -> Connector -> Adapter -> Recv. - Using real SharedMemoryConnector (inline mode for simplicity). + Using real SharedMemoryConnector and key-addressed SHM metadata. """ # 1. Setup Connector - config = {"shm_threshold_bytes": 1024, "inline_small_payloads": True} - connector = SharedMemoryConnector(config) + connector = SharedMemoryConnector({}) connectors_map = {("0", "1"): connector} # 2. Setup Data diff --git a/tests/distributed/omni_connectors/test_basic_connectors.py b/tests/distributed/omni_connectors/test_basic_connectors.py index 1518e98b1..787d5594b 100644 --- a/tests/distributed/omni_connectors/test_basic_connectors.py +++ b/tests/distributed/omni_connectors/test_basic_connectors.py @@ -60,10 +60,9 @@ def test_ndarray_serialization(): def test_create_shm_connector(): """Test creating SharedMemoryConnector via Factory.""" - spec = ConnectorSpec(name="SharedMemoryConnector", extra={"shm_threshold_bytes": 1024}) + spec = ConnectorSpec(name="SharedMemoryConnector") connector = OmniConnectorFactory.create_connector(spec) assert isinstance(connector, SharedMemoryConnector) - assert connector.threshold == 1024 def test_create_unknown_connector(): @@ -75,21 +74,20 @@ def test_create_unknown_connector(): @pytest.fixture def shm_connector(): - config = {"shm_threshold_bytes": 100, "inline_small_payloads": True} - return SharedMemoryConnector(config) + connector = SharedMemoryConnector({}) + yield connector + connector.close() -def test_put_get_inline(shm_connector): - """Test inline transfer for small data.""" +def test_put_get_small_payload_uses_shm(shm_connector): + """Small payloads use key-addressed SHM; inline metadata is retired.""" data = {"small": "data"} success, size, metadata = shm_connector.put("stage_0", "stage_1", "req_1", data) assert success is True - assert "inline_bytes" in metadata - assert "shm" not in metadata + assert "shm" in metadata + assert "inline_bytes" not in metadata assert "size" in metadata - assert shm_connector._metrics["inline_writes"] == 1 - assert shm_connector._metrics["shm_writes"] == 0 # Retrieve retrieved_data, ret_size = shm_connector.get("stage_0", "stage_1", "req_1", metadata) @@ -195,3 +193,42 @@ def test_mooncake_connector_defaults_missing_host_to_detected_ip(monkeypatch: py assert connector.get_connection_info()["host"] == "10.20.30.40" finally: connector.close() + + +def test_mooncake_get_counts_zmq_timeout_separately(mocker: MockerFixture): + import vllm_omni.distributed.omni_connectors.connectors.mooncake_transfer_engine_connector as mooncake_module + + connector = object.__new__(mooncake_module.MooncakeTransferEngineConnector) + connector._closed = False + connector._metrics = {"errors": 0, "timeouts": 0} + connector._resolve_metadata = mocker.Mock( + return_value={"source_host": "127.0.0.1", "source_port": 1234, "data_size": 8} + ) + recv_buffer = mocker.Mock() + connector._alloc_recv_buffer = mocker.Mock(return_value=(recv_buffer, 1000)) + connector._request_transfer = mocker.Mock(side_effect=mooncake_module.zmq.Again()) + + assert connector.get("s0", "s1", "req") is None + assert connector._metrics == {"errors": 0, "timeouts": 1} + recv_buffer.release.assert_called_once() + connector._closed = True + + +def test_mooncake_get_counts_cuda_sync_failure_as_error(mocker: MockerFixture): + import vllm_omni.distributed.omni_connectors.connectors.mooncake_transfer_engine_connector as mooncake_module + + connector = object.__new__(mooncake_module.MooncakeTransferEngineConnector) + connector._closed = False + connector._metrics = {"errors": 0, "timeouts": 0} + connector._resolve_metadata = mocker.Mock( + return_value={"source_host": "127.0.0.1", "source_port": 1234, "data_size": 8} + ) + recv_buffer = mocker.Mock() + connector._alloc_recv_buffer = mocker.Mock(return_value=(recv_buffer, 1000)) + connector._request_transfer = mocker.Mock(return_value=mooncake_module.TRANS_DONE) + connector._sync_receive_buffer_if_needed = mocker.Mock(side_effect=RuntimeError("CUDA sync failed")) + + assert connector.get("s0", "s1", "req") is None + assert connector._metrics == {"errors": 1, "timeouts": 0} + recv_buffer.release.assert_called_once() + connector._closed = True diff --git a/tests/distributed/omni_connectors/test_omni_connector_configs.py b/tests/distributed/omni_connectors/test_omni_connector_configs.py index 3dc80b57b..82f13c5cd 100644 --- a/tests/distributed/omni_connectors/test_omni_connector_configs.py +++ b/tests/distributed/omni_connectors/test_omni_connector_configs.py @@ -48,7 +48,6 @@ def test_load_qwen_yaml_configs(yaml_file): print(f"Testing config load: {yaml_file.name}") try: # Attempt to load the config - # default_shm_threshold doesn't matter much for loading correctness, using default config = load_omni_transfer_config(yaml_file) assert config is not None, "Config should not be None" diff --git a/tests/distributed/omni_connectors/test_shm_connector.py b/tests/distributed/omni_connectors/test_shm_connector.py index 572376aef..bba6002a0 100644 --- a/tests/distributed/omni_connectors/test_shm_connector.py +++ b/tests/distributed/omni_connectors/test_shm_connector.py @@ -2,7 +2,10 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """Unit tests for SharedMemoryConnector focusing on TP / CFG / metadata fallback.""" +import os + import pytest +import torch from vllm_omni.distributed.omni_connectors.connectors.shm_connector import ( SharedMemoryConnector, @@ -13,7 +16,7 @@ pytestmark = [pytest.mark.core_model, pytest.mark.cpu] @pytest.fixture() def connector(): - c = SharedMemoryConnector({"shm_threshold_bytes": 0}) + c = SharedMemoryConnector({}) yield c c.close() @@ -36,6 +39,30 @@ class TestKeyBasedReadWrite: assert obj == data assert rsize == size assert "test_key_1" not in connector._pending_keys + assert connector._metrics["gets"] == 1 + + def test_tensor_payload_removes_lock_file(self, connector): + key = "tensor_payload" + payload = torch.ones(2, 2) + ok, _, metadata = connector.put("s0", "s1", key, payload) + assert ok + + result = connector.get("s0", "s1", key, metadata=metadata) + + assert result is not None + assert torch.equal(result[0], payload) + assert not os.path.exists(f"/dev/shm/shm_{key}_lockfile.lock") + + def test_falsey_payload_removes_lock_file(self, connector): + key = "falsey_payload" + ok, _, metadata = connector.put("s0", "s1", key, 0) + assert ok + + result = connector.get("s0", "s1", key, metadata=metadata) + + assert result is not None + assert result[0] == 0 + assert not os.path.exists(f"/dev/shm/shm_{key}_lockfile.lock") def test_get_nonexistent_key_returns_none(self, connector): result = connector.get("s0", "s1", "no_such_key_xyz", metadata=None) diff --git a/tests/e2e/accuracy/test_hunyuan_image3.py b/tests/e2e/accuracy/test_hunyuan_image3.py index 2214b734d..e2d84b13d 100644 --- a/tests/e2e/accuracy/test_hunyuan_image3.py +++ b/tests/e2e/accuracy/test_hunyuan_image3.py @@ -141,7 +141,6 @@ _DEPLOY_CONFIG = { "connectors": { "shared_memory_connector": { "name": "SharedMemoryConnector", - "extra": {"shm_threshold_bytes": 65536}, }, }, "stages": [ diff --git a/tests/engine/test_async_omni_engine_abort.py b/tests/engine/test_async_omni_engine_abort.py index 794475388..dfe00732f 100644 --- a/tests/engine/test_async_omni_engine_abort.py +++ b/tests/engine/test_async_omni_engine_abort.py @@ -1,6 +1,5 @@ import asyncio import os -import sys from contextlib import ExitStack import pytest @@ -67,11 +66,9 @@ async def generate( @pytest.mark.asyncio async def test_abort(): with ExitStack() as after: - # Avoid SHM IPC in tests to prevent /dev/shm exhaustion and SIGBUS. engine = AsyncOmni( model=model, stage_configs_path=stage_config, - shm_threshold_bytes=sys.maxsize, ) after.callback(engine.shutdown) diff --git a/tests/entrypoints/test_pd_disaggregation.py b/tests/entrypoints/test_pd_disaggregation.py index 5ffabfbf2..57e484af1 100644 --- a/tests/entrypoints/test_pd_disaggregation.py +++ b/tests/entrypoints/test_pd_disaggregation.py @@ -121,9 +121,7 @@ class _FakeStage: self._in_q = in_q self._out_q = out_q - def init_stage_worker( - self, model: str, *, is_async=False, shm_threshold_bytes=65536, ctx=None, batch_timeout=10, **kwargs - ): + def init_stage_worker(self, model: str, *, is_async=False, ctx=None, batch_timeout=10, **kwargs): self._proc = _ns( start=lambda: None, join=lambda timeout=None: None, diff --git a/tests/helpers/runtime.py b/tests/helpers/runtime.py index 263f34da2..7c19c643d 100644 --- a/tests/helpers/runtime.py +++ b/tests/helpers/runtime.py @@ -2333,7 +2333,6 @@ class OmniRunner: # production default in AsyncOmniEngine remains 600s; this only # affects the test runner wrapper. init_timeout: int = 1800, - shm_threshold_bytes: int = 65536, log_stats: bool = False, stage_configs_path: str | None = None, **kwargs, @@ -2353,7 +2352,6 @@ class OmniRunner: stage_init_timeout=stage_init_timeout, batch_timeout=batch_timeout, init_timeout=init_timeout, - shm_threshold_bytes=shm_threshold_bytes, stage_configs_path=stage_configs_path, **kwargs, ) diff --git a/tests/worker/test_omni_connector_mixin.py b/tests/worker/test_omni_connector_mixin.py index b8eb899fa..87cf4ba73 100644 --- a/tests/worker/test_omni_connector_mixin.py +++ b/tests/worker/test_omni_connector_mixin.py @@ -110,7 +110,6 @@ class TestMixinAsyncChunkSendRecv(unittest.TestCase): sender = MixinHost() sender.init_omni_connectors( - vllm_config=None, model_config=_make_model_config(stage_id=0, async_chunk=True), ) sender._omni_connector = connector @@ -147,7 +146,6 @@ class TestMixinAsyncChunkSendRecv(unittest.TestCase): sender = MixinHost() sender.init_omni_connectors( - vllm_config=None, model_config=_make_model_config(stage_id=0, async_chunk=True), ) sender._omni_connector = connector @@ -180,7 +178,6 @@ class TestMixinKVCacheTransfer(unittest.TestCase): host = MixinHost() host.init_omni_connectors( - vllm_config=None, model_config=_make_model_config(), kv_transfer_manager=mock_kvm, ) @@ -202,7 +199,6 @@ class TestMixinKVCacheTransfer(unittest.TestCase): host = MixinHost() host.init_omni_connectors( - vllm_config=None, model_config=_make_model_config(), kv_transfer_manager=mock_kvm, ) @@ -219,7 +215,6 @@ class TestMixinKVCacheTransfer(unittest.TestCase): host = MixinHost() host.init_omni_connectors( - vllm_config=None, model_config=_make_model_config(), kv_transfer_manager=mock_kvm, ) @@ -257,7 +252,6 @@ class TestMixinKVCacheTransfer(unittest.TestCase): host = MixinHost() host.init_omni_connectors( - vllm_config=None, model_config=_make_model_config(), kv_transfer_manager=mock_kvm, ) @@ -281,7 +275,6 @@ class TestOmniConnectorOutput(unittest.TestCase): def test_output_aggregation(self): host = MixinHost() host.init_omni_connectors( - vllm_config=None, model_config=_make_model_config(), ) @@ -310,7 +303,6 @@ class TestMixinNoConnector(unittest.TestCase): def test_no_connector(self): host = MixinHost() host.init_omni_connectors( - vllm_config=None, model_config=_make_model_config(), ) self.assertIsNone(host._omni_connector) @@ -336,7 +328,6 @@ class TestFinishedLoadReqsDrain(unittest.TestCase): def test_finished_load_reqs_flow_to_chunk_ready(self): host = MixinHost() host.init_omni_connectors( - vllm_config=None, model_config=_make_model_config(), ) @@ -389,7 +380,6 @@ class TestFullPayloadSendWithCustomFunc(unittest.TestCase): host = MixinHost() host.init_omni_connectors( - vllm_config=None, model_config=_make_model_config(), ) host._omni_connector = MockConnector(stage_id=0) @@ -424,7 +414,6 @@ class TestFullPayloadSendWithCustomFunc(unittest.TestCase): host = MixinHost() host.init_omni_connectors( - vllm_config=None, model_config=_make_model_config(), ) host._omni_connector = MockConnector(stage_id=0) @@ -453,7 +442,6 @@ class TestKVSentReqIdsAccumulation(unittest.TestCase): host = MixinHost() host.init_omni_connectors( - vllm_config=None, model_config=_make_model_config(), kv_transfer_manager=mock_kvm, ) @@ -488,7 +476,6 @@ class TestChunkStreamCompletedGuard(unittest.TestCase): def _make_host(self, stage_id: int = 1) -> MixinHost: host = MixinHost() host.init_omni_connectors( - vllm_config=None, model_config=_make_model_config(stage_id=stage_id, async_chunk=True), ) host._omni_connector = MockConnector(stage_id=stage_id) @@ -587,7 +574,6 @@ class TestCleanupFinishedRequest(unittest.TestCase): def _make_host(self, stage_id: int = 1) -> MixinHost: host = MixinHost() host.init_omni_connectors( - vllm_config=None, model_config=_make_model_config(stage_id=stage_id, async_chunk=True), ) host._omni_connector = MockConnector(stage_id=stage_id) @@ -771,7 +757,6 @@ class TestSendChunkCachesMapping(unittest.TestCase): """send_chunk should cache the internal→external mapping.""" host = MixinHost() host.init_omni_connectors( - vllm_config=None, model_config=_make_model_config(stage_id=0, async_chunk=True), ) host._omni_connector = MockConnector(stage_id=0) @@ -802,7 +787,6 @@ class TestLocalPayloadCacheLifecycle(unittest.TestCase): def _make_host(self) -> MixinHost: host = MixinHost() host.init_omni_connectors( - vllm_config=None, model_config=_make_model_config(stage_id=0), ) host._omni_connector = MockConnector(stage_id=0) @@ -901,7 +885,6 @@ class TestTPAsyncChunkFanout(unittest.TestCase): def _make_host(self, rank: int) -> MixinHost: host = MixinHost() host.init_omni_connectors( - vllm_config=None, model_config=_make_model_config(stage_id=2, async_chunk=True, worker_type="gen"), ) host._omni_connector = MagicMock() @@ -984,7 +967,6 @@ class TestKVTransferLifecycle(unittest.TestCase): def _make_host(self) -> MixinHost: host = MixinHost() host.init_omni_connectors( - vllm_config=None, model_config=_make_model_config(stage_id=0), ) return host @@ -1039,7 +1021,6 @@ class TestAsyncPayloadLifecycle(unittest.TestCase): def test_send_side_request_payload_not_cleared_before_payload_is_consumable(self): host = MixinHost() host.init_omni_connectors( - vllm_config=None, model_config=_make_model_config(stage_id=1, async_chunk=True, worker_type="ar"), ) host._request_ids_mapping["r1"] = "r1" @@ -1063,7 +1044,6 @@ class TestAsyncPayloadLifecycle(unittest.TestCase): def test_payload_consumable_ignores_token_horizon_only_updates(self): host = MixinHost() host.init_omni_connectors( - vllm_config=None, model_config=_make_model_config(stage_id=1, async_chunk=True, worker_type="ar"), ) payload = { @@ -1084,7 +1064,6 @@ class TestAsyncPayloadLifecycle(unittest.TestCase): def test_payload_consumable_accepts_decode_embeddings(self): host = MixinHost() host.init_omni_connectors( - vllm_config=None, model_config=_make_model_config(stage_id=1, async_chunk=True, worker_type="ar"), ) payload = { @@ -1098,7 +1077,6 @@ class TestAsyncPayloadLifecycle(unittest.TestCase): def test_ar_metadata_only_followup_chunk_does_not_rewake_request(self): host = MixinHost() host.init_omni_connectors( - vllm_config=None, model_config=_make_model_config(stage_id=1, async_chunk=True, worker_type="ar"), ) host._omni_connector = MagicMock() @@ -1138,7 +1116,6 @@ class TestAsyncPayloadLifecycle(unittest.TestCase): def test_non_ar_recv_does_not_overwrite_unconsumed_staged_chunk(self): host = MixinHost() host.init_omni_connectors( - vllm_config=None, model_config=_make_model_config(stage_id=2, async_chunk=True, worker_type="gen"), ) host._omni_connector = MagicMock() @@ -1164,7 +1141,6 @@ class TestAsyncPayloadLifecycle(unittest.TestCase): def test_non_ar_recv_waits_for_scheduler_handoff_before_fetching_next_chunk(self): host = MixinHost() host.init_omni_connectors( - vllm_config=None, model_config=_make_model_config(stage_id=2, async_chunk=True, worker_type="gen"), ) host._omni_connector = MagicMock() @@ -1208,7 +1184,7 @@ class TestAsyncPayloadLifecycle(unittest.TestCase): class TestRankAwareKVRouting(unittest.TestCase): def _make_host(self, *, from_tp: int, to_tp: int, local_rank: int) -> MixinHost: host = MixinHost() - host.init_omni_connectors(vllm_config=None, model_config=_make_model_config(stage_id=1)) + host.init_omni_connectors(model_config=_make_model_config(stage_id=1)) host._from_tp = from_tp host._to_tp = to_tp host._local_rank = local_rank @@ -1287,7 +1263,7 @@ class TestConnectorConfigValidation(unittest.TestCase): model_config.stage_connector_config = {"name": " "} with self.assertRaisesRegex(RuntimeError, "missing connector name"): - host.init_omni_connectors(vllm_config=None, model_config=model_config) + host.init_omni_connectors(model_config=model_config) class _FailingConnector: @@ -1319,7 +1295,6 @@ class TestSendRetry(unittest.TestCase): def _make_sender(self, connector): sender = MixinHost() sender.init_omni_connectors( - vllm_config=None, model_config=_make_model_config(stage_id=0, async_chunk=True), ) sender._omni_connector = connector diff --git a/vllm_omni/config/omni_config.py b/vllm_omni/config/omni_config.py index 4c2154401..be89e1cae 100644 --- a/vllm_omni/config/omni_config.py +++ b/vllm_omni/config/omni_config.py @@ -821,7 +821,6 @@ class VllmOmniOrchestratorConfig: omni_dp_size_local: int = Field(default=1, ge=1) omni_lb_policy: str = "random" omni_heartbeat_timeout: float = Field(default=30.0, gt=0.0) - shm_threshold_bytes: int = Field(default=65536, ge=0) batch_timeout: int = Field(default=10, ge=0) diff --git a/vllm_omni/deploy/aura_omni.yaml b/vllm_omni/deploy/aura_omni.yaml index 486e44b57..cf6826fdd 100644 --- a/vllm_omni/deploy/aura_omni.yaml +++ b/vllm_omni/deploy/aura_omni.yaml @@ -12,7 +12,6 @@ connectors: connector_of_shared_memory: name: SharedMemoryConnector extra: - shm_threshold_bytes: 65536 codec_streaming: true connector_get_sleep_s: 0.01 connector_get_max_wait_first_chunk: 3000 diff --git a/vllm_omni/deploy/bagel.yaml b/vllm_omni/deploy/bagel.yaml index 8de6f9305..644e2c3c0 100644 --- a/vllm_omni/deploy/bagel.yaml +++ b/vllm_omni/deploy/bagel.yaml @@ -40,8 +40,6 @@ stages: connectors: shared_memory_connector: name: SharedMemoryConnector - extra: - shm_threshold_bytes: 65536 rdma_connector: name: MooncakeTransferEngineConnector diff --git a/vllm_omni/deploy/dynin_omni_multiconnector.yaml b/vllm_omni/deploy/dynin_omni_multiconnector.yaml index c863ec3d5..5abb4c5a3 100644 --- a/vllm_omni/deploy/dynin_omni_multiconnector.yaml +++ b/vllm_omni/deploy/dynin_omni_multiconnector.yaml @@ -31,8 +31,6 @@ connectors: # Alternative SHM connector with an explicit threshold. shared_memory_connector: name: SharedMemoryConnector - extra: - shm_threshold_bytes: 65536 # 64KB threshold stages: - stage_id: 0 diff --git a/vllm_omni/deploy/fish_qwen3_omni.yaml b/vllm_omni/deploy/fish_qwen3_omni.yaml index 539b37775..f10ed1656 100644 --- a/vllm_omni/deploy/fish_qwen3_omni.yaml +++ b/vllm_omni/deploy/fish_qwen3_omni.yaml @@ -9,7 +9,6 @@ connectors: connector_of_shared_memory: name: SharedMemoryConnector extra: - shm_threshold_bytes: 65536 codec_streaming: true connector_get_sleep_s: 0.01 connector_get_max_wait_first_chunk: 3000 diff --git a/vllm_omni/deploy/fish_qwen3_omni_2gpu.yaml b/vllm_omni/deploy/fish_qwen3_omni_2gpu.yaml index 9fec34d69..360b72974 100644 --- a/vllm_omni/deploy/fish_qwen3_omni_2gpu.yaml +++ b/vllm_omni/deploy/fish_qwen3_omni_2gpu.yaml @@ -20,7 +20,6 @@ connectors: connector_of_shared_memory: name: SharedMemoryConnector extra: - shm_threshold_bytes: 65536 codec_streaming: true # Tightened from 10ms to 1ms: chunks become available sooner under # cross-GPU pipelining, so connector idle waiting dominates if left high. diff --git a/vllm_omni/deploy/fish_qwen3_omni_high_concurrency_dual_gpu.yaml b/vllm_omni/deploy/fish_qwen3_omni_high_concurrency_dual_gpu.yaml index a65609c6c..57ea6e757 100644 --- a/vllm_omni/deploy/fish_qwen3_omni_high_concurrency_dual_gpu.yaml +++ b/vllm_omni/deploy/fish_qwen3_omni_high_concurrency_dual_gpu.yaml @@ -10,7 +10,6 @@ connectors: connector_of_shared_memory: name: SharedMemoryConnector extra: - shm_threshold_bytes: 65536 codec_streaming: true connector_get_sleep_s: 0.001 connector_get_max_wait_first_chunk: 3000 diff --git a/vllm_omni/deploy/fish_qwen3_omni_high_concurrency_single_gpu.yaml b/vllm_omni/deploy/fish_qwen3_omni_high_concurrency_single_gpu.yaml index d771e4163..0c9771b18 100644 --- a/vllm_omni/deploy/fish_qwen3_omni_high_concurrency_single_gpu.yaml +++ b/vllm_omni/deploy/fish_qwen3_omni_high_concurrency_single_gpu.yaml @@ -10,7 +10,6 @@ connectors: connector_of_shared_memory: name: SharedMemoryConnector extra: - shm_threshold_bytes: 65536 codec_streaming: true connector_get_sleep_s: 0.001 connector_get_max_wait_first_chunk: 3000 diff --git a/vllm_omni/deploy/higgs_audio_v2.yaml b/vllm_omni/deploy/higgs_audio_v2.yaml index 07e25d5c1..9c181400b 100644 --- a/vllm_omni/deploy/higgs_audio_v2.yaml +++ b/vllm_omni/deploy/higgs_audio_v2.yaml @@ -17,7 +17,6 @@ connectors: connector_of_shared_memory: name: SharedMemoryConnector extra: - shm_threshold_bytes: 65536 codec_streaming: true connector_get_sleep_s: 0.01 connector_get_max_wait_first_chunk: 5000 diff --git a/vllm_omni/deploy/higgs_multimodal_qwen3.yaml b/vllm_omni/deploy/higgs_multimodal_qwen3.yaml index 5fc44a295..da299415b 100644 --- a/vllm_omni/deploy/higgs_multimodal_qwen3.yaml +++ b/vllm_omni/deploy/higgs_multimodal_qwen3.yaml @@ -16,7 +16,6 @@ connectors: connector_of_shared_memory: name: SharedMemoryConnector extra: - shm_threshold_bytes: 65536 # Stream PCM bytes back to the client as Stage 1 emits each chunk. codec_streaming: true # Streaming chunk shape for talker2code2wav_async_chunk: diff --git a/vllm_omni/deploy/higgs_multimodal_qwen3_high_throughput.yaml b/vllm_omni/deploy/higgs_multimodal_qwen3_high_throughput.yaml index 86427be13..8a751483c 100644 --- a/vllm_omni/deploy/higgs_multimodal_qwen3_high_throughput.yaml +++ b/vllm_omni/deploy/higgs_multimodal_qwen3_high_throughput.yaml @@ -15,7 +15,6 @@ connectors: connector_of_shared_memory: name: SharedMemoryConnector extra: - shm_threshold_bytes: 65536 # Stream PCM bytes back to the client as Stage 1 emits each chunk. codec_streaming: true # Streaming chunk shape for talker2code2wav_async_chunk: diff --git a/vllm_omni/deploy/higgs_multimodal_qwen3_low_latency.yaml b/vllm_omni/deploy/higgs_multimodal_qwen3_low_latency.yaml index 8210db4a8..0d6ae8218 100644 --- a/vllm_omni/deploy/higgs_multimodal_qwen3_low_latency.yaml +++ b/vllm_omni/deploy/higgs_multimodal_qwen3_low_latency.yaml @@ -21,7 +21,6 @@ connectors: connector_of_shared_memory: name: SharedMemoryConnector extra: - shm_threshold_bytes: 65536 # Stream PCM bytes back to the client as Stage 1 emits each chunk. codec_streaming: true # Streaming chunk shape for talker2code2wav_async_chunk: diff --git a/vllm_omni/deploy/hunyuan_image_3_moe.yaml b/vllm_omni/deploy/hunyuan_image_3_moe.yaml index b661674c7..4ae4520a7 100644 --- a/vllm_omni/deploy/hunyuan_image_3_moe.yaml +++ b/vllm_omni/deploy/hunyuan_image_3_moe.yaml @@ -27,8 +27,6 @@ connectors: memory_pool_device: "npu" shared_memory_connector: name: SharedMemoryConnector - extra: - shm_threshold_bytes: 65536 stages: - stage_id: 0 diff --git a/vllm_omni/deploy/indextts2.yaml b/vllm_omni/deploy/indextts2.yaml index 75eb60188..c5fefcf30 100644 --- a/vllm_omni/deploy/indextts2.yaml +++ b/vllm_omni/deploy/indextts2.yaml @@ -4,7 +4,6 @@ connectors: connector_of_shared_memory: name: SharedMemoryConnector extra: - shm_threshold_bytes: 65536 codec_streaming: false connector_get_sleep_s: 0.01 connector_get_max_wait_first_chunk: 5000 diff --git a/vllm_omni/deploy/indextts2_low_latency.yaml b/vllm_omni/deploy/indextts2_low_latency.yaml index 621116ab2..678144a2b 100644 --- a/vllm_omni/deploy/indextts2_low_latency.yaml +++ b/vllm_omni/deploy/indextts2_low_latency.yaml @@ -17,7 +17,6 @@ connectors: connector_of_shared_memory: name: SharedMemoryConnector extra: - shm_threshold_bytes: 65536 codec_streaming: false connector_get_sleep_s: 0.01 connector_get_max_wait_first_chunk: 5000 diff --git a/vllm_omni/deploy/mimo_audio.yaml b/vllm_omni/deploy/mimo_audio.yaml index a50632964..daeedec99 100644 --- a/vllm_omni/deploy/mimo_audio.yaml +++ b/vllm_omni/deploy/mimo_audio.yaml @@ -13,7 +13,6 @@ connectors: connector_of_shared_memory: name: SharedMemoryConnector extra: - shm_threshold_bytes: 365536 codec_streaming: true connector_get_sleep_s: 0.001 connector_get_max_wait_first_chunk: 3000 diff --git a/vllm_omni/deploy/ming_flash_omni_image.yaml b/vllm_omni/deploy/ming_flash_omni_image.yaml index 9f7c6a34d..a02bd07e5 100644 --- a/vllm_omni/deploy/ming_flash_omni_image.yaml +++ b/vllm_omni/deploy/ming_flash_omni_image.yaml @@ -50,8 +50,6 @@ stages: connectors: shared_memory_connector: name: SharedMemoryConnector - extra: - shm_threshold_bytes: 65536 # 64KB threshold edges: - from: 0 diff --git a/vllm_omni/deploy/moss_sound_effect.yaml b/vllm_omni/deploy/moss_sound_effect.yaml index 0fe616290..1dcf0d42f 100644 --- a/vllm_omni/deploy/moss_sound_effect.yaml +++ b/vllm_omni/deploy/moss_sound_effect.yaml @@ -10,7 +10,6 @@ connectors: shm: name: SharedMemoryConnector extra: - shm_threshold_bytes: 65536 codec_streaming: true connector_get_sleep_s: 0.01 connector_get_max_wait_first_chunk: 3000 diff --git a/vllm_omni/deploy/moss_tts.yaml b/vllm_omni/deploy/moss_tts.yaml index b1549d0d0..92bd2f3ba 100644 --- a/vllm_omni/deploy/moss_tts.yaml +++ b/vllm_omni/deploy/moss_tts.yaml @@ -8,7 +8,6 @@ connectors: shm: name: SharedMemoryConnector extra: - shm_threshold_bytes: 65536 codec_streaming: true connector_get_sleep_s: 0.01 connector_get_max_wait_first_chunk: 3000 diff --git a/vllm_omni/deploy/moss_tts_80gb.yaml b/vllm_omni/deploy/moss_tts_80gb.yaml index e61d4cf5d..57dcbc1df 100644 --- a/vllm_omni/deploy/moss_tts_80gb.yaml +++ b/vllm_omni/deploy/moss_tts_80gb.yaml @@ -11,7 +11,6 @@ connectors: shm: name: SharedMemoryConnector extra: - shm_threshold_bytes: 65536 codec_streaming: true connector_get_sleep_s: 0.01 connector_get_max_wait_first_chunk: 3000 diff --git a/vllm_omni/deploy/moss_tts_local.yaml b/vllm_omni/deploy/moss_tts_local.yaml index 1455ccc09..ab66df300 100644 --- a/vllm_omni/deploy/moss_tts_local.yaml +++ b/vllm_omni/deploy/moss_tts_local.yaml @@ -9,7 +9,6 @@ connectors: shm: name: SharedMemoryConnector extra: - shm_threshold_bytes: 65536 codec_streaming: true connector_get_sleep_s: 0.005 connector_get_max_wait_first_chunk: 1000 diff --git a/vllm_omni/deploy/moss_tts_realtime.yaml b/vllm_omni/deploy/moss_tts_realtime.yaml index b7909661a..ca63161e2 100644 --- a/vllm_omni/deploy/moss_tts_realtime.yaml +++ b/vllm_omni/deploy/moss_tts_realtime.yaml @@ -11,7 +11,6 @@ connectors: shm: name: SharedMemoryConnector extra: - shm_threshold_bytes: 65536 codec_streaming: true connector_get_sleep_s: 0.005 connector_get_max_wait_first_chunk: 1000 diff --git a/vllm_omni/deploy/moss_ttsd.yaml b/vllm_omni/deploy/moss_ttsd.yaml index f0b5f6eae..3ffba8678 100644 --- a/vllm_omni/deploy/moss_ttsd.yaml +++ b/vllm_omni/deploy/moss_ttsd.yaml @@ -8,7 +8,6 @@ connectors: shm: name: SharedMemoryConnector extra: - shm_threshold_bytes: 65536 codec_streaming: true connector_get_sleep_s: 0.01 connector_get_max_wait_first_chunk: 3000 diff --git a/vllm_omni/deploy/moss_voice_generator.yaml b/vllm_omni/deploy/moss_voice_generator.yaml index 010f04205..fa3cd91af 100644 --- a/vllm_omni/deploy/moss_voice_generator.yaml +++ b/vllm_omni/deploy/moss_voice_generator.yaml @@ -10,7 +10,6 @@ connectors: shm: name: SharedMemoryConnector extra: - shm_threshold_bytes: 65536 codec_streaming: true connector_get_sleep_s: 0.01 connector_get_max_wait_first_chunk: 3000 diff --git a/vllm_omni/deploy/qwen3_tts.yaml b/vllm_omni/deploy/qwen3_tts.yaml index f27da1b5b..a12c920ca 100644 --- a/vllm_omni/deploy/qwen3_tts.yaml +++ b/vllm_omni/deploy/qwen3_tts.yaml @@ -17,7 +17,6 @@ connectors: connector_of_shared_memory: name: SharedMemoryConnector extra: - shm_threshold_bytes: 65536 codec_streaming: true connector_get_sleep_s: 0.01 connector_get_max_wait_first_chunk: 3000 diff --git a/vllm_omni/deploy/qwen3_tts_high_concurrency.yaml b/vllm_omni/deploy/qwen3_tts_high_concurrency.yaml index 939396840..847383955 100644 --- a/vllm_omni/deploy/qwen3_tts_high_concurrency.yaml +++ b/vllm_omni/deploy/qwen3_tts_high_concurrency.yaml @@ -15,7 +15,6 @@ connectors: connector_of_shared_memory: name: SharedMemoryConnector extra: - shm_threshold_bytes: 65536 codec_streaming: true connector_get_sleep_s: 0.01 connector_get_max_wait_first_chunk: 3000 diff --git a/vllm_omni/deploy/voxtral_tts.yaml b/vllm_omni/deploy/voxtral_tts.yaml index 929daddb1..d024c3900 100644 --- a/vllm_omni/deploy/voxtral_tts.yaml +++ b/vllm_omni/deploy/voxtral_tts.yaml @@ -10,7 +10,6 @@ connectors: connector_of_shared_memory: name: SharedMemoryConnector extra: - shm_threshold_bytes: 65536 codec_streaming: true connector_get_sleep_s: 0.01 connector_get_max_wait_first_chunk: 3000 diff --git a/vllm_omni/distributed/omni_connectors/connectors/mooncake_transfer_engine_connector.py b/vllm_omni/distributed/omni_connectors/connectors/mooncake_transfer_engine_connector.py index 34d19062a..2d0c1f11a 100644 --- a/vllm_omni/distributed/omni_connectors/connectors/mooncake_transfer_engine_connector.py +++ b/vllm_omni/distributed/omni_connectors/connectors/mooncake_transfer_engine_connector.py @@ -30,6 +30,9 @@ except ImportError: # Stale buffer TTL: buffers older than this are automatically reclaimed # to prevent memory leaks when receiver crashes or gives up. _BUFFER_TTL_SECONDS = 300 # 5 minutes +_BASE_TRANSFER_TIMEOUT_MS = 30_000 +_TRANSFER_TIMEOUT_STEP_BYTES = 100 * 1024 * 1024 +_TRANSFER_TIMEOUT_PER_STEP_MS = 5_000 # ZMQ Message constants TRANS_DONE = b"trans_done" @@ -119,6 +122,7 @@ class MooncakeTransferEngineConnector(OmniConnectorBase): self._worker_local = threading.local() self._last_ttl_check: float = _time_mod.monotonic() self._sender_endpoints: dict[int, tuple[str, int]] = {} + self._in_flight_buffers: set[str] = set() self._metrics = { "puts": 0, @@ -155,6 +159,7 @@ class MooncakeTransferEngineConnector(OmniConnectorBase): # --- Memory Pool Configuration --- self.pool_size = config.get("memory_pool_size", 1024**3) # Default 1GB self.pool_device = config.get("memory_pool_device", "cpu") + self.buffer_ttl_seconds = float(config.get("buffer_ttl_seconds", _BUFFER_TTL_SECONDS)) # --- Sender Configuration (for receiver to query without metadata) --- # When receiver doesn't have metadata, it uses these to connect to sender @@ -175,7 +180,12 @@ class MooncakeTransferEngineConnector(OmniConnectorBase): self.engine_id = str(uuid.uuid4()) - # --- Mooncake Engine Init --- + self._init_engine() + self._init_pool() + self._init_listener(config) + + def _init_engine(self) -> None: + """Initialize the Mooncake engine and discover the local RPC port.""" self.engine = TransferEngine() # Note: For P2P handshake mode, local_hostname should be just the IP address. # Mooncake will auto-assign an RPC port, retrievable via get_rpc_port(). @@ -186,7 +196,8 @@ class MooncakeTransferEngineConnector(OmniConnectorBase): self.rpc_port = self.engine.get_rpc_port() logger.info(f"MooncakeTransferEngineConnector initialized at {self.host}:{self.rpc_port}") - # --- Pool Allocation & Registration --- + def _init_pool(self) -> None: + """Allocate and register the memory pool used by TransferEngine.""" logger.info(f"Allocating RDMA Memory Pool: {self.pool_size / 1024**2:.2f} MB on {self.pool_device}") try: if self.pool_device == "cpu": @@ -207,9 +218,8 @@ class MooncakeTransferEngineConnector(OmniConnectorBase): self.allocator = BufferAllocator(self.pool_size, alignment=4096) # 4KB alignment for safety - # --- State Management & Background Threads --- - # Most fields already initialized at the top of __init__ for teardown - # safety. Only create the real ZMQ context and thread pool here. + def _init_listener(self, config: dict[str, Any]) -> None: + """Initialize control-plane sockets and start the sender listener.""" self.zmq_ctx = zmq.Context() self._sender_executor = ThreadPoolExecutor(max_workers=4, thread_name_prefix="mooncake-sender") @@ -642,44 +652,18 @@ class MooncakeTransferEngineConnector(OmniConnectorBase): raise RuntimeError("Cannot get data: MooncakeTransferEngineConnector is closed") get_key = self._make_key(get_key, from_stage, to_stage) - _t0 = _time_mod.perf_counter() - + metadata = self._resolve_metadata(get_key, metadata) if not metadata: - # Path 3: no metadata at all — query default sender - if not self.sender_host or not self.sender_zmq_port or str(self.sender_host).lower() == "auto": - raise RuntimeError( - f"get(metadata=None) requires sender info to be resolved, " - f"but sender_host={self.sender_host!r}, sender_zmq_port={self.sender_zmq_port!r}. " - f"Call update_sender_info(host, port) before using get() without metadata." - ) - metadata = self._query_metadata_from_sender(get_key) - if not metadata: - return None - elif "data_size" not in metadata: - # Path 2: partial metadata (host/port only) — query that sender - partial_host = metadata.get("source_host") - partial_port = metadata.get("source_port") - if not partial_host or not partial_port: - logger.warning( - "get(%s): partial metadata missing source_host/source_port, cannot resolve data_size. metadata=%s", - get_key, - metadata, - ) - return None - queried = self._query_metadata_at(get_key, str(partial_host), int(partial_port)) - if not queried: - return None - metadata = queried + return None _t1 = _time_mod.perf_counter() - _query_ms = (_t1 - _t0) * 1000 + query_ms = (_t1 - _t0) * 1000 src_host = metadata.get("source_host") src_port = metadata.get("source_port") - data_size = metadata.get("data_size", 0) - is_fast_path = metadata.get("is_fast_path", False) - + data_size = int(metadata.get("data_size", 0) or 0) + is_fast_path = bool(metadata.get("is_fast_path", False)) if not src_host or not src_port or str(src_host).lower() == "auto": logger.error( f"Invalid metadata for {get_key}: source_host={src_host!r}, " @@ -691,27 +675,116 @@ class MooncakeTransferEngineConnector(OmniConnectorBase): logger.warning(f"Skipping get for {get_key}: data_size is 0 (metadata={metadata})") return None - # 1. Allocate Destination Buffer from Pool - # For non-fast-path (serialized objects), allocate a few extra bytes - # as padding. Mooncake's TCP/memcpy transport may prepend a leading - # protocol byte (0x01) before the payload. The extra padding allows - # the receiver to detect and skip this byte without truncating the - # actual payload. The padding is harmless for fast-path (ManagedBuffer) - # because the caller reads exact-size slices. - _MOONCAKE_TCP_PADDING = 4 if not is_fast_path else 0 - alloc_size = data_size + _MOONCAKE_TCP_PADDING try: - offset = self.allocator.alloc(alloc_size) - recv_buffer = ManagedBuffer(self.allocator, offset, alloc_size, self.pool) - dst_ptr = self.base_ptr + offset + recv_buffer, dst_ptr = self._alloc_recv_buffer(data_size, is_fast_path) except MemoryError: - logger.error(f"Failed to allocate {alloc_size} bytes in receive pool") return None _t2 = _time_mod.perf_counter() - _alloc_ms = (_t2 - _t1) * 1000 + alloc_ms = (_t2 - _t1) * 1000 + try: + resp = self._request_transfer( + get_key=get_key, + source_host=str(src_host), + source_port=int(src_port), + dst_ptr=dst_ptr, + data_size=data_size, + ) + except zmq.Again as e: + self._metrics["timeouts"] += 1 + logger.error(f"RDMA Get timed out: {e}", exc_info=True) + recv_buffer.release() + return None + except Exception as e: + self._metrics["errors"] += 1 + logger.error(f"RDMA transfer request failed: {e}", exc_info=True) + recv_buffer.release() + return None - # 2. Prepare Handshake + _t3 = _time_mod.perf_counter() + rdma_ms = (_t3 - _t2) * 1000 + if resp != TRANS_DONE: + self._metrics["errors"] += 1 + logger.error(f"RDMA Get failed: received {resp} instead of TRANS_DONE") + recv_buffer.release() + return None + + try: + sync_ms = self._sync_receive_buffer_if_needed() + except Exception as e: + self._metrics["errors"] += 1 + logger.error(f"Failed to synchronize Mooncake receive buffer: {e}", exc_info=True) + recv_buffer.release() + return None + + try: + return self._decode_result( + get_key=get_key, + recv_buffer=recv_buffer, + data_size=data_size, + is_fast_path=is_fast_path, + query_ms=query_ms, + alloc_ms=alloc_ms, + rdma_ms=rdma_ms, + sync_ms=sync_ms, + start_time=_t0, + ) + except Exception as e: + self._metrics["errors"] += 1 + logger.error(f"RDMA result decode failed for {get_key}: {e}", exc_info=True) + return None + + def _resolve_metadata(self, get_key: str, metadata: dict[str, Any] | None) -> dict[str, Any] | None: + """Resolve full transfer metadata for a receive request.""" + if not metadata: + # Path 3: no metadata at all — query default sender + if not self.sender_host or not self.sender_zmq_port or str(self.sender_host).lower() == "auto": + raise RuntimeError( + f"get(metadata=None) requires sender info to be resolved, " + f"but sender_host={self.sender_host!r}, sender_zmq_port={self.sender_zmq_port!r}. " + f"Call update_sender_info(host, port) before using get() without metadata." + ) + return self._query_metadata_from_sender(get_key) + + if "data_size" in metadata: + # Path 1: metadata is complete + return metadata + + # Path 2: partial metadata (host/port only) — query that sender + partial_host = metadata.get("source_host") + partial_port = metadata.get("source_port") + if not partial_host or not partial_port: + logger.warning( + "get(%s): partial metadata missing source_host/source_port, cannot resolve data_size. metadata=%s", + get_key, + metadata, + ) + return None + return self._query_metadata_at(get_key, str(partial_host), int(partial_port)) + + def _alloc_recv_buffer(self, data_size: int, is_fast_path: bool) -> tuple[ManagedBuffer, int]: + """Allocate the receive-side buffer and return ``(buffer, dst_ptr)``.""" + # For non-fast-path serialized objects, allocate a few extra bytes as + # padding. Mooncake's TCP/memcpy transport may prepend a protocol byte. + padding = 4 if not is_fast_path else 0 + alloc_size = data_size + padding + try: + offset = self.allocator.alloc(alloc_size) + except MemoryError: + logger.error(f"Failed to allocate {alloc_size} bytes in receive pool") + raise + recv_buffer = ManagedBuffer(self.allocator, offset, alloc_size, self.pool) + return recv_buffer, self.base_ptr + offset + + def _request_transfer( + self, + get_key: str, + source_host: str, + source_port: int, + dst_ptr: int, + data_size: int, + ) -> bytes: + """Ask the sender to write this key into the local receive buffer.""" agent_meta = MooncakeAgentMetadata( remote_hostname=self.host, remote_port=self.rpc_port, @@ -720,109 +793,82 @@ class MooncakeTransferEngineConnector(OmniConnectorBase): lengths=[data_size], ) - # 3. ZMQ Transaction (uses cached socket to avoid TCP reconnection) # Timeout scales with data size: base 30s + 5s per 100MB. - # Large RDMA transfers (esp. loopback or cross-NIC) can take tens of - # seconds; if we time out too early the receiver closes and deregisters - # its memory, causing the sender's in-flight write to fail with ret=-1. - _base_timeout_ms = 30000 - _size_timeout_ms = max(0, (data_size // (100 * 1024 * 1024))) * 5000 - _total_timeout_ms = _base_timeout_ms + _size_timeout_ms - zmq_addr = f"tcp://{src_host}:{src_port}" - req_socket = self._get_req_socket(zmq_addr, timeout_ms=_total_timeout_ms) - + size_timeout_ms = (data_size // _TRANSFER_TIMEOUT_STEP_BYTES) * _TRANSFER_TIMEOUT_PER_STEP_MS + total_timeout_ms = _BASE_TRANSFER_TIMEOUT_MS + size_timeout_ms + zmq_addr = f"tcp://{source_host}:{source_port}" + req_socket = self._get_req_socket(zmq_addr, timeout_ms=total_timeout_ms) try: req_socket.send(msgspec.msgpack.encode(agent_meta)) - resp = req_socket.recv() - - _t3 = _time_mod.perf_counter() - _rdma_ms = (_t3 - _t2) * 1000 - - if resp == TRANS_DONE: - # Success - # Ensure data is visible on GPU - # Note: RDMA write visibility on GPU usually requires some form of fence/sync. - # torch.accelerator.synchronize() is a heavy hammer but safe. - # Ideally Mooncake engine provides a way to poll for completion that guarantees visibility. - # TODO(wzliu): Replace synchronize with cuda event in the future for better performance. - if self.pool.is_cuda: - with torch.cuda.device(self.pool.device): - torch.cuda.current_stream().synchronize() - - _t4 = _time_mod.perf_counter() - _sync_ms = (_t4 - _t3) * 1000 - - if is_fast_path: - # Return ManagedBuffer directly for ALL fast_path data - # (both bytes and tensor payloads). Caller is responsible - # for consuming the data and calling release() afterwards. - # This avoids the expensive to_bytes() copy (~54ms for 115MB). - # - # Usage patterns for callers: - # - Zero-copy read: mv = memoryview(buf.tensor.numpy()) - # - Copy to GPU: buf.tensor.to(device); buf.release() - # - Fallback: data = buf.to_bytes(); buf.release() - _t5 = _time_mod.perf_counter() - _total_ms = (_t5 - _t0) * 1000 - _mbps = (data_size / 1024 / 1024) / (_total_ms / 1000) if _total_ms > 0 else 0 - logger.info( - f"[RDMA GET] {get_key}: query={_query_ms:.1f}ms, alloc={_alloc_ms:.1f}ms, " - f"rdma={_rdma_ms:.1f}ms, sync={_sync_ms:.1f}ms, " - f"total={_total_ms:.1f}ms, {_mbps:.1f} MB/s (fast_path, zero-copy)" - ) - self._metrics["gets"] += 1 - self._metrics["bytes_transferred"] += data_size - return recv_buffer, data_size - else: - # If it was a serialized object or generic bytes, we assume standard Omni behavior: - # Deserialize and return object. This requires a copy (to_bytes). - # We MUST release the buffer after deserialization. - try: - _t_copy_start = _time_mod.perf_counter() - raw_bytes = recv_buffer.to_bytes() - _t_copy_end = _time_mod.perf_counter() - _copy_ms = (_t_copy_end - _t_copy_start) * 1000 - - # Try deserializing the first data_size bytes. Mooncake's - # TCP/memcpy transport may prepend a leading protocol byte - # (0x01) before the payload — handle this by retrying from - # offset 1 if the first attempt raises DecodeError. - _t_deser_start = _time_mod.perf_counter() - payload = raw_bytes[:data_size] - try: - val = OmniSerializer.deserialize(payload) - except msgspec.DecodeError: - # Likely a leading Mooncake TCP protocol byte — retry - # from offset 1, using the extra padding bytes if needed. - payload = raw_bytes[1 : data_size + 1] - val = OmniSerializer.deserialize(payload) - _t_deser_end = _time_mod.perf_counter() - _deser_ms = (_t_deser_end - _t_deser_start) * 1000 - - _total_ms = (_t_deser_end - _t0) * 1000 - _mbps = (data_size / 1024 / 1024) / (_total_ms / 1000) if _total_ms > 0 else 0 - logger.info( - f"[RDMA GET] {get_key}: query={_query_ms:.1f}ms, alloc={_alloc_ms:.1f}ms, " - f"rdma={_rdma_ms:.1f}ms, sync={_sync_ms:.1f}ms, copy={_copy_ms:.1f}ms, " - f"deser={_deser_ms:.1f}ms, total={_total_ms:.1f}ms, {_mbps:.1f} MB/s" - ) - self._metrics["gets"] += 1 - self._metrics["bytes_transferred"] += data_size - return val, data_size - finally: - recv_buffer.release() - else: - self._metrics["errors"] += 1 - logger.error(f"RDMA Get failed: received {resp} instead of TRANS_DONE") - recv_buffer.release() - return None - except Exception as e: - # Socket may be stuck after timeout; discard it + return req_socket.recv() + except Exception: + # Socket may be stuck after timeout; discard it. self._invalidate_req_socket(zmq_addr) - self._metrics["timeouts"] += 1 - logger.error(f"RDMA Get error: {e}", exc_info=True) + raise + + def _sync_receive_buffer_if_needed(self) -> float: + """Synchronize GPU-visible receive memory until Mooncake exposes a finer fence.""" + t0 = _time_mod.perf_counter() + if self.pool.is_cuda: + with torch.cuda.device(self.pool.device): + torch.cuda.current_stream().synchronize() + return (_time_mod.perf_counter() - t0) * 1000 + + def _decode_result( + self, + get_key: str, + recv_buffer: ManagedBuffer, + data_size: int, + is_fast_path: bool, + query_ms: float, + alloc_ms: float, + rdma_ms: float, + sync_ms: float, + start_time: float, + ) -> tuple[Any, int]: + """Decode a completed transfer result.""" + if is_fast_path: + try: + total_ms = (_time_mod.perf_counter() - start_time) * 1000 + mbps = (data_size / 1024 / 1024) / (total_ms / 1000) if total_ms > 0 else 0 + logger.info( + f"[RDMA GET] {get_key}: query={query_ms:.1f}ms, alloc={alloc_ms:.1f}ms, " + f"rdma={rdma_ms:.1f}ms, sync={sync_ms:.1f}ms, " + f"total={total_ms:.1f}ms, {mbps:.1f} MB/s (fast_path, zero-copy)" + ) + self._metrics["gets"] += 1 + self._metrics["bytes_transferred"] += data_size + return recv_buffer, data_size + except Exception: + recv_buffer.release() + raise + + try: + copy_start = _time_mod.perf_counter() + raw_bytes = recv_buffer.to_bytes() + copy_ms = (_time_mod.perf_counter() - copy_start) * 1000 + + deser_start = _time_mod.perf_counter() + payload = raw_bytes[:data_size] + try: + value = OmniSerializer.deserialize(payload) + except msgspec.DecodeError: + payload = raw_bytes[1 : data_size + 1] + value = OmniSerializer.deserialize(payload) + deser_ms = (_time_mod.perf_counter() - deser_start) * 1000 + + total_ms = (_time_mod.perf_counter() - start_time) * 1000 + mbps = (data_size / 1024 / 1024) / (total_ms / 1000) if total_ms > 0 else 0 + logger.info( + f"[RDMA GET] {get_key}: query={query_ms:.1f}ms, alloc={alloc_ms:.1f}ms, " + f"rdma={rdma_ms:.1f}ms, sync={sync_ms:.1f}ms, copy={copy_ms:.1f}ms, " + f"deser={deser_ms:.1f}ms, total={total_ms:.1f}ms, {mbps:.1f} MB/s" + ) + self._metrics["gets"] += 1 + self._metrics["bytes_transferred"] += data_size + return value, data_size + finally: recv_buffer.release() - return None def cleanup(self, request_id: str, from_stage: str | None = None, to_stage: str | None = None) -> None: """Release the producer-side buffer associated with the request. @@ -847,6 +893,7 @@ class MooncakeTransferEngineConnector(OmniConnectorBase): with self._local_buffers_lock: item = self._local_buffers.pop(request_id, None) if item: + self._in_flight_buffers.discard(request_id) # item is (src_addrs, lengths, holder, should_release, is_fast_path, created_at) _, _, holder, should_release, _, _ = item if should_release and isinstance(holder, ManagedBuffer): @@ -904,6 +951,7 @@ class MooncakeTransferEngineConnector(OmniConnectorBase): if should_release and isinstance(holder, ManagedBuffer): holder.release() self._local_buffers.clear() + self._in_flight_buffers.clear() # 5. Close thread-local cached REQ sockets (for the calling thread). # Sockets cached in *other* threads are not directly accessible here @@ -945,21 +993,26 @@ class MooncakeTransferEngineConnector(OmniConnectorBase): # ------------------------------------------------------- def _cleanup_stale_buffers(self) -> None: - """Reclaim buffers older than ``_BUFFER_TTL_SECONDS``. + """Reclaim stale buffers not currently used by a transfer. + Prevents permanent memory leaks when a receiver crashes or times out without ever pulling the data. - TODO(wzliu): In extreme rare case, long transfer time, there might exist - TTL cleanup vs in-flight transfer conflict, which will be handled in the next PR. """ + if self.buffer_ttl_seconds <= 0: + return now = _time_mod.monotonic() with self._local_buffers_lock: - stale_keys = [k for k, v in self._local_buffers.items() if now - v[5] > _BUFFER_TTL_SECONDS] + stale_keys = [ + k + for k, v in self._local_buffers.items() + if k not in self._in_flight_buffers and now - v[5] > self.buffer_ttl_seconds + ] for k in stale_keys: item = self._local_buffers.pop(k) _, _, holder, should_release, _, _ = item if should_release and isinstance(holder, ManagedBuffer): holder.release() - logger.warning(f"TTL expired ({_BUFFER_TTL_SECONDS}s): cleaned up stale buffer for {k}") + logger.warning(f"TTL expired ({self.buffer_ttl_seconds}s): cleaned up stale buffer for {k}") def _zmq_listener_loop(self): socket = self.zmq_ctx.socket(zmq.ROUTER) @@ -1058,30 +1111,36 @@ class MooncakeTransferEngineConnector(OmniConnectorBase): with self._local_buffers_lock: item = self._local_buffers.get(meta.request_id) + if item: + self._in_flight_buffers.add(meta.request_id) if not item: response_queue.put((identity, TRANS_ERROR)) self._notify_listener(notify_addr) return - src_addrs, src_lengths, _, _, _, _ = item - remote_session = f"{meta.remote_hostname}:{meta.remote_port}" + try: + src_addrs, src_lengths, _, _, _, _ = item + remote_session = f"{meta.remote_hostname}:{meta.remote_port}" - # RDMA Write - ret = self.engine.batch_transfer_sync_write(remote_session, src_addrs, meta.dst_addrs, src_lengths) + # RDMA Write + ret = self.engine.batch_transfer_sync_write(remote_session, src_addrs, meta.dst_addrs, src_lengths) - if ret == 0: - self.cleanup(meta.request_id) - response_queue.put((identity, TRANS_DONE)) - else: - # Keep buffer in _local_buffers so receiver can retry on transient failures. - # Buffer will be cleaned up when: (a) a retry succeeds, (b) close() is called, - # or (c) a future TTL mechanism reclaims stale entries. - logger.warning( - f"RDMA write failed for {meta.request_id} to {remote_session} " - f"(ret={ret}). Buffer retained for retry." - ) - response_queue.put((identity, TRANS_ERROR)) + if ret == 0: + self.cleanup(meta.request_id) + response_queue.put((identity, TRANS_DONE)) + else: + # Keep buffer in _local_buffers so receiver can retry on transient failures. + # Buffer will be cleaned up when: (a) a retry succeeds, (b) close() is called, + # or (c) TTL reclaims it after it is no longer in-flight. + logger.warning( + f"RDMA write failed for {meta.request_id} to {remote_session} " + f"(ret={ret}). Buffer retained for retry." + ) + response_queue.put((identity, TRANS_ERROR)) + finally: + with self._local_buffers_lock: + self._in_flight_buffers.discard(meta.request_id) except Exception as e: logger.error(f"Push failed: {e}") diff --git a/vllm_omni/distributed/omni_connectors/connectors/shm_connector.py b/vllm_omni/distributed/omni_connectors/connectors/shm_connector.py index 5de0d54db..3a51795c4 100644 --- a/vllm_omni/distributed/omni_connectors/connectors/shm_connector.py +++ b/vllm_omni/distributed/omni_connectors/connectors/shm_connector.py @@ -27,16 +27,11 @@ class SharedMemoryConnector(OmniConnectorBase): def __init__(self, config: dict[str, Any]): self.config = config self.stage_id = config.get("stage_id", -1) - self.device = config.get("device", "cuda:0") - self.threshold = int(config.get("shm_threshold_bytes", 65536)) - self.inline_small_payloads = bool(config.get("inline_small_payloads", False)) self._pending_keys: set[str] = set() self._metrics = { "puts": 0, "gets": 0, "bytes_transferred": 0, - "shm_writes": 0, - "inline_writes": 0, } def put( @@ -47,33 +42,18 @@ class SharedMemoryConnector(OmniConnectorBase): data: Any, ) -> tuple[bool, int, dict[str, Any] | None]: try: - # Always serialize first to check size (and for SHM writing) - # Note: For extremely large objects in "inline" mode (e.g. Ray), - # we might double-serialize if we're not careful, but here we assume - # if it's huge we use SHM, or if Ray, threshold is maxsize. payload = self.serialize_obj(data) size = len(payload) - # The legacy async-chunk adapter transfers only the connector key - # across stages; it cannot deliver inline metadata to the receiver. - # Keep key-addressed SHM as the default and only inline payloads - # when the caller explicitly enables the metadata-aware path. - if size >= self.threshold or not self.inline_small_payloads: - lock_file = f"/dev/shm/shm_{put_key}_lockfile.lock" - with open(lock_file, "wb+") as lockf: - fcntl.flock(lockf, fcntl.LOCK_EX) - meta = shm_write_bytes(payload, name=put_key) - fcntl.flock(lockf, fcntl.LOCK_UN) + lock_file = f"/dev/shm/shm_{put_key}_lockfile.lock" + with open(lock_file, "wb+") as lockf: + fcntl.flock(lockf, fcntl.LOCK_EX) + meta = shm_write_bytes(payload, name=put_key) + fcntl.flock(lockf, fcntl.LOCK_UN) - # meta contains {'name': ..., 'size': ...} - metadata = {"shm": meta, "size": size} - self._pending_keys.add(put_key) - self._metrics["shm_writes"] += 1 - else: - # Inline small payloads to avoid the mmap + file-lock overhead - # that dominates codec chunk transfers. - metadata = {"inline_bytes": payload, "size": size} - self._metrics["inline_writes"] += 1 + # meta contains {'name': ..., 'size': ...} + metadata = {"shm": meta, "size": size} + self._pending_keys.add(put_key) self._metrics["puts"] += 1 self._metrics["bytes_transferred"] += size @@ -84,22 +64,26 @@ class SharedMemoryConnector(OmniConnectorBase): logger.error(f"SharedMemoryConnector put failed for req {put_key}: {e}") return False, 0, None - def _get_data_with_lock(self, lock_file: str, shm_handle: dict): - obj = None + def _get_data_with_lock(self, lock_file: str, shm_handle: dict[str, Any]) -> tuple[Any, int] | None: + deserialized = False try: with open(lock_file, "rb+") as lockf: fcntl.flock(lockf, fcntl.LOCK_EX) data_bytes = shm_read_bytes(shm_handle) fcntl.flock(lockf, fcntl.LOCK_UN) obj = self.deserialize_obj(data_bytes) - return obj, int(shm_handle.get("size", 0)) + result = (obj, int(shm_handle.get("size", 0))) + deserialized = True + return result except Exception as e: logger.error(f"SharedMemoryConnector shm get failed for req : {e}") return None finally: - # If data has been received, delete lock_file. - if obj and os.path.exists(lock_file): - os.remove(lock_file) + if deserialized: + try: + os.remove(lock_file) + except FileNotFoundError: + pass def _get_by_key(self, get_key: str) -> tuple[Any, int] | None: """Read a SHM segment addressed purely by *get_key*.""" @@ -142,31 +126,21 @@ class SharedMemoryConnector(OmniConnectorBase): if isinstance(metadata, dict) and get_key in metadata: metadata = metadata.get(get_key) - if not isinstance(metadata, dict): - return self._get_by_key(get_key) - - if "inline_bytes" in metadata: - try: - obj = self.deserialize_obj(metadata["inline_bytes"]) - self._pending_keys.discard(get_key) - return obj, int(metadata.get("size", 0)) - except Exception as e: - logger.error(f"SharedMemoryConnector inline get failed for req {get_key}: {e}") - return None - - if "shm" in metadata: + if isinstance(metadata, dict) and "shm" in metadata: shm_handle = metadata["shm"] lock_file = f"/dev/shm/shm_{shm_handle['name']}_lockfile.lock" result = self._get_data_with_lock(lock_file, shm_handle) if result is not None: self._pending_keys.discard(get_key) - return result + else: + # Missing or non-SHM metadata falls back to key-based lookup. + result = self._get_by_key(get_key) + else: + result = self._get_by_key(get_key) - # Metadata is a dict but has no SHM-specific handle (e.g. RDMA- - # style source_host/source_port). Fall back to key-based read. - return self._get_by_key(get_key) - - return self._get_by_key(get_key) + if result is not None: + self._metrics["gets"] += 1 + return result def cleanup(self, request_id: str) -> None: """Best-effort cleanup of unconsumed SHM segments for *request_id*. @@ -217,4 +191,4 @@ class SharedMemoryConnector(OmniConnectorBase): self._pending_keys.clear() def health(self) -> dict[str, Any]: - return {"status": "healthy", "threshold": self.threshold, **self._metrics} + return {"status": "healthy", **self._metrics} diff --git a/vllm_omni/distributed/omni_connectors/utils/initialization.py b/vllm_omni/distributed/omni_connectors/utils/initialization.py index f7a464809..bbbda44e3 100644 --- a/vllm_omni/distributed/omni_connectors/utils/initialization.py +++ b/vllm_omni/distributed/omni_connectors/utils/initialization.py @@ -4,7 +4,6 @@ """Utilities for OmniConnector configuration and validation.""" import json -import sys import warnings from pathlib import Path from typing import TYPE_CHECKING, Any @@ -42,7 +41,6 @@ KV_REPLICA_PORT_STRIDE = 1024 def initialize_connectors_from_config( config_path: str | Path | None = None, - default_shm_threshold: int = 65536, purpose: str = "request_forwarding", caller_stage_id: int | str | None = None, is_sender: bool | None = None, @@ -53,7 +51,7 @@ def initialize_connectors_from_config( Returns: tuple: (OmniTransferConfig, dict of {(from, to): connector_instance}) """ - transfer_config = load_omni_transfer_config(config_path, default_shm_threshold=default_shm_threshold) + transfer_config = load_omni_transfer_config(config_path) if not transfer_config: logger.info("No OmniTransferConfig provided") @@ -190,7 +188,6 @@ def get_connectors_config_for_stage(transfer_config: OmniTransferConfig | None, def load_omni_transfer_config( config_path: str | Path | None = None, config_dict: dict[str, Any] | None = None, - default_shm_threshold: int = 65536, ) -> OmniTransferConfig | None: """Load OmniTransferConfig from file or dict.""" if config_path is None and config_dict is None: @@ -325,10 +322,7 @@ def load_omni_transfer_config( expected_edges.add(edge_key) if edge_key not in connectors: logger.info(f"Auto-configuring SharedMemoryConnector for edge {edge_key}") - connectors[edge_key] = ConnectorSpec( - name="SharedMemoryConnector", - extra={"shm_threshold_bytes": default_shm_threshold}, - ) + connectors[edge_key] = ConnectorSpec(name="SharedMemoryConnector") # Fallback: infer edges from engine_input_source for each stage for stage_config in stage_args: @@ -343,9 +337,7 @@ def load_omni_transfer_config( if edge_key not in connectors: logger.info(f"Auto-configuring SharedMemoryConnector for edge {edge_key}") - connectors[edge_key] = ConnectorSpec( - name="SharedMemoryConnector", extra={"shm_threshold_bytes": default_shm_threshold} - ) + connectors[edge_key] = ConnectorSpec(name="SharedMemoryConnector") except Exception as e: logger.warning(f"Failed to auto-configure SHM connectors: {e}") @@ -369,22 +361,16 @@ def load_omni_transfer_config( def initialize_orchestrator_connectors( - config_path: str | None, worker_backend: str | None = "multi_process", shm_threshold_bytes: int = 65536 + config_path: str | None, ) -> tuple[OmniTransferConfig | None, dict[tuple[str, str], OmniConnectorBase]]: """Initialize connectors shared at orchestrator level. Args: config_path: The path to the configuration file. - worker_backend: The backend to use for the worker. Returns: A tuple containing the OmniTransferConfig and a dictionary of connectors. """ - if worker_backend == "ray": - default_shm_threshold = sys.maxsize - else: - default_shm_threshold = max(0, shm_threshold_bytes) transfer_config, connectors = initialize_connectors_from_config( config_path, - default_shm_threshold=default_shm_threshold, purpose="request_forwarding", caller_stage_id="orchestrator", is_sender=True, diff --git a/vllm_omni/engine/arg_utils.py b/vllm_omni/engine/arg_utils.py index bdf43711c..19db55c63 100644 --- a/vllm_omni/engine/arg_utils.py +++ b/vllm_omni/engine/arg_utils.py @@ -423,7 +423,6 @@ class OrchestratorArgs: init_timeout: int = 600 # === Cross-stage Communication === - shm_threshold_bytes: int = 65536 batch_timeout: int = 10 # === Cluster / Backend === diff --git a/vllm_omni/platforms/xpu/stage_configs/voxtral_tts.yaml b/vllm_omni/platforms/xpu/stage_configs/voxtral_tts.yaml index 0820ab632..cf7937e52 100644 --- a/vllm_omni/platforms/xpu/stage_configs/voxtral_tts.yaml +++ b/vllm_omni/platforms/xpu/stage_configs/voxtral_tts.yaml @@ -93,7 +93,6 @@ runtime: connector_of_shared_memory: name: SharedMemoryConnector extra: - shm_threshold_bytes: 65536 codec_streaming: true connector_get_sleep_s: 0.01 connector_get_max_wait_first_chunk: 3000 diff --git a/vllm_omni/worker/gpu_ar_model_runner.py b/vllm_omni/worker/gpu_ar_model_runner.py index 6cd2b3566..411796d37 100644 --- a/vllm_omni/worker/gpu_ar_model_runner.py +++ b/vllm_omni/worker/gpu_ar_model_runner.py @@ -323,7 +323,6 @@ class GPUARModelRunner(OmniGPUModelRunner, OmniConnectorModelRunnerMixin): } if getattr(self.model_config, "model_arch", None) in _OMNI_CONNECTOR_INIT_ARCHS: self.init_omni_connectors( - vllm_config=self.vllm_config, model_config=self.model_config, kv_transfer_manager=self.kv_transfer_manager, ) diff --git a/vllm_omni/worker/gpu_generation_model_runner.py b/vllm_omni/worker/gpu_generation_model_runner.py index ec41900b1..bcb647605 100644 --- a/vllm_omni/worker/gpu_generation_model_runner.py +++ b/vllm_omni/worker/gpu_generation_model_runner.py @@ -70,7 +70,6 @@ class GPUGenerationModelRunner(OmniGPUModelRunner, OmniConnectorModelRunnerMixin } if getattr(self.model_config, "model_arch", None) in _OMNI_CONNECTOR_INIT_ARCHS: self.init_omni_connectors( - vllm_config=self.vllm_config, model_config=self.model_config, ) diff --git a/vllm_omni/worker/omni_connector_model_runner_mixin.py b/vllm_omni/worker/omni_connector_model_runner_mixin.py index b173f81ac..f9f13cbe7 100644 --- a/vllm_omni/worker/omni_connector_model_runner_mixin.py +++ b/vllm_omni/worker/omni_connector_model_runner_mixin.py @@ -17,7 +17,6 @@ import inspect import os import threading from collections import defaultdict, deque -from types import SimpleNamespace from typing import TYPE_CHECKING, Any import torch @@ -36,6 +35,7 @@ from vllm_omni.worker.payload_span import ( _EMBED_SPAN_GROUPS: tuple[tuple[str, str, str], ...] = (("decode", "decode_token_start", "decode_token_end"),) if TYPE_CHECKING: + from vllm_omni.config.model import OmniModelConfig from vllm_omni.distributed.omni_connectors.connectors.base import ( OmniConnectorBase, ) @@ -87,14 +87,12 @@ class OmniConnectorModelRunnerMixin: def init_omni_connectors( self, - vllm_config: Any, - model_config: Any, + model_config: OmniModelConfig, kv_transfer_manager: OmniKVTransferManager | None = None, ) -> None: """Initialize connectors and background threads. Args: - vllm_config: Full vLLM config object. model_config: Stage-level model config with connector settings. kv_transfer_manager: Existing KV transfer manager to delegate to. """ @@ -1024,31 +1022,6 @@ class OmniConnectorModelRunnerMixin: self._work_available.set() return sent_ids - def recv_stage_inputs(self, scheduler_output: Any) -> dict[str, Any] | None: - """Compatibility wrapper for ``recv_full_payload_inputs``.""" - return self.recv_full_payload_inputs(scheduler_output) - - def accumulate_batch_output( - self, - req_id: str, - pooler_output: Any, - request: Any, - ) -> None: - """Compatibility wrapper for ``accumulate_full_payload_output``.""" - self.accumulate_full_payload_output(req_id, pooler_output, request) - - def flush_batch_outputs(self, finished_req_ids: set[str]) -> None: - """Compatibility wrapper for ``flush_full_payload_outputs``.""" - self.flush_full_payload_outputs(finished_req_ids) - - def send_stage_outputs( - self, - scheduler_output: Any, - outputs: dict[str, tuple[Any, Any] | Any], - ) -> list[str]: - """Compatibility wrapper for ``send_full_payload_outputs``.""" - return self.send_full_payload_outputs(scheduler_output, outputs) - # ------------------------------------------------------------------ # # Streaming chunk mode (recv_chunk / send_chunk) # ------------------------------------------------------------------ # @@ -1505,20 +1478,6 @@ class OmniConnectorModelRunnerMixin: # Output aggregation # ------------------------------------------------------------------ # - def _empty_output_with_connector_signals(self) -> Any: - """Return a minimal ModelRunnerOutput carrying pending connector signals. - - Used by early-return paths (e.g. ``num_scheduled_tokens == 0``) - that still need to deliver ``omni_connector_output`` to the - Scheduler so that WAITING_FOR_INPUT / WAITING_FOR_CHUNK - transitions are not lost. - """ - from vllm_omni.outputs import OmniModelRunnerOutput - - output = OmniModelRunnerOutput(req_ids=[], req_id_to_index={}) - output.omni_connector_output = self.get_omni_connector_output() - return output - def get_omni_connector_output(self) -> OmniConnectorOutput: """Collect and reset transfer results for this execute_model cycle. @@ -2088,48 +2047,6 @@ class OmniConnectorModelRunnerMixin: # Helpers # ------------------------------------------------------------------ # - @staticmethod - def _freeze_request_attr(value: Any) -> Any: - if isinstance(value, list): - return list(value) - if isinstance(value, tuple): - return list(value) - if isinstance(value, torch.Tensor): - return value.clone() - raw_list = getattr(value, "_x", None) - if raw_list is not None: - return list(raw_list) - return value - - def _snapshot_request_for_send(self, request: Any, external_req_id: str) -> Any: - finished = bool(getattr(request, "is_finished", lambda: False)()) - attrs: dict[str, Any] = {} - try: - attrs.update(vars(request)) - except TypeError: - pass - - for name in ( - "request_id", - "req_id", - "external_req_id", - "prompt_token_ids", - "output_token_ids", - "all_token_ids", - "additional_information", - "sampling_params", - "multi_modal_data", - "mm_hashes", - ): - if hasattr(request, name): - attrs[name] = self._freeze_request_attr(getattr(request, name)) - - attrs["external_req_id"] = external_req_id - attrs["_frozen_is_finished"] = finished - snapshot = SimpleNamespace(**attrs) - snapshot.is_finished = lambda: finished - return snapshot - @staticmethod def _create_connector(model_config: Any) -> OmniConnectorBase | None: """Create a connector from model_config, or None if unconfigured.""" From f18924c1cad8d4aa0def12054d5717ef3c859f91 Mon Sep 17 00:00:00 2001 From: Alicia <115451386+congw729@users.noreply.github.com> Date: Wed, 22 Jul 2026 09:45:17 +0800 Subject: [PATCH 049/240] [Refactor][Misc] Remove unused code in diffusion hooks/model_loader (#5270) Signed-off-by: Alicia <115451386+congw729@users.noreply.github.com> --- vllm_omni/diffusion/hooks/__init__.py | 6 -- vllm_omni/diffusion/hooks/base.py | 33 ++------ .../diffusion/hooks/sequence_parallel.py | 84 ------------------- .../model_loader/diffusers_loader.py | 11 --- .../diffusion/model_loader/hub_prefetch.py | 33 +------- .../models/qwen_image/pipeline_qwen_image.py | 5 +- .../qwen_image/pipeline_qwen_image_edit.py | 5 +- .../pipeline_qwen_image_edit_plus.py | 5 +- .../qwen_image/pipeline_qwen_image_layered.py | 5 +- 9 files changed, 17 insertions(+), 170 deletions(-) diff --git a/vllm_omni/diffusion/hooks/__init__.py b/vllm_omni/diffusion/hooks/__init__.py index 68f918383..ea1a28a6c 100644 --- a/vllm_omni/diffusion/hooks/__init__.py +++ b/vllm_omni/diffusion/hooks/__init__.py @@ -3,7 +3,6 @@ """Hook mechanism for model forward interception.""" from vllm_omni.diffusion.hooks.base import ( - BaseState, HookRegistry, ModelHook, StateManager, @@ -12,14 +11,11 @@ from vllm_omni.diffusion.hooks.sequence_parallel import ( SequenceParallelGatherHook, SequenceParallelSplitHook, apply_sequence_parallel, - disable_sequence_parallel_for_model, - enable_sequence_parallel_for_model, remove_sequence_parallel, ) __all__ = [ # Base hooks - "BaseState", "StateManager", "ModelHook", "HookRegistry", @@ -28,6 +24,4 @@ __all__ = [ "SequenceParallelGatherHook", "apply_sequence_parallel", "remove_sequence_parallel", - "enable_sequence_parallel_for_model", - "disable_sequence_parallel_for_model", ] diff --git a/vllm_omni/diffusion/hooks/base.py b/vllm_omni/diffusion/hooks/base.py index 6863a4594..8857737ca 100644 --- a/vllm_omni/diffusion/hooks/base.py +++ b/vllm_omni/diffusion/hooks/base.py @@ -20,21 +20,18 @@ from vllm.logger import init_logger logger = init_logger(__name__) -class BaseState: - """Base class for hook state containers.""" - - def reset(self) -> None: # pragma: no cover - default is no-op - pass - - class StateManager: - """Manage per-context hook state instances.""" + """Manage per-context hook state instances. - def __init__(self, state_cls: Callable[[], BaseState], init_args: tuple = (), init_kwargs: dict | None = None): + State classes (e.g. TeaCacheState, MagCacheState) are plain classes; + they only need to be constructible with the given init args. + """ + + def __init__(self, state_cls: Callable[..., Any], init_args: tuple = (), init_kwargs: dict | None = None): self._state_cls = state_cls self._init_args = init_args self._init_kwargs = init_kwargs or {} - self._states: dict[str, BaseState] = {} + self._states: dict[str, Any] = {} self._context: str = "default" @property @@ -45,7 +42,7 @@ class StateManager: def set_context(self, name: str) -> None: self._context = name or "default" - def get_state(self) -> BaseState: + def get_state(self) -> Any: if self._context not in self._states: self._states[self._context] = self._state_cls(*self._init_args, **self._init_kwargs) return self._states[self._context] @@ -332,17 +329,3 @@ class HookRegistry: hook = self._hooks.get(name) if hook is not None: hook.reset_state(self.module) - - def reset(self) -> None: - """Reset all hooks and clear the registry. - - This removes all hooks from the registry and resets each hook's state. - Also restores module.forward to its original implementation. - """ - for name, hook in list(self._hooks.items()): - hook.reset_state(self.module) - self._hooks.clear() - - if hasattr(self.module, "_omni_original_forward"): - self.module.forward = self.module._omni_original_forward # type: ignore[attr-defined] - delattr(self.module, "_omni_original_forward") diff --git a/vllm_omni/diffusion/hooks/sequence_parallel.py b/vllm_omni/diffusion/hooks/sequence_parallel.py index 0c6cda334..e8f88c8a9 100644 --- a/vllm_omni/diffusion/hooks/sequence_parallel.py +++ b/vllm_omni/diffusion/hooks/sequence_parallel.py @@ -679,87 +679,3 @@ def remove_sequence_parallel( continue registry.remove_hook(hook_name) - - -def enable_sequence_parallel_for_model( - model: nn.Module, - config: SequenceParallelConfig | None = None, -) -> None: - """Enable sequence parallelism for a model using its _sp_plan. - - This is a convenience function that reads the model's _sp_plan attribute - and applies sequence parallelism automatically. - - Note: This corresponds to `enable_context_parallel_for_model` in diffusers, - but uses vLLM-Omni's _sp_plan instead of diffusers' _cp_plan. - - The function performs two main tasks: - 1. Applies _sp_plan hooks to shard inputs and gather outputs - 2. Ensures Attention layers are configured for the correct parallel mode - (handled automatically by vLLM-Omni's forward_context mechanism) - - Args: - model: The model to enable SP for. Must have a _sp_plan attribute. - config: Optional config. If None, uses default based on current - parallel state. - - Raises: - ValueError: If model has no _sp_plan defined. - - Note: - vLLM-Omni supports Ulysses + Ring hybrid parallelism: - - ulysses_degree > 1: Uses All-to-All communication over Q/K/V heads - - ring_degree > 1: Uses Ring attention with K/V passing - - Both > 1: Hybrid mode (Ulysses handles head redistribution, - Ring handles K/V circulation) - """ - from vllm_omni.diffusion.distributed.parallel_state import ( - get_ring_parallel_world_size, - get_ulysses_parallel_world_size, - ) - from vllm_omni.diffusion.distributed.sp_plan import get_sp_plan_from_model - - plan = get_sp_plan_from_model(model) - if plan is None: - raise ValueError( - f"Model {model.__class__.__name__} has no _sp_plan defined. " - f"Define _sp_plan as a class attribute or pass a plan explicitly." - ) - - if config is None: - # Create config from current parallel state - ulysses_degree = get_ulysses_parallel_world_size() - ring_degree = get_ring_parallel_world_size() - config = SequenceParallelConfig( - ulysses_degree=ulysses_degree, - ring_degree=ring_degree, - ) - if ulysses_degree > 1 and ring_degree > 1: - mode = "hybrid" - elif ulysses_degree > 1: - mode = "ulysses" - else: - mode = "ring" - logger.info( - f"Created SP config from parallel state: " - f"ulysses_degree={ulysses_degree}, ring_degree={ring_degree}, " - f"mode={mode}" - ) - - apply_sequence_parallel(model, config, plan) - logger.info(f"Enabled sequence parallelism for {model.__class__.__name__}") - - -def disable_sequence_parallel_for_model(model: nn.Module) -> None: - """Disable sequence parallelism for a model. - - Note: This corresponds to `disable_context_parallel_for_model` in diffusers. - - Args: - model: The model to disable SP for. - """ - from vllm_omni.diffusion.distributed.sp_plan import get_sp_plan_from_model - - plan = get_sp_plan_from_model(model) - if plan is not None: - remove_sequence_parallel(model, plan) diff --git a/vllm_omni/diffusion/model_loader/diffusers_loader.py b/vllm_omni/diffusion/model_loader/diffusers_loader.py index 8ee172e6b..bb048451a 100644 --- a/vllm_omni/diffusion/model_loader/diffusers_loader.py +++ b/vllm_omni/diffusion/model_loader/diffusers_loader.py @@ -12,7 +12,6 @@ from typing import cast import torch from torch import nn -from vllm.config import ModelConfig from vllm.config.load import LoadConfig from vllm.logger import init_logger from vllm.model_executor.layers.quantization.base_config import QuantizeMethodBase @@ -81,7 +80,6 @@ def _natural_sort_key(filepath: str) -> list: return [int(s) if s.isdigit() else s for s in re.split(r"(\d+)", os.path.basename(filepath))] -MODEL_INDEX = "model_index.json" DIFFUSION_MODEL_WEIGHTS_INDEX = "diffusion_pytorch_model.safetensors.index.json" TRANSFORMER_WEIGHTS_INDEX = "model.safetensors.index.json" INDEX_FILES = [DIFFUSION_MODEL_WEIGHTS_INDEX, TRANSFORMER_WEIGHTS_INDEX] @@ -326,15 +324,6 @@ class DiffusersPipelineLoader: return all_parameter_names return {name for name in all_parameter_names if name.startswith(source_prefixes)} - def download_model(self, model_config: ModelConfig) -> None: - self._prepare_weights( - model_name_or_path=model_config.model, - subfolder=None, - revision=model_config.revision, - fall_back_to_pt=True, - allow_patterns_overrides=None, - ) - def load_model( self, load_device: str, diff --git a/vllm_omni/diffusion/model_loader/hub_prefetch.py b/vllm_omni/diffusion/model_loader/hub_prefetch.py index a33dca74d..9a9295ab9 100644 --- a/vllm_omni/diffusion/model_loader/hub_prefetch.py +++ b/vllm_omni/diffusion/model_loader/hub_prefetch.py @@ -433,10 +433,7 @@ def from_pretrained_with_prefetch( ``factory`` is a bound ``SomeModel.from_pretrained`` (or any callable with the same ``(model, *, subfolder, local_files_only, **kwargs)`` signature). - This is a stronger sibling of :func:`retry_on_missing_shard`: that helper - only retries the missing-shard ``OSError`` and never re-prefetches, so it - cannot recover the second face of the same race. Two shapes of partial - -cache failure crash the diffusion server outright: + Two shapes of partial-cache failure crash the diffusion server outright: * ``OSError: does not appear to have a file named text_encoder/model-0000X-of-0000Y.safetensors`` - a shard is still under @@ -488,31 +485,3 @@ def from_pretrained_with_prefetch( assert last_exc is not None # loop only exits via return or a caught exc raise last_exc - - -def retry_on_missing_shard(load_fn, *, max_retries: int = 3, base_delay: float = 5.0): - """Call *load_fn* with retry on the transformers v5 shard-resolution race. - - When the prefetch lock cannot be acquired (e.g. flock unsupported on - the filesystem and dotfile lock times out), ``from_pretrained`` may - still hit the ``cached_files`` race. This wrapper retries with - exponential backoff when the OSError message matches the specific - "does not appear to have a file named" pattern. - """ - for attempt in range(max_retries): - try: - return load_fn() - except OSError as exc: - if "does not appear to have a file" not in str(exc): - raise - if attempt == max_retries - 1: - raise - delay = base_delay * (attempt + 1) - logger.warning( - "from_pretrained failed with shard-resolution race (%s); retrying in %.1fs (attempt %d/%d)", - exc, - delay, - attempt + 1, - max_retries, - ) - time.sleep(delay) diff --git a/vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image.py b/vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image.py index 986c2ef86..474ea946c 100644 --- a/vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image.py +++ b/vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image.py @@ -309,9 +309,8 @@ class QwenImagePipeline( model, subfolder="scheduler", local_files_only=local_files_only ) # ``from_pretrained_with_prefetch`` re-prefetches and retries on a - # half-written cache (missing-shard ``OSError`` *and* the default - # -config size-mismatch ``RuntimeError`` that ``retry_on_missing_shard`` - # could not recover) instead of crashing the worker. + # half-written cache (missing-shard ``OSError`` and the default + # -config size-mismatch ``RuntimeError``) instead of crashing the worker. self.text_encoder = from_pretrained_with_prefetch( Qwen2_5_VLForConditionalGeneration.from_pretrained, model, diff --git a/vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image_edit.py b/vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image_edit.py index 3d6e937c7..3f0d16002 100644 --- a/vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image_edit.py +++ b/vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image_edit.py @@ -263,9 +263,8 @@ class QwenImageEditPipeline( model, subfolder="scheduler", local_files_only=local_files_only ) # ``from_pretrained_with_prefetch`` re-prefetches and retries on a - # half-written cache (missing-shard ``OSError`` *and* the default - # -config size-mismatch ``RuntimeError`` that ``retry_on_missing_shard`` - # could not recover) instead of crashing the worker. + # half-written cache (missing-shard ``OSError`` and the default + # -config size-mismatch ``RuntimeError``) instead of crashing the worker. self.text_encoder = from_pretrained_with_prefetch( Qwen2_5_VLForConditionalGeneration.from_pretrained, model, diff --git a/vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image_edit_plus.py b/vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image_edit_plus.py index a11637163..8b5c4e3b1 100644 --- a/vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image_edit_plus.py +++ b/vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image_edit_plus.py @@ -230,9 +230,8 @@ class QwenImageEditPlusPipeline( model, subfolder="scheduler", local_files_only=local_files_only ) # ``from_pretrained_with_prefetch`` re-prefetches and retries on a - # half-written cache (missing-shard ``OSError`` *and* the default - # -config size-mismatch ``RuntimeError`` that ``retry_on_missing_shard`` - # could not recover) instead of crashing the worker. + # half-written cache (missing-shard ``OSError`` and the default + # -config size-mismatch ``RuntimeError``) instead of crashing the worker. self.text_encoder = from_pretrained_with_prefetch( Qwen2_5_VLForConditionalGeneration.from_pretrained, model, diff --git a/vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image_layered.py b/vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image_layered.py index e2fe67976..830c4d6c1 100644 --- a/vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image_layered.py +++ b/vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image_layered.py @@ -237,9 +237,8 @@ class QwenImageLayeredPipeline( model, subfolder="scheduler", local_files_only=local_files_only ) # ``from_pretrained_with_prefetch`` re-prefetches and retries on a - # half-written cache (missing-shard ``OSError`` *and* the default - # -config size-mismatch ``RuntimeError`` that ``retry_on_missing_shard`` - # could not recover) instead of crashing the worker. + # half-written cache (missing-shard ``OSError`` and the default + # -config size-mismatch ``RuntimeError``) instead of crashing the worker. self.text_encoder = from_pretrained_with_prefetch( Qwen2_5_VLForConditionalGeneration.from_pretrained, model, From e2115fb6425e4bea06f48bd958530d2fa805705b Mon Sep 17 00:00:00 2001 From: Yuekai Zhang Date: Wed, 22 Jul 2026 11:26:00 +0800 Subject: [PATCH 050/240] [Bugfix][CI] Relax higgs_audio_v2 PCM streaming HNR threshold (fixes #5045) (#5232) Signed-off-by: Yuekai Zhang Co-authored-by: Claude Fable 5 --- .../online_serving/test_higgs_audio_v2_expansion.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/e2e/online_serving/test_higgs_audio_v2_expansion.py b/tests/e2e/online_serving/test_higgs_audio_v2_expansion.py index e564baf9d..8927eb039 100644 --- a/tests/e2e/online_serving/test_higgs_audio_v2_expansion.py +++ b/tests/e2e/online_serving/test_higgs_audio_v2_expansion.py @@ -92,6 +92,15 @@ class TestHiggsAudioV2OnlineHappyPath: higgs_audio_v2.yaml pins ``codec_streaming: true`` + ``async_chunk: false``, so the only streaming surface exposed to clients is the WAV/PCM bytes served chunk-by-chunk from Stage 1 — exercise it directly. + + NOTE: ``min_hnr_db=-3.0`` keeps the check as a catastrophic-codec-failure + guard (white noise / sample scramble measure around -10 dB) while + tolerating Higgs sampling variance: legitimate speech for this prompt + measured -1.03 dB on L4 (weekly CI, issue #5045) and 1.34 dB on H100, so + the default 1.0 dB threshold flakes on hardware-dependent tail samples. + Same rationale as the higgs_audio_v3 streaming thresholds (issue #4411); + content correctness is separately guarded by the full_model transcript + check. """ openai_client.send_audio_speech_request( { @@ -102,6 +111,7 @@ class TestHiggsAudioV2OnlineHappyPath: "response_format": "pcm", "timeout": DEFAULT_SPEECH_TIMEOUT_S, "min_audio_bytes": _MIN_AUDIO_BYTES, + "min_hnr_db": -3.0, } ) From e39f4dddfb2319c1c19790ec4361a78cfb0534bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zeyu=20Huang=20=7C=20=E9=BB=83=E6=BE=A4=E5=AE=87?= <11222265+fhfuih@users.noreply.github.com> Date: Wed, 22 Jul 2026 14:05:49 +0800 Subject: [PATCH 051/240] [Refactor]: clean up diffusion scheduler interfaces (#5216) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Huang, Zeyu <11222265+fhfuih@users.noreply.github.com> Signed-off-by: Zeyu Huang | 黃澤宇 <11222265+fhfuih@users.noreply.github.com> Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> Co-authored-by: Cursor --- docs/api/README.md | 2 +- .../feature/diffusion_continuous_batching.md | 11 +- .../diffusion_request_level_batching.md | 4 +- .../feature/diffusion_step_execution.md | 2 +- docs/design/module/dit_module.md | 57 +++++---- tests/ar_diffusion/test_engine_routing.py | 6 +- tests/ar_diffusion/test_pipeline_bridge.py | 2 +- .../test_hunyuan_image3_step_execution.py | 8 +- .../test_diffusion_engine_rpc_routing.py | 4 +- tests/diffusion/test_diffusion_scheduler.py | 62 +++++----- .../diffusion/test_diffusion_step_pipeline.py | 6 +- .../test_input_batch_txt_seq_lens.py | 6 +- tests/diffusion/test_worker_wrapper_base.py | 2 +- vllm_omni/diffusion/diffusion_engine.py | 10 +- .../diffusion/executor/multiproc_executor.py | 2 +- .../models/helios/pipeline_helios.py | 24 ++-- .../hunyuan_image3/pipeline_hunyuan_image3.py | 40 +++---- vllm_omni/diffusion/models/interface.py | 8 +- .../models/qwen_image/pipeline_qwen_image.py | 10 +- vllm_omni/diffusion/request.py | 3 +- vllm_omni/diffusion/sched/__init__.py | 11 +- vllm_omni/diffusion/sched/base_scheduler.py | 89 ++++++++------ vllm_omni/diffusion/sched/interface.py | 109 +++++------------- .../diffusion/sched/request_scheduler.py | 28 +++-- vllm_omni/diffusion/sched/step_scheduler.py | 9 +- .../worker/diffusion_model_runner.py | 40 ++++--- .../diffusion/worker/diffusion_worker.py | 16 +-- vllm_omni/diffusion/worker/input_batch.py | 58 +++++----- vllm_omni/diffusion/worker/utils.py | 6 +- .../omni_connectors/kv_transfer_manager.py | 11 +- vllm_omni/experimental/ar_diffusion/runner.py | 11 +- 31 files changed, 319 insertions(+), 338 deletions(-) diff --git a/docs/api/README.md b/docs/api/README.md index cfea6d175..09f6a6e24 100644 --- a/docs/api/README.md +++ b/docs/api/README.md @@ -131,7 +131,7 @@ Worker classes and model runners for distributed inference. - [vllm_omni.diffusion.worker.diffusion_worker.DiffusionWorker][] - [vllm_omni.diffusion.worker.diffusion_worker.WorkerProc][] - [vllm_omni.diffusion.worker.diffusion_worker.WorkerWrapperBase][] -- [vllm_omni.diffusion.worker.utils.DiffusionRequestState][] +- [vllm_omni.diffusion.worker.utils.StepRequestState][] - [vllm_omni.diffusion.worker.utils.RunnerOutput][] - [vllm_omni.platforms.npu.worker.npu_ar_model_runner.ExecuteModelState][] - [vllm_omni.platforms.npu.worker.npu_ar_model_runner.NPUARModelRunner][] diff --git a/docs/design/feature/diffusion_continuous_batching.md b/docs/design/feature/diffusion_continuous_batching.md index f90969369..223b88518 100644 --- a/docs/design/feature/diffusion_continuous_batching.md +++ b/docs/design/feature/diffusion_continuous_batching.md @@ -71,7 +71,7 @@ The scheduler derives its batch capacity from `max_num_seqs` through `max_num_running_reqs`. Batch admission is gated by -[`SamplingParamsKey`](gh-file:vllm_omni/diffusion/sched/interface.py), +[`StepBatchSamplingParamsKey`](gh-file:vllm_omni/diffusion/sched/interface.py), which is built from shape-sensitive and CFG-sensitive sampling fields. This is the core correctness rule for batching: requests are only co-batched when they share the same denoise tensor contract. @@ -100,9 +100,10 @@ request-mode prompt semantics, see ## Runner The runner keeps persistent per-request execution state in -[`DiffusionRequestState`](gh-file:vllm_omni/diffusion/worker/utils.py), -while the scheduler owns a separate lightweight request state for queueing and -lifecycle tracking. +[`StepRequestState`](gh-file:vllm_omni/diffusion/worker/utils.py), +while the scheduler owns a separate lightweight +[`SchedulerRequestState`](gh-file:vllm_omni/diffusion/sched/interface.py) +for queueing and lifecycle tracking. For each step, the runner builds an [`InputBatch`](gh-file:vllm_omni/diffusion/worker/input_batch.py) from the @@ -140,7 +141,7 @@ compatibility gating and runner-side `StepInputBatch` packing. - Experimental feature; use `max_num_seqs=1` for the older conservative path. - Only native pipelines that already support `step_execution=True`. -- Only homogeneous batches keyed by `SamplingParamsKey` are supported. +- Only homogeneous batches keyed by `StepBatchSamplingParamsKey` are supported. - `cache_backend`, KV transfer, and other request-mode extras are not wired into the batched step-wise path yet. - Future work can relax the current same-shape restriction with richer diff --git a/docs/design/feature/diffusion_request_level_batching.md b/docs/design/feature/diffusion_request_level_batching.md index 57287ada4..8c1cdbd33 100644 --- a/docs/design/feature/diffusion_request_level_batching.md +++ b/docs/design/feature/diffusion_request_level_batching.md @@ -82,7 +82,7 @@ The scheduler derives its capacity from `max_num_seqs` through can decide whether admission wait is useful before scheduling a new wave. Batch compatibility is controlled by -[`SamplingParamsKey`](gh-file:vllm_omni/diffusion/sched/interface.py). The key +[`RequestBatchSamplingParamsKey`](gh-file:vllm_omni/diffusion/sched/interface.py). The key contains shape-sensitive and guidance-sensitive fields, including output count and LoRA identity. Requests with incompatible shapes, CFG settings, output counts, LoRA adapters, or LoRA scales are kept in separate batches. @@ -147,7 +147,7 @@ outputs do not fall back to pickle IPC for tensor payloads. - Only pipelines that declare the request-batch contract use fused batch execution. -- Batches are homogeneous under `SamplingParamsKey`; heterogeneous resolution or +- Batches are homogeneous under `RequestBatchSamplingParamsKey`; heterogeneous resolution or incompatible guidance settings do not co-batch yet. - FIFO scheduling can reduce batching opportunities when an incompatible request is at the front of the queue. diff --git a/docs/design/feature/diffusion_step_execution.md b/docs/design/feature/diffusion_step_execution.md index 797a1cb48..01936e223 100644 --- a/docs/design/feature/diffusion_step_execution.md +++ b/docs/design/feature/diffusion_step_execution.md @@ -49,7 +49,7 @@ state object: The state lives in [`vllm_omni/diffusion/worker/utils.py`](gh-file:vllm_omni/diffusion/worker/utils.py) -as `DiffusionRequestState`. Store request-scoped tensors there, or use +as `StepRequestState`. Store request-scoped tensors there, or use `state.extra` for model-specific fields that do not justify extending the core dataclass. diff --git a/docs/design/module/dit_module.md b/docs/design/module/dit_module.md index 02d50181b..97d1db499 100644 --- a/docs/design/module/dit_module.md +++ b/docs/design/module/dit_module.md @@ -135,28 +135,7 @@ The scheduler is a **request-state scheduler**. It owns request lifecycle manage ### Key Components -#### 2.1 Scheduler Interface - -```python -class SchedulerInterface(ABC): - def add_request(self, request: OmniDiffusionRequest) -> str: ... - def schedule(self) -> DiffusionSchedulerOutput: ... - def update_from_output( - self, - sched_output: DiffusionSchedulerOutput, - output: DiffusionOutput, - ) -> set[str]: ... -``` - -**Responsibilities**: - -- **Lifecycle contract**: Defines how the engine adds requests, triggers one scheduling cycle, and feeds executor results back. - -- **Stable boundary**: `DiffusionSchedulerOutput` is the only scheduling result consumed by `DiffusionEngine`. - -- **Pluggability**: Different scheduler policies can reuse the same engine integration path. - -#### 2.2 Request State Model +#### 2.1 Request State Model ```python class DiffusionRequestStatus(enum.IntEnum): @@ -168,10 +147,12 @@ class DiffusionRequestStatus(enum.IntEnum): FINISHED_ERROR = ... @dataclass -class DiffusionRequestState: +class SchedulerRequestState: request_id: str req: OmniDiffusionRequest + sampling_params_key: StepBatchSamplingParamsKey | RequestBatchSamplingParamsKey | None = None status: DiffusionRequestStatus = DiffusionRequestStatus.WAITING + error: str | None = None ``` **Design Features**: @@ -180,18 +161,34 @@ class DiffusionRequestState: - **Centralized error handling**: Completion, abort, and error states are all normalized in the scheduler layer. -#### 2.3 Shared Bookkeeping in `_BaseScheduler` +#### 2.2 Shared Bookkeeping in `BaseScheduler` ```python -class _BaseScheduler(SchedulerInterface): +class BaseScheduler: def __init__(self) -> None: self._request_states = {} self._waiting = deque() self._running = [] self._finished_req_ids = set() self.max_num_running_reqs = 1 + + def add_request(self, request: OmniDiffusionRequest) -> str: ... + def schedule(self) -> DiffusionSchedulerOutput: ... + def update_from_output( + self, + sched_output: DiffusionSchedulerOutput, + output: BaseRunnerOutput, + ) -> set[str]: ... ``` +**Responsibilities**: + +- **Lifecycle contract**: Defines how the engine adds requests, triggers one scheduling cycle, and feeds executor results back. + +- **Stable boundary**: `DiffusionSchedulerOutput` is the only scheduling result consumed by `DiffusionEngine`. + +- **Pluggability**: Different scheduler policies can reuse the same engine integration path. + **Design Features**: - **Common state storage**: Shared request maps and waiting/running sets live in the base class. @@ -199,12 +196,12 @@ class _BaseScheduler(SchedulerInterface): - **Shared cleanup logic**: Duplicate-request checks, finish handling, and state removal are centralized instead of duplicated in each policy. -- **Current constraint boundary**: `_BaseScheduler` derives `max_num_running_reqs` from `max_num_seqs`. Request-mode diffusion can use that capacity for compatible independent requests when the configured pipeline declares `supports_request_batch = True`; step-wise diffusion uses the same scheduler capacity for compatible step batches. +- **Current constraint boundary**: `BaseScheduler` derives `max_num_running_reqs` from `max_num_seqs`. Request-mode diffusion can use that capacity for compatible independent requests when the configured pipeline declares `supports_request_batch = True`; step-wise diffusion uses the same scheduler capacity for compatible step batches. -#### 2.4 Current `RequestScheduler` Policy +#### 2.3 Current `RequestScheduler` Policy ```python -class RequestScheduler(_BaseScheduler): +class RequestScheduler(BaseScheduler): def schedule(self) -> DiffusionSchedulerOutput: # 1. keep existing RUNNING requests in the scheduling result # 2. pull WAITING requests while capacity remains @@ -215,11 +212,11 @@ class RequestScheduler(_BaseScheduler): - **FIFO request scheduling**: Waiting requests are promoted in queue order. -- **Compatible request admission**: `RequestScheduler` admits waiting requests while capacity remains and the request's `SamplingParamsKey` is compatible with the active batch. Request-mode execution keeps each logical request independent, while batch-capable pipelines receive the scheduled requests as a runner-side `DiffusionRequestBatch`. +- **Compatible request admission**: `RequestScheduler` admits waiting requests while capacity remains and the request's `RequestBatchSamplingParamsKey` is compatible with the active batch. Request-mode execution keeps each logical request independent, while batch-capable pipelines receive the scheduled requests as a runner-side `DiffusionRequestBatch`. - **Executor result feedback**: `update_from_output()` converts executor output into `FINISHED_COMPLETED` or `FINISHED_ERROR` and returns finished request ids. -#### 2.5 Engine-Driven Execution Loop +#### 2.4 Engine-Driven Execution Loop ```python request_id = scheduler.add_request(request) diff --git a/tests/ar_diffusion/test_engine_routing.py b/tests/ar_diffusion/test_engine_routing.py index fb55e01e9..5972ffff4 100644 --- a/tests/ar_diffusion/test_engine_routing.py +++ b/tests/ar_diffusion/test_engine_routing.py @@ -136,8 +136,8 @@ def test_runner_rejects_multi_sequence_config(): ARDiffusionModelRunner._preallocate_kv_cache(fake) -def test_execute_model_accepts_kv_prefetch_jobs_kwarg(): - """Regression: the worker passes ``kv_prefetch_jobs`` to every runner +def test_execute_model_accepts_kv_prefetch_job_kwarg(): + """Regression: the worker passes ``kv_prefetch_job`` to every runner (added with the KV prefetch path, #4448); the AR-Diffusion override must accept and forward it or DreamZero crashes on the warm-up dummy run.""" import inspect @@ -147,4 +147,4 @@ def test_execute_model_accepts_kv_prefetch_jobs_kwarg(): for cls in (DiffusionModelRunner, ARDiffusionModelRunner): params = inspect.signature(cls.execute_model).parameters - assert "kv_prefetch_jobs" in params, cls + assert "kv_prefetch_job" in params, cls diff --git a/tests/ar_diffusion/test_pipeline_bridge.py b/tests/ar_diffusion/test_pipeline_bridge.py index 28c3c959d..2c4284531 100644 --- a/tests/ar_diffusion/test_pipeline_bridge.py +++ b/tests/ar_diffusion/test_pipeline_bridge.py @@ -203,7 +203,7 @@ def test_failed_forward_tears_down_session(monkeypatch): runner.device = None runner._perf_e2e_times = [] - def boom(self, req, kv_prefetch_jobs=None): + def boom(self, req, kv_prefetch_job=None): raise RuntimeError("layer 17 exploded") monkeypatch.setattr(DiffusionModelRunner, "execute_model", boom) diff --git a/tests/diffusion/models/hunyuan_image3/test_hunyuan_image3_step_execution.py b/tests/diffusion/models/hunyuan_image3/test_hunyuan_image3_step_execution.py index b4f1e7493..d9d310ee4 100644 --- a/tests/diffusion/models/hunyuan_image3/test_hunyuan_image3_step_execution.py +++ b/tests/diffusion/models/hunyuan_image3/test_hunyuan_image3_step_execution.py @@ -20,7 +20,7 @@ from vllm_omni.diffusion.models.hunyuan_image3.pipeline_hunyuan_image3 import ( ) from vllm_omni.diffusion.worker.input_batch import InputBatch from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch -from vllm_omni.diffusion.worker.utils import DiffusionRequestState +from vllm_omni.diffusion.worker.utils import StepRequestState pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu] @@ -38,8 +38,8 @@ def _pipeline(): return pipeline -def _state(request_id: str, step_index: int) -> DiffusionRequestState: - state = DiffusionRequestState( +def _state(request_id: str, step_index: int) -> StepRequestState: + state = StepRequestState( request_id=request_id, sampling=SimpleNamespace(), prompt="prompt", @@ -133,7 +133,7 @@ def test_prepare_encode_preserves_normal_hunyuan_bot_task_semantics( monkeypatch.setattr(hy3_module, "get_system_prompt", fake_get_system_prompt) pipeline.prepare_model_inputs = fake_prepare_model_inputs - state = DiffusionRequestState( + state = StepRequestState( request_id="req-bot-task", sampling=sampling, prompt=prompt_item, diff --git a/tests/diffusion/test_diffusion_engine_rpc_routing.py b/tests/diffusion/test_diffusion_engine_rpc_routing.py index 4b7d0efbb..8c4f08152 100644 --- a/tests/diffusion/test_diffusion_engine_rpc_routing.py +++ b/tests/diffusion/test_diffusion_engine_rpc_routing.py @@ -36,10 +36,10 @@ from vllm_omni.diffusion.sched.interface import RequestBatchSamplingParamsKey from vllm_omni.diffusion.worker.utils import RunnerOutput # Default values for every batch-key field, so SimpleNamespace-based -# sampling_params satisfy ``get_sampling_params_key``'s attribute lookups. +# sampling_params satisfy ``RequestScheduler._build_sampling_params_key``'s attribute lookups. _SAMPLING_KEY_DEFAULTS = {f.name: f.default for f in _dc_fields(RequestBatchSamplingParamsKey)} -pytestmark = [pytest.mark.diffusion, pytest.mark.cpu] +pytestmark = [pytest.mark.diffusion, pytest.mark.cpu, pytest.mark.core_model] # ───────────────────────────────────────── helpers ───────────────────────── diff --git a/tests/diffusion/test_diffusion_scheduler.py b/tests/diffusion/test_diffusion_scheduler.py index 7d6d2d11b..68b237c67 100644 --- a/tests/diffusion/test_diffusion_scheduler.py +++ b/tests/diffusion/test_diffusion_scheduler.py @@ -14,10 +14,10 @@ from vllm_omni.diffusion.data import DiffusionOutput, DiffusionRequestAbortedErr from vllm_omni.diffusion.diffusion_engine import DiffusionEngine from vllm_omni.diffusion.request import OmniDiffusionRequest from vllm_omni.diffusion.sched import ( + BaseScheduler, DiffusionRequestStatus, RequestScheduler, Scheduler, - SchedulerInterface, StepScheduler, ) from vllm_omni.diffusion.sched.interface import CachedRequestData, NewRequestData @@ -85,7 +85,7 @@ def _cached_ids(sched_output) -> list[str]: return list(sched_output.scheduled_cached_reqs.request_ids) -class _StubScheduler(SchedulerInterface): +class _StubScheduler: def __init__(self, request: OmniDiffusionRequest, output) -> None: self._request = request self._output = output @@ -154,8 +154,14 @@ class _StubScheduler(SchedulerInterface): return None -class TestGetSamplingParamsKey: - """Pure-function tests for the batch-compatibility key builder.""" +class _ConcreteScheduler(BaseScheduler): + def update_from_output(self, sched_output, output) -> set[str]: + del sched_output, output + return set() + + +class TestGetStepBatchSamplingParamsKey: + """Tests for the step-batch compatibility key builder on BaseScheduler.""" @staticmethod def _make(lora_int_id: int | None = None, lora_scale: float = 1.0) -> OmniDiffusionRequest: @@ -176,34 +182,32 @@ class TestGetSamplingParamsKey: ) def test_distinguishes_lora_id(self) -> None: - from vllm_omni.diffusion.sched.base_scheduler import get_sampling_params_key - - assert get_sampling_params_key(self._make(lora_int_id=1)) != get_sampling_params_key(self._make(lora_int_id=2)) + scheduler = _ConcreteScheduler() + assert scheduler._build_sampling_params_key(self._make(lora_int_id=1)) != scheduler._build_sampling_params_key( + self._make(lora_int_id=2) + ) def test_distinguishes_lora_scale(self) -> None: - from vllm_omni.diffusion.sched.base_scheduler import get_sampling_params_key - - assert get_sampling_params_key(self._make(lora_int_id=1, lora_scale=0.5)) != get_sampling_params_key( - self._make(lora_int_id=1, lora_scale=1.0) - ) + scheduler = _ConcreteScheduler() + assert scheduler._build_sampling_params_key( + self._make(lora_int_id=1, lora_scale=0.5) + ) != scheduler._build_sampling_params_key(self._make(lora_int_id=1, lora_scale=1.0)) def test_treats_no_lora_as_distinct_bucket(self) -> None: - from vllm_omni.diffusion.sched.base_scheduler import get_sampling_params_key - - assert get_sampling_params_key(self._make(lora_int_id=None)) != get_sampling_params_key( - self._make(lora_int_id=1) - ) + scheduler = _ConcreteScheduler() + assert scheduler._build_sampling_params_key( + self._make(lora_int_id=None) + ) != scheduler._build_sampling_params_key(self._make(lora_int_id=1)) def test_equal_for_same_lora_identity(self) -> None: - from vllm_omni.diffusion.sched.base_scheduler import get_sampling_params_key - - a = get_sampling_params_key(self._make(lora_int_id=1, lora_scale=0.5)) - b = get_sampling_params_key(self._make(lora_int_id=1, lora_scale=0.5)) + scheduler = _ConcreteScheduler() + a = scheduler._build_sampling_params_key(self._make(lora_int_id=1, lora_scale=0.5)) + b = scheduler._build_sampling_params_key(self._make(lora_int_id=1, lora_scale=0.5)) assert a == b class TestGetRequestBatchSamplingParamsKey: - """Pure-function tests for the request-batch compatibility key builder.""" + """Tests for the request-batch compatibility key builder on RequestScheduler.""" @staticmethod def _make( @@ -220,21 +224,19 @@ class TestGetRequestBatchSamplingParamsKey: return OmniDiffusionRequest(prompt="prompt", sampling_params=sp, request_id=f"req-{num_inference_steps}") def test_distinguishes_num_inference_steps(self) -> None: - from vllm_omni.diffusion.sched.base_scheduler import get_request_batch_sampling_params_key - - assert get_request_batch_sampling_params_key( + scheduler = RequestScheduler() + assert scheduler._build_sampling_params_key( self._make(num_inference_steps=2) - ) != get_request_batch_sampling_params_key(self._make(num_inference_steps=4)) + ) != scheduler._build_sampling_params_key(self._make(num_inference_steps=4)) def test_ignores_seed_and_generator(self) -> None: - from vllm_omni.diffusion.sched.base_scheduler import get_request_batch_sampling_params_key - + scheduler = RequestScheduler() gen_a = torch.Generator(device="cpu").manual_seed(1) gen_b = torch.Generator(device="cpu").manual_seed(2) - assert get_request_batch_sampling_params_key( + assert scheduler._build_sampling_params_key( self._make(seed=1, generator=gen_a) - ) == get_request_batch_sampling_params_key(self._make(seed=2, generator=gen_b)) + ) == scheduler._build_sampling_params_key(self._make(seed=2, generator=gen_b)) class TestRequestScheduler: diff --git a/tests/diffusion/test_diffusion_step_pipeline.py b/tests/diffusion/test_diffusion_step_pipeline.py index 7e18996ed..0964833d7 100644 --- a/tests/diffusion/test_diffusion_step_pipeline.py +++ b/tests/diffusion/test_diffusion_step_pipeline.py @@ -43,7 +43,7 @@ from vllm_omni.diffusion.sched.interface import ( from vllm_omni.diffusion.worker.diffusion_model_runner import DiffusionModelRunner from vllm_omni.diffusion.worker.diffusion_worker import DiffusionWorker from vllm_omni.diffusion.worker.input_batch import InputBatch -from vllm_omni.diffusion.worker.utils import DiffusionRequestState, RunnerOutput +from vllm_omni.diffusion.worker.utils import RunnerOutput, StepRequestState from vllm_omni.engine.async_omni_engine import AsyncOmniEngine from vllm_omni.inputs.data import OmniDiffusionSamplingParams from vllm_omni.platforms import current_omni_platform @@ -369,8 +369,8 @@ def _make_batch_scheduler_output(reqs, *, step_id=0, finished_req_ids=None): ) -def _make_input_batch_state(request_id: str, latent_value: float) -> DiffusionRequestState: - state = DiffusionRequestState( +def _make_input_batch_state(request_id: str, latent_value: float) -> StepRequestState: + state = StepRequestState( request_id=request_id, sampling=SimpleNamespace(), prompt=None, diff --git a/tests/diffusion/test_input_batch_txt_seq_lens.py b/tests/diffusion/test_input_batch_txt_seq_lens.py index 315f0f703..88fb60a5e 100644 --- a/tests/diffusion/test_input_batch_txt_seq_lens.py +++ b/tests/diffusion/test_input_batch_txt_seq_lens.py @@ -2,14 +2,14 @@ import pytest import torch from vllm_omni.diffusion.worker.input_batch import _prepare_request_prompt_field -from vllm_omni.diffusion.worker.utils import DiffusionRequestState +from vllm_omni.diffusion.worker.utils import StepRequestState from vllm_omni.inputs.data import OmniDiffusionSamplingParams -pytestmark = [pytest.mark.cpu] +pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu] def test_prepare_request_prompt_field_refreshes_txt_seq_lens_after_padding(): - state = DiffusionRequestState( + state = StepRequestState( request_id="req-0", sampling=OmniDiffusionSamplingParams(), prompt_embeds=torch.randn(1, 10, 4), diff --git a/tests/diffusion/test_worker_wrapper_base.py b/tests/diffusion/test_worker_wrapper_base.py index ba22743fc..e9c4d613f 100644 --- a/tests/diffusion/test_worker_wrapper_base.py +++ b/tests/diffusion/test_worker_wrapper_base.py @@ -184,7 +184,7 @@ class TestWorkerWrapperBaseDelegation: mock_reqs = [mocker.Mock()] result = wrapper.execute_model(mock_reqs, mock_od_config) - wrapper.worker.execute_model.assert_called_once_with(mock_reqs, mock_od_config, kv_prefetch_jobs=None) + wrapper.worker.execute_model.assert_called_once_with(mock_reqs, mock_od_config, kv_prefetch_job=None) assert result == mock_output def test_load_weights_delegation(self, mocker: MockerFixture, mock_od_config): diff --git a/vllm_omni/diffusion/diffusion_engine.py b/vllm_omni/diffusion/diffusion_engine.py index d6e7b35de..e8ab578db 100644 --- a/vllm_omni/diffusion/diffusion_engine.py +++ b/vllm_omni/diffusion/diffusion_engine.py @@ -44,7 +44,7 @@ from vllm_omni.diffusion.registry import ( get_diffusion_pre_process_func, ) from vllm_omni.diffusion.request import DUMMY_DIFFUSION_REQUEST_ID, OmniDiffusionRequest -from vllm_omni.diffusion.sched import RequestScheduler, SchedulerInterface, StepScheduler +from vllm_omni.diffusion.sched import BaseScheduler, RequestScheduler, StepScheduler from vllm_omni.diffusion.sched.interface import DiffusionRequestStatus from vllm_omni.diffusion.worker.utils import BaseRunnerOutput, BatchRunnerOutput, RunnerOutput from vllm_omni.errors import client_error_from_metadata, is_client_error_status @@ -144,7 +144,7 @@ class DiffusionEngine: def __init__( self, od_config: OmniDiffusionConfig, - scheduler: SchedulerInterface | None = None, + scheduler: BaseScheduler | None = None, ): """Initialize the diffusion engine. @@ -167,9 +167,7 @@ class DiffusionEngine: executor_class = DiffusionExecutor.get_class(od_config) self.executor = executor_class(od_config) - self.scheduler: SchedulerInterface = scheduler or ( - StepScheduler() if self.step_execution else RequestScheduler() - ) + self.scheduler: BaseScheduler = scheduler or (StepScheduler() if self.step_execution else RequestScheduler()) self.scheduler.initialize(od_config) self.supports_request_batch = False if self.step_execution else supports_request_batch(od_config) self.main_loop: asyncio.AbstractEventLoop | None = None @@ -618,7 +616,7 @@ class DiffusionEngine: @staticmethod def make_engine( config: OmniDiffusionConfig, - scheduler: SchedulerInterface | None = None, + scheduler: BaseScheduler | None = None, ) -> DiffusionEngine: """Factory method to create the engine selected by ``config.engine_backend``. diff --git a/vllm_omni/diffusion/executor/multiproc_executor.py b/vllm_omni/diffusion/executor/multiproc_executor.py index d0ed76bb9..f90624ed7 100644 --- a/vllm_omni/diffusion/executor/multiproc_executor.py +++ b/vllm_omni/diffusion/executor/multiproc_executor.py @@ -329,7 +329,7 @@ class MultiprocDiffusionExecutor(DiffusionExecutor): try: result = self.collective_rpc( "execute_model", - args=(req, self.od_config, scheduler_output.kv_prefetch_jobs), + args=(req, self.od_config, scheduler_output.kv_prefetch_job), unique_reply_rank=0, exec_all_ranks=True, ) diff --git a/vllm_omni/diffusion/models/helios/pipeline_helios.py b/vllm_omni/diffusion/models/helios/pipeline_helios.py index 12ecda368..655ef8821 100644 --- a/vllm_omni/diffusion/models/helios/pipeline_helios.py +++ b/vllm_omni/diffusion/models/helios/pipeline_helios.py @@ -37,7 +37,7 @@ if TYPE_CHECKING: from vllm.model_executor.layers.quantization.base_config import QuantizationConfig from vllm_omni.diffusion.worker.input_batch import InputBatch - from vllm_omni.diffusion.worker.utils import DiffusionRequestState + from vllm_omni.diffusion.worker.utils import StepRequestState logger = logging.getLogger(__name__) @@ -279,9 +279,9 @@ class HeliosPipeline( def prepare_encode( self, - state: DiffusionRequestState, + state: StepRequestState, **kwargs: Any, - ) -> DiffusionRequestState: + ) -> StepRequestState: """Initialize Helios request state for chunk-wise step execution.""" del kwargs # Wrap the single request in a DiffusionRequestBatch so the batch @@ -530,7 +530,7 @@ class HeliosPipeline( self._prepare_next_chunk(state) return state - def _prepare_next_chunk(self, state: DiffusionRequestState) -> None: + def _prepare_next_chunk(self, state: StepRequestState) -> None: extra = state.extra k = state.chunk_index is_first_chunk = k == 0 @@ -604,7 +604,7 @@ class HeliosPipeline( state.step_index = 0 self._num_timesteps = state.chunk_num_steps - def _prepare_stage2_chunk(self, state: DiffusionRequestState) -> None: + def _prepare_stage2_chunk(self, state: StepRequestState) -> None: extra = state.extra batch_size, num_channel, num_frames_lat, height, width = state.latents.shape latents_flat = state.latents.permute(0, 2, 1, 3, 4).reshape( @@ -634,7 +634,7 @@ class HeliosPipeline( return num_steps * 2 return num_steps - def _set_stage2_timesteps(self, state: DiffusionRequestState) -> None: + def _set_stage2_timesteps(self, state: StepRequestState) -> None: extra = state.extra patch_size = self.transformer.config.patch_size image_seq_len = (state.latents.shape[-1] * state.latents.shape[-2] * state.latents.shape[-3]) // ( @@ -655,7 +655,7 @@ class HeliosPipeline( def denoise_step( self, input_batch: InputBatch, - states: Sequence[DiffusionRequestState], + states: Sequence[StepRequestState], **kwargs: Any, ) -> torch.Tensor | None: del kwargs @@ -668,7 +668,7 @@ class HeliosPipeline( def _denoise_stage1_step( self, - state: DiffusionRequestState, + state: StepRequestState, latents: torch.Tensor, timesteps: torch.Tensor, ) -> torch.Tensor: @@ -727,7 +727,7 @@ class HeliosPipeline( cfg_normalize=False, ) - def _denoise_stage2_step(self, state: DiffusionRequestState) -> torch.Tensor: + def _denoise_stage2_step(self, state: StepRequestState) -> torch.Tensor: extra = state.extra latents = state.latents assert latents is not None @@ -773,7 +773,7 @@ class HeliosPipeline( def step_scheduler( self, - state: DiffusionRequestState, + state: StepRequestState, noise_pred: torch.Tensor, **kwargs: Any, ) -> None: @@ -800,7 +800,7 @@ class HeliosPipeline( state.step_in_chunk += 1 state.step_index = state.step_in_chunk - def _step_scheduler_stage2(self, state: DiffusionRequestState, noise_pred: torch.Tensor) -> None: + def _step_scheduler_stage2(self, state: StepRequestState, noise_pred: torch.Tensor) -> None: extra = state.extra t = state.current_timestep assert t is not None and state.latents is not None @@ -866,7 +866,7 @@ class HeliosPipeline( def post_decode( self, - state: DiffusionRequestState, + state: StepRequestState, **kwargs: Any, ) -> DiffusionOutput: del kwargs diff --git a/vllm_omni/diffusion/models/hunyuan_image3/pipeline_hunyuan_image3.py b/vllm_omni/diffusion/models/hunyuan_image3/pipeline_hunyuan_image3.py index c0476b879..095a420c8 100644 --- a/vllm_omni/diffusion/models/hunyuan_image3/pipeline_hunyuan_image3.py +++ b/vllm_omni/diffusion/models/hunyuan_image3/pipeline_hunyuan_image3.py @@ -54,7 +54,7 @@ from .system_prompt import get_system_prompt if TYPE_CHECKING: from vllm_omni.diffusion.worker.input_batch import InputBatch - from vllm_omni.diffusion.worker.utils import DiffusionRequestState + from vllm_omni.diffusion.worker.utils import StepRequestState logger = logging.getLogger(__name__) @@ -486,7 +486,7 @@ class HunyuanImage3Pipeline( self._pipeline = HunyuanImage3Text2ImagePipeline(model=self, scheduler=self.scheduler, vae=self.vae) return self._pipeline - def _validate_step_request(self, state: "DiffusionRequestState") -> None: + def _validate_step_request(self, state: "StepRequestState") -> None: prompt = state.prompt sampling = state.sampling if prompt is None: @@ -584,7 +584,7 @@ class HunyuanImage3Pipeline( def _extract_step_prompt_inputs( self, - state: "DiffusionRequestState", + state: "StepRequestState", ) -> tuple[list[str], list[str | None], str | None, list[list[JointImageInfo]] | None, str]: sampling = state.sampling return self._extract_prompt_inputs( @@ -610,7 +610,7 @@ class HunyuanImage3Pipeline( def _restore_injected_ar_kv( self, - states: list["DiffusionRequestState"], + states: list["StepRequestState"], row_state_indexes: list[int], row_branches: list[int], ) -> None: @@ -755,7 +755,7 @@ class HunyuanImage3Pipeline( def _prompt_kv_prefix_lens( self, - states: list["DiffusionRequestState"], + states: list["StepRequestState"], row_state_indexes: list[int], row_branches: list[int], ) -> list[int] | None: @@ -774,7 +774,7 @@ class HunyuanImage3Pipeline( def _merge_step_model_inputs( self, - states: list["DiffusionRequestState"], + states: list["StepRequestState"], row_state_indexes: list[int], row_branches: list[int], first_step: bool, @@ -828,7 +828,7 @@ class HunyuanImage3Pipeline( def _restore_prompt_kv_cache( self, - states: list["DiffusionRequestState"], + states: list["StepRequestState"], row_state_indexes: list[int], row_branches: list[int], ) -> None: @@ -855,7 +855,7 @@ class HunyuanImage3Pipeline( def _capture_prompt_kv_cache( self, - states: list["DiffusionRequestState"], + states: list["StepRequestState"], row_state_indexes: list[int], row_branches: list[int], ) -> None: @@ -1838,9 +1838,9 @@ class HunyuanImage3Pipeline( def prepare_encode( self, - state: "DiffusionRequestState", + state: "StepRequestState", **kwargs: Any, - ) -> "DiffusionRequestState": + ) -> "StepRequestState": del kwargs self._validate_step_request(state) pipe = self.pipeline @@ -1955,7 +1955,7 @@ class HunyuanImage3Pipeline( } return state - def _step_group_key(self, state: "DiffusionRequestState") -> tuple[Any, ...]: + def _step_group_key(self, state: "StepRequestState") -> tuple[Any, ...]: if _STEP_CFG_FACTOR not in state.extra: raise ValueError(f"Missing Hunyuan CFG factor for request {state.request_id}.") if _STEP_MODEL_KWARGS not in state.extra: @@ -1989,16 +1989,16 @@ class HunyuanImage3Pipeline( def _split_step_groups( self, - states: list["DiffusionRequestState"], - ) -> list[list["DiffusionRequestState"]]: - grouped: dict[tuple[Any, ...], list[DiffusionRequestState]] = {} + states: list["StepRequestState"], + ) -> list[list["StepRequestState"]]: + grouped: dict[tuple[Any, ...], list[StepRequestState]] = {} for state in states: grouped.setdefault(self._step_group_key(state), []).append(state) return list(grouped.values()) @staticmethod def _step_row_order( - states: list["DiffusionRequestState"], + states: list["StepRequestState"], cfg_factor: int, ) -> tuple[list[int], list[int]]: row_state_indexes = [state_idx for _ in range(cfg_factor) for state_idx in range(len(states))] @@ -2006,7 +2006,7 @@ class HunyuanImage3Pipeline( return row_state_indexes, row_branches @staticmethod - def _validate_step_group_states(states: list["DiffusionRequestState"]) -> tuple[bool, int]: + def _validate_step_group_states(states: list["StepRequestState"]) -> tuple[bool, int]: if not states: raise ValueError("HunyuanImage3 denoise_step received an empty group.") @@ -2043,7 +2043,7 @@ class HunyuanImage3Pipeline( def _split_merged_kwargs_to_states( self, - states: list["DiffusionRequestState"], + states: list["StepRequestState"], merged_kwargs: dict[str, Any], row_state_indexes: list[int], row_branches: list[int], @@ -2094,7 +2094,7 @@ class HunyuanImage3Pipeline( state.extra[_STEP_MODEL_KWARGS] = next_kwargs state.extra[_STEP_INPUT_IDS] = None - def _denoise_step_group(self, states: list["DiffusionRequestState"]) -> torch.Tensor: + def _denoise_step_group(self, states: list["StepRequestState"]) -> torch.Tensor: first_step, cfg_factor = self._validate_step_group_states(states) row_state_indexes, row_branches = self._step_row_order(states, cfg_factor) latents = torch.cat([state.latents for state in states], dim=0) @@ -2179,7 +2179,7 @@ class HunyuanImage3Pipeline( def step_scheduler( self, - state: "DiffusionRequestState", + state: "StepRequestState", noise_pred: torch.Tensor, **kwargs: Any, ) -> None: @@ -2200,7 +2200,7 @@ class HunyuanImage3Pipeline( def post_decode( self, - state: "DiffusionRequestState", + state: "StepRequestState", **kwargs: Any, ) -> DiffusionOutput: output_type = kwargs.get("output_type", "pil") diff --git a/vllm_omni/diffusion/models/interface.py b/vllm_omni/diffusion/models/interface.py index 657a45bf8..bd26c0395 100644 --- a/vllm_omni/diffusion/models/interface.py +++ b/vllm_omni/diffusion/models/interface.py @@ -17,7 +17,7 @@ if TYPE_CHECKING: from vllm_omni.diffusion.data import DiffusionOutput from vllm_omni.diffusion.worker.input_batch import InputBatch - from vllm_omni.diffusion.worker.utils import DiffusionRequestState + from vllm_omni.diffusion.worker.utils import StepRequestState @runtime_checkable @@ -54,7 +54,7 @@ class SupportsStepExecution(Protocol): supports_step_execution: ClassVar[bool] = True - def prepare_encode(self, state: DiffusionRequestState, **kwargs: Any) -> DiffusionRequestState: + def prepare_encode(self, state: StepRequestState, **kwargs: Any) -> StepRequestState: """Prepare request-level inputs and return initialized state.""" ... @@ -62,11 +62,11 @@ class SupportsStepExecution(Protocol): """Run one denoise forward on the runner-assembled batch.""" ... - def step_scheduler(self, state: DiffusionRequestState, noise_pred: torch.Tensor, **kwargs: Any) -> None: + def step_scheduler(self, state: StepRequestState, noise_pred: torch.Tensor, **kwargs: Any) -> None: """Run one scheduler step.""" ... - def post_decode(self, state: DiffusionRequestState, **kwargs: Any) -> DiffusionOutput: + def post_decode(self, state: StepRequestState, **kwargs: Any) -> DiffusionOutput: """Decode output after denoise loop or at a partial chunk boundary.""" ... diff --git a/vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image.py b/vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image.py index 474ea946c..0aeb8b85c 100644 --- a/vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image.py +++ b/vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image.py @@ -48,7 +48,7 @@ from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch, spli if TYPE_CHECKING: from vllm_omni.diffusion.worker.input_batch import InputBatch - from vllm_omni.diffusion.worker.utils import DiffusionRequestState + from vllm_omni.diffusion.worker.utils import StepRequestState from vllm_omni.model_executor.model_loader.weight_utils import ( download_weights_from_hf_specific, @@ -770,9 +770,9 @@ class QwenImagePipeline( def prepare_encode( self, - state: "DiffusionRequestState", + state: "StepRequestState", **kwargs: Any, - ) -> "DiffusionRequestState": + ) -> "StepRequestState": """Populate *state* with encoded prompts, latents, timesteps, and CFG config.""" sampling = state.sampling prompt, negative_prompt = self._extract_prompts([state.prompt] if state.prompt is not None else []) @@ -959,7 +959,7 @@ class QwenImagePipeline( def step_scheduler( self, - state: "DiffusionRequestState", + state: "StepRequestState", noise_pred: torch.Tensor, **kwargs: Any, ) -> None: @@ -980,7 +980,7 @@ class QwenImagePipeline( def post_decode( self, - state: "DiffusionRequestState", + state: "StepRequestState", **kwargs: Any, ) -> DiffusionOutput: """Decode final latents from *state*.""" diff --git a/vllm_omni/diffusion/request.py b/vllm_omni/diffusion/request.py index 3ad3f1541..0edb30d23 100644 --- a/vllm_omni/diffusion/request.py +++ b/vllm_omni/diffusion/request.py @@ -4,6 +4,7 @@ import random from dataclasses import dataclass +from typing import Any from vllm_omni.inputs.data import OmniDiffusionSamplingParams, OmniPromptType @@ -28,7 +29,7 @@ class OmniDiffusionRequest: prompt: OmniPromptType sampling_params: OmniDiffusionSamplingParams request_id: str - kv_sender_info: dict | None = None + kv_sender_info: dict[str, Any] | None = None def __post_init__(self): """Initialize dependent fields after dataclass initialization.""" diff --git a/vllm_omni/diffusion/sched/__init__.py b/vllm_omni/diffusion/sched/__init__.py index e02637338..29b9cc667 100644 --- a/vllm_omni/diffusion/sched/__init__.py +++ b/vllm_omni/diffusion/sched/__init__.py @@ -1,13 +1,15 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from vllm_omni.diffusion.sched.base_scheduler import BaseScheduler, SchedulerInterface from vllm_omni.diffusion.sched.interface import ( CachedRequestData, - DiffusionRequestState, DiffusionRequestStatus, DiffusionSchedulerOutput, + KVPrefetchJob, NewRequestData, - SchedulerInterface, + SchedulerRequestState, + StepBatchSamplingParamsKey, ) from vllm_omni.diffusion.sched.request_scheduler import RequestScheduler from vllm_omni.diffusion.sched.step_scheduler import StepScheduler @@ -17,10 +19,13 @@ Scheduler = RequestScheduler __all__ = [ "DiffusionRequestStatus", "CachedRequestData", - "DiffusionRequestState", "DiffusionSchedulerOutput", + "KVPrefetchJob", "NewRequestData", + "SchedulerRequestState", + "BaseScheduler", "SchedulerInterface", + "StepBatchSamplingParamsKey", "RequestScheduler", "StepScheduler", "Scheduler", diff --git a/vllm_omni/diffusion/sched/base_scheduler.py b/vllm_omni/diffusion/sched/base_scheduler.py index 9e4b6af5b..5ea91b167 100644 --- a/vllm_omni/diffusion/sched/base_scheduler.py +++ b/vllm_omni/diffusion/sched/base_scheduler.py @@ -3,6 +3,7 @@ from __future__ import annotations +from abc import ABC, abstractmethod from collections import deque from dataclasses import fields @@ -12,54 +13,37 @@ from vllm_omni.diffusion.data import OmniDiffusionConfig from vllm_omni.diffusion.request import OmniDiffusionRequest from vllm_omni.diffusion.sched.interface import ( CachedRequestData, - DiffusionRequestState, DiffusionRequestStatus, DiffusionSchedulerOutput, + KVPrefetchJob, NewRequestData, RequestBatchSamplingParamsKey, - SamplingParamsKey, - SchedulerInterface, + SchedulerRequestState, + StepBatchSamplingParamsKey, ) +from vllm_omni.diffusion.worker.utils import RunnerOutput logger = init_logger(__name__) +BatchSamplingParamsKey = StepBatchSamplingParamsKey | RequestBatchSamplingParamsKey + # LoRA identity is derived from `sampling.lora_request`, not a same-named field # on sampling params, so it must be resolved separately from the bulk lookup. -_SAMPLING_PARAMS_KEY_FIELD_NAMES = frozenset(f.name for f in fields(SamplingParamsKey)) - {"lora_int_id"} -_REQUEST_BATCH_SAMPLING_PARAMS_KEY_FIELD_NAMES = frozenset(f.name for f in fields(RequestBatchSamplingParamsKey)) - { +_STEP_BATCH_SAMPLING_PARAMS_KEY_FIELD_NAMES = frozenset(field.name for field in fields(StepBatchSamplingParamsKey)) - { "lora_int_id" } -def get_sampling_params_key(request: OmniDiffusionRequest) -> SamplingParamsKey: - """Build a batch-compatibility key from the request's sampling params.""" - sampling = request.sampling_params - lora_request = getattr(sampling, "lora_request", None) - return SamplingParamsKey( - lora_int_id=lora_request.lora_int_id if lora_request is not None else None, - **{name: getattr(sampling, name) for name in _SAMPLING_PARAMS_KEY_FIELD_NAMES}, - ) - - -def get_request_batch_sampling_params_key(request: OmniDiffusionRequest) -> RequestBatchSamplingParamsKey: - """Build a request-batch compatibility key from the request's sampling params.""" - sampling = request.sampling_params - lora_request = getattr(sampling, "lora_request", None) - key_kwargs = {name: getattr(sampling, name) for name in _REQUEST_BATCH_SAMPLING_PARAMS_KEY_FIELD_NAMES} - key_kwargs["lora_int_id"] = lora_request.lora_int_id if lora_request is not None else None - return RequestBatchSamplingParamsKey(**key_kwargs) - - -class _BaseScheduler(SchedulerInterface): +class BaseScheduler(ABC): """Shared queue/state bookkeeping for diffusion schedulers.""" def __init__(self) -> None: self.od_config: OmniDiffusionConfig | None = None - self._request_states: dict[str, DiffusionRequestState] = {} + self._request_states: dict[str, SchedulerRequestState] = {} self._step_id: int = 0 self._waiting: deque[str] = deque() self._running: list[str] = [] - self._running_sampling_params_key: SamplingParamsKey | RequestBatchSamplingParamsKey | None = None + self._running_sampling_params_key: BatchSamplingParamsKey | None = None self._finished_req_ids: set[str] = set() self.max_num_running_reqs: int = 1 self._prefetch_enabled: bool = False @@ -129,13 +113,13 @@ class _BaseScheduler(SchedulerInterface): # kv_sender_info (would target the wrong sender under multi-replica) or # one already finished/aborted (would consume its sender buffer for # nothing). - kv_prefetch_jobs: dict | None = None + kv_prefetch_job: KVPrefetchJob | None = None if self._prefetch_enabled and self._waiting: nxt = self._request_states.get(self._waiting[0]) if nxt is not None and not nxt.is_finished(): sender_info = getattr(nxt.req, "kv_sender_info", None) if sender_info: - kv_prefetch_jobs = { + kv_prefetch_job = { "request_id": nxt.request_id, "kv_sender_info": sender_info, } @@ -147,7 +131,7 @@ class _BaseScheduler(SchedulerInterface): finished_req_ids=set(self._finished_req_ids), num_running_reqs=len(self._running), num_waiting_reqs=len(self._waiting), - kv_prefetch_jobs=kv_prefetch_jobs, + kv_prefetch_job=kv_prefetch_job, ) # update after schedule @@ -155,6 +139,10 @@ class _BaseScheduler(SchedulerInterface): self._finished_req_ids.clear() return scheduler_output + @abstractmethod + def update_from_output(self, sched_output: DiffusionSchedulerOutput, output: RunnerOutput) -> set[str]: + pass + def has_requests(self) -> bool: return bool(self._waiting or self._running) @@ -164,10 +152,10 @@ class _BaseScheduler(SchedulerInterface): def num_running_requests(self) -> int: return len(self._running) - def get_request_state(self, request_id: str) -> DiffusionRequestState | None: + def get_request_state(self, request_id: str) -> SchedulerRequestState | None: return self._request_states.get(request_id) - def pop_request_state(self, request_id: str) -> DiffusionRequestState | None: + def pop_request_state(self, request_id: str) -> SchedulerRequestState | None: self._pop_extra_request_state(request_id) return self._request_states.pop(request_id, None) @@ -262,21 +250,21 @@ class _BaseScheduler(SchedulerInterface): def _pop_extra_request_state(self, request_id: str) -> None: """Remove subclass-owned per-request state before popping request state.""" - def _make_request_state(self, request_id: str, request: OmniDiffusionRequest) -> DiffusionRequestState: - return DiffusionRequestState( + def _make_request_state(self, request_id: str, request: OmniDiffusionRequest) -> SchedulerRequestState: + return SchedulerRequestState( request_id=request_id, req=request, sampling_params_key=self._build_sampling_params_key(request), ) - def _can_schedule_waiting(self, state: DiffusionRequestState) -> bool: + def _can_schedule_waiting(self, state: SchedulerRequestState) -> bool: if not self._running: return True current_key = self._current_sampling_params_key() return current_key is not None and current_key == state.sampling_params_key - def _current_sampling_params_key(self) -> SamplingParamsKey | RequestBatchSamplingParamsKey | None: + def _current_sampling_params_key(self) -> BatchSamplingParamsKey | None: if self._running_sampling_params_key is not None or not self._running: return self._running_sampling_params_key state = self._request_states.get(self._running[0]) @@ -285,5 +273,30 @@ class _BaseScheduler(SchedulerInterface): def _build_sampling_params_key( self, request: OmniDiffusionRequest - ) -> SamplingParamsKey | RequestBatchSamplingParamsKey | None: - return get_sampling_params_key(request) + ) -> StepBatchSamplingParamsKey | RequestBatchSamplingParamsKey: # return type loosened for subclassing + """Build a step-batch compatibility key from sampling parameters.""" + sampling = request.sampling_params + # LoRA identity is optional on sampling params (and on test stubs). + lora_request = getattr(sampling, "lora_request", None) + return StepBatchSamplingParamsKey( + lora_int_id=lora_request.lora_int_id if lora_request is not None else None, + **{name: getattr(sampling, name) for name in _STEP_BATCH_SAMPLING_PARAMS_KEY_FIELD_NAMES}, + ) + + +class SchedulerInterface(BaseScheduler): + """Deprecated compatibility base for custom scheduler injection. + + Prefer subclassing :class:`BaseScheduler` directly. Subclassing this name + still works but emits a :class:`DeprecationWarning`. + """ + + def __init_subclass__(cls, **kwargs) -> None: + import warnings + + warnings.warn( + "SchedulerInterface is deprecated; subclass BaseScheduler instead.", + DeprecationWarning, + stacklevel=2, + ) + super().__init_subclass__(**kwargs) diff --git a/vllm_omni/diffusion/sched/interface.py b/vllm_omni/diffusion/sched/interface.py index 695641b5d..25ec722d5 100644 --- a/vllm_omni/diffusion/sched/interface.py +++ b/vllm_omni/diffusion/sched/interface.py @@ -4,17 +4,12 @@ from __future__ import annotations import enum -from abc import ABC, abstractmethod from dataclasses import dataclass from functools import cached_property -from typing import TYPE_CHECKING +from typing import Any, TypedDict -from vllm_omni.diffusion.data import OmniDiffusionConfig from vllm_omni.diffusion.request import OmniDiffusionRequest -if TYPE_CHECKING: - from vllm_omni.diffusion.worker.utils import RunnerOutput - class DiffusionRequestStatus(enum.IntEnum): """Request status tracked by diffusion scheduler.""" @@ -34,7 +29,7 @@ class DiffusionRequestStatus(enum.IntEnum): @dataclass(frozen=True, eq=True) -class SamplingParamsKey: +class StepBatchSamplingParamsKey: """Denoise step level Batch-compatibility key derived from ``OmniDiffusionSamplingParams``. Only requests with the same key can be batched together. @@ -81,33 +76,33 @@ class RequestBatchSamplingParamsKey: """ # Spatial / temporal shape. - height: object = None - width: object = None + height: int | None = None + width: int | None = None num_frames: int = 1 - resolution: object = 640 - fps: object = None - frame_rate: object = None - boundary_ratio: object = None + resolution: int = 640 + fps: int | None = None + frame_rate: float | None = None + boundary_ratio: float | None = None # CFG / guidance. do_classifier_free_guidance: bool = False guidance_scale: float = 0.0 guidance_scale_provided: bool = False - guidance_scale_2: object = None + guidance_scale_2: float | None = None guidance_rescale: float = 0.0 - true_cfg_scale: object = None + true_cfg_scale: float | None = None cfg_normalize: bool = False - strength: object = None + strength: float | None = None # Scheduling / output shape. - num_inference_steps: object = None - sigmas: object = None - max_sequence_length: object = None + num_inference_steps: int | None = None + sigmas: list[float] | None = None + max_sequence_length: int | None = None num_outputs_per_prompt: int = 1 eta: float = 0.0 - decode_timestep: object = None - decode_noise_scale: object = None - output_type: object = None + decode_timestep: float | list[float] | None = None + decode_noise_scale: float | list[float] | None = None + output_type: str | None = None # Model-specific batch defaults used by request-mode pipelines. layers: int = 4 @@ -119,12 +114,12 @@ class RequestBatchSamplingParamsKey: @dataclass -class DiffusionRequestState: +class SchedulerRequestState: """Scheduler-owned state for one queued OmniDiffusionRequest.""" request_id: str req: OmniDiffusionRequest - sampling_params_key: SamplingParamsKey | RequestBatchSamplingParamsKey | None = None + sampling_params_key: StepBatchSamplingParamsKey | RequestBatchSamplingParamsKey | None = None status: DiffusionRequestStatus = DiffusionRequestStatus.WAITING error: str | None = None @@ -145,7 +140,7 @@ class NewRequestData: req: OmniDiffusionRequest @classmethod - def from_state(cls, state: DiffusionRequestState) -> NewRequestData: + def from_state(cls, state: SchedulerRequestState) -> NewRequestData: return cls(request_id=state.request_id, req=state.req) @@ -160,10 +155,20 @@ class CachedRequestData: return cls(request_ids=[]) +class KVPrefetchJob(TypedDict): + """Descriptor for prefetching the next request's received KV cache.""" + + request_id: str + kv_sender_info: dict[str, Any] + + @dataclass class DiffusionSchedulerOutput: """Output of a single scheduling cycle.""" + # Stable scheduler-cycle diagnostics for engines, injected schedulers, and + # instrumentation; they are intentionally retained even without a direct + # production consumer for every field. step_id: int # global step index scheduled_new_reqs: list[NewRequestData] scheduled_cached_reqs: CachedRequestData @@ -171,7 +176,7 @@ class DiffusionSchedulerOutput: num_running_reqs: int num_waiting_reqs: int # next request to background-prefetch KV - kv_prefetch_jobs: dict | None = None + kv_prefetch_job: KVPrefetchJob | None = None @cached_property def scheduled_request_ids(self) -> list[str]: @@ -190,55 +195,3 @@ class DiffusionSchedulerOutput: @property def is_empty(self) -> bool: return self.num_scheduled_reqs == 0 - - -class SchedulerInterface(ABC): - """Abstract lifecycle contract for diffusion schedulers.""" - - @abstractmethod - def initialize(self, od_config: OmniDiffusionConfig) -> None: - """Initialize or reset scheduler state.""" - - @abstractmethod - def add_request(self, request: OmniDiffusionRequest) -> str: - """Add a request and return the scheduler-owned request id.""" - - @abstractmethod - def schedule(self) -> DiffusionSchedulerOutput: - """Run one scheduling cycle.""" - - @abstractmethod - def update_from_output(self, sched_output: DiffusionSchedulerOutput, output: RunnerOutput) -> set[str]: - """Update scheduler state from executor output.""" - - @abstractmethod - def get_request_state(self, request_id: str) -> DiffusionRequestState | None: - """Return request state if present.""" - - @abstractmethod - def has_requests(self) -> bool: - """Return whether the scheduler still owns runnable requests.""" - - @abstractmethod - def num_waiting_requests(self) -> int: - """Return the number of requests waiting to be scheduled.""" - - @abstractmethod - def num_running_requests(self) -> int: - """Return the number of requests currently running.""" - - @abstractmethod - def pop_request_state(self, request_id: str) -> DiffusionRequestState | None: - """Remove and return request state if present.""" - - @abstractmethod - def preempt_request(self, request_id: str) -> bool: - """Preempt a running request back to waiting.""" - - @abstractmethod - def finish_requests(self, request_ids: str | list[str], status: DiffusionRequestStatus) -> None: - """Mark one or more requests finished.""" - - @abstractmethod - def close(self) -> None: - """Release scheduler-owned state.""" diff --git a/vllm_omni/diffusion/sched/request_scheduler.py b/vllm_omni/diffusion/sched/request_scheduler.py index 9a02eb8f3..9e7442d3a 100644 --- a/vllm_omni/diffusion/sched/request_scheduler.py +++ b/vllm_omni/diffusion/sched/request_scheduler.py @@ -3,30 +3,38 @@ from __future__ import annotations +from dataclasses import fields from typing import TYPE_CHECKING from vllm_omni.diffusion.request import OmniDiffusionRequest -from vllm_omni.diffusion.sched.base_scheduler import _BaseScheduler, get_request_batch_sampling_params_key +from vllm_omni.diffusion.sched.base_scheduler import BaseScheduler from vllm_omni.diffusion.sched.interface import ( DiffusionRequestStatus, DiffusionSchedulerOutput, + RequestBatchSamplingParamsKey, ) if TYPE_CHECKING: from vllm_omni.diffusion.worker.utils import RunnerOutput +# LoRA identity is derived from `sampling.lora_request`, not a same-named field +# on sampling params, so it must be resolved separately from the bulk lookup. +_REQUEST_BATCH_SAMPLING_PARAMS_KEY_FIELD_NAMES = frozenset( + field.name for field in fields(RequestBatchSamplingParamsKey) +) - {"lora_int_id"} -class RequestScheduler(_BaseScheduler): + +class RequestScheduler(BaseScheduler): """Diffusion scheduler with vLLM-style waiting/running queues.""" - def _build_sampling_params_key(self, request: OmniDiffusionRequest): - return get_request_batch_sampling_params_key(request) - - def add_request(self, request: OmniDiffusionRequest) -> str: - return super().add_request(request) - - def schedule(self) -> DiffusionSchedulerOutput: - return super().schedule() + def _build_sampling_params_key(self, request: OmniDiffusionRequest) -> RequestBatchSamplingParamsKey: + """Build a request-batch compatibility key from sampling parameters.""" + sampling = request.sampling_params + # LoRA identity is optional on sampling params (and on test stubs). + lora_request = getattr(sampling, "lora_request", None) + key_kwargs = {name: getattr(sampling, name) for name in _REQUEST_BATCH_SAMPLING_PARAMS_KEY_FIELD_NAMES} + key_kwargs["lora_int_id"] = lora_request.lora_int_id if lora_request is not None else None + return RequestBatchSamplingParamsKey(**key_kwargs) def update_from_output(self, sched_output: DiffusionSchedulerOutput, output: RunnerOutput) -> set[str]: scheduled_request_ids = sched_output.scheduled_request_ids diff --git a/vllm_omni/diffusion/sched/step_scheduler.py b/vllm_omni/diffusion/sched/step_scheduler.py index 6cb6a9a28..a5a7a04ae 100644 --- a/vllm_omni/diffusion/sched/step_scheduler.py +++ b/vllm_omni/diffusion/sched/step_scheduler.py @@ -9,7 +9,7 @@ from typing import TYPE_CHECKING, Any from vllm.logger import init_logger from vllm_omni.diffusion.request import OmniDiffusionRequest -from vllm_omni.diffusion.sched.base_scheduler import _BaseScheduler +from vllm_omni.diffusion.sched.base_scheduler import BaseScheduler from vllm_omni.diffusion.sched.interface import ( DiffusionRequestStatus, DiffusionSchedulerOutput, @@ -27,8 +27,8 @@ class _StepProgress: total_steps: int -class StepScheduler(_BaseScheduler): - """Placeholder scheduler that advances a request one denoise step per update.""" +class StepScheduler(BaseScheduler): + """Scheduler that advances each request by one denoise step per update.""" def __init__(self) -> None: super().__init__() @@ -62,9 +62,6 @@ class StepScheduler(_BaseScheduler): ) return request_id - def schedule(self) -> DiffusionSchedulerOutput: - return super().schedule() - def update_from_output(self, sched_output: DiffusionSchedulerOutput, output: RunnerOutput) -> set[str]: scheduled_request_ids = sched_output.scheduled_request_ids if not scheduled_request_ids: diff --git a/vllm_omni/diffusion/worker/diffusion_model_runner.py b/vllm_omni/diffusion/worker/diffusion_model_runner.py index cb16bdeac..4ac9afe7a 100644 --- a/vllm_omni/diffusion/worker/diffusion_model_runner.py +++ b/vllm_omni/diffusion/worker/diffusion_model_runner.py @@ -36,13 +36,13 @@ from vllm_omni.diffusion.models.interface import supports_step_execution from vllm_omni.diffusion.offloader import get_offload_backend from vllm_omni.diffusion.registry import _NO_CACHE_ACCELERATION from vllm_omni.diffusion.request import OmniDiffusionRequest -from vllm_omni.diffusion.sched.interface import DiffusionSchedulerOutput +from vllm_omni.diffusion.sched.interface import DiffusionSchedulerOutput, KVPrefetchJob from vllm_omni.diffusion.worker.input_batch import InputBatch, scatter_latents from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch from vllm_omni.diffusion.worker.utils import ( BatchRunnerOutput, - DiffusionRequestState, RunnerOutput, + StepRequestState, attach_stage_durations, clear_pipeline_stage_durations, consume_pipeline_stage_durations, @@ -124,7 +124,7 @@ class DiffusionModelRunner(OmniConnectorModelRunnerMixin): self.prompt_embed_cache = None # Cache for per-request stepwise state. - self.state_cache: dict[str, DiffusionRequestState] = {} + self.state_cache: dict[str, StepRequestState] = {} # Initialize KV cache manager for connector management. self.kv_transfer_manager = OmniKVTransferManager.from_od_config(od_config) @@ -336,7 +336,7 @@ class DiffusionModelRunner(OmniConnectorModelRunnerMixin): req: OmniDiffusionRequest, *, od_config: OmniDiffusionConfig, - kv_prefetch_jobs: dict | None = None, + kv_prefetch_job: KVPrefetchJob | None = None, use_prefetch: bool = False, ) -> None: # Receive AR KV. Single-request execution can use the prefetch path: @@ -358,8 +358,8 @@ class DiffusionModelRunner(OmniConnectorModelRunnerMixin): logger.debug("KV recv for %s %.1fms", req.request_id, kv_recv_ms) # Kick off the next request's prefetch (+ H2D) to overlap this forward. - if use_prefetch and self._kv_prefetch_enabled and kv_prefetch_jobs is not None: - self.kv_transfer_manager.start_prefetch(kv_prefetch_jobs, self.target_device) + if use_prefetch and self._kv_prefetch_enabled and kv_prefetch_job is not None: + self.kv_transfer_manager.start_prefetch(kv_prefetch_job, self.target_device) if req.sampling_params.generator is None and req.sampling_params.seed is not None: if req.sampling_params.generator_device is not None: @@ -385,7 +385,7 @@ class DiffusionModelRunner(OmniConnectorModelRunnerMixin): return # Refresh cache context if needed. Batch admission groups requests by - # SamplingParamsKey, so the first request's num_inference_steps applies + # RequestBatchSamplingParamsKey, so the first request's num_inference_steps applies # to the whole runner batch. num_inference_steps = first_req.sampling_params.num_inference_steps if num_inference_steps is None and od_config.cache_backend in ( @@ -432,7 +432,7 @@ class DiffusionModelRunner(OmniConnectorModelRunnerMixin): od_config: OmniDiffusionConfig, allow_single_output: bool, require_request_batch_support: bool, - kv_prefetch_jobs: dict | None = None, + kv_prefetch_job: KVPrefetchJob | None = None, record_name: str, ) -> BatchRunnerOutput: assert self.pipeline is not None, "Model not loaded. Call load_model() first." @@ -454,7 +454,7 @@ class DiffusionModelRunner(OmniConnectorModelRunnerMixin): self._prepare_request_for_forward( req, od_config=od_config, - kv_prefetch_jobs=kv_prefetch_jobs, + kv_prefetch_job=kv_prefetch_job, use_prefetch=allow_single_output, ) @@ -497,7 +497,7 @@ class DiffusionModelRunner(OmniConnectorModelRunnerMixin): def _attach_stepwise_metrics( self, - state: DiffusionRequestState, + state: StepRequestState, output: DiffusionOutput, *, is_primary: bool, @@ -508,7 +508,11 @@ class DiffusionModelRunner(OmniConnectorModelRunnerMixin): ) attach_stage_durations(state, output) - def execute_model(self, req: OmniDiffusionRequest, kv_prefetch_jobs: dict | None = None) -> DiffusionOutput: + def execute_model( + self, + req: OmniDiffusionRequest, + kv_prefetch_job: KVPrefetchJob | None = None, + ) -> DiffusionOutput: """ Execute a forward pass for the given requests. @@ -529,7 +533,7 @@ class DiffusionModelRunner(OmniConnectorModelRunnerMixin): od_config=self.od_config, allow_single_output=True, require_request_batch_support=False, - kv_prefetch_jobs=kv_prefetch_jobs, + kv_prefetch_job=kv_prefetch_job, record_name="pipeline_forward", ) output = runner_output.runner_outputs[0].result @@ -564,14 +568,12 @@ class DiffusionModelRunner(OmniConnectorModelRunnerMixin): """Return whether current pipeline supports step execution.""" return self.pipeline is not None and supports_step_execution(self.pipeline) - def _update_states( - self, scheduler_output: DiffusionSchedulerOutput - ) -> tuple[list[DiffusionRequestState], list[str]]: + def _update_states(self, scheduler_output: DiffusionSchedulerOutput) -> tuple[list[StepRequestState], list[str]]: """Step-before update: cleanup finished requests and get/create one running state.""" for request_id in scheduler_output.finished_req_ids: self.state_cache.pop(request_id, None) - resolved: list[DiffusionRequestState] = [] + resolved: list[StepRequestState] = [] new_request_ids: list[str] = [] try: # process new requests @@ -580,7 +582,7 @@ class DiffusionModelRunner(OmniConnectorModelRunnerMixin): new_request_ids.append(request_id) if request_id in self.state_cache: raise ValueError(f"Received duplicate new-request payload for cached request {request_id}.") - new_state = DiffusionRequestState( + new_state = StepRequestState( request_id=request_id, sampling=copy.deepcopy(sched_new_req.req.sampling_params), prompt=sched_new_req.req.prompt, @@ -609,7 +611,7 @@ class DiffusionModelRunner(OmniConnectorModelRunnerMixin): return resolved, new_request_ids - def _prepare_batch_inputs(self, states: list[DiffusionRequestState], new_request_ids: list[str]) -> InputBatch: + def _prepare_batch_inputs(self, states: list[StepRequestState], new_request_ids: list[str]) -> InputBatch: # process new reqs for state in states: if state.request_id in new_request_ids: @@ -639,7 +641,7 @@ class DiffusionModelRunner(OmniConnectorModelRunnerMixin): def _update_states_after( self, - states: list[DiffusionRequestState], + states: list[StepRequestState], input_batch: InputBatch, interrupted: bool = False, ): diff --git a/vllm_omni/diffusion/worker/diffusion_worker.py b/vllm_omni/diffusion/worker/diffusion_worker.py index 006bbae48..1500ccd3f 100644 --- a/vllm_omni/diffusion/worker/diffusion_worker.py +++ b/vllm_omni/diffusion/worker/diffusion_worker.py @@ -46,7 +46,7 @@ from vllm_omni.diffusion.ipc import DIFFUSION_RPC_RESULT_ENVELOPE, pack_diffusio from vllm_omni.diffusion.lora.manager import DiffusionLoRAManager from vllm_omni.diffusion.registry import get_diffusion_ir_op_priority_func from vllm_omni.diffusion.request import OmniDiffusionRequest -from vllm_omni.diffusion.sched.interface import DiffusionSchedulerOutput +from vllm_omni.diffusion.sched.interface import DiffusionSchedulerOutput, KVPrefetchJob from vllm_omni.diffusion.worker.diffusion_model_runner import DiffusionModelRunner from vllm_omni.diffusion.worker.utils import BaseRunnerOutput, BatchRunnerOutput from vllm_omni.engine.stage_init_utils import set_death_signal @@ -416,7 +416,7 @@ class DiffusionWorker: self, req: OmniDiffusionRequest, od_config: OmniDiffusionConfig, - kv_prefetch_jobs: dict | None = None, + kv_prefetch_job: KVPrefetchJob | None = None, ) -> DiffusionOutput: """Execute a forward pass by delegating to the model runner.""" assert self.model_runner is not None, "Model runner not initialized" @@ -430,7 +430,7 @@ class DiffusionWorker: profiler = self._get_profiler() ctx = profiler.annotate_context_manager("diffusion_forward") if profiler else nullcontext() with ctx: - output = self.model_runner.execute_model(req, kv_prefetch_jobs=kv_prefetch_jobs) + output = self.model_runner.execute_model(req, kv_prefetch_job=kv_prefetch_job) if profiler: profiler.step() return output @@ -440,7 +440,7 @@ class DiffusionWorker: ) -> BatchRunnerOutput: """Batch forward: LoRA activate once, delegate to model runner.""" assert self.model_runner is not None, "Model runner not initialized" - # LoRA: same adapter/scale within batch guaranteed by SamplingParamsKey + # LoRA: same adapter/scale within batch guaranteed by RequestBatchSamplingParamsKey. if self.lora_manager is not None and scheduler_output.scheduled_new_reqs: sp = scheduler_output.scheduled_new_reqs[0].req.sampling_params try: @@ -475,7 +475,7 @@ class DiffusionWorker: Newly scheduled requests register their (lora_request, lora_scale) in ``_step_lora_state`` so cached requests can resolve to the same adapter on later ticks. Finished requests are evicted. Batch - homogeneity is enforced by the scheduler via ``SamplingParamsKey``, + homogeneity is enforced by the scheduler via ``StepBatchSamplingParamsKey``, so any scheduled request id resolves to the active LoRA identity. """ for request_id in scheduler_output.finished_req_ids: @@ -1155,7 +1155,7 @@ class WorkerWrapperBase: self, reqs: list[OmniDiffusionRequest], od_config: OmniDiffusionConfig, - kv_prefetch_jobs: dict | None = None, + kv_prefetch_job: KVPrefetchJob | None = None, ) -> DiffusionOutput: """ Execute a forward pass. @@ -1163,12 +1163,12 @@ class WorkerWrapperBase: Args: reqs: List of diffusion requests od_config: OmniDiffusionConfig configuration - kv_prefetch_jobs: Optional next-request KV prefetch descriptor. + kv_prefetch_job: Optional next-request KV prefetch descriptor. Returns: DiffusionOutput with generated results """ - return self.worker.execute_model(reqs, od_config, kv_prefetch_jobs=kv_prefetch_jobs) + return self.worker.execute_model(reqs, od_config, kv_prefetch_job=kv_prefetch_job) def execute_stepwise(self, scheduler_output: DiffusionSchedulerOutput) -> BaseRunnerOutput: """Execute one diffusion step.""" diff --git a/vllm_omni/diffusion/worker/input_batch.py b/vllm_omni/diffusion/worker/input_batch.py index 511f2dd69..c0d10198a 100644 --- a/vllm_omni/diffusion/worker/input_batch.py +++ b/vllm_omni/diffusion/worker/input_batch.py @@ -17,7 +17,7 @@ from dataclasses import dataclass, field import numpy as np import torch -from vllm_omni.diffusion.worker.utils import DiffusionRequestState +from vllm_omni.diffusion.worker.utils import StepRequestState def _normalize_prompt_embeds(x: torch.Tensor) -> torch.Tensor: @@ -59,9 +59,9 @@ def _pad_mask(x: torch.Tensor, target_seq_len: int) -> torch.Tensor: def _select_states( - states: Sequence[DiffusionRequestState], + states: Sequence[StepRequestState], idx_mapping: torch.Tensor | None, -) -> tuple[list[DiffusionRequestState], torch.Tensor, np.ndarray]: +) -> tuple[list[StepRequestState], torch.Tensor, np.ndarray]: if not states: raise ValueError("Cannot build InputBatch from empty states.") @@ -73,7 +73,7 @@ def _select_states( raise ValueError("idx_mapping must be a 1D tensor.") idx_mapping = idx_mapping.to(dtype=torch.int32) - selected_states: list[DiffusionRequestState] = [] + selected_states: list[StepRequestState] = [] for batch_idx, state_idx in enumerate(idx_mapping.tolist()): if state_idx < 0 or state_idx >= len(states): raise ValueError(f"idx_mapping[{batch_idx}]={state_idx} is out of range for states.") @@ -81,12 +81,12 @@ def _select_states( return selected_states, idx_mapping, idx_mapping.detach().cpu().numpy() -def _prepare_request_ids(states: Sequence[DiffusionRequestState]) -> list[str]: +def _prepare_request_ids(states: Sequence[StepRequestState]) -> list[str]: return [state.request_id for state in states] def _prepare_prompt_field_on_state( - state: DiffusionRequestState, + state: StepRequestState, *, embeds_attr: str, mask_attr: str, @@ -180,7 +180,7 @@ def _get_seq_lens_from_mask(mask: torch.Tensor) -> list[int]: def _get_request_prompt_seq_lens( - state: DiffusionRequestState, + state: StepRequestState, *, embeds_attr: str, mask_attr: str, @@ -209,7 +209,7 @@ def _get_request_prompt_seq_lens( def _prepare_request_prompt_field( - state: DiffusionRequestState, + state: StepRequestState, *, embeds_attr: str, mask_attr: str, @@ -266,7 +266,7 @@ def _prepare_request_prompt_field( def _prepare_padded_prompt_fields( - states: Sequence[DiffusionRequestState], + states: Sequence[StepRequestState], *, embeds_attr: str, mask_attr: str, @@ -337,7 +337,7 @@ def _prepare_padded_prompt_fields( def _prepare_latents( - states: Sequence[DiffusionRequestState], + states: Sequence[StepRequestState], *, out: torch.Tensor | None = None, ) -> torch.Tensor: @@ -352,7 +352,7 @@ def _prepare_latents( def _require_state_latents( - state: DiffusionRequestState, + state: StepRequestState, *, for_field: str, ) -> torch.Tensor: @@ -382,7 +382,7 @@ def _expand_scalar_or_vector( def _prepare_timesteps( - states: Sequence[DiffusionRequestState], + states: Sequence[StepRequestState], *, out: torch.Tensor | None = None, ) -> torch.Tensor: @@ -410,9 +410,9 @@ def _prepare_timesteps( def _prepare_cfg_scalars( - states: Sequence[DiffusionRequestState], + states: Sequence[StepRequestState], ) -> tuple[bool, float, bool]: - def _cfg_scalars(state: DiffusionRequestState) -> tuple[bool, float, bool]: + def _cfg_scalars(state: StepRequestState) -> tuple[bool, float, bool]: true_cfg_scale = getattr(state.sampling, "true_cfg_scale", None) or 4.0 cfg_normalize = bool(getattr(state.sampling, "cfg_normalize", False)) return state.do_true_cfg, true_cfg_scale, cfg_normalize @@ -425,7 +425,7 @@ def _prepare_cfg_scalars( def _prepare_guidance( - states: Sequence[DiffusionRequestState], + states: Sequence[StepRequestState], *, out: torch.Tensor | None = None, ) -> torch.Tensor | None: @@ -461,7 +461,7 @@ def _prepare_guidance( def _prepare_image_latents( - states: Sequence[DiffusionRequestState], + states: Sequence[StepRequestState], *, out: torch.Tensor | None = None, ) -> torch.Tensor | None: @@ -478,7 +478,7 @@ def _prepare_image_latents( def _prepare_seq_lens( - states: Sequence[DiffusionRequestState], + states: Sequence[StepRequestState], attr_name: str, ) -> list[int] | None: values = [getattr(state, attr_name) for state in states] @@ -489,7 +489,7 @@ def _prepare_seq_lens( return [int(value[0]) for value in values if value is not None] -def _prepare_img_shapes(states: Sequence[DiffusionRequestState]) -> list | None: +def _prepare_img_shapes(states: Sequence[StepRequestState]) -> list | None: values = [state.img_shapes for state in states] if all(value is None for value in values): return None @@ -499,7 +499,7 @@ def _prepare_img_shapes(states: Sequence[DiffusionRequestState]) -> list | None: def _prepare_prompt_embeds( - states: Sequence[DiffusionRequestState], + states: Sequence[StepRequestState], *, embeds_out: torch.Tensor | None = None, mask_out: torch.Tensor | None = None, @@ -516,7 +516,7 @@ def _prepare_prompt_embeds( def _prepare_negative_prompt_embeds( - states: Sequence[DiffusionRequestState], + states: Sequence[StepRequestState], *, embeds_out: torch.Tensor | None = None, mask_out: torch.Tensor | None = None, @@ -544,7 +544,7 @@ def _same_composition( def _scatter_batch_tensor_by_mapping( - states: Sequence[DiffusionRequestState], + states: Sequence[StepRequestState], idx_mapping_np: np.ndarray, *, attr_name: str, @@ -582,7 +582,7 @@ class InputBatch: """Ephemeral step-level batch view. Static request-local tensors are normalized and padded onto - ``DiffusionRequestState`` itself, making the request state the persistent + ``StepRequestState`` itself, making the request state the persistent source of truth. ``InputBatch`` only assembles a contiguous view for the current step and refreshes dynamic fields in-place when composition is unchanged. @@ -613,7 +613,7 @@ class InputBatch: img_shapes: list | None = None txt_seq_lens: list[int] | None = None negative_txt_seq_lens: list[int] | None = None - states: Sequence[DiffusionRequestState] = field(default_factory=tuple) + states: Sequence[StepRequestState] = field(default_factory=tuple) def __post_init__(self) -> None: if len(self.request_ids) != int(self.idx_mapping.numel()): @@ -625,14 +625,14 @@ class InputBatch: def _refresh_dynamic_fields( self, - selected_states: Sequence[DiffusionRequestState], + selected_states: Sequence[StepRequestState], ) -> None: self.latents = _prepare_latents(selected_states, out=self.latents) self.timesteps = _prepare_timesteps(selected_states, out=self.timesteps) def _refresh_static_fields( self, - states: Sequence[DiffusionRequestState], + states: Sequence[StepRequestState], ) -> None: self.do_true_cfg, self.true_cfg_scale, self.cfg_normalize = _prepare_cfg_scalars(states) self.guidance = _prepare_guidance(states, out=self.guidance) @@ -656,7 +656,7 @@ class InputBatch: def _repack_dynamic_fields( self, - selected_states: Sequence[DiffusionRequestState], + selected_states: Sequence[StepRequestState], ) -> None: # Same-composition cache hits reuse static prompt fields; request ids # must keep the same encoded prompt metadata for the batch lifetime. @@ -665,7 +665,7 @@ class InputBatch: def _rebuild( self, - selected_states: Sequence[DiffusionRequestState], + selected_states: Sequence[StepRequestState], idx_mapping: torch.Tensor, idx_mapping_np: np.ndarray, request_ids: list[str], @@ -685,7 +685,7 @@ class InputBatch: @classmethod def make_batch( cls, - states: Sequence[DiffusionRequestState], + states: Sequence[StepRequestState], idx_mapping: torch.Tensor | None = None, cached_batch: InputBatch | None = None, ) -> InputBatch: @@ -737,7 +737,7 @@ class InputBatch: def scatter_latents( - states: Sequence[DiffusionRequestState], + states: Sequence[StepRequestState], input_batch: InputBatch, ) -> None: """Scatter the step-updated latents back into persistent request states. diff --git a/vllm_omni/diffusion/worker/utils.py b/vllm_omni/diffusion/worker/utils.py index ba6165a9d..8d85fae70 100644 --- a/vllm_omni/diffusion/worker/utils.py +++ b/vllm_omni/diffusion/worker/utils.py @@ -33,7 +33,7 @@ def consume_pipeline_stage_durations(pipeline: Any) -> dict[str, float]: def merge_stage_durations( - state: DiffusionRequestState, + state: StepRequestState, stage_durations: dict[str, float], ) -> None: if not stage_durations: @@ -43,7 +43,7 @@ def merge_stage_durations( def attach_stage_durations( - state: DiffusionRequestState, + state: StepRequestState, output: DiffusionOutput, ) -> None: if state.stage_durations: @@ -51,7 +51,7 @@ def attach_stage_durations( @dataclass -class DiffusionRequestState: +class StepRequestState: """Per-request mutable state across all pipeline stages. Owned by Runner and passed through all step-execution stages: diff --git a/vllm_omni/distributed/omni_connectors/kv_transfer_manager.py b/vllm_omni/distributed/omni_connectors/kv_transfer_manager.py index 51e604336..b75a7089a 100644 --- a/vllm_omni/distributed/omni_connectors/kv_transfer_manager.py +++ b/vllm_omni/distributed/omni_connectors/kv_transfer_manager.py @@ -15,6 +15,7 @@ import torch from vllm.logger import init_logger from vllm_omni.diffusion.request import OmniDiffusionRequest +from vllm_omni.diffusion.sched.interface import KVPrefetchJob from vllm_omni.platforms import current_omni_platform from .factory import OmniConnectorFactory @@ -1205,16 +1206,14 @@ class OmniKVTransferManager: logger.exception("Failed to release KV pool buffer") buffers.clear() - def start_prefetch( - self, kv_prefetch_jobs: dict[str, Any] | None, target_device: torch.device | None = None - ) -> None: + def start_prefetch(self, kv_prefetch_job: KVPrefetchJob | None, target_device: torch.device | None = None) -> None: """Kick off a background KV load (non-blocking). No-op unless prefetch enabled.""" - if not (self._async_prefetch and self.config.need_recv_cache) or not kv_prefetch_jobs: + if not (self._async_prefetch and self.config.need_recv_cache) or not kv_prefetch_job: return # Followers receive via collective distribute; bg pull would consume the owner's payload. if self.topo_config.is_follower: return - rid = kv_prefetch_jobs.get("request_id") + rid = kv_prefetch_job.get("request_id") if not rid: return # Memory pressure → skip prefetch; sync receive handles it. @@ -1228,7 +1227,7 @@ class OmniKVTransferManager: self._discard_future(stale_rid) if rid in self._prefetch_futures: return - sender_info = kv_prefetch_jobs.get("kv_sender_info") + sender_info = kv_prefetch_job.get("kv_sender_info") if not sender_info: # No explicit endpoint → bg receive would target wrong sender under multi-replica. logger.debug("Skip KV prefetch for %s: stub has no kv_sender_info", rid) diff --git a/vllm_omni/experimental/ar_diffusion/runner.py b/vllm_omni/experimental/ar_diffusion/runner.py index 862b3d9e8..8c968f8ce 100644 --- a/vllm_omni/experimental/ar_diffusion/runner.py +++ b/vllm_omni/experimental/ar_diffusion/runner.py @@ -25,6 +25,7 @@ from vllm.logger import init_logger from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig from vllm_omni.diffusion.models.dreamzero.pipeline_dreamzero import MAX_DREAMZERO_SESSIONS from vllm_omni.diffusion.request import OmniDiffusionRequest +from vllm_omni.diffusion.sched.interface import KVPrefetchJob from vllm_omni.diffusion.worker.diffusion_model_runner import DiffusionModelRunner from vllm_omni.experimental.ar_diffusion.kv_cache.config import ARDiffusionKVConfig from vllm_omni.experimental.ar_diffusion.kv_cache.manager import ARDiffusionKVCache @@ -234,10 +235,14 @@ class ARDiffusionModelRunner(DiffusionModelRunner): num_frame_per_block=num_frame_per_block, ) - def execute_model(self, req: OmniDiffusionRequest, kv_prefetch_jobs: dict | None = None) -> DiffusionOutput: + def execute_model( + self, + req: OmniDiffusionRequest, + kv_prefetch_job: KVPrefetchJob | None = None, + ) -> DiffusionOutput: # KV disabled -> base behavior, unchanged. if self.kv_cache is None: - return super().execute_model(req, kv_prefetch_jobs=kv_prefetch_jobs) + return super().execute_model(req, kv_prefetch_job=kv_prefetch_job) kv = self.kv_cache # DreamZero KV is session-scoped (the model state persists across a @@ -273,7 +278,7 @@ class ARDiffusionModelRunner(DiffusionModelRunner): # sync below makes the stop reflect completed GPU work, not just dispatch. _e2e_t0 = time.perf_counter() try: - out = super().execute_model(req, kv_prefetch_jobs=kv_prefetch_jobs) + out = super().execute_model(req, kv_prefetch_job=kv_prefetch_job) except Exception: # Transactional containment: a forward that died partway may have # written some layers' K/V into allocated-but-uncommitted blocks From 24eb4ab89cc2ad3f8cdd83f57b66ce7786a0d65c Mon Sep 17 00:00:00 2001 From: Yifan Tian Date: Wed, 22 Jul 2026 14:06:12 +0800 Subject: [PATCH 052/240] [Model] Add Cache-DiT support for FLUX.1-Kontext-dev (#4205) Signed-off-by: yifan Signed-off-by: Didan Deng <33117903+wtomin@users.noreply.github.com> Co-authored-by: Didan Deng <33117903+wtomin@users.noreply.github.com> --- docs/user_guide/diffusion_features.md | 2 +- .../online_serving/test_flux_kontext_expansion.py | 13 +++++++++++++ .../diffusion/models/flux/pipeline_flux_kontext.py | 5 +++++ 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/docs/user_guide/diffusion_features.md b/docs/user_guide/diffusion_features.md index c34c1338c..e5da3c294 100644 --- a/docs/user_guide/diffusion_features.md +++ b/docs/user_guide/diffusion_features.md @@ -116,7 +116,7 @@ The following tables show which models support each feature: | **FLUX.1-dev** | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ | ❌ | ❌ | ✅ | ❌ | | **FLUX.1-schnell** | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ | ❌ | ❌ | ✅ | ❌ | | **FLUX.2-klein** | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ❌ | ❌ | ✅ | ❌ | -| **FLUX.1-Kontext-dev** | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | +| **FLUX.1-Kontext-dev** | ❌ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | | **FLUX.2-dev** | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | | **GLM-Image** | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | | **Hidream-I1-Full** | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | diff --git a/tests/e2e/online_serving/test_flux_kontext_expansion.py b/tests/e2e/online_serving/test_flux_kontext_expansion.py index 17d4b0263..4a8a9ab98 100644 --- a/tests/e2e/online_serving/test_flux_kontext_expansion.py +++ b/tests/e2e/online_serving/test_flux_kontext_expansion.py @@ -45,6 +45,19 @@ def _get_diffusion_feature_cases(model: str): id="parallel_cfg_2", marks=PARALLEL_FEATURE_MARKS, ), + pytest.param( + OmniServerParams( + model=model, + server_args=[ + "--tensor-parallel-size", + "2", + "--enable-cpu-offload", + "--cache-backend", + "cache_dit", + ], + ), + id="parallel_002", + ), ] diff --git a/vllm_omni/diffusion/models/flux/pipeline_flux_kontext.py b/vllm_omni/diffusion/models/flux/pipeline_flux_kontext.py index 15f0e4e0c..5aff911c6 100644 --- a/vllm_omni/diffusion/models/flux/pipeline_flux_kontext.py +++ b/vllm_omni/diffusion/models/flux/pipeline_flux_kontext.py @@ -641,6 +641,11 @@ class FluxKontextPipeline( num_inference_steps = req.sampling_params.num_inference_steps or num_inference_steps sigmas = req.sampling_params.sigmas or sigmas guidance_scale = req.sampling_params.guidance_scale or guidance_scale + num_images_per_prompt = ( + req.sampling_params.num_outputs_per_prompt + if req.sampling_params.num_outputs_per_prompt > 0 + else num_images_per_prompt + ) true_cfg_scale = ( req.sampling_params.true_cfg_scale if req.sampling_params.true_cfg_scale is not None else true_cfg_scale ) From 39e83e27f09474d0d5c007b4cf93dc810584eb23 Mon Sep 17 00:00:00 2001 From: Mu GuanLin <1203789601@qq.com> Date: Wed, 22 Jul 2026 14:36:32 +0800 Subject: [PATCH 053/240] [Bug][CI] Pin the LTX FP8 quality gate to its established sigma schedule (#5302) Signed-off-by: mglyn <1203789601@qq.com> --- .../examples/online_serving/text_to_video.md | 4 +- .../online_serving/text_to_video/README.md | 2 +- .../models/ltx2/test_ltx2_pipeline.py | 46 ++++++++++++++++-- .../quantization/test_quantization_quality.py | 42 +++++++++++++++++ .../diffusion/models/ltx2/ltx2_denoise.py | 47 ++++++++++++++----- .../diffusion/models/ltx2/ltx2_request.py | 11 +++++ .../diffusion/models/ltx2/pipeline_ltx2.py | 1 + 7 files changed, 134 insertions(+), 19 deletions(-) diff --git a/docs/user_guide/examples/online_serving/text_to_video.md b/docs/user_guide/examples/online_serving/text_to_video.md index 0f1f33a25..e30b627ee 100644 --- a/docs/user_guide/examples/online_serving/text_to_video.md +++ b/docs/user_guide/examples/online_serving/text_to_video.md @@ -296,14 +296,14 @@ done ```bash vllm serve Lightricks/LTX-2 --omni --port 8098 \ - --enforce-eager --flow-shift 1.0 --boundary-ratio 1.0 + --enforce-eager --boundary-ratio 1.0 ``` For multi-GPU memory reduction, you can enable HSDP: ```bash vllm serve Lightricks/LTX-2 --omni --port 8098 \ - --enforce-eager --flow-shift 1.0 --boundary-ratio 1.0 \ + --enforce-eager --boundary-ratio 1.0 \ --use-hsdp --hsdp-shard-size 2 ``` diff --git a/examples/online_serving/text_to_video/README.md b/examples/online_serving/text_to_video/README.md index 93de44a88..71afd9e7c 100644 --- a/examples/online_serving/text_to_video/README.md +++ b/examples/online_serving/text_to_video/README.md @@ -254,7 +254,7 @@ done ```bash vllm serve Lightricks/LTX-2 --omni --port 8098 \ - --enforce-eager --flow-shift 1.0 --boundary-ratio 1.0 + --enforce-eager --boundary-ratio 1.0 ``` #### Start with Optimization Presets diff --git a/tests/diffusion/models/ltx2/test_ltx2_pipeline.py b/tests/diffusion/models/ltx2/test_ltx2_pipeline.py index 2f9155fe0..41b1d5a84 100644 --- a/tests/diffusion/models/ltx2/test_ltx2_pipeline.py +++ b/tests/diffusion/models/ltx2/test_ltx2_pipeline.py @@ -17,7 +17,11 @@ from vllm_omni.diffusion.models.ltx2.ltx2_components import ( LTX23_COMPONENT_PROFILE, ) from vllm_omni.diffusion.models.ltx2.ltx2_conditioning import LTXI2VConditioningMixin -from vllm_omni.diffusion.models.ltx2.ltx2_denoise import LTXDenoiseExecutor, LTXPhaseResult +from vllm_omni.diffusion.models.ltx2.ltx2_denoise import ( + LTXDenoiseExecutor, + LTXPhaseResult, + prepare_scheduler_stage, +) from vllm_omni.diffusion.models.ltx2.ltx2_guidance import ( LTX_LEGACY_VELOCITY_GUIDANCE, LTX_OFFICIAL_X0_GUIDANCE, @@ -324,6 +328,38 @@ def test_ltx2_two_stage_reuses_prompt_context_between_phases(): torch.testing.assert_close(output.output[1], torch.tensor([4.0])) +def test_ltx_custom_sigmas_bypass_scheduler_shifting(): + from diffusers import FlowMatchEulerDiscreteScheduler + + scheduler = FlowMatchEulerDiscreteScheduler( + use_dynamic_shifting=True, + base_shift=0.95, + max_shift=2.05, + shift_terminal=0.1, + ) + pipeline = SimpleNamespace( + scheduler=scheduler, + _make_video_audio_scheduler=lambda *args, **kwargs: object(), + ) + sigmas = [1.0, 0.75, 0.25, 0.0] + + audio_scheduler, _, timesteps = prepare_scheduler_stage( + pipeline, + SimpleNamespace(num_inference_steps=3), + device=torch.device("cpu"), + sigmas=sigmas, + timesteps=None, + latent_num_frames=1, + latent_height=1, + latent_width=1, + ) + + expected = torch.tensor(sigmas, dtype=torch.float32) + torch.testing.assert_close(pipeline.scheduler.sigmas, expected) + torch.testing.assert_close(audio_scheduler.sigmas, expected) + torch.testing.assert_close(timesteps, expected[:-1] * scheduler.config.num_train_timesteps) + + class TestLTXRequestParsing: @pytest.mark.parametrize( ("t2v_cls", "i2v_cls"), @@ -513,6 +549,7 @@ class TestLTX23ForwardStages: from vllm_omni.inputs.data import OmniDiffusionSamplingParams pipe = _make_ltx_request_pipe(LTX23Pipeline) + request_sigmas = [1.0, 0.4] req = DiffusionRequestBatch( [ OmniDiffusionRequest( @@ -525,12 +562,13 @@ class TestLTX23ForwardStages: num_inference_steps=3, guidance_scale=4.5, output_type="latent", + sigmas=request_sigmas, ), request_id="ltx23-t2v-forward-stage-delegation", ) ] ) - sigmas = [1.0, 0.5] + fallback_sigmas = [1.0, 0.5] timesteps = [1000, 500] attention_kwargs = {"scale": 1.0} seen = {} @@ -545,7 +583,7 @@ class TestLTX23ForwardStages: output = pipe.forward( req, - sigmas=sigmas, + sigmas=fallback_sigmas, timesteps=timesteps, noise_scale=0.25, attention_kwargs=attention_kwargs, @@ -561,7 +599,7 @@ class TestLTX23ForwardStages: assert seen["request_inputs"].output_type == "latent" assert seen["kwargs"] == { "noise_scale": 0.25, - "sigmas": sigmas, + "sigmas": request_sigmas, "timesteps": timesteps, "attention_kwargs": attention_kwargs, } diff --git a/tests/diffusion/quantization/test_quantization_quality.py b/tests/diffusion/quantization/test_quantization_quality.py index 9b53d8941..c2807797d 100644 --- a/tests/diffusion/quantization/test_quantization_quality.py +++ b/tests/diffusion/quantization/test_quantization_quality.py @@ -82,6 +82,7 @@ class QualityTestConfig: gpu: str = "H100" # minimum GPU requirement negative_prompt: str = "" guidance_scale: float | None = None + sigmas: list[float] | None = None enable_cpu_offload: bool = False def baseline_ref(self) -> str: @@ -215,6 +216,19 @@ QUALITY_CONFIGS = [ width=256, num_frames=25, num_inference_steps=8, + # Preserve the final scheduler trajectory used when this FP8 quality + # gate was established. Explicit LTX sigmas bypass dynamic shifting. + sigmas=[ + 1.0, + 0.92185378074646, + 0.8327768445014954, + 0.7303033471107483, + 0.6111654043197632, + 0.4709382653236389, + 0.30347853899002075, + 0.10000002384185791, + 0.0, + ], ), ] @@ -322,6 +336,7 @@ def _generate_video(omni, config: QualityTestConfig): num_inference_steps=config.num_inference_steps, num_frames=config.num_frames, guidance_scale=config.guidance_scale, + sigmas=config.sigmas, ), ) @@ -446,6 +461,33 @@ def test_benchmark_generate_image_unwraps_nested_omni_request_output(monkeypatch assert peak_mem == 0.0 +def test_generate_video_forwards_sigmas(monkeypatch): + from vllm_omni.platforms import current_omni_platform + + monkeypatch.setattr(current_omni_platform, "device_type", "cpu", raising=False) + monkeypatch.setattr(torch.accelerator, "reset_peak_memory_stats", lambda: None, raising=False) + monkeypatch.setattr(torch.accelerator, "max_memory_allocated", lambda: 0, raising=False) + captured = SimpleNamespace(sampling=None) + + class DummyOmni: + def generate(self, _prompt, sampling): + captured.sampling = sampling + return [SimpleNamespace(images=[np.zeros((1, 2, 2, 3), dtype=np.float32)])] + + config = QualityTestConfig( + id="ltx-sigmas", + model="unused", + quantization="fp8", + task="t2v", + prompt="test", + max_lpips=0.1, + sigmas=[1.0, 0.5], + ) + _generate_video(DummyOmni(), config) + + assert captured.sampling.sigmas == [1.0, 0.5] + + _marks = hardware_marks(res={"cuda": "H100"}) _OUTPUT_DIR = Path(os.environ["VLLM_OMNI_QUALITY_OUTPUT_DIR"]) if "VLLM_OMNI_QUALITY_OUTPUT_DIR" in os.environ else None diff --git a/vllm_omni/diffusion/models/ltx2/ltx2_denoise.py b/vllm_omni/diffusion/models/ltx2/ltx2_denoise.py index af8df4b41..d0dfa3e54 100644 --- a/vllm_omni/diffusion/models/ltx2/ltx2_denoise.py +++ b/vllm_omni/diffusion/models/ltx2/ltx2_denoise.py @@ -177,6 +177,17 @@ class I2VVideoAudioScheduler: return ((video_out, audio_out),) +def _set_scheduler_sigmas(scheduler: Any, sigmas: torch.Tensor) -> torch.Tensor: + sigmas = sigmas.to(torch.float32) + timesteps = sigmas[:-1] * scheduler.config.get("num_train_timesteps", 1000) + scheduler.sigmas = sigmas + scheduler.timesteps = timesteps + scheduler.num_inference_steps = len(timesteps) + scheduler._step_index = None + scheduler._begin_index = None + return timesteps + + def prepare_scheduler_stage( pipeline: Any, request_inputs: LTXRequestInputs, @@ -188,18 +199,9 @@ def prepare_scheduler_stage( latent_height: int, latent_width: int, ) -> tuple[Any, Any, torch.Tensor]: - sigmas = ( - np.linspace(1.0, 1 / request_inputs.num_inference_steps, request_inputs.num_inference_steps) - if sigmas is None - else sigmas - ) - mu = calculate_shift( - pipeline.scheduler.config.get("max_image_seq_len", 4096), - pipeline.scheduler.config.get("base_image_seq_len", 1024), - pipeline.scheduler.config.get("max_image_seq_len", 4096), - pipeline.scheduler.config.get("base_shift", 0.95), - pipeline.scheduler.config.get("max_shift", 2.05), - ) + if sigmas is not None and timesteps is not None: + raise ValueError("Only one of `sigmas` or `timesteps` may be provided.") + audio_scheduler = copy.deepcopy(pipeline.scheduler) video_audio_scheduler = pipeline._make_video_audio_scheduler( audio_scheduler, @@ -207,6 +209,27 @@ def prepare_scheduler_stage( latent_height, latent_width, ) + if sigmas is not None: + scheduler_sigmas = torch.as_tensor(sigmas, dtype=torch.float32, device=device) + if scheduler_sigmas.ndim != 1 or scheduler_sigmas.numel() < 2: + raise ValueError("An LTX custom sigma schedule must contain at least two boundary values.") + if scheduler_sigmas[-1] != 0: + scheduler_sigmas = torch.cat([scheduler_sigmas, scheduler_sigmas.new_zeros(1)]) + timesteps_tensor = _set_scheduler_sigmas(pipeline.scheduler, scheduler_sigmas) + _set_scheduler_sigmas(audio_scheduler, scheduler_sigmas.clone()) + return audio_scheduler, video_audio_scheduler, timesteps_tensor + + sigmas = np.linspace(1.0, 1 / request_inputs.num_inference_steps, request_inputs.num_inference_steps) + # Official LTX2 one-stage omits the latent, so mu stays at max_shift across resolutions. + # Mu varies with latent token count only when the scheduler receives a latent, as in official HQ two-stage; + # explicit sigma schedules bypass mu entirely. + mu = calculate_shift( + pipeline.scheduler.config.get("max_image_seq_len", 4096), + pipeline.scheduler.config.get("base_image_seq_len", 1024), + pipeline.scheduler.config.get("max_image_seq_len", 4096), + pipeline.scheduler.config.get("base_shift", 0.95), + pipeline.scheduler.config.get("max_shift", 2.05), + ) retrieve_timesteps( audio_scheduler, request_inputs.num_inference_steps, diff --git a/vllm_omni/diffusion/models/ltx2/ltx2_request.py b/vllm_omni/diffusion/models/ltx2/ltx2_request.py index 02cab34d2..d8a4356b3 100644 --- a/vllm_omni/diffusion/models/ltx2/ltx2_request.py +++ b/vllm_omni/diffusion/models/ltx2/ltx2_request.py @@ -71,6 +71,17 @@ class LTXRequestMixin: supports_request_batch = False supports_guidance_rescale = False + @staticmethod + def _resolve_request_sigmas( + req: DiffusionRequestBatch, + fallback: list[float] | None, + ) -> list[float] | None: + request_sigmas = [sampling.sigmas for sampling in req.sampling_params_list] + first = request_sigmas[0] if request_sigmas else None + if any(item != first for item in request_sigmas[1:]): + raise ValueError("Batched LTX requests must use identical custom sigmas.") + return first if first is not None else fallback + def check_inputs( self, prompt: str | list[str] | None, diff --git a/vllm_omni/diffusion/models/ltx2/pipeline_ltx2.py b/vllm_omni/diffusion/models/ltx2/pipeline_ltx2.py index c32a6918a..5de1f4862 100644 --- a/vllm_omni/diffusion/models/ltx2/pipeline_ltx2.py +++ b/vllm_omni/diffusion/models/ltx2/pipeline_ltx2.py @@ -96,6 +96,7 @@ class LTXOneStagePipeline(LTXPipelineRuntime): image: Any | None = None, ) -> DiffusionOutput | list[DiffusionOutput]: del return_dict + sigmas = self._resolve_request_sigmas(req, sigmas) request_inputs = self._resolve_request_inputs( req, prompt=prompt, From f43d761c26ed2e7031ebb43c4b83429526d76efb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zeyu=20Huang=20=7C=20=E9=BB=83=E6=BE=A4=E5=AE=87?= <11222265+fhfuih@users.noreply.github.com> Date: Wed, 22 Jul 2026 17:07:09 +0800 Subject: [PATCH 054/240] [Refactor]: clean up diffusion worker interfaces (#5217) Signed-off-by: Huang, Zeyu <11222265+fhfuih@users.noreply.github.com> Co-authored-by: Cursor Co-authored-by: Hongsheng Liu --- docs/design/module/dit_module.md | 47 +++++----- tests/diffusion/test_diffusion_worker.py | 33 ------- .../test_multiproc_engine_concurrency.py | 8 +- tests/diffusion/test_worker_wrapper_base.py | 32 +------ .../diffusion/worker/diffusion_worker.py | 85 ++++++------------- vllm_omni/diffusion/worker/utils.py | 6 -- 6 files changed, 58 insertions(+), 153 deletions(-) diff --git a/docs/design/module/dit_module.md b/docs/design/module/dit_module.md index 97d1db499..9634adad3 100644 --- a/docs/design/module/dit_module.md +++ b/docs/design/module/dit_module.md @@ -267,7 +267,7 @@ class WorkerProc: self.result_mq = MessageQueue(n_reader=1, ...) # Initialize GPU worker - self.worker = GPUWorker(local_rank=gpu_id, rank=gpu_id, od_config=od_config) + self.worker = self._create_worker(gpu_id, od_config, ...) ``` **Initialization Steps**: @@ -289,13 +289,18 @@ class WorkerProc: #### 3.2 GPU Worker ```python -class GPUWorker: - def init_device_and_model(self): +class DiffusionWorker: + def init_device(self): # Set distributed environment variables os.environ["RANK"] = str(rank) os.environ["WORLD_SIZE"] = str(world_size) - # Initialize PyTorch distributed + # Select the device and initialize distributed execution + self.device = current_omni_platform.get_torch_device(rank) + current_omni_platform.set_device(self.device) + self.vllm_config = _create_diffusion_worker_vllm_config( + self.device, self.od_config + ) init_distributed_environment(world_size, rank) parallel_config = self.od_config.parallel_config initialize_model_parallel( @@ -306,16 +311,18 @@ class GPUWorker: pipeline_parallel_size=parallel_config.pipeline_parallel_size, ) - # Load model - model_loader = DiffusersPipelineLoader(load_config, self.od_config) - self.pipeline = model_loader.load_model(load_device=f"cuda:{rank}") + def load_model(self, load_format="default", custom_pipeline_name=None): + self.model_runner.load_model( + load_format=load_format, + custom_pipeline_name=custom_pipeline_name, + ) - # Setup cache backend - from vllm_omni.diffusion.cache.selector import get_cache_backend - self.cache_backend = get_cache_backend(od_config.cache_backend, od_config.cache_config) - - if self.cache_backend is not None: - self.cache_backend.enable(self.pipeline) + def init_lora_manager(self): + self.lora_manager = DiffusionLoRAManager( + pipeline=self.model_runner.pipeline, + device=self.device, + dtype=self.od_config.dtype, + ) ``` **Key Features**: @@ -329,17 +336,17 @@ class GPUWorker: #### 3.3 Worker Busy Loop ```python -def worker_busy_loop(self): +def _worker_busy_loop(self): while self._running: # 1. Receive unified message (generation request, RPC request, or shutdown) - msg = self.recv_message() + msg = self.mq.dequeue(timeout=1.0) # 2. Route message based on type if isinstance(msg, dict) and msg.get("type") == "rpc": # Handle RPC request - result, should_reply = self.execute_rpc(msg) + result, should_reply = self._execute_rpc(msg) if should_reply: - self.return_result(result) + self._return_result(result) elif isinstance(msg, dict) and msg.get("type") == "shutdown": # Handle shutdown message @@ -348,7 +355,7 @@ def worker_busy_loop(self): else: # Handle generation request (OmniDiffusionRequest list) output = self.worker.execute_model(msg, self.od_config) - self.return_result(output) + self._return_result(output) ``` **Execution Flow**: @@ -920,8 +927,8 @@ def initialize_model_parallel( └─> DiffusionEngine submits scheduled request to executor.add_req(req) 4. Worker Execution - └─> WorkerProc.worker_busy_loop() - └─> GPUWorker.execute_model(reqs) + └─> WorkerProc._worker_busy_loop() + └─> DiffusionWorker.execute_model(req) └─> Pipeline.forward(req) ├─> encode_prompt() ├─> prepare_latents() diff --git a/tests/diffusion/test_diffusion_worker.py b/tests/diffusion/test_diffusion_worker.py index fb45cd41d..6a49d0909 100644 --- a/tests/diffusion/test_diffusion_worker.py +++ b/tests/diffusion/test_diffusion_worker.py @@ -5,7 +5,6 @@ Unit tests for DiffusionWorker class. This module tests the DiffusionWorker implementation: -- load_weights: Loading model weights - sleep: Putting worker into sleep mode (levels 1 and 2) - wake_up: Waking worker from sleep mode """ @@ -59,38 +58,6 @@ def mock_gpu_worker(mocker: MockerFixture, mock_od_config): return worker -class TestDiffusionWorkerLoadWeights: - """Test DiffusionWorker.load_weights method.""" - - def test_load_weights_calls_pipeline(self, mocker: MockerFixture, mock_gpu_worker): - """Test that load_weights delegates to model_runner.load_weights.""" - # Setup mock weights - mock_weights = [ - ("layer1.weight", torch.randn(10, 10)), - ("layer2.weight", torch.randn(20, 20)), - ] - expected_loaded = {"layer1.weight", "layer2.weight"} - - # Configure model_runner mock - mock_gpu_worker.model_runner.load_weights = mocker.Mock(return_value=expected_loaded) - - # Call load_weights - result = mock_gpu_worker.load_weights(mock_weights) - - # Verify model_runner.load_weights was called with the weights - mock_gpu_worker.model_runner.load_weights.assert_called_once_with(mock_weights) - assert result == expected_loaded - - def test_load_weights_empty_iterable(self, mocker: MockerFixture, mock_gpu_worker): - """Test load_weights with empty weights iterable.""" - mock_gpu_worker.model_runner.load_weights = mocker.Mock(return_value=set()) - - result = mock_gpu_worker.load_weights([]) - - mock_gpu_worker.model_runner.load_weights.assert_called_once_with([]) - assert result == set() - - def test_diffusion_vllm_model_config_supplies_dtype_for_quant_methods(): from types import SimpleNamespace diff --git a/tests/diffusion/test_multiproc_engine_concurrency.py b/tests/diffusion/test_multiproc_engine_concurrency.py index 9b8e08a74..5f5d90f64 100644 --- a/tests/diffusion/test_multiproc_engine_concurrency.py +++ b/tests/diffusion/test_multiproc_engine_concurrency.py @@ -526,7 +526,7 @@ class TestWorkerProcRpcRankStatus: monkeypatch.setattr(torch.distributed, "all_gather_object", _all_gather_object) - result, should_reply = proc.execute_rpc( + result, should_reply = proc._execute_rpc( { "method": "remove_lora", "args": (), @@ -549,7 +549,7 @@ class TestWorkerProcRpcRankStatus: proc.worker.execute_method = Mock(side_effect=RuntimeError("local boom")) monkeypatch.setattr(torch.distributed, "is_initialized", lambda: False) - result, should_reply = proc.execute_rpc( + result, should_reply = proc._execute_rpc( { "method": "add_lora", "args": (), @@ -575,7 +575,7 @@ class TestWorkerProcRpcRankStatus: proc = self._make_worker_proc() with pytest.raises(ValueError, match="collect_rank_status requires exec_all_ranks=True"): - proc.execute_rpc( + proc._execute_rpc( { "method": "ping", "args": (), @@ -594,7 +594,7 @@ class TestWorkerProcRpcRankStatus: proc.worker.execute_method = Mock(side_effect=original) with pytest.raises(ValueError) as excinfo: - proc.execute_rpc( + proc._execute_rpc( { "method": "bad", "args": (), diff --git a/tests/diffusion/test_worker_wrapper_base.py b/tests/diffusion/test_worker_wrapper_base.py index e9c4d613f..170faec80 100644 --- a/tests/diffusion/test_worker_wrapper_base.py +++ b/tests/diffusion/test_worker_wrapper_base.py @@ -161,19 +161,6 @@ class TestWorkerWrapperBaseExtension: class TestWorkerWrapperBaseDelegation: """Test WorkerWrapperBase delegation to wrapped worker.""" - def test_generate_delegation(self, mocker: MockerFixture, mock_od_config): - """Test that generate() delegates to worker.generate().""" - mocker.patch.object(DiffusionWorker, "__init__", return_value=None) - wrapper = WorkerWrapperBase(gpu_id=0, od_config=mock_od_config, base_worker_class=DiffusionWorker) - mock_output = mocker.Mock() - wrapper.worker.generate = mocker.Mock(return_value=mock_output) - - mock_requests = [mocker.Mock()] - result = wrapper.generate(mock_requests) - - wrapper.worker.generate.assert_called_once_with(mock_requests) - assert result == mock_output - def test_execute_model_delegation(self, mocker: MockerFixture, mock_od_config): """Test that execute_model() delegates to worker.execute_model().""" mocker.patch.object(DiffusionWorker, "__init__", return_value=None) @@ -181,25 +168,12 @@ class TestWorkerWrapperBaseDelegation: mock_output = mocker.Mock() wrapper.worker.execute_model = mocker.Mock(return_value=mock_output) - mock_reqs = [mocker.Mock()] - result = wrapper.execute_model(mock_reqs, mock_od_config) + mock_req = mocker.Mock() + result = wrapper.execute_model(mock_req, mock_od_config) - wrapper.worker.execute_model.assert_called_once_with(mock_reqs, mock_od_config, kv_prefetch_job=None) + wrapper.worker.execute_model.assert_called_once_with(mock_req, mock_od_config, kv_prefetch_job=None) assert result == mock_output - def test_load_weights_delegation(self, mocker: MockerFixture, mock_od_config): - """Test that load_weights() delegates to worker.load_weights().""" - mocker.patch.object(DiffusionWorker, "__init__", return_value=None) - wrapper = WorkerWrapperBase(gpu_id=0, od_config=mock_od_config, base_worker_class=DiffusionWorker) - expected_result = {"weight1", "weight2"} - wrapper.worker.load_weights = mocker.Mock(return_value=expected_result) - - mock_weights = [("weight1", mocker.Mock()), ("weight2", mocker.Mock())] - result = wrapper.load_weights(mock_weights) - - wrapper.worker.load_weights.assert_called_once_with(mock_weights) - assert result == expected_result - def test_sleep_delegation(self, mocker: MockerFixture, mock_od_config): """Test that sleep() delegates to worker.sleep().""" mocker.patch.object(DiffusionWorker, "__init__", return_value=None) diff --git a/vllm_omni/diffusion/worker/diffusion_worker.py b/vllm_omni/diffusion/worker/diffusion_worker.py index 1500ccd3f..df3ee5910 100644 --- a/vllm_omni/diffusion/worker/diffusion_worker.py +++ b/vllm_omni/diffusion/worker/diffusion_worker.py @@ -13,7 +13,7 @@ import multiprocessing as mp import os import signal import traceback -from collections.abc import Iterable, Iterator +from collections.abc import Iterator from contextlib import AbstractContextManager, contextmanager, nullcontext from dataclasses import dataclass from typing import Any @@ -387,10 +387,6 @@ class DiffusionWorker: lora_scale=self.od_config.lora_scale, ) - def generate(self, request: OmniDiffusionRequest) -> DiffusionOutput: - """Generate output for the given requests.""" - return self.execute_model(request, self.od_config) - def profile(self, is_start: bool = True, profile_prefix: str | None = None) -> None: """Start or stop profiling for this GPU worker. @@ -506,11 +502,6 @@ class DiffusionWorker: raise logger.warning("LoRA activation skipped: %s", exc) - def load_weights(self, weights) -> set[str]: - """Load weights by delegating to the model runner.""" - assert self.model_runner is not None, "Model runner not initialized" - return self.model_runner.load_weights(weights) - def remove_lora(self, adapter_id: int) -> bool: return self.lora_manager.remove_adapter(adapter_id) @@ -525,7 +516,7 @@ class DiffusionWorker: def pin_lora(self, adapter_id: int) -> bool: return self.lora_manager.pin_adapter(adapter_id) - def sleep(self, level: int = 1) -> bool: + def sleep(self, level: int = 1) -> int: """ Put the worker to sleep, offloading model weights. @@ -598,7 +589,7 @@ class DiffusionWorker: logger.info(f"[Worker {self.rank}] Wake-up complete.") return True - def handle_sleep_task(self, task: OmniSleepTask) -> OmniACK: + def handle_sleep_task(self, task: OmniSleepTask | dict) -> OmniACK | None: from vllm_omni.platforms import current_omni_platform try: @@ -655,7 +646,7 @@ class DiffusionWorker: pass return OmniACK(task_id=task.task_id, status="ERROR", error_msg=str(e)) - def handle_wake_task(self, task: OmniWakeTask) -> OmniACK: + def handle_wake_task(self, task: OmniWakeTask | dict) -> OmniACK | None: from vllm_omni.platforms import current_omni_platform try: @@ -798,7 +789,7 @@ class WorkerProc: od_config: OmniDiffusionConfig, worker_extension_cls: str | None, custom_pipeline_args: dict[str, Any] | None = None, - ) -> DiffusionWorker: + ) -> "WorkerWrapperBase": """Create a worker instance. Override in subclasses for different worker types.""" worker_cls_path = current_omni_platform.get_diffusion_worker_cls() base_worker_class = resolve_obj_by_qualname(worker_cls_path) @@ -811,7 +802,7 @@ class WorkerProc: ) return wrapper - def return_result(self, output: Any): + def _return_result(self, output: Any) -> None: """Reply to client, only on rank 0.""" if self.result_mq is not None: if isinstance(output, OmniACK): @@ -824,10 +815,6 @@ class WorkerProc: logger.warning("SHM pack failed for model output: %s", e) self.result_mq.enqueue(output) - def recv_message(self): - """Receive messages from broadcast queue.""" - return self.mq.dequeue(indefinite=True) - def _gather_rpc_rank_statuses(self, status: dict[str, Any]) -> list[dict[str, Any]]: if not torch.distributed.is_initialized(): return [status] @@ -840,7 +827,7 @@ class WorkerProc: logger.warning("RPC rank status gather returned missing entries for ranks: %s", missing_ranks) return [s for s in statuses if s is not None] - def execute_rpc(self, rpc_request: dict) -> tuple[object | None, bool]: + def _execute_rpc(self, rpc_request: dict) -> tuple[object | None, bool]: """Execute an RPC request and indicate whether to reply.""" method = rpc_request["method"] args = rpc_request.get("args", ()) @@ -905,7 +892,7 @@ class WorkerProc: raise rpc_exception return result, should_reply - def worker_busy_loop(self) -> None: + def _worker_busy_loop(self) -> None: """Main busy loop for Multiprocessing Workers.""" logger.info(f"Worker {self.gpu_id} ready to receive requests via shared memory") @@ -930,21 +917,21 @@ class WorkerProc: if isinstance(msg, dict) and msg.get("type") == "sleep": task = OmniSleepTask(level=msg.get("level", 2), task_id=msg.get("task_id", "local")) ack = self.worker.handle_sleep_task(task) - self.return_result(ack) + self._return_result(ack) elif isinstance(msg, dict) and msg.get("type") == "wake_up": task = OmniWakeTask(tags=msg.get("tags"), task_id=msg.get("task_id", "local")) ack = self.worker.handle_wake_task(task) - self.return_result(ack) + self._return_result(ack) # Route message based on type elif isinstance(msg, dict) and msg.get("type") == "rpc": try: - result, should_reply = self.execute_rpc(msg) + result, should_reply = self._execute_rpc(msg) if should_reply: - self.return_result(result) + self._return_result(result) except Exception as e: logger.error(f"Error processing RPC: {e}", exc_info=True) if self.result_mq is not None: - self.return_result({"status": "error", "error": str(e)}) + self._return_result({"status": "error", "error": str(e)}) elif isinstance(msg, dict) and msg.get("type") == "shutdown": logger.info("Worker %s: Received shutdown message", self.gpu_id) @@ -963,7 +950,7 @@ class WorkerProc: output = DiffusionOutput.from_exception(e) try: - self.return_result(output) + self._return_result(output) except zmq.ZMQError as e: logger.error(f"ZMQ error sending reply: {e}") continue @@ -1022,7 +1009,7 @@ class WorkerProc: "result_handle": worker_proc.result_mq_handle if rank == 0 else None, } ) - worker_proc.worker_busy_loop() + worker_proc._worker_busy_loop() except SystemExit: logger.info("Worker %d: Shutdown signal received, starting cleanup.", rank) raise @@ -1139,21 +1126,9 @@ class WorkerWrapperBase: return worker_class - def generate(self, requests: list[OmniDiffusionRequest]) -> DiffusionOutput: - """ - Generate output for the given requests. - - Args: - requests: List of diffusion requests - - Returns: - DiffusionOutput with generated results - """ - return self.worker.generate(requests) - def execute_model( self, - reqs: list[OmniDiffusionRequest], + req: OmniDiffusionRequest, od_config: OmniDiffusionConfig, kv_prefetch_job: KVPrefetchJob | None = None, ) -> DiffusionOutput: @@ -1161,32 +1136,20 @@ class WorkerWrapperBase: Execute a forward pass. Args: - reqs: List of diffusion requests + req: Diffusion request. od_config: OmniDiffusionConfig configuration kv_prefetch_job: Optional next-request KV prefetch descriptor. Returns: DiffusionOutput with generated results """ - return self.worker.execute_model(reqs, od_config, kv_prefetch_job=kv_prefetch_job) + return self.worker.execute_model(req, od_config, kv_prefetch_job=kv_prefetch_job) def execute_stepwise(self, scheduler_output: DiffusionSchedulerOutput) -> BaseRunnerOutput: """Execute one diffusion step.""" return self.worker.execute_stepwise(scheduler_output) - def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - """ - Load model weights. - - Args: - weights: Iterable of (name, tensor) tuples - - Returns: - Set of loaded weight names - """ - return self.worker.load_weights(weights) - - def sleep(self, level: int = 1) -> bool: + def sleep(self, level: int = 1) -> int: """ Put the worker to sleep. The worker should not process any requests. The caller should guarantee that no requests are being processed @@ -1194,11 +1157,11 @@ class WorkerWrapperBase: Args: level: The sleep level. Level 1 sleep will offload the model - weights and discard the kv cache. - Currently only support level 1. + weights and discard the kv cache. Level 2 also saves buffers. + Currently only support level 1 and level 2. Returns: - True on success + Bytes held by the allocator before sleeping. """ return self.worker.sleep(level) @@ -1219,10 +1182,10 @@ class WorkerWrapperBase: """ return self.worker.wake_up(tags) - def handle_sleep_task(self, task): + def handle_sleep_task(self, task: OmniSleepTask | dict) -> OmniACK | None: return self.worker.handle_sleep_task(task) - def handle_wake_task(self, task): + def handle_wake_task(self, task: OmniWakeTask | dict) -> OmniACK | None: return self.worker.handle_wake_task(task) def shutdown(self) -> None: diff --git a/vllm_omni/diffusion/worker/utils.py b/vllm_omni/diffusion/worker/utils.py index 8d85fae70..b49388fee 100644 --- a/vllm_omni/diffusion/worker/utils.py +++ b/vllm_omni/diffusion/worker/utils.py @@ -163,12 +163,6 @@ class StepRequestState: return self.denoise_completed return self.chunk_index >= self.total_chunks - @property - def new_request(self) -> bool: - # TODO: this is only an approximation for current stepwise mode. - # A real "new request" signal should eventually come from scheduler/runner state transitions. - return self.step_index == 0 or self.timesteps is None - class BaseRunnerOutput(ABC): @abstractmethod From 9479720f81bc931bfc53e601ec619713e87b281c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zeyu=20Huang=20=7C=20=E9=BB=83=E6=BE=A4=E5=AE=87?= <11222265+fhfuih@users.noreply.github.com> Date: Wed, 22 Jul 2026 17:07:36 +0800 Subject: [PATCH 055/240] [Refactor]: clean up diffusion model runner surface (#5218) Signed-off-by: Huang, Zeyu <11222265+fhfuih@users.noreply.github.com> Co-authored-by: Hongsheng Liu --- .../diffusion/test_diffusion_model_runner.py | 26 +++-- .../worker/diffusion_model_runner.py | 101 ++++++++---------- 2 files changed, 58 insertions(+), 69 deletions(-) diff --git a/tests/diffusion/test_diffusion_model_runner.py b/tests/diffusion/test_diffusion_model_runner.py index 5d4ef5d72..e82b8f7f0 100644 --- a/tests/diffusion/test_diffusion_model_runner.py +++ b/tests/diffusion/test_diffusion_model_runner.py @@ -98,7 +98,7 @@ class _ChunkStepPipeline: return output -def _make_request(skip_cache_refresh: bool = True): +def _make_request(): sampling_params = SimpleNamespace( generator=None, seed=None, @@ -109,7 +109,6 @@ def _make_request(skip_cache_refresh: bool = True): request_id="req-test", prompt="a prompt", sampling_params=sampling_params, - skip_cache_refresh=skip_cache_refresh, kv_sender_info=None, ) @@ -120,7 +119,6 @@ def _make_request_with_params(req_id: str, sampling_params): prompt=f"prompt-{req_id}", prompts=[f"prompt-{req_id}"], sampling_params=sampling_params, - skip_cache_refresh=True, ) @@ -199,7 +197,7 @@ def test_execute_stepwise_streaming_returns_chunks_at_boundaries(monkeypatch): runner.pipeline = _ChunkStepPipeline(chunks) runner.od_config.streaming_output = True runner.od_config.step_execution = True - req = _make_request(skip_cache_refresh=True) + req = _make_request() req.request_id = "req" monkeypatch.setattr(model_runner_module, "set_forward_context", _noop_forward_context) @@ -235,7 +233,7 @@ def test_execute_stepwise_streaming_returns_chunks_at_boundaries(monkeypatch): def test_execute_model_skips_cache_summary_without_active_cache_backend(monkeypatch): """Guard cache diagnostics with runtime backend state to avoid stale-config crashes.""" runner = _make_runner(cache_backend=None, cache_backend_name="cache_dit") - req = _make_request(skip_cache_refresh=True) + req = _make_request() cache_summary_calls = [] @@ -259,11 +257,18 @@ def test_execute_model_skips_cache_summary_without_active_cache_backend(monkeypa @hardware_test(res={"cuda": "L4"}, num_cards=1) def test_execute_model_emits_cache_summary_with_active_cache_dit_backend(monkeypatch): class _EnabledCacheBackend: + def __init__(self): + self.refresh_calls = [] + def is_enabled(self): return True - runner = _make_runner(cache_backend=_EnabledCacheBackend(), cache_backend_name="cache_dit") - req = _make_request(skip_cache_refresh=True) + def refresh(self, pipeline, num_inference_steps, verbose=True): + self.refresh_calls.append((pipeline, num_inference_steps, verbose)) + + cache_backend = _EnabledCacheBackend() + runner = _make_runner(cache_backend=cache_backend, cache_backend_name="cache_dit") + req = _make_request() cache_summary_calls = [] @@ -281,6 +286,7 @@ def test_execute_model_emits_cache_summary_with_active_cache_dit_backend(monkeyp assert output.output == "ok" assert cache_summary_calls == [(runner.pipeline, True)] + assert cache_backend.refresh_calls == [(runner.pipeline, 4, True)] @pytest.mark.core_model @@ -288,7 +294,7 @@ def test_execute_model_emits_cache_summary_with_active_cache_dit_backend(monkeyp def test_execute_model_passes_single_request_batch_to_non_admission_pipeline(monkeypatch): runner = _make_runner(cache_backend=None, cache_backend_name="none") runner.pipeline = _SingleRequestBatchPipeline() - req = _make_request(skip_cache_refresh=True) + req = _make_request() monkeypatch.setattr(model_runner_module, "set_forward_context", _noop_forward_context) @@ -304,7 +310,7 @@ def test_execute_model_passes_single_request_batch_to_non_admission_pipeline(mon def test_execute_model_accepts_bare_diffusion_output_from_single_request_pipeline(monkeypatch): runner = _make_runner(cache_backend=None, cache_backend_name="none") runner.pipeline = _SingleRequestDiffusionOutputPipeline() - req = _make_request(skip_cache_refresh=True) + req = _make_request() monkeypatch.setattr(model_runner_module, "set_forward_context", _noop_forward_context) @@ -515,7 +521,7 @@ def test_execute_model_runs_forward_after_kv_receive(monkeypatch): """execute_model runs the pipeline forward after the KV receive step.""" runner = _make_runner(cache_backend=None, cache_backend_name=None) runner.kv_transfer_manager.receive_multi_kv_cache_distributed = lambda *a, **k: True - req = _make_request(skip_cache_refresh=True) + req = _make_request() monkeypatch.setattr(model_runner_module, "set_forward_context", _noop_forward_context) monkeypatch.setattr(model_runner_module.current_omni_platform, "reset_peak_memory_stats", lambda: None) diff --git a/vllm_omni/diffusion/worker/diffusion_model_runner.py b/vllm_omni/diffusion/worker/diffusion_model_runner.py index 4ac9afe7a..34a191506 100644 --- a/vllm_omni/diffusion/worker/diffusion_model_runner.py +++ b/vllm_omni/diffusion/worker/diffusion_model_runner.py @@ -12,13 +12,13 @@ from __future__ import annotations import copy import time -from collections.abc import Iterable -from contextlib import nullcontext +from collections.abc import Callable +from contextlib import AbstractContextManager, nullcontext from typing import Any import torch from torch.profiler import record_function -from vllm.config import LoadConfig +from vllm.config import LoadConfig, VllmConfig from vllm.logger import init_logger from vllm.utils.mem_utils import DeviceMemoryProfiler, GiB_bytes @@ -49,6 +49,7 @@ from vllm_omni.diffusion.worker.utils import ( merge_stage_durations, ) from vllm_omni.distributed.omni_connectors.kv_transfer_manager import OmniKVTransferManager +from vllm_omni.inputs.data import OmniDiffusionSamplingParams from vllm_omni.platforms import current_omni_platform from vllm_omni.worker.omni_connector_model_runner_mixin import OmniConnectorModelRunnerMixin @@ -103,7 +104,7 @@ class DiffusionModelRunner(OmniConnectorModelRunnerMixin): def __init__( self, - vllm_config, + vllm_config: VllmConfig, od_config: OmniDiffusionConfig, device: torch.device, ): @@ -118,10 +119,11 @@ class DiffusionModelRunner(OmniConnectorModelRunnerMixin): self.vllm_config = vllm_config self.od_config = od_config self.device = device - self.pipeline = None - self.cache_backend = None - self.offload_backend = None - self.prompt_embed_cache = None + self.pipeline: Any | None = None + self.cache_backend: Any | None = None + self.offload_backend: Any | None = None + self.prompt_embed_cache: Any | None = None + self.input_batch: InputBatch | None = None # Cache for per-request stepwise state. self.state_cache: dict[str, StepRequestState] = {} @@ -140,7 +142,7 @@ class DiffusionModelRunner(OmniConnectorModelRunnerMixin): ) @property - def target_device(self) -> torch.device | None: + def _target_device(self) -> torch.device | None: return getattr(self.pipeline, "device", None) def _compile_transformer(self, attr_name: str) -> None: @@ -161,12 +163,12 @@ class DiffusionModelRunner(OmniConnectorModelRunnerMixin): def load_model( self, - memory_pool_context_fn: callable | None = None, + memory_pool_context_fn: Callable[[str], AbstractContextManager[Any]] | None = None, load_format: str = "default", custom_pipeline_name: str | None = None, ) -> None: """ - Load the diffusion model, apply compilation and offloading. + Launch the diffusion pipeline, applying compilation, offloading, and caching. Args: memory_pool_context_fn: Optional function that returns a context manager @@ -186,9 +188,9 @@ class DiffusionModelRunner(OmniConnectorModelRunnerMixin): "cpu" if self.od_config.enable_cpu_offload or self.od_config.enable_layerwise_offload else str(self.device) ) - def get_memory_context(): + def get_memory_context() -> AbstractContextManager[Any]: if memory_pool_context_fn is not None: - return memory_pool_context_fn(tag="weights") + return memory_pool_context_fn("weights") return nullcontext() # Load model within forward context @@ -217,13 +219,13 @@ class DiffusionModelRunner(OmniConnectorModelRunnerMixin): logger.warning("streaming_output=True requires step_execution=True; enabling step execution.") self.od_config.step_execution = True - if getattr(self.od_config, "step_execution", False) and not self.supports_step_mode(): + if getattr(self.od_config, "step_execution", False) and not self._supports_step_mode(): raise ValueError( "step_execution=True requires a pipeline implementing " "prepare_encode(), denoise_step(), step_scheduler(), and post_decode(); " f"{self.od_config.model_class_name} does not support that contract." ) - if self.od_config.streaming_output and not self.supports_step_mode(): + if self.od_config.streaming_output and not self._supports_step_mode(): raise ValueError( "streaming_output=True requires step execution support; " f"{self.od_config.model_class_name} does not support that contract." @@ -287,17 +289,19 @@ class DiffusionModelRunner(OmniConnectorModelRunnerMixin): logger.info("Model runner: Initialization complete.") - def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - """Load weights into the pipeline.""" - return self.pipeline.load_weights(weights) - def clear_prompt_embed_cache(self) -> None: - """Evict all cached text-encoder outputs (e.g. between training epochs).""" + """Evict all cached text-encoder outputs (e.g. between training epochs). + + Kept primarily for extension purposes. + """ if self.prompt_embed_cache is not None: self.prompt_embed_cache.clear() def get_prompt_embed_cache_stats(self) -> dict | None: - """Return hit/miss statistics for the prompt-embedding cache, if enabled.""" + """Return hit/miss statistics for the prompt-embedding cache, if enabled. + + Kept primarily for extension purposes. + """ if self.prompt_embed_cache is None: return None return self.prompt_embed_cache.stats() @@ -346,29 +350,32 @@ class DiffusionModelRunner(OmniConnectorModelRunnerMixin): if use_prefetch and self._kv_prefetch_enabled: self.kv_transfer_manager.consume_and_distribute_kv_cache( req, - target_device=self.target_device, + target_device=self._target_device, ) else: self.kv_transfer_manager.receive_multi_kv_cache_distributed( req, cfg_kv_collect_func=getattr(od_config, "cfg_kv_collect_func", None), - target_device=self.target_device if use_prefetch else getattr(self.pipeline, "device", None), + target_device=self._target_device if use_prefetch else getattr(self.pipeline, "device", None), ) kv_recv_ms = (time.perf_counter() - kv_recv_t0) * 1000 logger.debug("KV recv for %s %.1fms", req.request_id, kv_recv_ms) # Kick off the next request's prefetch (+ H2D) to overlap this forward. if use_prefetch and self._kv_prefetch_enabled and kv_prefetch_job is not None: - self.kv_transfer_manager.start_prefetch(kv_prefetch_job, self.target_device) + self.kv_transfer_manager.start_prefetch(kv_prefetch_job, self._target_device) - if req.sampling_params.generator is None and req.sampling_params.seed is not None: - if req.sampling_params.generator_device is not None: - gen_device = req.sampling_params.generator_device + self._initialize_generator(req.sampling_params) + + def _initialize_generator(self, sampling_params: OmniDiffusionSamplingParams) -> None: + if sampling_params.generator is None and sampling_params.seed is not None: + if sampling_params.generator_device is not None: + gen_device = sampling_params.generator_device elif self.device.type == "cpu": gen_device = "cpu" else: gen_device = self.device - req.sampling_params.generator = torch.Generator(device=gen_device).manual_seed(req.sampling_params.seed) + sampling_params.generator = torch.Generator(device=gen_device).manual_seed(sampling_params.seed) def _refresh_cache_for_requests( self, @@ -377,11 +384,7 @@ class DiffusionModelRunner(OmniConnectorModelRunnerMixin): od_config: OmniDiffusionConfig, ) -> None: first_req = reqs[0] - if ( - getattr(first_req, "skip_cache_refresh", False) - or self.cache_backend is None - or not self.cache_backend.is_enabled() - ): + if self.cache_backend is None or not self.cache_backend.is_enabled(): return # Refresh cache context if needed. Batch admission groups requests by @@ -499,8 +502,6 @@ class DiffusionModelRunner(OmniConnectorModelRunnerMixin): self, state: StepRequestState, output: DiffusionOutput, - *, - is_primary: bool, ) -> None: merge_stage_durations( state, @@ -564,7 +565,7 @@ class DiffusionModelRunner(OmniConnectorModelRunnerMixin): # Step-wise execution # ------------------------------------------------------------------ - def supports_step_mode(self) -> bool: + def _supports_step_mode(self) -> bool: """Return whether current pipeline supports step execution.""" return self.pipeline is not None and supports_step_execution(self.pipeline) @@ -593,7 +594,7 @@ class DiffusionModelRunner(OmniConnectorModelRunnerMixin): self.kv_transfer_manager.receive_multi_kv_cache_distributed( state_req, cfg_kv_collect_func=getattr(self.od_config, "cfg_kv_collect_func", None), - target_device=self.target_device, + target_device=self._target_device, ) self.state_cache[request_id] = new_state resolved.append(new_state) @@ -615,15 +616,7 @@ class DiffusionModelRunner(OmniConnectorModelRunnerMixin): # process new reqs for state in states: if state.request_id in new_request_ids: - # set generator - if state.sampling.generator is None and state.sampling.seed is not None: - if state.sampling.generator_device is not None: - gen_device = state.sampling.generator_device - elif self.device.type == "cpu": - gen_device = "cpu" - else: - gen_device = self.device - state.sampling.generator = torch.Generator(device=gen_device).manual_seed(state.sampling.seed) + self._initialize_generator(state.sampling) clear_pipeline_stage_durations(self.pipeline) # encode self.pipeline.prepare_encode(state) @@ -644,7 +637,7 @@ class DiffusionModelRunner(OmniConnectorModelRunnerMixin): states: list[StepRequestState], input_batch: InputBatch, interrupted: bool = False, - ): + ) -> None: """Step-after update: clear cached state for completed request.""" gathered_latents = torch.cat([state.latents for state in states], dim=0) if ( @@ -663,19 +656,10 @@ class DiffusionModelRunner(OmniConnectorModelRunnerMixin): if interrupted or state.request_denoise_completed: self.state_cache.pop(state.request_id, None) - def _prepare_attn_metadata(self, input_batch: InputBatch) -> Any: - model_state = getattr(self, "model_state", None) - if model_state is None: - return {} - prepare_attn = getattr(model_state, "prepare_attn", None) - if not callable(prepare_attn): - return {} - return prepare_attn(input_batch) - def execute_stepwise(self, scheduler_output: DiffusionSchedulerOutput) -> BatchRunnerOutput: """Execute one step for one scheduled request and return runner output.""" assert self.pipeline is not None, "Model not loaded. Call load_model() first." - if not self.supports_step_mode(): + if not self._supports_step_mode(): raise ValueError("Current pipeline does not support step execution.") # Stepwise mode only supports the basic state-driven denoise path for now. # Request-mode extras such as cache backends, editing inputs, and @@ -692,7 +676,7 @@ class DiffusionModelRunner(OmniConnectorModelRunnerMixin): if new_request_ids and not had_active_states and is_primary and current_omni_platform.is_available(): current_omni_platform.reset_peak_memory_stats() input_batch = self._prepare_batch_inputs(states, new_request_ids) - attn_metadata = self._prepare_attn_metadata(input_batch) + attn_metadata = {} with set_forward_context( vllm_config=self.vllm_config, @@ -741,7 +725,6 @@ class DiffusionModelRunner(OmniConnectorModelRunnerMixin): self._attach_stepwise_metrics( req, result, - is_primary=is_primary, ) else: result = None From 8fc9df4331eea40bc1741a34e5cb61a1af3d04b0 Mon Sep 17 00:00:00 2001 From: wangyu <53896905+yenuo26@users.noreply.github.com> Date: Wed, 22 Jul 2026 17:29:59 +0800 Subject: [PATCH 056/240] [CI][Bugfix] remove module-level skip_if_gated_repo_inaccessible in flux kontext e2e (#5250) (#5297) Signed-off-by: wangyu <410167048@qq.com> --- tests/e2e/online_serving/test_flux_kontext_expansion.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/e2e/online_serving/test_flux_kontext_expansion.py b/tests/e2e/online_serving/test_flux_kontext_expansion.py index 4a8a9ab98..5714ef711 100644 --- a/tests/e2e/online_serving/test_flux_kontext_expansion.py +++ b/tests/e2e/online_serving/test_flux_kontext_expansion.py @@ -5,7 +5,6 @@ and are supported by the FluxKontext model. import pytest -from tests.helpers import skip_if_gated_repo_inaccessible from tests.helpers.mark import hardware_marks from tests.helpers.media import generate_synthetic_image from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler, dummy_messages_from_mix_data @@ -15,7 +14,6 @@ pytestmark = [pytest.mark.diffusion, pytest.mark.slow] EDIT_PROMPT = "Transform this modern, geometrist image into a Vincent van Gogh style impressionist painting." NEGATIVE_PROMPT = "blurry, low quality, modern, geometrist" MODEL = "black-forest-labs/FLUX.1-Kontext-dev" -skip_if_gated_repo_inaccessible(MODEL) PARALLEL_FEATURE_MARKS = hardware_marks(res={"cuda": "L4"}, num_cards=2) From 633ebbd8a41c149b673eb06ec461bae5294fc071 Mon Sep 17 00:00:00 2001 From: Yukim1 <121286183+zwhzzz0821@users.noreply.github.com> Date: Wed, 22 Jul 2026 17:44:27 +0800 Subject: [PATCH 057/240] [Refactor] Migrate Step-Audio2 online test to deploy config (#5309) Signed-off-by: zwhzzz0821 <2831474076@qq.com> --- .../stage_configs/step_audio2_ci.yaml | 81 ------------------- .../test_step_audio2_expansion.py | 47 ++++------- tests/test_config_factory.py | 28 +++++++ vllm_omni/deploy/step_audio2_ci.yaml | 31 +++++++ 4 files changed, 74 insertions(+), 113 deletions(-) delete mode 100644 tests/e2e/online_serving/stage_configs/step_audio2_ci.yaml create mode 100644 vllm_omni/deploy/step_audio2_ci.yaml diff --git a/tests/e2e/online_serving/stage_configs/step_audio2_ci.yaml b/tests/e2e/online_serving/stage_configs/step_audio2_ci.yaml deleted file mode 100644 index 703a7cdcc..000000000 --- a/tests/e2e/online_serving/stage_configs/step_audio2_ci.yaml +++ /dev/null @@ -1,81 +0,0 @@ -# Stage config for Step-Audio2 CI tests -# Stage 0: Thinker (audio understanding → text + audio tokens) -# Stage 1: Token2Wav (audio tokens → waveform) - -# The following config has been verified on 2x 24GB GPU (L4/RTX3090/RTX4090). -stage_args: - - stage_id: 0 - runtime: - process: true - devices: "0" - max_batch_size: 1 - engine_args: - model_stage: thinker - model_arch: StepAudio2ThinkerForConditionalGeneration - tensor_parallel_size: 1 - hf_overrides: - architectures: ["StepAudio2ThinkerForConditionalGeneration"] - - worker_cls: vllm_omni.worker.gpu_ar_worker.GPUARWorker - scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler - max_model_len: 1024 - max_num_batched_tokens: 1024 - max_num_seqs: 1 - gpu_memory_utilization: 0.7 - skip_mm_profiling: true - enforce_eager: true - trust_remote_code: true - enable_prefix_caching: false - engine_output_type: text # Output text + audio tokens - is_comprehension: true - final_output: true - final_output_type: text - default_sampling_params: - temperature: 0.7 - top_p: 0.9 - top_k: -1 - max_tokens: 256 - seed: 42 - detokenize: true - repetition_penalty: 1.05 - - - stage_id: 1 - runtime: - process: true - devices: "0" - max_batch_size: 1 - engine_args: - - model_stage: token2wav - model_arch: StepAudio2Token2WavModel - worker_cls: vllm_omni.worker.gpu_generation_worker.GPUGenerationWorker - scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler - hf_overrides: - architectures: ["StepAudio2Token2WavModel"] - gpu_memory_utilization: 0.2 - enforce_eager: true - trust_remote_code: true - enable_prefix_caching: false - max_num_batched_tokens: 1024 - engine_output_type: audio # Output waveform - engine_input_source: [0] - custom_process_input_func: vllm_omni.model_executor.stage_input_processors.step_audio2.thinker2token2wav - final_output: true - final_output_type: audio - default_sampling_params: - temperature: 0.0 - top_p: 1.0 - top_k: -1 - max_tokens: 1 - seed: 42 - detokenize: false - -runtime: - enabled: true - defaults: - window_size: -1 - max_inflight: 1 - edges: - - from: 0 - to: 1 - window_size: -1 diff --git a/tests/e2e/online_serving/test_step_audio2_expansion.py b/tests/e2e/online_serving/test_step_audio2_expansion.py index 05beb1131..9c8280b59 100644 --- a/tests/e2e/online_serving/test_step_audio2_expansion.py +++ b/tests/e2e/online_serving/test_step_audio2_expansion.py @@ -3,34 +3,32 @@ """ E2E expansion tests for Step-Audio2 online serving (weekly CI). -Tests speech-to-speech translation via /v1/chat/completions. +Tests audio-to-text generation via /v1/chat/completions. Request shape follows examples/online_serving/step_audio2/openai_chat_completion_client.py. """ import os -from pathlib import Path import pytest from tests.helpers.mark import hardware_test from tests.helpers.media import generate_synthetic_audio from tests.helpers.runtime import OmniServerParams +from tests.helpers.stage_config import get_deploy_config_path os.environ.setdefault("VLLM_WORKER_MULTIPROC_METHOD", "spawn") MODEL = "stepfun-ai/Step-Audio-2-mini" -STAGE_CONFIG = str(Path(__file__).parent / "stage_configs" / "step_audio2_ci.yaml") +DEPLOY_CONFIG = get_deploy_config_path("step_audio2_ci.yaml") SAMPLE_RATE = 16000 SEED = 42 -# pyttsx3 synthetic input phrase; Step-Audio2 text output includes codec tokens -# () so we validate audio only, not text↔audio similarity. SYNTH_PHRASE = "how are you" TEST_PARAMS = [ pytest.param( OmniServerParams( model=MODEL, - stage_config_path=STAGE_CONFIG, + server_args=["--deploy-config", DEPLOY_CONFIG, "--trust-remote-code"], env_dict={"VLLM_IMAGE_FETCH_TIMEOUT": "60"}, ), id="step_audio2", @@ -64,19 +62,11 @@ def _default_sampling_params_list() -> list[dict]: "detokenize": True, "repetition_penalty": 1.05, }, - { - "temperature": 0.0, - "top_p": 1.0, - "top_k": -1, - "max_tokens": 1, - "seed": SEED, - "detokenize": False, - }, ] -def _build_s2st_request_config(omni_server, audio_base64: str) -> dict: - """Build audio-to-audio (S2ST) chat completion request config.""" +def _build_audio_to_text_request_config(omni_server, audio_base64: str) -> dict: + """Build an audio-to-text chat completion request config.""" audio_url = f"data:audio/wav;base64,{audio_base64}" return { "model": omni_server.model, @@ -86,7 +76,7 @@ def _build_s2st_request_config(omni_server, audio_base64: str) -> dict: "content": [ { "type": "text", - "text": "You are an audio processing assistant. Listen and repeat the audio content.", + "text": "You are a speech recognition assistant. Transcribe the audio accurately.", } ], }, @@ -94,26 +84,19 @@ def _build_s2st_request_config(omni_server, audio_base64: str) -> dict: "role": "user", "content": [ {"type": "audio_url", "audio_url": {"url": audio_url}}, - {"type": "text", "text": "Please listen to this audio and repeat its content."}, + {"type": "text", "text": "Please transcribe the audio content."}, ], }, - {"role": "assistant", "content": ""}, ], - # Thinker text includes undetokenized audio codec placeholders; assert audio only. - "modalities": ["audio"], - "audio_ref_text": SYNTH_PHRASE, + "modalities": ["text"], "sampling_params_list": _default_sampling_params_list(), - "extra_body": { - "continue_final_message": True, - "add_generation_prompt": False, - }, } @hardware_test(res={"cuda": "H100"}, num_cards=1) -def test_single_s2st_request(omni_server, openai_client) -> None: - """Test single speech-to-speech request via chat completions API.""" - request_config = _build_s2st_request_config( +def test_single_audio_to_text_request(omni_server, openai_client) -> None: + """Test a single audio-to-text request via the chat completions API.""" + request_config = _build_audio_to_text_request_config( omni_server, _synthetic_audio_base64(duration_sec=2), ) @@ -122,9 +105,9 @@ def test_single_s2st_request(omni_server, openai_client) -> None: @hardware_test(res={"cuda": "H100"}, num_cards=1) @pytest.mark.parametrize("request_num", [2, 4]) -def test_concurrent_s2st_requests(omni_server, openai_client, request_num: int) -> None: - """Test concurrent speech-to-speech requests.""" - request_config = _build_s2st_request_config( +def test_concurrent_audio_to_text_requests(omni_server, openai_client, request_num: int) -> None: + """Test concurrent audio-to-text requests.""" + request_config = _build_audio_to_text_request_config( omni_server, _synthetic_audio_base64(duration_sec=2), ) diff --git a/tests/test_config_factory.py b/tests/test_config_factory.py index e08349140..c70b8c487 100644 --- a/tests/test_config_factory.py +++ b/tests/test_config_factory.py @@ -1174,6 +1174,7 @@ stages: ("step_audio_2.yaml", "step_audio_2", 2, "audio", None), ("step_audio_2_asr.yaml", "step_audio_2_asr", 1, "text", "step_audio_2_asr"), ("step_audio_2_async_chunk.yaml", "step_audio_2", 2, "audio", None), + ("step_audio2_ci.yaml", "step_audio_2_asr", 1, "text", "step_audio_2_asr"), ("hunyuan_video_15.yaml", "hunyuan_video_15", 1, "video", None), ("wan2_2_ti2v.yaml", "wan2_2_ti2v", 1, "video", None), ], @@ -1241,6 +1242,33 @@ stages: assert deploy.stages[0].tensor_parallel_size == 2 assert deploy.stages[1].devices == "1" + def test_step_audio2_online_ci_deploy_is_single_thinker(self): + deploy = load_deploy_config(Path(get_deploy_config_path("step_audio2_ci.yaml"))) + + assert deploy.pipeline == "step_audio_2_asr" + assert deploy.async_chunk is False + assert deploy.trust_remote_code is True + assert deploy.enable_prefix_caching is False + assert len(deploy.stages) == 1 + + stage = deploy.stages[0] + assert stage.devices == "0" + assert stage.tensor_parallel_size == 1 + assert stage.max_model_len == 1024 + assert stage.max_num_batched_tokens == 1024 + assert stage.max_num_seqs == 1 + assert stage.gpu_memory_utilization == 0.7 + assert stage.skip_mm_profiling is True + assert stage.enforce_eager is True + assert stage.async_scheduling is False + assert stage.engine_extras["hf_overrides"]["architectures"] == ["StepAudio2ThinkerForConditionalGeneration"] + + stages = merge_pipeline_deploy(OMNI_PIPELINES[deploy.pipeline], deploy) + assert len(stages) == 1 + assert stages[0].final_output is True + assert stages[0].final_output_type == "text" + assert stages[0].yaml_engine_args["engine_output_type"] == "text" + def test_step_audio2_dispatches_sync_and_async_chunk_processors(self): pipeline = resolve_pipeline_config("step_audio_2") assert isinstance(pipeline, PipelineConfig) diff --git a/vllm_omni/deploy/step_audio2_ci.yaml b/vllm_omni/deploy/step_audio2_ci.yaml new file mode 100644 index 000000000..4373dfaa8 --- /dev/null +++ b/vllm_omni/deploy/step_audio2_ci.yaml @@ -0,0 +1,31 @@ +# Step-Audio2 online CI deploy: thinker only (audio/text -> text). +# +# The two-stage topology can deadlock when an audio-to-text request skips the +# Token2Wav stage, so this config explicitly selects the ASR pipeline. +# Keeps the legacy 1x 24GB CI sizing; validated here on 1x H100-80GB. +pipeline: step_audio_2_asr +async_chunk: false +trust_remote_code: true +enable_prefix_caching: false + +stages: + - stage_id: 0 + devices: "0" + tensor_parallel_size: 1 + max_model_len: 1024 + max_num_batched_tokens: 1024 + max_num_seqs: 1 + gpu_memory_utilization: 0.7 + skip_mm_profiling: true + enforce_eager: true + async_scheduling: false + hf_overrides: + architectures: ["StepAudio2ThinkerForConditionalGeneration"] + default_sampling_params: + temperature: 0.7 + top_p: 0.9 + top_k: -1 + max_tokens: 256 + seed: 42 + detokenize: true + repetition_penalty: 1.05 From 1984346bc7e53100f751fcf749d967306105fbf5 Mon Sep 17 00:00:00 2001 From: rein yang <73573651+R2-Y@users.noreply.github.com> Date: Wed, 22 Jul 2026 17:59:44 +0800 Subject: [PATCH 058/240] [Refactor] [0/N] Scheduler clean dead code (#5312) Signed-off-by: R2-Y --- docs/api/README.md | 1 - .../sched/test_omni_scheduling_coordinator.py | 453 ++---------------- vllm_omni/core/sched/omni_ar_scheduler.py | 25 +- .../core/sched/omni_generation_scheduler.py | 35 +- vllm_omni/core/sched/omni_scheduler_mixin.py | 1 - .../core/sched/omni_scheduling_coordinator.py | 240 ++-------- vllm_omni/core/sched/output.py | 2 +- 7 files changed, 93 insertions(+), 664 deletions(-) diff --git a/docs/api/README.md b/docs/api/README.md index 09f6a6e24..4a34e3f31 100644 --- a/docs/api/README.md +++ b/docs/api/README.md @@ -62,7 +62,6 @@ Engine classes for offline and online inference. Core scheduling and caching components. -- [vllm_omni.core.sched.omni_ar_scheduler.KVCacheTransferData][] - [vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler][] - [vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler][] - [vllm_omni.core.sched.output.OmniCachedRequestData][] diff --git a/tests/core/sched/test_omni_scheduling_coordinator.py b/tests/core/sched/test_omni_scheduling_coordinator.py index 068cbd362..3685fa5a2 100644 --- a/tests/core/sched/test_omni_scheduling_coordinator.py +++ b/tests/core/sched/test_omni_scheduling_coordinator.py @@ -4,6 +4,9 @@ These tests use mock request objects and mock queues. They do not require GPU, vLLM runtime, or any connector. + +Chunk waiting (WAITING_FOR_CHUNK / process_pending_chunks) lives on +OmniChunkTransferAdapter — see tests/distributed/omni_connectors/. """ from __future__ import annotations @@ -11,6 +14,7 @@ from __future__ import annotations import unittest from types import SimpleNamespace +import pytest import torch import vllm_omni.core.sched.omni_scheduling_coordinator as coord_mod @@ -19,6 +23,8 @@ from vllm_omni.core.sched.omni_scheduling_coordinator import ( uses_full_payload_input_coordinator, ) +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + # ------------------------------------------------------------------ # # Mock helpers # ------------------------------------------------------------------ # @@ -27,7 +33,6 @@ from vllm_omni.core.sched.omni_scheduling_coordinator import ( class _RequestStatus: WAITING = "waiting" RUNNING = "running" - WAITING_FOR_CHUNK = "waiting_for_chunk" WAITING_FOR_INPUT = "waiting_for_input" FINISHED_STOPPED = "finished_stopped" @@ -185,105 +190,12 @@ class TestFullPayloadCoordinatorSelection(unittest.TestCase): ) -class TestChunkCoordinatorStateTransition(unittest.TestCase): - """Test 5: process_pending_chunks transitions WAITING_FOR_CHUNK → target.""" - - def test_ready_request_transitions_to_waiting(self): - coord = OmniSchedulingCoordinator(scheduler_max_num_seqs=10, stage_id=1, async_chunk=True) - - req = _make_request("r1", status=RequestStatus.WAITING_FOR_CHUNK) - waiting = MockQueue([req]) - running: list = [] - - coord.process_pending_chunks( - waiting, - running, - chunk_ready_req_ids={"r1"}, - chunk_finished_req_ids=set(), - ) - - self.assertEqual(req.status, RequestStatus.WAITING) - self.assertIn("r1", coord.requests_with_ready_chunks) - - def test_non_ready_stays_waiting_for_chunk(self): - coord = OmniSchedulingCoordinator(scheduler_max_num_seqs=10, stage_id=1, async_chunk=True) - - req = _make_request("r1", status=RequestStatus.WAITING_FOR_CHUNK) - waiting = MockQueue([req]) - running: list = [] - - coord.process_pending_chunks( - waiting, - running, - chunk_ready_req_ids=set(), - chunk_finished_req_ids=set(), - ) - - self.assertEqual(req.status, RequestStatus.WAITING_FOR_CHUNK) - - def test_stage_0_is_noop(self): - coord = OmniSchedulingCoordinator(scheduler_max_num_seqs=10, stage_id=0) - req = _make_request("r1") - waiting = MockQueue([req]) - running: list = [] - - coord.process_pending_chunks( - waiting, - running, - chunk_ready_req_ids={"r1"}, - chunk_finished_req_ids=set(), - ) - self.assertNotEqual(req.status, RequestStatus.WAITING_FOR_CHUNK) - - -class TestChunkCoordinatorRestoreQueues(unittest.TestCase): - """Test 6: restore_queues returns waiting-for-chunk requests.""" - - def test_restore(self): - coord = OmniSchedulingCoordinator(scheduler_max_num_seqs=10, stage_id=1) - - r1 = _make_request("r1") - r2 = _make_request("r2") - coord._waiting_for_chunk_waiting.append(r1) - coord._waiting_for_chunk_running.append(r2) - - waiting = MockQueue() - running: list = [] - - coord.restore_queues(waiting, running) - - self.assertIn(r1, waiting) - self.assertIn(r2, running) - self.assertEqual(len(coord._waiting_for_chunk_waiting), 0) - self.assertEqual(len(coord._waiting_for_chunk_running), 0) - - -class TestChunkCoordinatorFinishedSignal(unittest.TestCase): - """Test 8: chunk_finished_req_ids → finished_requests.""" - - def test_finished_signal(self): - coord = OmniSchedulingCoordinator(scheduler_max_num_seqs=10, stage_id=1, async_chunk=True) - - req = _make_request("r1", status=RequestStatus.WAITING_FOR_CHUNK) - waiting = MockQueue([req]) - running: list = [] - - coord.process_pending_chunks( - waiting, - running, - chunk_ready_req_ids={"r1"}, - chunk_finished_req_ids={"r1"}, - ) - - self.assertIn("r1", coord.finished_requests) - - -class TestChunkCoordinatorUpdateRequestMetadata(unittest.TestCase): +class TestCoordinatorUpdateRequestMetadata(unittest.TestCase): """Test update_request_metadata applies scheduling metadata to requests.""" def test_ar_mode_no_longer_sets_additional_information(self): """AR mode only processes scheduling metadata, not full payloads.""" - coord = OmniSchedulingCoordinator(scheduler_max_num_seqs=10, stage_id=1) + coord = OmniSchedulingCoordinator(stage_id=1) req = _make_request("r1") requests = {"r1": req} @@ -300,7 +212,7 @@ class TestChunkCoordinatorUpdateRequestMetadata(unittest.TestCase): self.assertIsNone(getattr(req, "additional_information", None)) def test_generation_mode(self): - coord = OmniSchedulingCoordinator(scheduler_max_num_seqs=10, stage_id=1) + coord = OmniSchedulingCoordinator(stage_id=1) req = _make_request("r1") req.prompt_token_ids = [0, 0, 0] @@ -313,7 +225,6 @@ class TestChunkCoordinatorUpdateRequestMetadata(unittest.TestCase): request_metadata = { "r1": { "code_predictor_codes": [10, 20, 30], - "left_context_size": 25, } } @@ -325,10 +236,9 @@ class TestChunkCoordinatorUpdateRequestMetadata(unittest.TestCase): self.assertEqual(req._all_token_ids, [10, 20, 30]) self.assertEqual(req._output_token_ids, []) self.assertIsNone(req.additional_information) - self.assertEqual(req._omni_initial_model_buffer, {"meta": {"left_context_size": 25}}) def test_generation_mode_flattens_tensor_code_predictor_codes(self): - coord = OmniSchedulingCoordinator(scheduler_max_num_seqs=10, stage_id=1) + coord = OmniSchedulingCoordinator(stage_id=1) req = _make_request("r1") req.prompt_token_ids = [9] @@ -349,7 +259,7 @@ class TestChunkCoordinatorUpdateRequestMetadata(unittest.TestCase): self.assertEqual(req._output_token_ids, []) def test_generation_mode_flattens_nested_code_predictor_codes(self): - coord = OmniSchedulingCoordinator(scheduler_max_num_seqs=10, stage_id=1) + coord = OmniSchedulingCoordinator(stage_id=1) req = _make_request("r1") req.prompt_token_ids = [9] @@ -370,53 +280,30 @@ class TestChunkCoordinatorUpdateRequestMetadata(unittest.TestCase): self.assertEqual(req._output_token_ids, []) -class TestChunkCoordinatorPostprocess(unittest.TestCase): - """Test postprocess_scheduler_output clears ready chunks.""" - - def test_clear_ready(self): - coord = OmniSchedulingCoordinator(scheduler_max_num_seqs=10, stage_id=1) - coord.requests_with_ready_chunks = {"r1", "r2"} - - new_req = SimpleNamespace(req_id="r1") - cached_reqs = SimpleNamespace(req_ids=["r2"]) - scheduler_output = SimpleNamespace( - scheduled_new_reqs=[new_req], - scheduled_cached_reqs=cached_reqs, - ) - - coord.postprocess_scheduler_output(scheduler_output) - - self.assertEqual(coord.requests_with_ready_chunks, set()) - - class TestWaitingForInputTransition(unittest.TestCase): - """Test B8: process_pending_full_payload_inputs transitions WAITING_FOR_INPUT.""" + """Test process_pending_full_payload_inputs transitions WAITING_FOR_INPUT.""" def test_transition_on_recv(self): - coord = OmniSchedulingCoordinator(scheduler_max_num_seqs=10, stage_id=1) + coord = OmniSchedulingCoordinator(stage_id=1) req = _make_request("r1", status=RequestStatus.WAITING_FOR_INPUT) waiting = MockQueue([req]) - running: list = [] coord.process_pending_full_payload_inputs( waiting, - running, stage_recv_req_ids={"r1"}, ) self.assertEqual(req.status, RequestStatus.WAITING) def test_stays_waiting_for_input_if_not_received(self): - coord = OmniSchedulingCoordinator(scheduler_max_num_seqs=10, stage_id=1) + coord = OmniSchedulingCoordinator(stage_id=1) req = _make_request("r1", status=RequestStatus.WAITING_FOR_INPUT) waiting = MockQueue([req]) - running: list = [] coord.process_pending_full_payload_inputs( waiting, - running, stage_recv_req_ids=set(), ) @@ -424,49 +311,39 @@ class TestWaitingForInputTransition(unittest.TestCase): self.assertEqual(len(coord._waiting_for_input), 1) def test_stage_0_is_noop(self): - coord = OmniSchedulingCoordinator(scheduler_max_num_seqs=10, stage_id=0) + coord = OmniSchedulingCoordinator(stage_id=0) req = _make_request("r1", status=RequestStatus.WAITING_FOR_INPUT) waiting = MockQueue([req]) - running: list = [] coord.process_pending_full_payload_inputs( waiting, - running, stage_recv_req_ids={"r1"}, ) self.assertEqual(req.status, RequestStatus.WAITING_FOR_INPUT) def test_restore_queues_includes_waiting_for_input(self): - coord = OmniSchedulingCoordinator(scheduler_max_num_seqs=10, stage_id=1) + coord = OmniSchedulingCoordinator(stage_id=1) r1 = _make_request("r1") coord._waiting_for_input.append(r1) waiting = MockQueue() - running: list = [] - coord.restore_queues(waiting, running) + coord.restore_queues(waiting) self.assertIn(r1, waiting) self.assertEqual(len(coord._waiting_for_input), 0) def test_full_payload_mode_auto_transitions_waiting_to_waiting_for_input(self): - """In full_payload_mode (async_chunk=False), fresh WAITING requests on - non-Stage-0 should be transitioned to WAITING_FOR_INPUT.""" - coord = OmniSchedulingCoordinator( - scheduler_max_num_seqs=10, - stage_id=1, - async_chunk=False, - ) + """Fresh downstream WAITING requests enter WAITING_FOR_INPUT.""" + coord = OmniSchedulingCoordinator(stage_id=1) req = _make_request("r1", status=RequestStatus.WAITING) waiting = MockQueue([req]) - running: list = [] coord.process_pending_full_payload_inputs( waiting, - running, stage_recv_req_ids=set(), ) @@ -474,37 +351,14 @@ class TestWaitingForInputTransition(unittest.TestCase): self.assertEqual(len(coord._waiting_for_input), 1) self.assertEqual(len(coord.pending_input_registrations), 1) - def test_async_chunk_mode_does_not_auto_transition(self): - """In async_chunk mode, fresh WAITING requests should NOT be - transitioned to WAITING_FOR_INPUT.""" - coord = OmniSchedulingCoordinator( - scheduler_max_num_seqs=10, - stage_id=1, - async_chunk=True, - ) - - req = _make_request("r1", status=RequestStatus.WAITING) - waiting = MockQueue([req]) - running: list = [] - - coord.process_pending_full_payload_inputs( - waiting, - running, - stage_recv_req_ids=set(), - ) - - self.assertEqual(req.status, RequestStatus.WAITING) - def test_pending_input_registrations(self): - coord = OmniSchedulingCoordinator(scheduler_max_num_seqs=10, stage_id=1) + coord = OmniSchedulingCoordinator(stage_id=1) req = _make_request("r1", status=RequestStatus.WAITING_FOR_INPUT) waiting = MockQueue([req]) - running: list = [] coord.process_pending_full_payload_inputs( waiting, - running, stage_recv_req_ids=set(), ) @@ -512,18 +366,13 @@ class TestWaitingForInputTransition(unittest.TestCase): self.assertEqual(coord.pending_input_registrations[0].request_id, "r1") def test_idle_cycles_retain_received_marker_before_request_appears(self): - coord = OmniSchedulingCoordinator( - scheduler_max_num_seqs=10, - stage_id=1, - async_chunk=False, - ) + coord = OmniSchedulingCoordinator(stage_id=1) coord._full_payload_input_received.add("late") coord.finished_requests.add("late") waiting = MockQueue() - running: list = [] - coord.process_pending_full_payload_inputs(waiting, running, stage_recv_req_ids=set()) + coord.process_pending_full_payload_inputs(waiting, stage_recv_req_ids=set()) self.assertIn("late", coord._full_payload_input_received) self.assertIn("late", coord.finished_requests) @@ -531,7 +380,7 @@ class TestWaitingForInputTransition(unittest.TestCase): late_req = _make_request("late", status=RequestStatus.WAITING) waiting.add_request(late_req) - coord.process_pending_full_payload_inputs(waiting, running, stage_recv_req_ids=set()) + coord.process_pending_full_payload_inputs(waiting, stage_recv_req_ids=set()) self.assertEqual(late_req.status, RequestStatus.WAITING) self.assertEqual(coord.pending_input_registrations, []) @@ -542,65 +391,18 @@ class TestWaitingForInputTransition(unittest.TestCase): class TestTimeoutDetection(unittest.TestCase): """Regression tests for orphaned pending-recv timeout detection. - Covers the full lifecycle: - 1. Request enters WAITING_FOR_CHUNK from either waiting or running queue - 2. restore_queues() moves it back to the scheduler queue - 3. Timeout fires via collect_timed_out_request_ids() - 4. Scheduler removes from both queues and calls _free_request() + Covers WAITING_FOR_INPUT lifecycle timeouts. Chunk waiting timeouts are + covered by OmniChunkTransferAdapter tests. """ - def test_waiting_since_recorded_on_chunk_wait(self): - """_waiting_since is set when a request enters WAITING_FOR_CHUNK.""" - coord = OmniSchedulingCoordinator( - scheduler_max_num_seqs=10, - stage_id=1, - async_chunk=True, - ) - req = _make_request("r1", status=RequestStatus.WAITING) - waiting = MockQueue([req]) - - coord.process_pending_chunks( - waiting, - [], - chunk_ready_req_ids=set(), - chunk_finished_req_ids=set(), - ) - - self.assertIn("r1", coord._waiting_since) - self.assertEqual(req.status, RequestStatus.WAITING_FOR_CHUNK) - - def test_waiting_since_cleared_on_chunk_arrival(self): - """_waiting_since is cleared when a chunk arrives.""" - coord = OmniSchedulingCoordinator( - scheduler_max_num_seqs=10, - stage_id=1, - async_chunk=True, - ) - req = _make_request("r1", status=RequestStatus.WAITING_FOR_CHUNK) - waiting = MockQueue([req]) - - coord.process_pending_chunks( - waiting, - [], - chunk_ready_req_ids={"r1"}, - chunk_finished_req_ids=set(), - ) - - self.assertNotIn("r1", coord._waiting_since) - def test_waiting_since_recorded_on_input_wait(self): """_waiting_since is set when a request enters WAITING_FOR_INPUT.""" - coord = OmniSchedulingCoordinator( - scheduler_max_num_seqs=10, - stage_id=1, - async_chunk=False, - ) + coord = OmniSchedulingCoordinator(stage_id=1) req = _make_request("r1", status=RequestStatus.WAITING) waiting = MockQueue([req]) coord.process_pending_full_payload_inputs( waiting, - [], stage_recv_req_ids=set(), ) @@ -608,11 +410,7 @@ class TestTimeoutDetection(unittest.TestCase): def test_waiting_since_cleared_on_input_arrival(self): """_waiting_since is cleared when input data arrives.""" - coord = OmniSchedulingCoordinator( - scheduler_max_num_seqs=10, - stage_id=1, - async_chunk=False, - ) + coord = OmniSchedulingCoordinator(stage_id=1) req = _make_request("r1", status=RequestStatus.WAITING_FOR_INPUT) coord._waiting_for_input.append(req) coord._waiting_since["r1"] = 0.0 @@ -620,7 +418,6 @@ class TestTimeoutDetection(unittest.TestCase): waiting = MockQueue() coord.process_pending_full_payload_inputs( waiting, - [], stage_recv_req_ids={"r1"}, ) @@ -629,10 +426,7 @@ class TestTimeoutDetection(unittest.TestCase): def test_collect_timed_out_request_ids_no_timeout(self): """No IDs returned when nothing has timed out.""" - coord = OmniSchedulingCoordinator( - scheduler_max_num_seqs=10, - stage_id=1, - ) + coord = OmniSchedulingCoordinator(stage_id=1) import time coord._waiting_since["r1"] = time.monotonic() @@ -642,10 +436,7 @@ class TestTimeoutDetection(unittest.TestCase): def test_collect_timed_out_request_ids_expired(self): """Timed-out IDs are returned and _waiting_since is cleared.""" - coord = OmniSchedulingCoordinator( - scheduler_max_num_seqs=10, - stage_id=1, - ) + coord = OmniSchedulingCoordinator(stage_id=1) coord._waiting_since["r1"] = 0.0 # epoch → definitely expired coord._waiting_since["r2"] = 0.0 @@ -662,29 +453,19 @@ class TestTimeoutDetection(unittest.TestCase): def test_collect_removes_from_coordinator_queues(self): """Timed-out requests are defensively removed from internal queues.""" - coord = OmniSchedulingCoordinator( - scheduler_max_num_seqs=10, - stage_id=1, - ) + coord = OmniSchedulingCoordinator(stage_id=1) r1 = _make_request("r1") - r2 = _make_request("r2") - coord._waiting_for_chunk_waiting.append(r1) - coord._waiting_for_input.append(r2) + coord._waiting_for_input.append(r1) coord._waiting_since["r1"] = 0.0 - coord._waiting_since["r2"] = 0.0 result = coord.collect_timed_out_request_ids(timeout_s=1.0) - self.assertEqual(result, {"r1", "r2"}) - self.assertEqual(len(coord._waiting_for_chunk_waiting), 0) + self.assertEqual(result, {"r1"}) self.assertEqual(len(coord._waiting_for_input), 0) def test_free_finished_request_clears_waiting_since(self): """free_finished_request clears coordinator lifecycle markers.""" - coord = OmniSchedulingCoordinator( - scheduler_max_num_seqs=10, - stage_id=1, - ) + coord = OmniSchedulingCoordinator(stage_id=1) coord._waiting_since["r1"] = 0.0 coord._full_payload_input_received.add("r1") coord.finished_requests.add("r1") @@ -693,174 +474,6 @@ class TestTimeoutDetection(unittest.TestCase): self.assertNotIn("r1", coord._full_payload_input_received) self.assertNotIn("r1", coord.finished_requests) - def test_timeout_from_running_queue_full_lifecycle(self): - """End-to-end: request from running → WAITING_FOR_CHUNK → restore → - timeout → removed from running list. - - This is the critical regression case: WAITING_FOR_CHUNK requests - that originated from self.running are placed back into self.running - by restore_queues(), but their status remains WAITING_FOR_CHUNK. - The scheduler must remove from BOTH queues unconditionally. - """ - coord = OmniSchedulingCoordinator( - scheduler_max_num_seqs=10, - stage_id=1, - async_chunk=True, - ) - - # 1) Request starts in running queue with WAITING status - req = _make_request("r1", status=RequestStatus.WAITING) - running = [req] - waiting = MockQueue() - - # 2) process_pending_chunks: moves to WAITING_FOR_CHUNK - coord.process_pending_chunks( - waiting, - running, - chunk_ready_req_ids=set(), - chunk_finished_req_ids=set(), - ) - self.assertEqual(req.status, RequestStatus.WAITING_FOR_CHUNK) - self.assertIn("r1", coord._waiting_since) - self.assertEqual(len(coord._waiting_for_chunk_running), 1) - - # 3) restore_queues: back to running (status stays WAITING_FOR_CHUNK) - coord.restore_queues(waiting, running) - self.assertIn(req, running) - self.assertEqual(len(coord._waiting_for_chunk_running), 0) - self.assertEqual(req.status, RequestStatus.WAITING_FOR_CHUNK) - - # 4) Force timeout by setting _waiting_since to epoch - coord._waiting_since["r1"] = 0.0 - - timed_out_ids = coord.collect_timed_out_request_ids(timeout_s=1.0) - self.assertEqual(timed_out_ids, {"r1"}) - - # 5) Scheduler removes from both queues (simulating the scheduler path) - timed_out_id_set = {id(req)} - running = [r for r in running if id(r) not in timed_out_id_set] - waiting.remove_requests([req]) - - self.assertNotIn(req, running) - self.assertEqual(len(waiting), 0) - - def test_timeout_from_waiting_queue_full_lifecycle(self): - """End-to-end: request from waiting → WAITING_FOR_CHUNK → restore → - timeout → removed from waiting queue.""" - coord = OmniSchedulingCoordinator( - scheduler_max_num_seqs=10, - stage_id=1, - async_chunk=True, - ) - - req = _make_request("r1", status=RequestStatus.WAITING) - waiting = MockQueue([req]) - running: list = [] - - coord.process_pending_chunks( - waiting, - running, - chunk_ready_req_ids=set(), - chunk_finished_req_ids=set(), - ) - self.assertEqual(len(coord._waiting_for_chunk_waiting), 1) - - coord.restore_queues(waiting, running) - self.assertIn(req, waiting) - - coord._waiting_since["r1"] = 0.0 - timed_out_ids = coord.collect_timed_out_request_ids(timeout_s=1.0) - self.assertEqual(timed_out_ids, {"r1"}) - - waiting.remove_requests([req]) - self.assertEqual(len(waiting), 0) - - -class TestOverflowPreemption(unittest.TestCase): - """Tests for P1-1: overflow requests must get WAITING status. - - Overflow happens when multiple WAITING_FOR_CHUNK requests in - ``_waiting_for_chunk_running`` receive their chunk in the same cycle. - ``_process_chunk_queue`` restores them to RUNNING (``continue`` - path) while RUNNING requests without chunks are moved out. If the - net result exceeds ``scheduler_max_num_seqs``, the tail is pushed - to ``waiting_queue`` and must have status == WAITING. - """ - - def test_overflow_sets_waiting_status(self): - coord = OmniSchedulingCoordinator( - scheduler_max_num_seqs=1, - stage_id=1, - async_chunk=True, - ) - - # r1 is currently RUNNING in the queue. - # r2, r3 were previously moved to _waiting_for_chunk_running. - r1 = _make_request("r1", status=RequestStatus.RUNNING) - r2 = _make_request("r2", status=RequestStatus.WAITING_FOR_CHUNK) - r3 = _make_request("r3", status=RequestStatus.WAITING_FOR_CHUNK) - - running = [r1] - waiting = MockQueue([]) - coord._waiting_for_chunk_running.extend([r2, r3]) - - # restore_queues puts r2, r3 back into running - coord.restore_queues(waiting, running) - self.assertEqual(len(running), 3) - - # Now process_pending_chunks with r2, r3 chunks ready: - # _process_chunk_queue will: - # r1 (RUNNING) → no chunk → move to _waiting_for_chunk_running - # r2 (WAITING_FOR_CHUNK, chunk ready) → set RUNNING, stay in running - # r3 (WAITING_FOR_CHUNK, chunk ready) → set RUNNING, stay in running - # running = [r2, r3], len=2 > max=1 → overflow - coord.process_pending_chunks( - waiting, - running, - chunk_ready_req_ids={"r2", "r3"}, - chunk_finished_req_ids=set(), - ) - - self.assertEqual(len(running), 1) - self.assertEqual(len(waiting), 1) - overflow_req = list(waiting)[0] - self.assertEqual( - overflow_req.status, - RequestStatus.WAITING, - f"Overflowed request should have WAITING status, got {overflow_req.status}", - ) - - def test_overflow_does_not_strand_request(self): - """Without the fix, the overflowed request would keep its - RUNNING status in the waiting queue and never be re-scheduled.""" - coord = OmniSchedulingCoordinator( - scheduler_max_num_seqs=1, - stage_id=1, - async_chunk=True, - ) - - r1 = _make_request("r1", status=RequestStatus.WAITING_FOR_CHUNK) - r2 = _make_request("r2", status=RequestStatus.WAITING_FOR_CHUNK) - coord._waiting_for_chunk_running.extend([r1, r2]) - - running: list = [] - waiting = MockQueue([]) - - coord.restore_queues(waiting, running) - self.assertEqual(len(running), 2) - - coord.process_pending_chunks( - waiting, - running, - chunk_ready_req_ids={"r1", "r2"}, - chunk_finished_req_ids=set(), - ) - - self.assertEqual(len(running), 1) - self.assertEqual(len(waiting), 1) - for req in waiting: - self.assertNotEqual(req.status, RequestStatus.RUNNING, "Overflowed request must not keep RUNNING status") - if __name__ == "__main__": unittest.main() diff --git a/vllm_omni/core/sched/omni_ar_scheduler.py b/vllm_omni/core/sched/omni_ar_scheduler.py index 74d09ab60..912b67bd7 100644 --- a/vllm_omni/core/sched/omni_ar_scheduler.py +++ b/vllm_omni/core/sched/omni_ar_scheduler.py @@ -1,7 +1,6 @@ from __future__ import annotations from collections import defaultdict -from dataclasses import asdict, dataclass from time import time from typing import Any @@ -36,17 +35,6 @@ from vllm_omni.outputs import OmniConnectorOutput logger = init_logger(__name__) -@dataclass -class KVCacheTransferData: - request_id: str - layer_blocks: dict[str, Any] - block_ids: list[int] - metadata: dict[str, Any] - - def to_dict(self) -> dict[str, Any]: - return asdict(self) - - class OmniARScheduler(OmniSchedulerMixin, VLLMScheduler): """Synchronous AutoRegressive scheduler for vLLM-Omni. This class is also used as a base class for the OmniARAsyncScheduler and holds most of the @@ -87,9 +75,7 @@ class OmniARScheduler(OmniSchedulerMixin, VLLMScheduler): self.input_coordinator: OmniSchedulingCoordinator | None = None if uses_full_payload_input_coordinator(model_config): self.input_coordinator = OmniSchedulingCoordinator( - scheduler_max_num_seqs=self.vllm_config.scheduler_config.max_num_seqs, stage_id=getattr(model_config, "stage_id", 0), - async_chunk=False, ) self._latest_omni_connector_output: OmniConnectorOutput | None = None # Snapshot prompt length for each streaming input update @@ -175,6 +161,7 @@ class OmniARScheduler(OmniSchedulerMixin, VLLMScheduler): if criteria_type == "prefill_finished": if confirmed_computed >= request.num_prompt_tokens: self.transfer_triggered_requests.add(request.request_id) + self._mark_request_for_kv_transfer(request.request_id, confirmed_computed) actually_queued = request.request_id in self.requests_needing_kv_transfer @@ -220,7 +207,6 @@ class OmniARScheduler(OmniSchedulerMixin, VLLMScheduler): queue.remove(req) self._consume_pending_connector_output(model_mode="ar") self._process_pending_input_timeouts() - if self.chunk_transfer_adapter: self.chunk_transfer_adapter.process_pending_chunks( self.waiting, self.running, scheduler_requests=self.requests @@ -247,7 +233,7 @@ class OmniARScheduler(OmniSchedulerMixin, VLLMScheduler): scheduler_requests=self.requests, ) if self.input_coordinator: - self.input_coordinator.restore_queues(self.waiting, self.running) + self.input_coordinator.restore_queues(self.waiting) try: # Late import to avoid circulars in some launch modes from .output import OmniNewRequestData @@ -655,7 +641,7 @@ class OmniARScheduler(OmniSchedulerMixin, VLLMScheduler): Tuple of (req_id, client_index) for requests that were aborted. Will not include any that were already finished. """ - + # TODO(yrr): chunk transfer adapter & input_coordinator unified to one if self.chunk_transfer_adapter: self.chunk_transfer_adapter.finish_requests(request_ids, finished_status, self.requests) @@ -823,11 +809,6 @@ class OmniARScheduler(OmniSchedulerMixin, VLLMScheduler): finally: self._free_input_coordinator_request(request_id) - def _free_blocks(self, request: Request): - # Helper to match base class structure if not directly available - # VLLMScheduler has _free_blocks - super()._free_blocks(request) - def _mark_request_for_kv_transfer(self, req_id: str, seq_len: int) -> None: """Mark a request as needing KV cache transfer when it finishes.""" # Avoid duplicate marking (if already pending in queue) diff --git a/vllm_omni/core/sched/omni_generation_scheduler.py b/vllm_omni/core/sched/omni_generation_scheduler.py index 973b8c55a..77a07be7d 100644 --- a/vllm_omni/core/sched/omni_generation_scheduler.py +++ b/vllm_omni/core/sched/omni_generation_scheduler.py @@ -50,14 +50,12 @@ class OmniGenerationScheduler(OmniSchedulerMixin, VLLMScheduler): self.input_coordinator: OmniSchedulingCoordinator | None = None if uses_full_payload_input_coordinator(model_config): self.input_coordinator = OmniSchedulingCoordinator( - scheduler_max_num_seqs=self.vllm_config.scheduler_config.max_num_seqs, stage_id=getattr(model_config, "stage_id", 0), - async_chunk=False, ) self._latest_omni_connector_output: OmniConnectorOutput | None = None def schedule(self, throttle_prefills: bool = False) -> SchedulerOutput: - """Diffusion fast path: + """One-shot generation fast path: - Feed all input tokens of the request at once (if 0, allocate 1 placeholder token). - If the token budget cannot be satisfied at once, fall back to the @@ -81,7 +79,7 @@ class OmniGenerationScheduler(OmniSchedulerMixin, VLLMScheduler): cached_prompt_token_ids: dict[str, list[int]] = {} cached_additional_information: dict[str, dict | None] = {} - # Temporary queue: preserve waiting order, do not disturb non-diffusion requests + # Temporary queue: preserve waiting order while requests await input. skipped_waiting_requests = create_request_queue(self.policy) req_index = 0 self._consume_pending_connector_output(model_mode="generation") @@ -144,8 +142,8 @@ class OmniGenerationScheduler(OmniSchedulerMixin, VLLMScheduler): if already_finished_reqs: self.running = remove_all(self.running, already_finished_reqs) - # Fast path selection and scheduling (treat all as diffusion requests, - # independent of pooling_params) + # Fast path selection and scheduling for one-shot generation requests, + # independent of pooling_params. while ( self.waiting and token_budget > 0 @@ -172,9 +170,6 @@ class OmniGenerationScheduler(OmniSchedulerMixin, VLLMScheduler): skipped_waiting_requests.prepend_request(request) continue - # Uniformly treat as diffusion. A feature flag can be added later - # via config or request tag. - # Allocate all input tokens for the request in one shot # (allocate 1 placeholder if zero) required_tokens = max(len(request.prompt_token_ids), 1) @@ -219,7 +214,7 @@ class OmniGenerationScheduler(OmniSchedulerMixin, VLLMScheduler): else: res = super().schedule(throttle_prefills) if self.input_coordinator: - self.input_coordinator.restore_queues(self.waiting, self.running) + self.input_coordinator.restore_queues(self.waiting) return self._wrap_omni_scheduler_output(res) # Compute common prefix blocks (aligned with v1) @@ -346,7 +341,7 @@ class OmniGenerationScheduler(OmniSchedulerMixin, VLLMScheduler): scheduler_requests=self.requests, ) if self.input_coordinator: - self.input_coordinator.restore_queues(self.waiting, self.running) + self.input_coordinator.restore_queues(self.waiting) return self._wrap_omni_scheduler_output(scheduler_output) @@ -397,24 +392,12 @@ class OmniGenerationScheduler(OmniSchedulerMixin, VLLMScheduler): finally: self._free_input_coordinator_request(request.request_id) - """ - Scheduler for the diffusion model. - This scheduler is modified to stop the request immediately for the diffusion model. - This is because the diffusion model can generate the final image/audio in one step. - Note: This is just a minimal modification to the original scheduler, - and there should be some further efforts to optimize the scheduler. - The original scheduler is still used for the AR model. - """ - def update_from_output( self, scheduler_output: SchedulerOutput, model_runner_output: OmniModelRunnerOutput, ) -> dict[int, EngineCoreOutputs]: - """Update the scheduler state based on the model runner output. - - This method is modified to stop the request immediately for the diffusion model. - """ + """Update scheduler state and finish completed one-shot work.""" sampled_token_ids = model_runner_output.sampled_token_ids logprobs = model_runner_output.logprobs prompt_logprobs_dict = model_runner_output.prompt_logprobs_dict @@ -496,7 +479,8 @@ class OmniGenerationScheduler(OmniSchedulerMixin, VLLMScheduler): finish_reason = None routed_experts = None - # Diffusion request: completes in one step; mark finished and free resources + # One-shot generation request: finish after its current input unit + # has been fully processed. if ( request.status == RequestStatus.FINISHED_STOPPED or (self.chunk_transfer_adapter is None and request.num_computed_tokens >= request.num_prompt_tokens) @@ -509,7 +493,6 @@ class OmniGenerationScheduler(OmniSchedulerMixin, VLLMScheduler): request.status = RequestStatus.FINISHED_STOPPED # Optional: set a stop_reason for front-end clarity # (does not affect protocol) - request.stop_reason = request.stop_reason # or "generation_done" stopped = True if stopped: diff --git a/vllm_omni/core/sched/omni_scheduler_mixin.py b/vllm_omni/core/sched/omni_scheduler_mixin.py index be73db9c3..fc7e1b626 100644 --- a/vllm_omni/core/sched/omni_scheduler_mixin.py +++ b/vllm_omni/core/sched/omni_scheduler_mixin.py @@ -68,7 +68,6 @@ class OmniSchedulerMixin: ) input_coordinator.process_pending_full_payload_inputs( self.waiting, - self.running, connector_output.stage_recv_req_ids if connector_output else set(), ) diff --git a/vllm_omni/core/sched/omni_scheduling_coordinator.py b/vllm_omni/core/sched/omni_scheduling_coordinator.py index af6c73d6d..fc21be991 100644 --- a/vllm_omni/core/sched/omni_scheduling_coordinator.py +++ b/vllm_omni/core/sched/omni_scheduling_coordinator.py @@ -1,13 +1,11 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Scheduling-side coordination for chunk and full_payload input waiting. +"""Scheduling-side coordination for full_payload input waiting. -Manages WAITING_FOR_CHUNK and WAITING_FOR_INPUT state transitions -based on readiness signals from OmniConnectorOutput, without ever -calling connector.put()/get(). +Manages WAITING_FOR_INPUT state transitions based on readiness signals +from OmniConnectorOutput, without ever calling connector.put()/get(). -This replaces the scheduling half of OmniChunkTransferAdapter; the -transport half lives in OmniConnectorModelRunnerMixin. +Chunk waiting (WAITING_FOR_CHUNK) lives on OmniChunkTransferAdapter. """ from __future__ import annotations @@ -64,7 +62,7 @@ _FULL_PAYLOAD_INPUT_STAGES: frozenset[tuple[str, str]] = frozenset( def uses_full_payload_input_coordinator(model_config: Any) -> bool: - """Returns True iff this stage parks pending requests in + """Returns True if this stage parks pending requests in WAITING_FOR_INPUT awaiting a full_payload delivery on the worker connector. Gated by (model_arch, model_stage) — see _FULL_PAYLOAD_INPUT_STAGES for the @@ -83,31 +81,20 @@ def uses_full_payload_input_coordinator(model_config: Any) -> bool: class OmniSchedulingCoordinator: - """Pure-scheduling coordinator for chunk and full_payload input waiting. + """Pure-scheduling coordinator for full_payload input waiting. The Scheduler owns an instance of this class. It consumes readiness signals produced by the Model Runner's ``OmniConnectorModelRunnerMixin`` - (via ``OmniConnectorOutput``) and manages ``WAITING_FOR_CHUNK`` and - ``WAITING_FOR_INPUT`` state transitions accordingly. + (via ``OmniConnectorOutput``) and manages ``WAITING_FOR_INPUT`` state + transitions accordingly. """ - def __init__(self, scheduler_max_num_seqs: int, stage_id: int = 0, async_chunk: bool = False): + def __init__(self, stage_id: int = 0): self._stage_id = stage_id - self._scheduler_max_num_seqs = scheduler_max_num_seqs - self._async_chunk = async_chunk self.finished_requests: set[str] = set() - self.requests_with_ready_chunks: set[str] = set() self._full_payload_input_received: set[str] = set() - self._waiting_for_chunk_waiting: deque[Any] = deque() - self._waiting_for_chunk_running: deque[Any] = deque() - - # Request IDs that were newly registered for chunk recv this cycle. - # The engine/Model Runner should call register_chunk_recv() for these - # so the bg thread starts polling. - self.pending_chunk_registrations: list[Any] = [] - # Requests waiting for full_payload stage input (WAITING_FOR_INPUT). self._waiting_for_input: deque[Any] = deque() # Per-cycle list of minimal handles to ship to the model runner so it @@ -117,67 +104,22 @@ class OmniSchedulingCoordinator: self.pending_input_registrations: list[OmniChunkRecvHandle] = [] # Monotonic timestamp recording when each request first entered - # WAITING_FOR_CHUNK or WAITING_FOR_INPUT. Used by - # collect_timed_out_request_ids() to detect orphaned waits. + # WAITING_FOR_INPUT. Used by collect_timed_out_request_ids() to + # detect orphaned waits. self._waiting_since: dict[str, float] = {} # ------------------------------------------------------------------ # # Core scheduling methods # ------------------------------------------------------------------ # - def process_pending_chunks( - self, - waiting_queue: Any, - running_queue: list[Request], - chunk_ready_req_ids: set[str], - chunk_finished_req_ids: set[str], - ) -> None: - """Transition requests whose chunks have arrived. - - Args: - waiting_queue: Scheduler's waiting request queue. - running_queue: Scheduler's running request list. - chunk_ready_req_ids: IDs with a newly arrived chunk this cycle. - chunk_finished_req_ids: IDs whose final chunk has arrived. - """ - if self._stage_id == 0 or not self._async_chunk: - return - - terminal_ready_req_ids = chunk_ready_req_ids.intersection(chunk_finished_req_ids) - self.finished_requests.update(chunk_finished_req_ids - terminal_ready_req_ids) - self.pending_chunk_registrations = [] - - self._process_chunk_queue( - waiting_queue, - self._waiting_for_chunk_waiting, - RequestStatus.WAITING, - chunk_ready_req_ids, - ) - self._process_chunk_queue( - running_queue, - self._waiting_for_chunk_running, - RequestStatus.RUNNING, - chunk_ready_req_ids, - ) - self.finished_requests.update(terminal_ready_req_ids) - - while len(running_queue) > self._scheduler_max_num_seqs: - request = running_queue.pop() - # Must reset status to WAITING so the scheduler treats it as - # schedulable work. KV blocks are NOT freed here (unlike a - # real preemption), so PREEMPTED would be incorrect. - request.status = RequestStatus.WAITING - waiting_queue.prepend_requests([request]) - def process_pending_full_payload_inputs( self, waiting_queue: Any, - running_queue: list[Request], stage_recv_req_ids: set[str], ) -> None: """Manage WAITING_FOR_INPUT lifecycle for full_payload_mode. - For non-Stage-0 stages in full_payload_mode (``async_chunk=False``): + For non-Stage-0 stages in full_payload mode: 1. Fresh WAITING requests are transitioned to WAITING_FOR_INPUT and registered for bg-thread polling. 2. WAITING_FOR_INPUT requests whose data has arrived (in @@ -187,7 +129,7 @@ class OmniSchedulingCoordinator: return self._full_payload_input_received.update(stage_recv_req_ids) - if not self._async_chunk and stage_recv_req_ids: + if stage_recv_req_ids: self.finished_requests.update(stage_recv_req_ids) logger.debug( "[Coordinator stage-%s] full_payload recv -> finished_requests: %s", @@ -206,19 +148,29 @@ class OmniSchedulingCoordinator: remaining.append(request) self._waiting_for_input = remaining - if not self._async_chunk: - to_remove: list[Any] = [] - queue_snapshot = list(waiting_queue) - for request in queue_snapshot: - if request.status == RequestStatus.WAITING: - if request.request_id in self._full_payload_input_received: - continue - if request.request_id in self.requests_with_ready_chunks: - continue - if request.request_id in self.finished_requests: - continue - request.status = RequestStatus.WAITING_FOR_INPUT - self._waiting_since.setdefault(request.request_id, time.monotonic()) + to_remove: list[Any] = [] + queue_snapshot = list(waiting_queue) + for request in queue_snapshot: + if request.status == RequestStatus.WAITING: + if request.request_id in self._full_payload_input_received: + continue + if request.request_id in self.finished_requests: + continue + request.status = RequestStatus.WAITING_FOR_INPUT + self._waiting_since.setdefault(request.request_id, time.monotonic()) + to_remove.append(request) + self._waiting_for_input.append(request) + self.pending_input_registrations.append( + OmniChunkRecvHandle( + request_id=request.request_id, + external_req_id=getattr(request, "external_req_id", None), + ) + ) + elif request.status == RequestStatus.WAITING_FOR_INPUT: + if request.request_id in stage_recv_req_ids: + request.status = RequestStatus.WAITING + self._waiting_since.pop(request.request_id, None) + else: to_remove.append(request) self._waiting_for_input.append(request) self.pending_input_registrations.append( @@ -227,38 +179,15 @@ class OmniSchedulingCoordinator: external_req_id=getattr(request, "external_req_id", None), ) ) - elif request.status == RequestStatus.WAITING_FOR_INPUT: - if request.request_id in stage_recv_req_ids: - request.status = RequestStatus.WAITING - self._waiting_since.pop(request.request_id, None) - else: - to_remove.append(request) - self._waiting_for_input.append(request) - self.pending_input_registrations.append( - OmniChunkRecvHandle( - request_id=request.request_id, - external_req_id=getattr(request, "external_req_id", None), - ) - ) - if to_remove: - # Use the bulk-remove helper: one O(N) sweep instead of N - # repeated O(N) removes from a list-backed queue. - waiting_queue.remove_requests(to_remove) - - def process_pending_full_payload_inputs_legacy( - self, - waiting_queue: Any, - running_queue: list[Request], - stage_recv_req_ids: set[str], - ) -> None: - """Compatibility wrapper for ``process_pending_full_payload_inputs``.""" - self.process_pending_full_payload_inputs(waiting_queue, running_queue, stage_recv_req_ids) + if to_remove: + # Use the bulk-remove helper: one O(N) sweep instead of N + # repeated O(N) removes from a list-backed queue. + waiting_queue.remove_requests(to_remove) def free_finished_request(self, request_id: str) -> None: """Prune internal tracking sets for a freed request to prevent unbounded growth.""" self._full_payload_input_received.discard(request_id) self.finished_requests.discard(request_id) - self.requests_with_ready_chunks.discard(request_id) self._waiting_since.pop(request_id, None) def collect_timed_out_request_ids( @@ -290,22 +219,16 @@ class OmniSchedulingCoordinator: # Defensively remove from coordinator internal queues (may already # be empty if restore_queues() has run). - for queue_attr in ( - "_waiting_for_chunk_waiting", - "_waiting_for_chunk_running", - "_waiting_for_input", - ): - queue = getattr(self, queue_attr) - remaining: deque[Any] = deque() - for request in queue: - if request.request_id not in timed_out_ids: - remaining.append(request) - setattr(self, queue_attr, remaining) + remaining: deque[Any] = deque() + for request in self._waiting_for_input: + if request.request_id not in timed_out_ids: + remaining.append(request) + self._waiting_for_input = remaining for req_id in timed_out_ids: self._waiting_since.pop(req_id, None) logger.warning( - "[Coordinator stage-%s] Request %s timed out waiting for chunk/input (waited > %.0fs)", + "[Coordinator stage-%s] Request %s timed out waiting for input (waited > %.0fs)", self._stage_id, req_id, timeout_s, @@ -316,17 +239,8 @@ class OmniSchedulingCoordinator: def restore_queues( self, waiting_queue: Any, - running_queue: list[Request], ) -> None: - """Return waiting-for-chunk/input requests to scheduling queues.""" - for request in self._waiting_for_chunk_waiting: - waiting_queue.add_request(request) - self._waiting_for_chunk_waiting = deque() - - if self._waiting_for_chunk_running: - running_queue.extend(self._waiting_for_chunk_running) - self._waiting_for_chunk_running = deque() - + """Return waiting-for-input requests to the waiting queue.""" for request in self._waiting_for_input: waiting_queue.add_request(request) self._waiting_for_input = deque() @@ -411,12 +325,6 @@ class OmniSchedulingCoordinator: if model_mode != "ar": new_ids = self._flatten_prompt_token_ids(metadata.get("code_predictor_codes")) - runtime_seed = None - if "left_context_size" in metadata: - runtime_seed = { - "meta": {"left_context_size": metadata["left_context_size"]}, - } - request._omni_initial_model_buffer = runtime_seed if new_ids: request.prompt_token_ids = new_ids request.num_prompt_tokens = len(new_ids) @@ -424,57 +332,3 @@ class OmniSchedulingCoordinator: request._all_token_ids.extend(new_ids) request._output_token_ids.clear() request.num_computed_tokens = 0 - - def postprocess_scheduler_output( - self, - scheduler_output: Any, - requests: dict[str, Request] | None = None, - ) -> None: - """Clear per-cycle ready state after scheduler output is materialized.""" - self._clear_chunk_ready(scheduler_output) - - # ------------------------------------------------------------------ # - # Internal helpers - # ------------------------------------------------------------------ # - - def _process_chunk_queue( - self, - queue: Any, - waiting_for_chunk_list: deque[Any], - target_status: RequestStatus, - chunk_ready_req_ids: set[str], - ) -> None: - queue_snapshot = list(queue) - for request in queue_snapshot: - if request.status != RequestStatus.WAITING_FOR_CHUNK: - if request.request_id in self.requests_with_ready_chunks: - continue - if request.request_id in self.finished_requests: - continue - if request.status == RequestStatus.WAITING_FOR_INPUT: - continue - if request.request_id in chunk_ready_req_ids: - self.requests_with_ready_chunks.add(request.request_id) - continue - self.pending_chunk_registrations.append(request) - request.status = RequestStatus.WAITING_FOR_CHUNK - self._waiting_since.setdefault(request.request_id, time.monotonic()) - else: - if request.request_id in chunk_ready_req_ids: - request.status = target_status - self.requests_with_ready_chunks.add(request.request_id) - self._waiting_since.pop(request.request_id, None) - continue - queue.remove(request) - waiting_for_chunk_list.append(request) - - def _clear_chunk_ready(self, scheduler_output: Any) -> None: - if scheduler_output.scheduled_new_reqs: - for req_data in scheduler_output.scheduled_new_reqs: - self.requests_with_ready_chunks.discard( - getattr(req_data, "req_id", None), - ) - - if scheduler_output.scheduled_cached_reqs: - for req_id in scheduler_output.scheduled_cached_reqs.req_ids: - self.requests_with_ready_chunks.discard(req_id) diff --git a/vllm_omni/core/sched/output.py b/vllm_omni/core/sched/output.py index 29cd87299..ec017b11c 100644 --- a/vllm_omni/core/sched/output.py +++ b/vllm_omni/core/sched/output.py @@ -73,7 +73,7 @@ class OmniCachedRequestData(CachedRequestData): @dataclass class OmniChunkRecvHandle: - """Minimal identifier carried from scheduler to runner for chunk-recv + """Minimal identifier carried from scheduler to runner for input-receive registration. The runner's ``register_chunk_recv`` only consumes ``request_id`` and From 68ee52f4e739e31ff9fcab3f3a76aafde5614429 Mon Sep 17 00:00:00 2001 From: Fyrgo <182356120+Fyrgo8@users.noreply.github.com> Date: Wed, 22 Jul 2026 18:35:37 +0800 Subject: [PATCH 059/240] [Model] MiniCPM-o 4.5: optimize SigLIP position IDs (#5130) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 道路自信 <182356120+daoluzixin@users.noreply.github.com> Co-authored-by: 道路自信 <182356120+daoluzixin@users.noreply.github.com> Co-authored-by: amy-why-3459 --- .../minicpmo_4_5/test_siglip_position_ids.py | 214 ++++++++++++++++++ .../minicpmo_4_5/minicpmo_4_5_omni_llm.py | 99 +++++--- 2 files changed, 283 insertions(+), 30 deletions(-) create mode 100644 tests/model_executor/models/minicpmo_4_5/test_siglip_position_ids.py diff --git a/tests/model_executor/models/minicpmo_4_5/test_siglip_position_ids.py b/tests/model_executor/models/minicpmo_4_5/test_siglip_position_ids.py new file mode 100644 index 000000000..ba2544c8d --- /dev/null +++ b/tests/model_executor/models/minicpmo_4_5/test_siglip_position_ids.py @@ -0,0 +1,214 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from __future__ import annotations + +from dataclasses import dataclass + +import pytest +import torch + +from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni_llm import SiglipVisionEmbeddings + +pytestmark = [pytest.mark.core_model] + + +@dataclass(frozen=True) +class _VisionConfig: + hidden_size: int = 4 + image_size: int = 8 + patch_size: int = 2 + num_channels: int = 3 + + +def _reference_position_ids( + embeddings: SiglipVisionEmbeddings, + patch_attention_mask: torch.BoolTensor, + tgt_sizes: torch.IntTensor | None, +) -> torch.Tensor: + batch_size = patch_attention_mask.size(0) + position_ids = torch.zeros((batch_size, patch_attention_mask[0].numel()), dtype=torch.long) + boundaries = torch.arange( + 1 / embeddings.num_patches_per_side, + 1.0, + 1 / embeddings.num_patches_per_side, + ) + + for batch_idx, patch_mask in enumerate(patch_attention_mask): + if tgt_sizes is None: + patch_grid_height = int(patch_mask[:, 0].sum()) + patch_grid_width = int(patch_mask[0].sum()) + else: + patch_grid_height = int(tgt_sizes[batch_idx, 0]) + patch_grid_width = int(tgt_sizes[batch_idx, 1]) + + fractional_coords_h = torch.arange(0, 1 - 1e-6, 1 / patch_grid_height) + fractional_coords_w = torch.arange(0, 1 - 1e-6, 1 / patch_grid_width) + bucket_coords_h = torch.bucketize(fractional_coords_h, boundaries, right=True) + bucket_coords_w = torch.bucketize(fractional_coords_w, boundaries, right=True) + grid_ids = (bucket_coords_h[:, None] * embeddings.num_patches_per_side + bucket_coords_w).flatten() + position_ids[batch_idx, patch_mask.flatten()] = grid_ids + + return position_ids + + +def _mixed_patch_masks() -> torch.BoolTensor: + return torch.tensor( + [ + [ + [True, True, True, False], + [True, True, True, False], + [False, False, False, False], + ], + [ + [True, True, True, True], + [False, False, False, False], + [False, False, False, False], + ], + [ + [True, False, True, True], + [False, True, False, True], + [True, False, False, False], + ], + ], + dtype=torch.bool, + ) + + +@pytest.mark.parametrize("num_patches_per_side", [1, 2, 7, 16, 70]) +@pytest.mark.cpu +def test_grid_ids_match_bucketized_reference(num_patches_per_side: int) -> None: + embeddings = SiglipVisionEmbeddings(_VisionConfig(image_size=num_patches_per_side * 2)) + boundaries = torch.arange( + 1 / num_patches_per_side, + 1.0, + 1 / num_patches_per_side, + ) + target_shapes = [ + (1, 1), + (max(1, num_patches_per_side - 1), num_patches_per_side), + (num_patches_per_side, num_patches_per_side + 1), + (num_patches_per_side * 7, num_patches_per_side * 7), + (num_patches_per_side * 2 + 1, 3), + ] + + for patch_grid_height, patch_grid_width in target_shapes: + fractional_coords_h = torch.arange(0, 1 - 1e-6, 1 / patch_grid_height) + fractional_coords_w = torch.arange(0, 1 - 1e-6, 1 / patch_grid_width) + bucket_coords_h = torch.bucketize(fractional_coords_h, boundaries, right=True) + bucket_coords_w = torch.bucketize(fractional_coords_w, boundaries, right=True) + expected = (bucket_coords_h[:, None] * num_patches_per_side + bucket_coords_w).flatten() + + actual = embeddings._create_grid_position_ids( + patch_grid_height, + patch_grid_width, + boundaries, + ) + + torch.testing.assert_close(actual, expected, rtol=0, atol=0) + + +@pytest.mark.cpu +def test_position_ids_match_reference_for_mixed_and_repeated_grids(mocker) -> None: + embeddings = SiglipVisionEmbeddings(_VisionConfig()) + patch_attention_mask = _mixed_patch_masks() + tgt_sizes = torch.tensor([[2, 3], [1, 4], [2, 3]], dtype=torch.int32) + grid_builder = mocker.spy(embeddings, "_create_grid_position_ids") + + actual = embeddings._create_position_ids(patch_attention_mask, tgt_sizes, device=torch.device("cpu")) + expected = _reference_position_ids(embeddings, patch_attention_mask, tgt_sizes) + + torch.testing.assert_close(actual, expected, rtol=0, atol=0) + assert grid_builder.call_count == 2 + + +@pytest.mark.cpu +def test_position_ids_match_reference_without_target_sizes() -> None: + embeddings = SiglipVisionEmbeddings(_VisionConfig()) + patch_attention_mask = torch.tensor( + [ + [ + [True, True, True, False], + [True, True, True, False], + [False, False, False, False], + ], + [ + [True, True, False, False], + [True, True, False, False], + [True, True, False, False], + ], + ], + dtype=torch.bool, + ) + + actual = embeddings._create_position_ids(patch_attention_mask, None, device=torch.device("cpu")) + expected = _reference_position_ids(embeddings, patch_attention_mask, None) + + torch.testing.assert_close(actual, expected, rtol=0, atol=0) + + +@pytest.mark.cpu +def test_forward_matches_reference_position_embedding() -> None: + torch.manual_seed(0) + embeddings = SiglipVisionEmbeddings(_VisionConfig()) + pixel_values = torch.randn(3, 3, 6, 8) + patch_attention_mask = _mixed_patch_masks() + tgt_sizes = torch.tensor([[2, 3], [1, 4], [2, 3]], dtype=torch.int32) + + with torch.no_grad(): + patch_embeds = embeddings.patch_embedding(pixel_values).flatten(2).transpose(1, 2) + reference_ids = _reference_position_ids(embeddings, patch_attention_mask, tgt_sizes) + expected = patch_embeds + embeddings.position_embedding(reference_ids) + actual = embeddings(pixel_values, patch_attention_mask, tgt_sizes) + + torch.testing.assert_close(actual, expected, rtol=0, atol=0) + + +@pytest.mark.cuda +@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required") +def test_cuda_position_ids_match_reference_for_processor_shapes(mocker) -> None: + embeddings = SiglipVisionEmbeddings(_VisionConfig(image_size=140)).to(device="cuda") + target_shapes = [(40, 25), (32, 32), (40, 25), (28, 37)] + max_num_patches = max(height * width for height, width in target_shapes) + patch_attention_mask = torch.zeros((len(target_shapes), 1, max_num_patches), dtype=torch.bool) + for batch_idx, (height, width) in enumerate(target_shapes): + patch_attention_mask[batch_idx, 0, : height * width] = True + + # Keep the valid-patch count while exercising non-contiguous padding. + patch_attention_mask[0, 0, 5] = False + patch_attention_mask[0, 0, -1] = True + tgt_sizes = torch.tensor(target_shapes, dtype=torch.int32) + expected = _reference_position_ids(embeddings, patch_attention_mask, tgt_sizes) + grid_builder = mocker.spy(embeddings, "_create_grid_position_ids") + + actual = embeddings._create_position_ids( + patch_attention_mask.to(device="cuda"), + tgt_sizes.to(device="cuda"), + device=torch.device("cuda"), + ) + + torch.testing.assert_close(actual.cpu(), expected, rtol=0, atol=0) + assert grid_builder.call_count == 3 + assert all(call.args[2].device.type == "cpu" for call in grid_builder.call_args_list) + + +@pytest.mark.cuda +@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required") +def test_cuda_forward_matches_reference_for_processor_shape() -> None: + torch.manual_seed(0) + embeddings = SiglipVisionEmbeddings(_VisionConfig(image_size=140)).to(device="cuda") + pixel_values = torch.randn(1, 3, 80, 50, device="cuda") + patch_attention_mask = torch.ones((1, 1, 40 * 25), dtype=torch.bool, device="cuda") + tgt_sizes = torch.tensor([[40, 25]], dtype=torch.int32, device="cuda") + + with torch.no_grad(): + patch_embeds = embeddings.patch_embedding(pixel_values).flatten(2).transpose(1, 2) + reference_ids = _reference_position_ids( + embeddings, + patch_attention_mask.cpu(), + tgt_sizes.cpu(), + ).to(device="cuda") + expected = patch_embeds + embeddings.position_embedding(reference_ids) + actual = embeddings(pixel_values, patch_attention_mask, tgt_sizes) + + torch.testing.assert_close(actual, expected) diff --git a/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py b/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py index 5d05e4756..34a4da1ac 100644 --- a/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py +++ b/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py @@ -936,47 +936,86 @@ class SiglipVisionEmbeddings(nn.Module): self.num_positions = self.num_patches self.position_embedding = nn.Embedding(self.num_positions, self.embed_dim) + def _create_grid_position_ids( + self, + patch_grid_height: int, + patch_grid_width: int, + boundaries: torch.Tensor, + ) -> torch.Tensor: + fractional_coords_h = torch.arange( + 0, + 1 - 1e-6, + 1 / patch_grid_height, + device=boundaries.device, + ) + fractional_coords_w = torch.arange( + 0, + 1 - 1e-6, + 1 / patch_grid_width, + device=boundaries.device, + ) + bucket_coords_h = torch.bucketize(fractional_coords_h, boundaries, right=True) + bucket_coords_w = torch.bucketize(fractional_coords_w, boundaries, right=True) + return (bucket_coords_h[:, None] * self.num_patches_per_side + bucket_coords_w).flatten() + + def _create_position_ids( + self, + patch_attention_mask: torch.BoolTensor, + tgt_sizes: torch.IntTensor | None, + device: torch.device, + ) -> torch.Tensor: + batch_size = patch_attention_mask.size(0) + flat_patch_attention_mask = patch_attention_mask.reshape(batch_size, -1).to(device="cpu") + + if tgt_sizes is None: + # As in the legacy path, infer rectangular grid sizes from a (B, H, W) mask. + target_sizes = torch.stack( + ( + patch_attention_mask[:, :, 0].sum(dim=1), + patch_attention_mask[:, 0, :].sum(dim=1), + ), + dim=1, + ) + else: + target_sizes = tgt_sizes + + # Group construction on the host preserves the legacy floating-point + # mapping and is faster than device-side construction for vision grids. + target_sizes_list = target_sizes.detach().to(device="cpu").tolist() + position_ids = torch.zeros(flat_patch_attention_mask.shape, dtype=torch.long) + boundaries = torch.arange( + 1 / self.num_patches_per_side, + 1.0, + 1 / self.num_patches_per_side, + ) + grid_position_ids: dict[tuple[int, int], torch.Tensor] = {} + + for batch_idx, (patch_grid_height, patch_grid_width) in enumerate(target_sizes_list): + grid_shape = (int(patch_grid_height), int(patch_grid_width)) + grid_ids = grid_position_ids.get(grid_shape) + if grid_ids is None: + grid_ids = self._create_grid_position_ids(*grid_shape, boundaries) + grid_position_ids[grid_shape] = grid_ids + + position_ids[batch_idx, flat_patch_attention_mask[batch_idx]] = grid_ids + + return position_ids.to(device=device) + def forward( self, pixel_values: torch.FloatTensor, patch_attention_mask: torch.BoolTensor, tgt_sizes: torch.IntTensor | None = None, ) -> torch.Tensor: - batch_size = pixel_values.size(0) - patch_embeds = self.patch_embedding(pixel_values) embeddings = patch_embeds.flatten(2).transpose(1, 2) - max_im_h, max_im_w = pixel_values.size(2), pixel_values.size(3) - max_nb_patches_h, max_nb_patches_w = max_im_h // self.patch_size, max_im_w // self.patch_size - boundaries = torch.arange(1 / self.num_patches_per_side, 1.0, 1 / self.num_patches_per_side) - position_ids = torch.full( - size=( - batch_size, - max_nb_patches_h * max_nb_patches_w, - ), - fill_value=0, + position_ids = self._create_position_ids( + patch_attention_mask, + tgt_sizes, + device=self.position_embedding.weight.device, ) - for batch_idx, p_attn_mask in enumerate(patch_attention_mask): - if tgt_sizes is not None: - nb_patches_h = tgt_sizes[batch_idx][0] - nb_patches_w = tgt_sizes[batch_idx][1] - else: - nb_patches_h = p_attn_mask[:, 0].sum() - nb_patches_w = p_attn_mask[0].sum() - - fractional_coords_h = torch.arange(0, 1 - 1e-6, 1 / nb_patches_h) - fractional_coords_w = torch.arange(0, 1 - 1e-6, 1 / nb_patches_w) - - bucket_coords_h = torch.bucketize(fractional_coords_h, boundaries, right=True) - bucket_coords_w = torch.bucketize(fractional_coords_w, boundaries, right=True) - - pos_ids = (bucket_coords_h[:, None] * self.num_patches_per_side + bucket_coords_w).flatten() - position_ids[batch_idx][p_attn_mask.view(-1).cpu()] = pos_ids - - position_ids = position_ids.to(self.position_embedding.weight.device) - embeddings = embeddings + self.position_embedding(position_ids) return embeddings From fb4dd41d793da4c1f1e0383b1e7557ae7bab5cf7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B1=AA=E5=BF=97=E9=B9=8F?= Date: Wed, 22 Jul 2026 18:48:05 +0800 Subject: [PATCH 060/240] [Refactor]: Remove unnecessary config getattr (#5199) Signed-off-by: princepride --- vllm_omni/engine/async_omni_engine.py | 1 + vllm_omni/engine/stage_init_utils.py | 28 +++++++++++++-------------- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/vllm_omni/engine/async_omni_engine.py b/vllm_omni/engine/async_omni_engine.py index fd3d63d7e..459057486 100644 --- a/vllm_omni/engine/async_omni_engine.py +++ b/vllm_omni/engine/async_omni_engine.py @@ -1089,6 +1089,7 @@ class AsyncOmniEngine: "devices": devices, }, "engine_args": stage_engine_args, + "engine_input_source": [], "default_sampling_params": stage_default_sampling_params, "final_output": True, "final_output_type": final_output_type, diff --git a/vllm_omni/engine/stage_init_utils.py b/vllm_omni/engine/stage_init_utils.py index d43160895..d741e9038 100644 --- a/vllm_omni/engine/stage_init_utils.py +++ b/vllm_omni/engine/stage_init_utils.py @@ -349,7 +349,7 @@ class StageMetadata: def extract_stage_metadata(stage_config: Any) -> StageMetadata: """Pure data extraction from a stage_config object.""" stage_id: int = stage_config.stage_id - stage_type: Literal["llm", "diffusion"] = getattr(stage_config, "stage_type", "llm") + stage_type: Literal["llm", "diffusion"] = _get_attr_or_item(stage_config, "stage_type", "llm") engine_args = stage_config.engine_args if current_omni_platform.is_rocm(): @@ -365,34 +365,34 @@ def extract_stage_metadata(stage_config: Any) -> StageMetadata: # when the selected_backend is not specified. engine_args["attention_backend"] = "TRITON_ATTN" - runtime_cfg = getattr(stage_config, "runtime", {}) - engine_input_source: list[int] = getattr(stage_config, "engine_input_source", []) - final_output: bool = getattr(stage_config, "final_output", False) - final_output_type: str | None = getattr(stage_config, "final_output_type", None) + runtime_cfg = stage_config.runtime + engine_input_source: list[int] = stage_config.engine_input_source + final_output: bool = stage_config.final_output + final_output_type: str | None = stage_config.final_output_type - default_sp = _to_dict(getattr(stage_config, "default_sampling_params", {})) + default_sp = _to_dict(_get_attr_or_item(stage_config, "default_sampling_params", {})) SPClass = SamplingParams if stage_type == "llm" else OmniDiffusionSamplingParams default_sampling_params: OmniSamplingParams = SPClass(**default_sp) custom_process_input_func: Callable | None = None - _cpif_path = getattr(stage_config, "custom_process_input_func", None) + _cpif_path = _get_attr_or_item(stage_config, "custom_process_input_func") if _cpif_path: mod_path, fn_name = _cpif_path.rsplit(".", 1) custom_process_input_func = getattr(importlib.import_module(mod_path), fn_name) prompt_expand_func: Callable | None = None - _pef_path = getattr(stage_config, "prompt_expand_func", None) + _pef_path = _get_attr_or_item(stage_config, "prompt_expand_func") if _pef_path: _mod, _fn = _pef_path.rsplit(".", 1) prompt_expand_func = getattr(importlib.import_module(_mod), _fn) cfg_kv_collect_func: Callable | None = None - _ckf_path = getattr(stage_config, "cfg_kv_collect_func", None) + _ckf_path = _get_attr_or_item(stage_config, "cfg_kv_collect_func") if _ckf_path: _mod, _fn = _ckf_path.rsplit(".", 1) cfg_kv_collect_func = getattr(importlib.import_module(_mod), _fn) - model_stage = getattr(engine_args, "model_stage", None) + model_stage = engine_args.get("model_stage") if stage_type == "diffusion": return StageMetadata( @@ -411,8 +411,8 @@ def extract_stage_metadata(stage_config: Any) -> StageMetadata: cfg_kv_collect_func=cfg_kv_collect_func, ) - engine_output_type = getattr(engine_args, "engine_output_type", None) - is_comprehension = getattr(stage_config, "is_comprehension", False) + engine_output_type = engine_args.get("engine_output_type") + is_comprehension = stage_config.is_comprehension requires_multimodal_data = getattr(runtime_cfg, "requires_multimodal_data", False) return StageMetadata( @@ -695,7 +695,7 @@ def build_engine_args_dict( if "tensor_parallel_size" in engine_args and engine_args["tensor_parallel_size"] is None: del engine_args["tensor_parallel_size"] - stage_type = getattr(stage_config, "stage_type", "llm") + stage_type = _get_attr_or_item(stage_config, "stage_type", "llm") stage_id = stage_config.stage_id engine_args_dict = _to_dict(engine_args) @@ -735,7 +735,7 @@ def build_engine_args_dict( engine_args_dict.setdefault("enable_prefix_caching", False) # Check whether the stage's default_sampling_params defines extra_args. - default_sp = _to_dict(getattr(stage_config, "default_sampling_params", {})) + default_sp = _to_dict(_get_attr_or_item(stage_config, "default_sampling_params", {})) engine_args_dict["has_sampling_extra_args"] = bool(default_sp.get("extra_args")) # TODO: Remove this after the performance regression is fixed From 23d0ee5c0cfdadcd8b413f2de1625d26a54611e4 Mon Sep 17 00:00:00 2001 From: Ricardo Noriega Date: Wed, 22 Jul 2026 13:00:01 +0200 Subject: [PATCH 061/240] [Bugfix] Return error for unsupported output modalities instead of empty choices (#4720) Signed-off-by: Ricardo Noriega De Soto --- .../openai_api/test_modality_validation.py | 146 ++++++++++++++++++ vllm_omni/entrypoints/openai/serving_chat.py | 12 ++ 2 files changed, 158 insertions(+) create mode 100644 tests/entrypoints/openai_api/test_modality_validation.py diff --git a/tests/entrypoints/openai_api/test_modality_validation.py b/tests/entrypoints/openai_api/test_modality_validation.py new file mode 100644 index 000000000..fde0c0ca1 --- /dev/null +++ b/tests/entrypoints/openai_api/test_modality_validation.py @@ -0,0 +1,146 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Regression tests for output modality validation in chat completions. + +Covers: +- #4719: unsupported modality must return an error, not empty choices +- Single-stage diffusion text-output path must remain accessible +""" + +from __future__ import annotations + +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest +from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest +from vllm.entrypoints.openai.engine.protocol import ErrorResponse + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +def _make_serving_chat(output_modalities: list[str], stage_configs=None): + """Build a minimal OmniOpenAIServingChat with mocked internals.""" + from vllm_omni.entrypoints.openai.serving_chat import OmniOpenAIServingChat + + instance = object.__new__(OmniOpenAIServingChat) + instance.engine_client = SimpleNamespace( + output_modalities=output_modalities, + errored=False, + stage_configs=stage_configs or [], + ) + instance._diffusion_mode = False + instance.models = MagicMock() + instance.models.model_name.return_value = "test-model" + instance.enable_prompt_tokens_details = False + + renderer = MagicMock() + renderer.get_tokenizer.return_value = MagicMock() + instance.renderer = renderer + instance.online_renderer = MagicMock() + instance.online_renderer.validate_chat_template.return_value = None + + instance.parser_cls = None + instance.use_harmony = False + instance.trust_request_chat_template = True + instance.chat_template = None + instance.chat_template_content_format = "auto" + instance.default_chat_template_kwargs = {} + instance.enable_auto_tools = False + instance.exclude_tools_when_tool_choice_none = False + instance._check_model = AsyncMock(return_value=None) + return instance + + +def _make_request(modalities): + """Build a ChatCompletionRequest with the given modalities.""" + req = ChatCompletionRequest( + model="test-model", + messages=[{"role": "user", "content": "hello"}], + ) + req.modalities = modalities + return req + + +@pytest.mark.asyncio +async def test_unsupported_modality_returns_error(): + """#4719: requesting 'video' on a text/audio model must return an error.""" + serving = _make_serving_chat(["text", "audio"]) + request = _make_request(["video"]) + + with patch.object(serving, "_preprocess_chat", new_callable=AsyncMock, return_value=([], [{}])): + result = await serving._create_chat_completion(request) + + assert isinstance(result, ErrorResponse) + assert "video" in result.error.message + assert "Unsupported" in result.error.message + + +@pytest.mark.asyncio +async def test_supported_modality_passes_validation(): + """Requesting a supported modality must not trigger an error.""" + serving = _make_serving_chat(["text", "audio"]) + request = _make_request(["text"]) + + with patch.object(serving, "_preprocess_chat", new_callable=AsyncMock, return_value=([], [{}])): + try: + result = await serving._create_chat_completion(request) + except (AttributeError, TypeError): + return + + assert not isinstance(result, ErrorResponse) + + +@pytest.mark.asyncio +async def test_invalid_modalities_type_returns_error(): + """Modalities must be a list of strings.""" + serving = _make_serving_chat(["text", "audio"]) + request = _make_request("audio") # bare string, not a list + + with patch.object(serving, "_preprocess_chat", new_callable=AsyncMock, return_value=([], [{}])): + result = await serving._create_chat_completion(request) + + assert isinstance(result, ErrorResponse) + assert "list of strings" in result.error.message + + +@pytest.mark.asyncio +async def test_modalities_with_non_string_element_returns_error(): + """Modalities list with non-string elements must be rejected.""" + serving = _make_serving_chat(["text", "audio"]) + request = _make_request(["text", 123]) + + with patch.object(serving, "_preprocess_chat", new_callable=AsyncMock, return_value=([], [{}])): + result = await serving._create_chat_completion(request) + + assert isinstance(result, ErrorResponse) + assert "list of strings" in result.error.message + + +@pytest.mark.asyncio +async def test_single_stage_diffusion_allows_text_modality(): + """Single-stage diffusion engines that advertise 'image' must also accept 'text'.""" + stage_configs = [SimpleNamespace(stage_type="diffusion", is_comprehension=False)] + serving = _make_serving_chat(["image"], stage_configs=stage_configs) + request = _make_request(["text"]) + + with patch.object(serving, "_preprocess_chat", new_callable=AsyncMock, return_value=([], [{}])): + try: + result = await serving._create_chat_completion(request) + except (AttributeError, TypeError): + return + + assert not isinstance(result, ErrorResponse) + + +@pytest.mark.asyncio +async def test_single_stage_diffusion_rejects_unsupported_modality(): + """Single-stage diffusion must still reject truly unsupported modalities.""" + stage_configs = [SimpleNamespace(stage_type="diffusion", is_comprehension=False)] + serving = _make_serving_chat(["image"], stage_configs=stage_configs) + request = _make_request(["video"]) + + with patch.object(serving, "_preprocess_chat", new_callable=AsyncMock, return_value=([], [{}])): + result = await serving._create_chat_completion(request) + + assert isinstance(result, ErrorResponse) + assert "video" in result.error.message diff --git a/vllm_omni/entrypoints/openai/serving_chat.py b/vllm_omni/entrypoints/openai/serving_chat.py index 3f6250be8..f819c7b57 100755 --- a/vllm_omni/entrypoints/openai/serving_chat.py +++ b/vllm_omni/entrypoints/openai/serving_chat.py @@ -455,6 +455,18 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin): output_modalities = getattr(request, "modalities", engine_output_modalities) request.modalities = output_modalities if output_modalities is not None else engine_output_modalities + if not isinstance(request.modalities, list) or not all(isinstance(m, str) for m in request.modalities): + return self.create_error_response("'modalities' must be a list of strings.") + allowed_modalities = set(engine_output_modalities) + if is_single_stage_diffusion(self.engine_client): + allowed_modalities.add("text") + unsupported = set(request.modalities) - allowed_modalities + if unsupported: + return self.create_error_response( + f"Unsupported output modalities {', '.join(sorted(unsupported))} for this model. " + f"Supported modalities: {', '.join(sorted(allowed_modalities))}", + ) + if request.modalities and "audio" in request.modalities: audio_format_check = self._resolve_audio_format(request) if isinstance(audio_format_check, ErrorResponse): From 8219c01b57ce41a5c15fa8af5a780f9d7899e0a3 Mon Sep 17 00:00:00 2001 From: Weiming Liao Date: Wed, 22 Jul 2026 21:53:04 +0800 Subject: [PATCH 062/240] [Bugfix][NPU] Port OmniConnectorModelRunnerMixin to NPU runners to fix async_chunk=false full-payload hang (#5234) (#5290) Signed-off-by: Weiming Liao --- .../npu/worker/npu_ar_model_runner.py | 47 ++++++++++++++++-- .../npu/worker/npu_generation_model_runner.py | 49 ++++++++++++++++--- 2 files changed, 85 insertions(+), 11 deletions(-) diff --git a/vllm_omni/platforms/npu/worker/npu_ar_model_runner.py b/vllm_omni/platforms/npu/worker/npu_ar_model_runner.py index 9a8a55709..b7b275b12 100644 --- a/vllm_omni/platforms/npu/worker/npu_ar_model_runner.py +++ b/vllm_omni/platforms/npu/worker/npu_ar_model_runner.py @@ -46,6 +46,7 @@ from vllm_omni.distributed.omni_connectors.kv_transfer_manager import OmniKVTran from vllm_omni.outputs import OmniModelRunnerOutput from vllm_omni.platforms.npu.worker.npu_model_runner import OmniNPUModelRunner from vllm_omni.utils.mm_outputs import build_mm_cpu, partition_payload_list, to_payload_element +from vllm_omni.worker.omni_connector_model_runner_mixin import OmniConnectorModelRunnerMixin from vllm_omni.worker.sampling_utils import sanitize_min_tokens_stop_ids @@ -100,7 +101,7 @@ class ExecuteModelState(NamedTuple): batch_desc: BatchDescriptor multimodal_outputs: Any # Omni-Specific -class NPUARModelRunner(OmniNPUModelRunner): +class NPUARModelRunner(OmniNPUModelRunner, OmniConnectorModelRunnerMixin): """Autoregressive NPU model runner that returns hidden states per request.""" def __init__(self, *args, **kwargs): @@ -112,6 +113,22 @@ class NPUARModelRunner(OmniNPUModelRunner): # Initialize KV cache manager (preserve vllm_config fallback behavior) self.kv_transfer_manager = OmniKVTransferManager.from_vllm_config(self.vllm_config, self.model_config) self._async_chunk = getattr(self.model_config, "async_chunk", False) + _OMNI_CONNECTOR_INIT_ARCHS = { + "Qwen3OmniMoeForConditionalGeneration", + "Qwen2_5OmniForConditionalGeneration", + "CovoAudioForConditionalGeneration", + "MiMoAudioModel", + "Qwen3TTSTalkerForConditionalGeneration", + "Qwen3TTSCode2Wav", + "CosyVoice3Model", + "DyninOmniForConditionalGeneration", + "IndexTTS2TalkerForConditionalGeneration", + } + if getattr(self.model_config, "model_arch", None) in _OMNI_CONNECTOR_INIT_ARCHS: + self.init_omni_connectors( + model_config=self.model_config, + kv_transfer_manager=self.kv_transfer_manager, + ) self._downstream_payload_cache: dict[str, bool] = {} def _make_buffer(self, *size, dtype, numpy=True): @@ -380,6 +397,15 @@ class NPUARModelRunner(OmniNPUModelRunner): request_id_resolver=self._resolve_global_request_id, ) # -------------------------------------- Omni-new ------------------------------------------------- + if hasattr(self, "_omni_connector"): + for request in getattr(scheduler_output, "pending_input_registrations", []): + self.register_chunk_recv(request) + self.recv_full_payload_inputs(scheduler_output) + if self._pending_full_payload_send: + flush_ids = set(getattr(scheduler_output, "finished_req_ids", set())) + flush_ids.update({rid for rid in self._pending_full_payload_send if rid not in self.requests}) + if flush_ids: + self.flush_full_payload_outputs(flush_ids) # self._draft_token_ids is None when `input_fits_in_drafter=False` # and there is no draft tokens scheduled. so it need to update the # spec_decoding info in scheduler_output with async_scheduling. @@ -435,7 +461,7 @@ class NPUARModelRunner(OmniNPUModelRunner): if kv_ids: output = copy(output) output.kv_extracted_req_ids = kv_ids - return output + return self.attach_omni_connector_output(output) if not num_scheduled_tokens: if ( @@ -462,7 +488,7 @@ class NPUARModelRunner(OmniNPUModelRunner): output = copy(output) output.kv_extracted_req_ids = kv_ids - return output + return self.attach_omni_connector_output(output) if self.cache_config.kv_sharing_fast_prefill: assert not self.num_prompt_logprobs, ( "--kv-sharing-fast-prefill produces incorrect " @@ -879,11 +905,11 @@ class NPUARModelRunner(OmniNPUModelRunner): # In case of PP with kv transfer, we need to pass through the # kv_connector_output if kv_connector_output.is_empty(): - return EMPTY_MODEL_RUNNER_OUTPUT + return self.attach_omni_connector_output(EMPTY_MODEL_RUNNER_OUTPUT) output = copy(EMPTY_MODEL_RUNNER_OUTPUT) output.kv_connector_output = kv_connector_output - return output + return self.attach_omni_connector_output(output) # Unpack ephemeral state. ( @@ -1166,6 +1192,15 @@ class NPUARModelRunner(OmniNPUModelRunner): # inter_stage_outputs (the NPU runner has no separate full-payload # accumulate). #4527's (None, pooler_output) starved it. (PR #4792) pooler_inter, pooler_client = pooler_output, pooler_output + + # [Omni] Full-payload send-side accumulation. Mirrors gpu_ar_model_runner.py. + if pooler_inter and self._should_accumulate_full_payload_output(): + with record_function_or_nullcontext("omni_output_builder:accumulate_full_payload_output"): + for i, rid in enumerate(req_ids_output_copy): + req_state = self.requests.get(rid) + if req_state is not None and pooler_inter[i]: + self.accumulate_full_payload_output(rid, pooler_inter[i], req_state) + inter_stage_outputs = self._build_multimodal_outputs(pooler_inter) multimodal_outputs = self._build_multimodal_outputs(pooler_client) model_runner_output = OmniModelRunnerOutput( @@ -1183,6 +1218,8 @@ class NPUARModelRunner(OmniNPUModelRunner): ) model_runner_output.kv_extracted_req_ids = kv_extracted_req_ids model_runner_output.routed_experts = routed_experts_lists + with record_function_or_nullcontext("omni_output_builder:get_omni_connector_output"): + model_runner_output.omni_connector_output = self.get_omni_connector_output() # -------------------------------------- Omni-new ------------------------------------------------- if self.ascend_config.profiling_chunk_config.enabled and hasattr(self, "_execution_start_time"): diff --git a/vllm_omni/platforms/npu/worker/npu_generation_model_runner.py b/vllm_omni/platforms/npu/worker/npu_generation_model_runner.py index ea03842b4..d43867c97 100644 --- a/vllm_omni/platforms/npu/worker/npu_generation_model_runner.py +++ b/vllm_omni/platforms/npu/worker/npu_generation_model_runner.py @@ -36,14 +36,30 @@ from vllm_omni.outputs import OmniModelRunnerOutput from vllm_omni.platforms.npu.worker.npu_ar_model_runner import ExecuteModelState, _ensure_tensor_values from vllm_omni.platforms.npu.worker.npu_model_runner import OmniNPUModelRunner from vllm_omni.utils.mm_outputs import partition_payload_list +from vllm_omni.worker.omni_connector_model_runner_mixin import OmniConnectorModelRunnerMixin -class NPUGenerationModelRunner(OmniNPUModelRunner): +class NPUGenerationModelRunner(OmniNPUModelRunner, OmniConnectorModelRunnerMixin): """Generation model runner for vLLM-omni on NPU (non-autoregressive).""" def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self._async_chunk = getattr(self.model_config, "async_chunk", False) + _OMNI_CONNECTOR_INIT_ARCHS = { + "Qwen3OmniMoeForConditionalGeneration", + "Qwen2_5OmniForConditionalGeneration", + "CovoAudioForConditionalGeneration", + "MiMoAudioModel", + "Qwen3TTSTalkerForConditionalGeneration", + "Qwen3TTSCode2Wav", + "CosyVoice3Model", + "DyninOmniForConditionalGeneration", + "IndexTTS2S2MelDecoder", + } + if getattr(self.model_config, "model_arch", None) in _OMNI_CONNECTOR_INIT_ARCHS: + self.init_omni_connectors( + model_config=self.model_config, + ) def _update_request_states(self, scheduler_output: SchedulerOutput): # remove requests @@ -113,6 +129,16 @@ class NPUGenerationModelRunner(OmniNPUModelRunner): if kv_connector_metadata is not None: get_kv_transfer_group().handle_preemptions(kv_connector_metadata) + if hasattr(self, "_omni_connector"): + for request in getattr(scheduler_output, "pending_input_registrations", []): + self.register_chunk_recv(request) + self.recv_full_payload_inputs(scheduler_output) + if self._pending_full_payload_send: + flush_ids = set(getattr(scheduler_output, "finished_req_ids", set())) + flush_ids.update({rid for rid in self._pending_full_payload_send if rid not in self.requests}) + if flush_ids: + self.flush_full_payload_outputs(flush_ids) + num_scheduled_tokens = scheduler_output.total_num_scheduled_tokens with record_function_or_nullcontext("prepare input"): # -------------------------------------- Omni-new ------------------------------------------------- @@ -129,7 +155,9 @@ class NPUGenerationModelRunner(OmniNPUModelRunner): encoder_cache=self.encoder_cache, ) as ec_connector_output: self._execute_mm_encoder(scheduler_output) - return make_empty_encoder_model_runner_output(scheduler_output) + return self.attach_omni_connector_output( + make_empty_encoder_model_runner_output(scheduler_output) + ) if not num_scheduled_tokens: if ( @@ -145,8 +173,10 @@ class NPUGenerationModelRunner(OmniNPUModelRunner): self._dummy_run(1) if not has_kv_transfer_group(): # Return empty ModelRunnerOutput if no work to do. - return EMPTY_MODEL_RUNNER_OUTPUT - return self.kv_connector_no_forward(scheduler_output, self.vllm_config) + return self.attach_omni_connector_output(EMPTY_MODEL_RUNNER_OUTPUT) + return self.attach_omni_connector_output( + self.kv_connector_no_forward(scheduler_output, self.vllm_config) + ) if self.cache_config.kv_sharing_fast_prefill: assert not self.num_prompt_logprobs, ( "--kv-sharing-fast-prefill produces incorrect " @@ -416,11 +446,11 @@ class NPUGenerationModelRunner(OmniNPUModelRunner): # In case of PP with kv transfer, we need to pass through the # kv_connector_output if kv_connector_output.is_empty(): - return EMPTY_MODEL_RUNNER_OUTPUT + return self.attach_omni_connector_output(EMPTY_MODEL_RUNNER_OUTPUT) output = copy(EMPTY_MODEL_RUNNER_OUTPUT) output.kv_connector_output = kv_connector_output - return output + return self.attach_omni_connector_output(output) # Unpack ephemeral state. ( @@ -489,6 +519,12 @@ class NPUGenerationModelRunner(OmniNPUModelRunner): # [Omni] Copy req_id mappings to avoid async scheduling mutation. req_ids_output_copy = self.input_batch.req_ids.copy() req_id_to_index_output_copy = self.input_batch.req_id_to_index.copy() + # [Omni] Full-payload send-side accumulation. Mirrors gpu_generation_model_runner.py. + if inter_stage_outputs and self._should_accumulate_full_payload_output(): + for i, rid in enumerate(req_ids_output_copy): + req_state = self.requests.get(rid) + if req_state is not None and inter_stage_outputs[i]: + self.accumulate_full_payload_output(rid, inter_stage_outputs[i], req_state) routed_experts_lists = None if self.vllm_config.model_config.enable_return_routed_experts and hasattr( self.input_batch, "num_tokens_no_spec" @@ -509,6 +545,7 @@ class NPUGenerationModelRunner(OmniNPUModelRunner): ec_connector_output=ec_connector_output if self.supports_mm_inputs else None, ) output.routed_experts = routed_experts_lists + output.omni_connector_output = self.get_omni_connector_output() # -------------------------------------- Omni-new ------------------------------------------------- if self.speculative_config is not None: From f44649ffd0d84dcec228a615b9ccff8000894115 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B1=AA=E5=BF=97=E9=B9=8F?= Date: Wed, 22 Jul 2026 23:36:00 +0800 Subject: [PATCH 063/240] [Refactor]: Remove sensenova examples (#5201) Signed-off-by: princepride --- .../image_to_image/image_edit.py | 4 + .../offline_inference/sensenova_u1/README.md | 214 ------------ .../offline_inference/sensenova_u1/end2end.py | 307 ------------------ .../text_to_image/text_to_image.py | 37 +-- recipes/SenseNova/SenseNova-U1.md | 76 +++-- .../test_diffusion_output_formatter.py | 2 + .../test_sensenova_u1_img2img_expansion.py | 20 +- .../test_sensenova_u1_text2img_expansion.py | 8 +- vllm_omni/diffusion/output_formatter.py | 6 +- vllm_omni/diffusion/utils/image_output.py | 112 +++++++ 10 files changed, 182 insertions(+), 604 deletions(-) delete mode 100644 examples/offline_inference/sensenova_u1/README.md delete mode 100644 examples/offline_inference/sensenova_u1/end2end.py create mode 100644 vllm_omni/diffusion/utils/image_output.py diff --git a/examples/offline_inference/image_to_image/image_edit.py b/examples/offline_inference/image_to_image/image_edit.py index 12bd978a3..10cf91088 100644 --- a/examples/offline_inference/image_to_image/image_edit.py +++ b/examples/offline_inference/image_to_image/image_edit.py @@ -97,6 +97,7 @@ import torch from PIL import Image from vllm_omni.diffusion.data import DiffusionParallelConfig +from vllm_omni.diffusion.utils.image_output import extract_images_from_outputs from vllm_omni.diffusion.utils.param_utils import apply_declared_extra_args from vllm_omni.entrypoints.omni import Omni from vllm_omni.entrypoints.openai.stage_params import clone_sampling_params @@ -625,6 +626,9 @@ def main(): if images: break + if not images: + images = extract_images_from_outputs(outputs) + if not images: raise ValueError("No images found in request_output") diff --git a/examples/offline_inference/sensenova_u1/README.md b/examples/offline_inference/sensenova_u1/README.md deleted file mode 100644 index 9fdc23a00..000000000 --- a/examples/offline_inference/sensenova_u1/README.md +++ /dev/null @@ -1,214 +0,0 @@ -# SenseNova-U1: Offline inference - -## Architecture - -SenseNova-U1 is a unified Qwen3-based LLM with Mixture-of-Tokenizers (MoT) attention. Unlike two-stage pipelines (e.g., BAGEL), it handles text encoding, optional reasoning ("think mode"), and flow-matching-based image denoising entirely within a single diffusion stage. - -| Feature | Description | -| :------ | :---------- | -| **Model type** | Unified LLM (single-stage diffusion pipeline) | -| **Base LLM** | Qwen3 with MoT attention and 3D RoPE | -| **Image generation** | Flow-matching Euler sampler, no separate VAE | -| **Think mode** | Optional chain-of-thought reasoning before image generation | -| **Parallelism** | Tensor Parallelism (TP) with fused QKV and fused gate/up projections | -| **Modalities** | text2img, img2img, img2text (understanding), text2text | - -## Quick Start - -```bash -cd examples/offline_inference/sensenova_u1 - -# Text-to-image -python end2end.py --prompt "A cute cat" --think - -# Image-to-image editing -python end2end.py --prompt "Turn this into an oil painting" \ - --image input.png --think - -# Image understanding (img2text) -python end2end.py --modality img2text \ - --prompt "Describe this image in detail" \ - --image photo.jpg - -# Text-to-text (pure chat) -python end2end.py --modality text2text \ - --prompt "What is the capital of France?" - -# Custom resolution (image generation) -python end2end.py --prompt "A futuristic cityscape at sunset" \ - --width 2048 --height 1024 --think - -# Text-to-image with cache acceleration (cache_dit or tea_cache) -python end2end.py --prompt "A cute cat" \ - --cache-backend tea_cache -``` - -> **Note**: Default configuration works on a single **NVIDIA A100 (80GB)** or **H100** GPU. - -## Text-to-Image (t2i) - -Standard text-to-image with optional think mode: - -```bash -python end2end.py \ - --prompt "Close portrait of an elderly woman by a farmhouse window, textured skin, gentle smile, warm natural light" \ - --width 1536 --height 2720 \ - --think --print-think -``` - -## Image-to-Image Editing (img2img) - -Pass one or more `--image` paths to trigger img2img mode. The model uses the -input image(s) as visual context for editing: - -```bash -# Single input image -python end2end.py \ - --prompt "Add a sunset sky in the background" \ - --image photo.jpg \ - --width 2048 --height 2048 \ - --think - -# Multiple reference images -python end2end.py \ - --prompt "Combine the style of Image-1 with the content of Image-2" \ - --image style_ref.png content_ref.png \ - --width 2048 --height 2048 \ - --think -``` - -### Dual CFG for img2img - -img2img supports dual classifier-free guidance (text CFG + image CFG): - -```bash -python end2end.py \ - --prompt "Make the person smile" \ - --image portrait.jpg \ - --cfg-scale 4.0 \ - --img-cfg-scale 2.0 \ - --think -``` - -| `img_cfg_scale` | Behavior | -| :-------------- | :------- | -| `1.0` (default) | Text-only CFG: guidance = image-condition → full-condition | -| `== cfg_scale` | Standard CFG: guidance = unconditional → full-condition | -| Other value | Dual CFG: separate text and image guidance strengths | - -## Full Parameter Reference - -### General Parameters - -| Parameter | Default | Description | -| :-------- | :------ | :---------- | -| `--modality` | `auto` | Task modality: `auto`, `text2img`, `img2img`, `img2text`, `text2text` | -| `--prompt` | "A cute cat..." | Text prompt / editing instruction / question | -| `--image` | None | Input image path(s) for img2img or img2text | - -### Image Generation Parameters (text2img / img2img) - -| Parameter | Default | Description | -| :-------- | :------ | :---------- | -| `--height` | 2048 | Height of generated image (pixels) | -| `--width` | 2048 | Width of generated image (pixels) | -| `--seed` | 42 | Random seed for reproducibility | -| `--num-steps` | 50 | Number of denoising steps | -| `--cfg-scale` | 4.0 | Text classifier-free guidance scale | -| `--img-cfg-scale` | 1.0 | Image CFG scale (img2img only, 1.0 = disabled) | -| `--cfg-norm` | "none" | CFG normalization: none, global, channel, cfg_zero_star (t2i only) | -| `--timestep-shift` | 3.0 | Timestep shift for flow-matching schedule | -| `--t-eps` | 0.02 | Epsilon for timestep schedule | -| `--think` | False | Enable think mode | -| `--print-think` | False | Print think text to stdout | - -### Text Generation Parameters (text2text / img2text) - -| Parameter | Default | Description | -| :-------- | :------ | :---------- | -| `--max-tokens` | 512 | Maximum number of tokens to generate | -| `--do-sample` | False | Use sampling instead of greedy decoding | -| `--temperature` | 0.7 | Sampling temperature (higher = more diverse) | - -### Infrastructure Parameters - -| Parameter | Default | Description | -| :-------- | :------ | :---------- | -| `--model` | `SenseNova/SenseNova-U1-8B-MoT` | HuggingFace model ID or local path | -| `--output` | `.` | Output directory for saved images | -| `--tensor-parallel-size` | 1 | Number of GPUs for tensor parallelism | -| `--enforce-eager` | False | Disable torch.compile | -| `--enable-cpu-offload` | False | Enable module-wise (sequential) CPU offload to reduce peak VRAM | -| `--cache-backend` | None | Set to `cache_dit` or `tea_cache` for image generation acceleration | -| `--enable-cache-dit-summary` | False | Print Cache-DiT cache statistics after generation | - -## Reducing GPU Memory Usage - -For hardware with limited VRAM, enable **module-wise CPU offload** with -`--enable-cpu-offload`. The pipeline implements `SupportsComponentDiscovery`, so the -vision encoder (`vision_model`) and the Qwen3 LLM (`language_model`) are -swapped between CPU and GPU on demand: - -- During text/vision encoding, the LLM is on CPU. -- During the diffusion loop, the vision encoder is on CPU. -- Lightweight FM modules stay resident on GPU. - -This lowers peak VRAM at the cost of extra CPU<->GPU transfers (use pinned -memory) — useful for running SenseNova-U1 on consumer-grade GPUs. - -```bash -# Text-to-image with CPU offload -python end2end.py \ - --prompt "A cute cat sitting on a windowsill" \ - --width 2048 --height 2048 \ - --enable-cpu-offload --think - -# Image-to-image editing with CPU offload -python end2end.py \ - --prompt "Turn this into an oil painting" \ - --image input.png \ - --width 2048 --height 2048 \ - --enable-cpu-offload --think - -# Image understanding (img2text) with CPU offload -python end2end.py \ - --modality img2text \ - --prompt "Describe this image in detail" \ - --image photo.jpg \ - --enable-cpu-offload -``` - -> **Notes** -> - CPU offload is **single-GPU only** (incompatible with `--tensor-parallel-size > 1`). -> - First-step latency is higher because of the cold-start CPU<->GPU transfers. -> - For more details and other offloading strategies, see -> [CPU Offloading for Diffusion Models](../../../docs/user_guide/diffusion/cpu_offload_diffusion.md). - -## Reproducing the E2E Test - -The following command reproduces the pixel-validated CI test case: - -```bash -python end2end.py \ - --prompt "Close portrait of an elderly woman by a farmhouse window, textured skin, gentle smile, warm natural light, emotional documentary look. The portrait should feel polished and natural, with sharp eyes, realistic skin texture, accurate facial anatomy, and premium lighting that keeps the face as the main focus." \ - --width 1536 --height 2720 \ - --seed 42 --num-steps 50 \ - --cfg-scale 4.0 --timestep-shift 3.0 --cfg-norm none \ - --think --print-think \ - --output outputs -``` - -The corresponding pytest: - -```bash -pytest -s -v tests/e2e/offline_inference/test_sensenova_u1_text2img.py \ - -m "advanced_model" --run-level "advanced_model" -``` - -The img2img counterpart lives at -`tests/e2e/offline_inference/test_sensenova_u1_img2img.py`. - -## Online Serving - -For OpenAI-compatible API serving, see -[`examples/online_serving/sensenova_u1/`](../../online_serving/sensenova_u1/README.md). diff --git a/examples/offline_inference/sensenova_u1/end2end.py b/examples/offline_inference/sensenova_u1/end2end.py deleted file mode 100644 index 9547c40b9..000000000 --- a/examples/offline_inference/sensenova_u1/end2end.py +++ /dev/null @@ -1,307 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -""" -End-to-end offline inference for SenseNova-U1-8B-MoT. - -Supports four modalities: text2img, img2img, img2text, text2text. - -Text-to-image: - python end2end.py --prompt "A cute cat" --think - -Image-to-image (editing): - python end2end.py --prompt "Turn this into an oil painting" \ - --image input.png --think - -Image understanding: - python end2end.py --modality img2text \ - --prompt "Describe this image" --image photo.jpg - -Text chat: - python end2end.py --modality text2text \ - --prompt "What is the capital of France?" - -See README.md for more examples. -""" - -import argparse -import os - -from PIL import Image - -from vllm_omni.entrypoints.omni import Omni -from vllm_omni.inputs.data import OmniDiffusionSamplingParams - - -def parse_args(): - parser = argparse.ArgumentParser( - description="SenseNova-U1 text-to-image / image-to-image / text / understanding via vLLM-Omni.", - ) - parser.add_argument( - "--model", - default="SenseNova/SenseNova-U1-8B-MoT", - help="HuggingFace model ID or local path.", - ) - parser.add_argument( - "--modality", - default="auto", - choices=["auto", "text2img", "img2img", "img2text", "text2text"], - help="Task modality. 'auto' infers from --image (img2img/img2text if images, else t2i/t2t).", - ) - parser.add_argument( - "--prompt", - default="A cute cat sitting on a windowsill, soft natural light", - help="Text prompt for generation or editing instruction.", - ) - parser.add_argument( - "--image", - nargs="+", - metavar="PATH", - default=None, - help="Input image path(s) for img2img or img2text.", - ) - parser.add_argument( - "--output", - type=str, - default=".", - help="Output directory to save generated images.", - ) - # Image dimensions - parser.add_argument("--height", type=int, default=2048, help="Height of generated image.") - parser.add_argument("--width", type=int, default=2048, help="Width of generated image.") - - # Generation parameters - parser.add_argument("--seed", type=int, default=42, help="Random seed for deterministic results.") - parser.add_argument("--num-steps", type=int, default=50, help="Number of denoising steps.") - parser.add_argument("--cfg-scale", type=float, default=4.0, help="Text classifier-free guidance scale.") - parser.add_argument( - "--img-cfg-scale", - type=float, - default=1.0, - help="Image CFG scale for img2img (1.0 = disabled). " - "When both --cfg-scale and --img-cfg-scale differ, dual CFG is used.", - ) - parser.add_argument( - "--cfg-norm", - type=str, - default="none", - choices=["none", "global", "channel", "cfg_zero_star"], - help="CFG normalization mode. cfg_zero_star is only for t2i.", - ) - parser.add_argument( - "--timestep-shift", - type=float, - default=3.0, - help="Timestep shift for flow-matching schedule.", - ) - parser.add_argument( - "--t-eps", - type=float, - default=0.02, - help="Epsilon for flow-matching timestep schedule.", - ) - - # Text generation parameters (for text2text / img2text) - parser.add_argument( - "--max-tokens", - type=int, - default=2048, - help="Max tokens for text generation (text2text / img2text).", - ) - parser.add_argument( - "--do-sample", - action="store_true", - help="Use sampling instead of greedy decoding for text generation.", - ) - parser.add_argument( - "--temperature", - type=float, - default=0.7, - help="Sampling temperature for text generation.", - ) - - # Think mode - parser.add_argument( - "--think", - action="store_true", - help="Enable think mode: the model reasons about the prompt before generating the image.", - ) - parser.add_argument( - "--print-think", - action="store_true", - help="Print the generated think text to stdout.", - ) - - # Advanced - parser.add_argument( - "--tensor-parallel-size", - type=int, - default=1, - help="Number of GPUs for tensor parallelism.", - ) - parser.add_argument( - "--cfg-parallel-size", - type=int, - default=1, - help="Number of GPUs for CFG parallelism.", - ) - parser.add_argument( - "--enforce-eager", - action="store_true", - help="Disable torch.compile and force eager execution.", - ) - parser.add_argument( - "--enable-cpu-offload", - action="store_true", - help=( - "Enable model-level (sequential) CPU offloading. " - "Mutually exclusive between vision_model and language_model: " - "only one is on GPU at a time, reducing peak VRAM at the cost " - "of extra CPU<->GPU transfers." - ), - ) - parser.add_argument( - "--cache-backend", - type=str, - default=None, - choices=["cache_dit", "tea_cache"], - help="Cache backend to use for image generation acceleration.", - ) - parser.add_argument( - "--enable-cache-dit-summary", - action="store_true", - help="Enable cache-dit summary logging after diffusion forward passes.", - ) - - return parser.parse_args() - - -def _resolve_modality(args): - """Determine the effective modality from CLI args.""" - if args.modality != "auto": - return args.modality - has_images = args.image is not None - # Default: image-producing tasks for backward compatibility - return "img2img" if has_images else "text2img" - - -def main(): - args = parse_args() - os.makedirs(args.output, exist_ok=True) - - modality = _resolve_modality(args) - is_text_output = modality in ("text2text", "img2text") - has_images = modality in ("img2img", "img2text") - - omni = Omni( - model=args.model, - tensor_parallel_size=args.tensor_parallel_size, - enforce_eager=args.enforce_eager, - enable_cpu_offload=args.enable_cpu_offload, - cache_backend=args.cache_backend, - enable_cache_dit_summary=args.enable_cache_dit_summary, - cfg_parallel_size=args.cfg_parallel_size, - ) - - extra_args = { - "cfg_scale": args.cfg_scale, - "cfg_norm": args.cfg_norm, - "timestep_shift": args.timestep_shift, - "cfg_interval": (0.0, 1.0), - "batch_size": 1, - "think": args.think, - "t_eps": args.t_eps, - } - if modality == "img2img": - extra_args["img_cfg_scale"] = args.img_cfg_scale - if is_text_output: - extra_args["max_tokens"] = args.max_tokens - extra_args["do_sample"] = args.do_sample - extra_args["temperature"] = args.temperature - - sampling_params = OmniDiffusionSamplingParams( - height=args.height, - width=args.width, - seed=args.seed, - num_inference_steps=args.num_steps, - extra_args=extra_args, - ) - - print(f"\n{'=' * 60}") - print(f"SenseNova-U1 Configuration ({modality}):") - print(f" Model : {args.model}") - if not is_text_output: - print(f" Image size : {args.width}x{args.height}") - print(f" Steps : {args.num_steps}") - print(f" CFG scale : {args.cfg_scale}") - if modality == "img2img": - print(f" Img CFG scale : {args.img_cfg_scale}") - if has_images: - print(f" Input images : {args.image}") - if is_text_output: - print(f" Max tokens : {args.max_tokens}") - print(f" Temperature : {args.temperature}") - print(f" Seed : {args.seed}") - print(f" Think mode : {args.think}") - print(f" TP size : {args.tensor_parallel_size}") - print(f" CFG size : {args.cfg_parallel_size}") - print(f" Cache backend : {args.cache_backend or 'none'}") - print(f"{'=' * 60}\n") - - # Build prompt dict - if has_images: - if not args.image: - raise ValueError(f"{modality} requires --image.") - input_images = [Image.open(p).convert("RGB") for p in args.image] - if is_text_output: - prompt_dict = { - "prompt": args.prompt, - "multi_modal_data": {"image": input_images}, - "modalities": ["text"], - } - else: - prompt_dict = { - "prompt": args.prompt, - "multi_modal_data": {"image": input_images}, - "modalities": ["img2img"], - } - elif is_text_output: - prompt_dict = {"prompt": args.prompt, "modalities": ["text"]} - else: - prompt_dict = {"prompt": args.prompt, "modalities": ["image"]} - - outputs = list( - omni.generate( - prompts=prompt_dict, - sampling_params_list=sampling_params, - ) - ) - - for req_output in outputs: - multimodal_output = getattr(req_output, "multimodal_output", {}) or {} - metadata = multimodal_output.get("metadata", {}) if isinstance(multimodal_output, dict) else {} - text_metadata = metadata.get("text", {}) if isinstance(metadata, dict) else {} - think_text = text_metadata.get("think_text") if isinstance(text_metadata, dict) else None - if args.print_think and think_text: - print(f"[Think]\n{think_text}\n") - - if is_text_output: - text = multimodal_output.get("text", "") if isinstance(multimodal_output, dict) else "" - if not text: - text = text_metadata.get("text_output", "") if isinstance(text_metadata, dict) else "" - if not text: - text = getattr(req_output, "text", "") or "" - print(f"[Response]\n{text}") - else: - images = getattr(req_output, "images", None) or [] - if not images: - print("[Warning] No images generated.") - continue - for j, img in enumerate(images): - save_path = os.path.join(args.output, f"sensenova_u1_output_{j}.png") - img.save(save_path) - print(f"[Output] Saved {img.size[0]}x{img.size[1]} image to {save_path}") - - -if __name__ == "__main__": - main() diff --git a/examples/offline_inference/text_to_image/text_to_image.py b/examples/offline_inference/text_to_image/text_to_image.py index 61dd29261..d01b2973f 100644 --- a/examples/offline_inference/text_to_image/text_to_image.py +++ b/examples/offline_inference/text_to_image/text_to_image.py @@ -11,6 +11,7 @@ from typing import Any import torch from vllm_omni.diffusion.data import logger +from vllm_omni.diffusion.utils.image_output import extract_images_from_outputs from vllm_omni.diffusion.utils.param_utils import apply_declared_extra_args from vllm_omni.entrypoints.omni import Omni from vllm_omni.entrypoints.openai.stage_params import clone_sampling_params @@ -601,11 +602,8 @@ def main(): if images: break - # Fallback: generation-stage pipelines (e.g. MammothModa2's AR->DiT) return the - # generated image as a tensor under multimodal_output instead of populating the - # `images` field that diffusion-stage pipelines fill. if not images: - images = _images_from_multimodal_output(outputs) + images = extract_images_from_outputs(outputs) if not images: raise ValueError("No images found in request_output") @@ -624,36 +622,5 @@ def main(): print(f"Saved generated image to {save_path}") -def _images_from_multimodal_output(outputs: list[Any]) -> list[Any]: - """Extract PIL images from multimodal_output tensors. - - Generation-stage pipelines (e.g. MammothModa2's AR->DiT) return the generated - image as a tensor (normalized to [-1, 1], CHW) under ``multimodal_output`` - rather than populating the ``images`` field. Convert any such tensors to PIL. - """ - from PIL import Image - - pil_images: list[Any] = [] - for output in outputs: - req_out = getattr(output, "request_output", output) - for completion in getattr(req_out, "outputs", None) or []: - # multimodal_output is a MultimodalPayload (a Mapping) keyed by modality, - # matching how omni examples (ming_flash_omni / magi_human / dynin) read it. - mm = getattr(completion, "multimodal_output", None) or {} - if "image" not in mm: - continue - payload = mm["image"] - for tensor in payload if isinstance(payload, list) else [payload]: - if not isinstance(tensor, torch.Tensor): - continue - img = tensor.detach().to("cpu", dtype=torch.float32) - if img.ndim == 4: - img = img[0] - img = (img / 2 + 0.5).clamp(0, 1).mul(255).to(torch.uint8) - img = img.permute(1, 2, 0).contiguous().numpy() - pil_images.append(Image.fromarray(img)) - return pil_images - - if __name__ == "__main__": main() diff --git a/recipes/SenseNova/SenseNova-U1.md b/recipes/SenseNova/SenseNova-U1.md index 5cb30a21d..be25df570 100644 --- a/recipes/SenseNova/SenseNova-U1.md +++ b/recipes/SenseNova/SenseNova-U1.md @@ -22,8 +22,10 @@ text-to-text chat. ## References -- Offline example: - [`examples/offline_inference/sensenova_u1/end2end.py`](../../examples/offline_inference/sensenova_u1/end2end.py) +- Offline text-to-image: + [`examples/offline_inference/text_to_image/text_to_image.py`](../../examples/offline_inference/text_to_image/text_to_image.py) +- Offline image-to-image: + [`examples/offline_inference/image_to_image/image_edit.py`](../../examples/offline_inference/image_to_image/image_edit.py) - Online serving: [`examples/online_serving/sensenova_u1/`](../../examples/online_serving/sensenova_u1/) - E2E tests: @@ -48,13 +50,14 @@ text-to-text chat. #### Command ```bash -python examples/offline_inference/sensenova_u1/end2end.py \ +python examples/offline_inference/text_to_image/text_to_image.py \ + --model SenseNova/SenseNova-U1-8B-MoT \ --prompt "Close portrait of an elderly woman by a farmhouse window, textured skin, gentle smile, warm natural light, emotional documentary look. The portrait should feel polished and natural, with sharp eyes, realistic skin texture, accurate facial anatomy, and premium lighting that keeps the face as the main focus." \ --width 1536 --height 2720 \ - --seed 42 --num-steps 50 \ - --cfg-scale 4.0 --timestep-shift 3.0 --cfg-norm none \ - --think --print-think \ - --output outputs + --seed 42 --num-inference-steps 50 \ + --cfg-scale 4.0 \ + --extra-body '{"think": true, "cfg_norm": "none", "timestep_shift": 3.0, "t_eps": 0.02}' \ + --output outputs/sensenova_u1_output.png ``` #### Verification @@ -74,8 +77,7 @@ pytest -s -v tests/e2e/offline_inference/test_sensenova_u1_text2img.py \ #### Text-to-Image via the shared example -SenseNova-U1 is also registered with the shared text-to-image example, so you -can run it through `text_to_image.py` instead of `end2end.py`. Forward the +SenseNova-U1 is registered with the shared text-to-image example. Forward SenseNova-specific generation parameters as a JSON object through `--extra-body`: ```bash @@ -94,48 +96,49 @@ The `--extra-body` keys are filtered against the model's declared [`vllm_omni/model_extras/sensenova_u1.py`](../../vllm_omni/model_extras/sensenova_u1.py)): `think`, `cfg_scale`, `cfg_norm`, `timestep_shift`, `t_eps`, `img_cfg_scale`, and `max_tokens`. No deploy YAML is needed — the engine auto-generates a -single-stage diffusion config. For richer task modalities (img2text, text2text) -and CPU offload, use the dedicated `end2end.py` script described above. +single-stage diffusion config. #### Image-to-Image Editing (img2img) ```bash -python examples/offline_inference/sensenova_u1/end2end.py \ +python examples/offline_inference/image_to_image/image_edit.py \ + --model SenseNova/SenseNova-U1-8B-MoT \ --prompt "Turn this into an oil painting" \ --image input.png \ - --width 2048 --height 2048 \ - --seed 42 --num-steps 50 \ - --cfg-scale 4.0 --img-cfg-scale 1.0 --cfg-norm none \ - --think --print-think \ - --output outputs + --resolution 2048 \ + --seed 42 --num-inference-steps 50 \ + --cfg-scale 4.0 \ + --extra-args '{"think": true, "img_cfg_scale": 1.0, "cfg_norm": "none", "timestep_shift": 3.0}' \ + --output outputs/sensenova_u1_edit.png ``` -- img2img uses dual CFG: `--cfg-scale` controls text guidance, `--img-cfg-scale` - controls image guidance (1.0 = image CFG disabled). -- Pass multiple `--image` paths for multi-reference editing. +- img2img uses dual CFG: `--cfg-scale` controls text guidance, while + `img_cfg_scale` in `--extra-args` controls image guidance (1.0 = image CFG + disabled). +- Pass multiple `--image` paths for multi-reference editing when the underlying + pipeline supports them. #### Image Understanding (img2text) ```bash -python examples/offline_inference/sensenova_u1/end2end.py \ - --modality img2text \ +vllm serve SenseNova/SenseNova-U1-8B-MoT --omni --port 8091 +python examples/online_serving/sensenova_u1/openai_chat_client.py \ --prompt "Describe this image in detail" \ - --image photo.jpg \ - --max-tokens 512 + --modality img2text \ + --image-url photo.jpg ``` #### Text-to-Text Chat (text2text) ```bash -python examples/offline_inference/sensenova_u1/end2end.py \ - --modality text2text \ +vllm serve SenseNova/SenseNova-U1-8B-MoT --omni --port 8091 +python examples/online_serving/sensenova_u1/openai_chat_client.py \ --prompt "Explain the theory of relativity in simple terms" \ - --max-tokens 256 + --modality text2text ``` -- For img2text and text2text, image generation parameters (height, width, - num-steps, cfg-scale) are ignored. -- Use `--do-sample --temperature 0.7` for more diverse text responses. +- For img2text and text2text, use the online chat-compatible example. The + offline generic image examples intentionally focus on image-producing tasks. ### 2x H200 (144GB) — TP=2 @@ -149,19 +152,20 @@ python examples/offline_inference/sensenova_u1/end2end.py \ #### Command ```bash -python examples/offline_inference/sensenova_u1/end2end.py \ +python examples/offline_inference/text_to_image/text_to_image.py \ + --model SenseNova/SenseNova-U1-8B-MoT \ --prompt "Close portrait of an elderly woman by a farmhouse window, textured skin, gentle smile, warm natural light, emotional documentary look. The portrait should feel polished and natural, with sharp eyes, realistic skin texture, accurate facial anatomy, and premium lighting that keeps the face as the main focus." \ --width 1536 --height 2720 \ - --seed 42 --num-steps 50 \ - --cfg-scale 4.0 --timestep-shift 3.0 --cfg-norm none \ - --think --print-think \ + --seed 42 --num-inference-steps 50 \ + --cfg-scale 4.0 \ + --extra-body '{"think": true, "cfg_norm": "none", "timestep_shift": 3.0, "t_eps": 0.02}' \ --tensor-parallel-size 2 \ - --output outputs + --output outputs/sensenova_u1_output.png ``` #### Verification -Verify the output image is generated at `outputs/sensenova_u1_output_0.png` +Verify the output image is generated at `outputs/sensenova_u1_output.png` with the expected 1536×2720 resolution. #### Notes diff --git a/tests/diffusion/test_diffusion_output_formatter.py b/tests/diffusion/test_diffusion_output_formatter.py index 11354bed6..5524e9686 100644 --- a/tests/diffusion/test_diffusion_output_formatter.py +++ b/tests/diffusion/test_diffusion_output_formatter.py @@ -107,6 +107,8 @@ def test_formatter_preserves_single_video_audio_actions_and_metadata( "diffusion_engine_total_time_ms": 60.0, "image_num": 1, "resolution": 512, + "width": None, + "height": None, "postprocess_time_ms": 30.0, } diff --git a/tests/e2e/offline_inference/test_sensenova_u1_img2img_expansion.py b/tests/e2e/offline_inference/test_sensenova_u1_img2img_expansion.py index 242d6a12e..1ac76da28 100644 --- a/tests/e2e/offline_inference/test_sensenova_u1_img2img_expansion.py +++ b/tests/e2e/offline_inference/test_sensenova_u1_img2img_expansion.py @@ -9,13 +9,14 @@ an input image and text prompt that match expected reference pixel values within a ±10 tolerance. Equivalent to running: - python examples/offline_inference/sensenova_u1/end2end.py \ + python examples/offline_inference/image_to_image/image_edit.py \ + --model SenseNova/SenseNova-U1-8B-MoT \ --prompt "Turn this into an oil painting" \ --image 2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg \ - --width 2560 --height 1669 \ - --seed 42 --num-steps 50 \ - --cfg-scale 4.0 --img-cfg-scale 1.0 --cfg-norm none \ - --think --print-think + --resolution 2048 \ + --seed 42 --num-inference-steps 50 \ + --cfg-scale 4.0 \ + --extra-args '{"think": true, "img_cfg_scale": 1.0, "cfg_norm": "none", "timestep_shift": 3.0}' """ import os @@ -33,11 +34,14 @@ from vllm_omni.entrypoints.omni import Omni from vllm_omni.inputs.data import OmniDiffusionSamplingParams # Reference pixel data extracted from the known-good output image generated by: -# python examples/offline_inference/sensenova_u1/end2end.py \ +# python examples/offline_inference/image_to_image/image_edit.py \ +# --model SenseNova/SenseNova-U1-8B-MoT \ # --prompt "Turn this into an oil painting" \ # --image 2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg \ -# --width 2560 --height 1669 --seed 42 --num-steps 50 \ -# --cfg-scale 4.0 --img-cfg-scale 1.0 --cfg-norm none --think +# --width 2560 --height 1669 \ +# --seed 42 --num-inference-steps 50 \ +# --cfg-scale 4.0 \ +# --extra-args '{"think": true, "img_cfg_scale": 1.0, "cfg_norm": "none", "timestep_shift": 3.0}' REFERENCE_PIXELS = [ {"position": (100, 100), "rgb": (95, 131, 174)}, {"position": (400, 50), "rgb": (142, 153, 174)}, diff --git a/tests/e2e/offline_inference/test_sensenova_u1_text2img_expansion.py b/tests/e2e/offline_inference/test_sensenova_u1_text2img_expansion.py index 65b11d3ee..092da16b9 100644 --- a/tests/e2e/offline_inference/test_sensenova_u1_text2img_expansion.py +++ b/tests/e2e/offline_inference/test_sensenova_u1_text2img_expansion.py @@ -30,10 +30,12 @@ from vllm_omni.entrypoints.omni import Omni from vllm_omni.inputs.data import OmniDiffusionSamplingParams # Reference pixel data extracted from the known-good output image generated by: -# python examples/offline_inference/sensenova_u1/end2end.py \ +# python examples/offline_inference/text_to_image/text_to_image.py \ +# --model SenseNova/SenseNova-U1-8B-MoT \ # --prompt "Close portrait of an elderly woman ..." \ -# --width 1536 --height 2720 --seed 42 --num-steps 50 \ -# --cfg-scale 4.0 --timestep-shift 3.0 --cfg-norm none --think +# --width 1536 --height 2720 --seed 42 --num-inference-steps 50 \ +# --cfg-scale 4.0 \ +# --extra-body '{"think": true, "cfg_norm": "none", "timestep_shift": 3.0, "t_eps": 0.02}' REFERENCE_PIXELS = [ {"position": (100, 100), "rgb": (247, 249, 250)}, {"position": (768, 200), "rgb": (176, 135, 97)}, diff --git a/vllm_omni/diffusion/output_formatter.py b/vllm_omni/diffusion/output_formatter.py index e6e2fb966..05b5b440d 100644 --- a/vllm_omni/diffusion/output_formatter.py +++ b/vllm_omni/diffusion/output_formatter.py @@ -145,7 +145,11 @@ def format_diffusion_outputs( "diffusion_engine_exec_time_ms": timings.exec_time_s * 1000, "diffusion_engine_total_time_ms": timings.total_time_ms, "image_num": int(request.sampling_params.num_outputs_per_prompt), - "resolution": int(request.sampling_params.resolution), + "resolution": ( + int(request.sampling_params.resolution) if request.sampling_params.resolution is not None else None + ), + "width": request.sampling_params.width, + "height": request.sampling_params.height, "postprocess_time_ms": timings.postprocess_time_s * 1000, } diff --git a/vllm_omni/diffusion/utils/image_output.py b/vllm_omni/diffusion/utils/image_output.py new file mode 100644 index 000000000..5f8c9903a --- /dev/null +++ b/vllm_omni/diffusion/utils/image_output.py @@ -0,0 +1,112 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from collections.abc import Iterable, Mapping, Sequence +from typing import Any + +import torch +from PIL import Image +from vllm.outputs import RequestOutput + +from vllm_omni.outputs import OmniRequestOutput + +ImageOutput = OmniRequestOutput | RequestOutput +ImageOutputs = ImageOutput | Sequence[ImageOutput] | None + + +def extract_images_from_outputs(outputs: ImageOutputs) -> list[Image.Image]: + """Extract PIL images from known Omni output types. + + Supported inputs: + - OmniRequestOutput / list[OmniRequestOutput] + - RequestOutput / list[RequestOutput] (compatibility fallback) + """ + for output in _iter_known_outputs(outputs): + if isinstance(output, OmniRequestOutput): + images = _coerce_images(output.images) + if images: + return images + + for payload in _iter_multimodal_image_payloads(output): + images = _coerce_images(payload) + if images: + return images + + return [] + + +def _iter_known_outputs(value: ImageOutputs) -> Iterable[ImageOutput]: + if value is None: + return + if isinstance(value, OmniRequestOutput | RequestOutput): + yield value + return + if isinstance(value, Sequence): + for item in value: + if isinstance(item, OmniRequestOutput | RequestOutput): + yield item + + +def _iter_multimodal_image_payloads(output: OmniRequestOutput | RequestOutput) -> Iterable[Any]: + if isinstance(output, OmniRequestOutput): + if output.multimodal_output is not None: + yield from _image_values_from_mapping_like(output.multimodal_output) + if output.request_output is not None: + yield from _iter_request_output_payloads(output.request_output) + else: + yield from _iter_request_output_payloads(output) + + +def _iter_request_output_payloads(request_output: RequestOutput) -> Iterable[Any]: + mm = getattr(request_output, "multimodal_output", None) + if mm is not None: + yield from _image_values_from_mapping_like(mm) + for completion in getattr(request_output, "outputs", []): + mm = getattr(completion, "multimodal_output", None) + if mm is not None: + yield from _image_values_from_mapping_like(mm) + + +def _image_values_from_mapping_like(value: Any) -> Iterable[Any]: + if hasattr(value, "to_dict") and callable(value.to_dict): + value = value.to_dict() + if not isinstance(value, Mapping): + return + for key in ("image", "images", "model_outputs"): + if key in value: + yield value[key] + + +def _coerce_images(payload: Any) -> list[Image.Image]: + if payload is None: + return [] + if isinstance(payload, Image.Image): + return [payload] + if isinstance(payload, torch.Tensor): + return _tensor_to_images(payload) + if isinstance(payload, list | tuple): + images: list[Image.Image] = [] + for item in payload: + images.extend(_coerce_images(item)) + return images + return [] + + +def _tensor_to_images(tensor: torch.Tensor) -> list[Image.Image]: + img = tensor.detach().to("cpu", dtype=torch.float32) + if img.ndim == 4: + images: list[Image.Image] = [] + for single in img: + images.extend(_tensor_to_images(single)) + return images + if img.ndim != 3: + return [] + + if img.shape[0] in (1, 3, 4): + img = img.permute(1, 2, 0) + + # Some diffusion models emit image tensors normalized in [-1, 1]. + if img.min().item() < 0.0: + img = img / 2 + 0.5 + img = img.clamp(0, 1).mul(255).to(torch.uint8).contiguous().numpy() + return [Image.fromarray(img)] From cc1939120f3dc06c9456aa1165833545feccab5d Mon Sep 17 00:00:00 2001 From: vOv <124694036+cr-gao@users.noreply.github.com> Date: Thu, 23 Jul 2026 00:01:45 +0800 Subject: [PATCH 064/240] [Bugfix] Skip non-positive scheduled token spans in omni stage runners (#5269) Signed-off-by: cr-gao Co-authored-by: Claude Opus 4.8 --- .../test_gpu_generation_model_runner.py | 43 +++++++++++++++++++ .../npu/worker/npu_ar_model_runner.py | 3 +- .../npu/worker/npu_generation_model_runner.py | 3 +- vllm_omni/worker/gpu_ar_model_runner.py | 3 +- .../worker/gpu_generation_model_runner.py | 3 +- 5 files changed, 51 insertions(+), 4 deletions(-) diff --git a/tests/worker/test_gpu_generation_model_runner.py b/tests/worker/test_gpu_generation_model_runner.py index 0cd9aeb26..e64eecb57 100644 --- a/tests/worker/test_gpu_generation_model_runner.py +++ b/tests/worker/test_gpu_generation_model_runner.py @@ -1,6 +1,12 @@ +import contextlib +from types import SimpleNamespace + import pytest import torch +from vllm.config import CacheConfig +from vllm.v1.worker.gpu_model_runner import EMPTY_MODEL_RUNNER_OUTPUT +import vllm_omni.worker.gpu_generation_model_runner as gen_runner_module from vllm_omni.worker.gpu_generation_model_runner import ( ExecuteModelState, GPUGenerationModelRunner, @@ -84,3 +90,40 @@ def test_sample_tokens_dict_output(): assert "audio" in output.multimodal_outputs[0] assert "unused" not in output.multimodal_outputs[0] assert output.multimodal_outputs[0]["audio"].shape == (1, 4) + + +class _StubSchedulerOutput: + def __init__(self, total_num_scheduled_tokens): + self.total_num_scheduled_tokens = total_num_scheduled_tokens + self.num_scheduled_tokens = {"req-1": total_num_scheduled_tokens} + self.finished_req_ids = set() + self.kv_connector_metadata = None + + +def _make_guard_runner(): + # Stubbed far enough that a span escaping the guard reaches the real + # `_prepare_inputs`, i.e. fails the way the reported crash does. + runner = object.__new__(GPUGenerationModelRunner) + runner.execute_model_state = None + runner.routed_experts_initialized = False + runner.speculative_config = None + runner.model_config = SimpleNamespace(async_chunk=False) + runner.cache_config = CacheConfig() + runner.input_batch = _DummyInputBatch() + runner.model = object() + runner._update_states = lambda scheduler_output: None + runner.synchronize_input_prep = contextlib.nullcontext + runner.attach_omni_connector_output = lambda result: result + return runner + + +@pytest.mark.parametrize("total", [-1, -512, 0]) +def test_execute_model_skips_non_positive_scheduled_span(monkeypatch, total): + """#5196: a negative span is truthy, so it used to reach `_prepare_inputs`, + whose `assert total_num_scheduled_tokens > 0` killed the stage EngineCore.""" + monkeypatch.setattr(gen_runner_module, "has_kv_transfer_group", lambda: False) + runner = _make_guard_runner() + + output = GPUGenerationModelRunner.execute_model(runner, _StubSchedulerOutput(total)) + + assert output is EMPTY_MODEL_RUNNER_OUTPUT diff --git a/vllm_omni/platforms/npu/worker/npu_ar_model_runner.py b/vllm_omni/platforms/npu/worker/npu_ar_model_runner.py index b7b275b12..0817d7bbf 100644 --- a/vllm_omni/platforms/npu/worker/npu_ar_model_runner.py +++ b/vllm_omni/platforms/npu/worker/npu_ar_model_runner.py @@ -463,7 +463,8 @@ class NPUARModelRunner(OmniNPUModelRunner, OmniConnectorModelRunnerMixin): output.kv_extracted_req_ids = kv_ids return self.attach_omni_connector_output(output) - if not num_scheduled_tokens: + # `<= 0`: upstream can schedule a negative span, which is truthy (#5196). + if num_scheduled_tokens <= 0: if ( self.parallel_config.distributed_executor_backend == "external_launcher" and self.parallel_config.data_parallel_size > 1 diff --git a/vllm_omni/platforms/npu/worker/npu_generation_model_runner.py b/vllm_omni/platforms/npu/worker/npu_generation_model_runner.py index d43867c97..f4fc05574 100644 --- a/vllm_omni/platforms/npu/worker/npu_generation_model_runner.py +++ b/vllm_omni/platforms/npu/worker/npu_generation_model_runner.py @@ -159,7 +159,8 @@ class NPUGenerationModelRunner(OmniNPUModelRunner, OmniConnectorModelRunnerMixin make_empty_encoder_model_runner_output(scheduler_output) ) - if not num_scheduled_tokens: + # `<= 0`: upstream can schedule a negative span, which is truthy (#5196). + if num_scheduled_tokens <= 0: if ( self.parallel_config.distributed_executor_backend == "external_launcher" and self.parallel_config.data_parallel_size > 1 diff --git a/vllm_omni/worker/gpu_ar_model_runner.py b/vllm_omni/worker/gpu_ar_model_runner.py index 411796d37..67ea34b10 100644 --- a/vllm_omni/worker/gpu_ar_model_runner.py +++ b/vllm_omni/worker/gpu_ar_model_runner.py @@ -1057,7 +1057,8 @@ class GPUARModelRunner(OmniGPUModelRunner, OmniConnectorModelRunnerMixin): output.kv_extracted_req_ids = kv_ids return self.attach_omni_connector_output(output) - if not num_scheduled_tokens: + # `<= 0`: upstream can schedule a negative span, which is truthy (#5196). + if num_scheduled_tokens <= 0: if ( self.parallel_config.distributed_executor_backend == "external_launcher" and self.parallel_config.data_parallel_size > 1 diff --git a/vllm_omni/worker/gpu_generation_model_runner.py b/vllm_omni/worker/gpu_generation_model_runner.py index bcb647605..6208f73e9 100644 --- a/vllm_omni/worker/gpu_generation_model_runner.py +++ b/vllm_omni/worker/gpu_generation_model_runner.py @@ -159,7 +159,8 @@ class GPUGenerationModelRunner(OmniGPUModelRunner, OmniConnectorModelRunnerMixin if scheduler_output.finished_req_ids and hasattr(self.model, "on_requests_finished"): self.model.on_requests_finished(scheduler_output.finished_req_ids) - if not scheduler_output.total_num_scheduled_tokens: + # `<= 0`: upstream can schedule a negative span, which is truthy (#5196). + if scheduler_output.total_num_scheduled_tokens <= 0: return self.attach_omni_connector_output(EMPTY_MODEL_RUNNER_OUTPUT) if has_ec_transfer() and not get_ec_transfer().is_consumer: From f10320f53fbe9290e4155f8e878febd9c18be71c Mon Sep 17 00:00:00 2001 From: Nick Cao Date: Wed, 22 Jul 2026 13:11:53 -0400 Subject: [PATCH 065/240] [Tests] Config driven tiny model builder (#5090) Signed-off-by: Nick Cao --- tests/helpers/tiny_model.py | 87 +++++++++++++++++++ tests/model_tests/diffusion/config_types.py | 3 +- .../diffusion/diff_model_builders.py | 68 +++------------ tests/model_tests/diffusion/model_settings.py | 14 ++- tests/model_tests/diffusion/task_runners.py | 67 ++++++++++++++ tests/model_tests/diffusion/test_alignment.py | 1 - .../diffusion/test_common_offline.py | 3 + .../diffusion/test_common_online.py | 3 + .../Flux2KleinPipeline/model_index.json | 25 ++++++ .../text_encoder/config.json | 60 +++++++++++++ .../transformer/config.json | 24 +++++ .../Flux2KleinPipeline/vae/config.json | 31 +++++++ .../LTX2Pipeline/audio_vae/config.json | 25 ++++++ .../LTX2Pipeline/connectors/config.json | 21 +++++ .../LTX2Pipeline/model_index.json | 36 ++++++++ .../LTX2Pipeline/text_encoder/config.json | 68 +++++++++++++++ .../LTX2Pipeline/transformer/config.json | 44 ++++++++++ .../tiny_configs/LTX2Pipeline/vae/config.json | 81 +++++++++++++++++ 18 files changed, 601 insertions(+), 60 deletions(-) create mode 100644 tests/helpers/tiny_model.py create mode 100644 tests/model_tests/diffusion/tiny_configs/Flux2KleinPipeline/model_index.json create mode 100644 tests/model_tests/diffusion/tiny_configs/Flux2KleinPipeline/text_encoder/config.json create mode 100644 tests/model_tests/diffusion/tiny_configs/Flux2KleinPipeline/transformer/config.json create mode 100644 tests/model_tests/diffusion/tiny_configs/Flux2KleinPipeline/vae/config.json create mode 100644 tests/model_tests/diffusion/tiny_configs/LTX2Pipeline/audio_vae/config.json create mode 100644 tests/model_tests/diffusion/tiny_configs/LTX2Pipeline/connectors/config.json create mode 100644 tests/model_tests/diffusion/tiny_configs/LTX2Pipeline/model_index.json create mode 100644 tests/model_tests/diffusion/tiny_configs/LTX2Pipeline/text_encoder/config.json create mode 100644 tests/model_tests/diffusion/tiny_configs/LTX2Pipeline/transformer/config.json create mode 100644 tests/model_tests/diffusion/tiny_configs/LTX2Pipeline/vae/config.json diff --git a/tests/helpers/tiny_model.py b/tests/helpers/tiny_model.py new file mode 100644 index 000000000..b212a8e7f --- /dev/null +++ b/tests/helpers/tiny_model.py @@ -0,0 +1,87 @@ +"""Reusable utility for building tiny diffusion models with random weights. + +Mirrors the component loading loop in DiffusionPipeline.from_pretrained, +but monkeypatches from_pretrained on ModelMixin and PreTrainedModel to +initialize with random weights instead of loading checkpoint files. +Components with vendored configs use those; others load configs from +the upstream HF model. +""" + +import os +import tempfile +from pathlib import Path +from unittest.mock import patch + +import torch +from diffusers import ModelMixin +from diffusers.pipelines.pipeline_loading_utils import _get_pipeline_class, simple_get_class_obj +from diffusers.pipelines.pipeline_utils import DiffusionPipeline +from transformers import AutoConfig, PreTrainedModel + +TINY_MODEL_DIR = os.path.join(tempfile.gettempdir(), "vllm-omni-tiny-models") + + +def _get_tiny_model_path(name: str) -> str: + path = os.path.join(TINY_MODEL_DIR, name) + os.makedirs(path, exist_ok=True) + return path + + +def _diffusers_from_config(cls, pretrained_model_name_or_path, **kwargs): + """Replacement for ModelMixin.from_pretrained that initializes random weights.""" + subfolder = kwargs.get("subfolder") + load_kwargs = {"subfolder": subfolder} if subfolder else {} + config = cls.load_config(pretrained_model_name_or_path, **load_kwargs) + return cls.from_config(config) + + +def _transformers_from_config(cls, pretrained_model_name_or_path, **kwargs): + """Replacement for PreTrainedModel.from_pretrained that initializes random weights.""" + subfolder = kwargs.get("subfolder") + load_kwargs = {"subfolder": subfolder} if subfolder else {} + config = AutoConfig.from_pretrained(pretrained_model_name_or_path, **load_kwargs) + return cls(config) + + +def build_tiny_from_configs(pipeline_name: str, model_id: str, configs_dir: str | Path) -> str: + """Build a tiny diffusion model from vendored configs with random weights. + + Args: + pipeline_name: Name of the pipeline (used as output directory name). + model_id: HuggingFace model ID for loading upstream components + (tokenizer, scheduler) that don't have vendored configs. + configs_dir: Path to the directory containing vendored config files + (model_index.json and per-component config.json). + + Returns: + Path to the saved tiny model directory with safetensors weights. + """ + model_dir = _get_tiny_model_path(pipeline_name) + config_dir = Path(configs_dir) + + config_dict = DiffusionPipeline.load_config(config_dir) + pipeline_cls = _get_pipeline_class(DiffusionPipeline, config=config_dict) + + init_dict, _, _ = pipeline_cls.extract_init_dict(config_dict) + + # Pop non-component entries (optional pipeline kwargs like is_distilled), + # same as DiffusionPipeline.from_pretrained lines 345-350 + _, optional_kwargs = DiffusionPipeline._get_signature_keys(pipeline_cls) + init_kwargs = {k: init_dict.pop(k) for k in optional_kwargs if k in init_dict} + + with ( + patch.object(ModelMixin, "from_pretrained", classmethod(_diffusers_from_config)), + patch.object(PreTrainedModel, "from_pretrained", classmethod(_transformers_from_config)), + ): + for name, (library_name, class_name) in init_dict.items(): + cls = simple_get_class_obj(library_name, class_name) + + # Use vendored config dir if available, otherwise upstream model + if (config_dir / name).exists(): + init_kwargs[name] = cls.from_pretrained(config_dir / name) + else: + init_kwargs[name] = cls.from_pretrained(model_id, subfolder=name) + + pipe = pipeline_cls(**init_kwargs) + pipe.to(torch.bfloat16).save_pretrained(model_dir) + return model_dir diff --git a/tests/model_tests/diffusion/config_types.py b/tests/model_tests/diffusion/config_types.py index 4120900fe..da599c2c2 100644 --- a/tests/model_tests/diffusion/config_types.py +++ b/tests/model_tests/diffusion/config_types.py @@ -34,7 +34,8 @@ class DiffusionTasks(StrEnum): TEXT_TO_IMAGE = auto() IMAGE_TO_IMAGE = auto() - # Text to video, text to audio, etc should be added here as needed + TEXT_TO_VIDEO = auto() + # Text to audio, etc should be added here as needed class DiffusionModelTestOpts(NamedTuple): diff --git a/tests/model_tests/diffusion/diff_model_builders.py b/tests/model_tests/diffusion/diff_model_builders.py index 9bee71f9b..54e7a4016 100644 --- a/tests/model_tests/diffusion/diff_model_builders.py +++ b/tests/model_tests/diffusion/diff_model_builders.py @@ -1,63 +1,17 @@ -import os -import tempfile +from pathlib import Path -import torch -from diffusers.models.autoencoders.autoencoder_kl_flux2 import AutoencoderKLFlux2 -from diffusers.models.transformers.transformer_flux2 import Flux2Transformer2DModel -from diffusers.pipelines.flux2.pipeline_flux2_klein import Flux2KleinPipeline -from diffusers.schedulers.scheduling_flow_match_euler_discrete import FlowMatchEulerDiscreteScheduler -from transformers import AutoTokenizer, Qwen3Config, Qwen3ForCausalLM +from tests.helpers.tiny_model import build_tiny_from_configs -TINY_MODEL_DIR = os.path.join(tempfile.gettempdir(), "vllm-omni-tiny-models") - - -def _get_tiny_model_path(name: str) -> str: - path = os.path.join(TINY_MODEL_DIR, name) - os.makedirs(path, exist_ok=True) - return path +TINY_CONFIGS_DIR = Path(__file__).parent / "tiny_configs" def tiny_flux2_klein_builder() -> str: - """Build a tiny Flux2Klein model.""" - model_dir = _get_tiny_model_path("Flux2KleinPipeline") - - pipe = Flux2KleinPipeline( - scheduler=FlowMatchEulerDiscreteScheduler(), - vae=AutoencoderKLFlux2( - down_block_types=("DownEncoderBlock2D",), - up_block_types=("UpDecoderBlock2D",), - block_out_channels=(32,), - layers_per_block=1, - latent_channels=16, - norm_num_groups=16, - ), - # NOTE: For now we need 28 layers because of hardcoded stuff in the model :( - text_encoder=Qwen3ForCausalLM( - Qwen3Config( - hidden_size=32, - intermediate_size=64, - num_hidden_layers=28, - num_attention_heads=2, - num_key_value_heads=2, - head_dim=16, - vocab_size=151936, - max_position_embeddings=512, - ) - ), - tokenizer=AutoTokenizer.from_pretrained("Qwen/Qwen2.5-0.5B"), - # NOTE: For now we need at least 2 layers for the transformer - # due to hardcoded hacks in CacheDiT for Flux2Klein specifically. - transformer=Flux2Transformer2DModel( - in_channels=64, - num_layers=2, - num_single_layers=2, - attention_head_dim=16, - num_attention_heads=2, - joint_attention_dim=96, - timestep_guidance_channels=32, - axes_dims_rope=(4, 4, 4, 4), - ), + """Build a tiny Flux2Klein model from vendored configs.""" + return build_tiny_from_configs( + "Flux2KleinPipeline", "black-forest-labs/FLUX.2-klein-4B", TINY_CONFIGS_DIR / "Flux2KleinPipeline" ) - # Need dtypes to be consistent; for now we just put it on bfloat16 - pipe.to(torch.bfloat16).save_pretrained(model_dir) - return model_dir + + +def tiny_ltx2_builder() -> str: + """Build a tiny LTX2 model from vendored configs.""" + return build_tiny_from_configs("LTX2Pipeline", "Lightricks/LTX-2", TINY_CONFIGS_DIR / "LTX2Pipeline") diff --git a/tests/model_tests/diffusion/model_settings.py b/tests/model_tests/diffusion/model_settings.py index 3f44afcae..c6c1cb1d9 100644 --- a/tests/model_tests/diffusion/model_settings.py +++ b/tests/model_tests/diffusion/model_settings.py @@ -38,5 +38,17 @@ DIFFUSION_TEST_SETTINGS = { [DiffusionAccs.SEQUENCE_PARALLEL, DiffusionAccs.CACHE_DIT, DiffusionAccs.LAYERWISE_OFFLOAD], [DiffusionAccs.CFG_PARALLEL, DiffusionAccs.TENSOR_PARALLEL, DiffusionAccs.CPU_OFFLOAD], ], - ) + ), + "LTX2Pipeline": DiffusionModelTestOpts( + model="Lightricks/LTX-2", + builder=diff_model_builders.tiny_ltx2_builder, + supported_tasks=[DiffusionTasks.TEXT_TO_VIDEO], + check_determinism=False, + check_multi_output=False, + extra_test_groups=[ + [DiffusionAccs.HSDP, DiffusionAccs.CACHE_DIT], + [DiffusionAccs.SEQUENCE_PARALLEL, DiffusionAccs.CACHE_DIT, DiffusionAccs.LAYERWISE_OFFLOAD], + [DiffusionAccs.CFG_PARALLEL, DiffusionAccs.TENSOR_PARALLEL, DiffusionAccs.CPU_OFFLOAD], + ], + ), } diff --git a/tests/model_tests/diffusion/task_runners.py b/tests/model_tests/diffusion/task_runners.py index 6b9c7ff99..23ff60eeb 100644 --- a/tests/model_tests/diffusion/task_runners.py +++ b/tests/model_tests/diffusion/task_runners.py @@ -28,6 +28,15 @@ IMAGE_GEN_SAMPLING_PARAMS = OmniDiffusionSamplingParams( seed=42, ) +VIDEO_NUM_FRAMES = 9 +VIDEO_GEN_SAMPLING_PARAMS = OmniDiffusionSamplingParams( + num_inference_steps=4, + height=HEIGHT, + width=WIDTH, + num_frames=VIDEO_NUM_FRAMES, + seed=42, +) + # Online extra_body for diffusion requests IMAGE_GEN_EXTRA_BODY = { "height": HEIGHT, @@ -36,6 +45,15 @@ IMAGE_GEN_EXTRA_BODY = { "seed": 42, } +# Online form_data for video generation requests (multipart /v1/videos API) +VIDEO_GEN_FORM_DATA = { + "height": HEIGHT, + "width": WIDTH, + "num_inference_steps": 4, + "num_frames": VIDEO_NUM_FRAMES, + "seed": 42, +} + ### Shared validation def _validate_images(images: list[Image.Image], expected_n: int = 1): @@ -70,11 +88,31 @@ def _get_online_images(responses: list[DiffusionResponse]) -> list[Image.Image]: return images +def _validate_video(outputs: list[OmniRequestOutput], expected_n: int = 1): + """Given a set of outputs, ensure we got video frames with the expected shape.""" + assert len(outputs) == expected_n + for output in outputs: + # Video models return numpy arrays via output.images + images = output.images + assert len(images) > 0 + for frame_data in images: + assert isinstance(frame_data, np.ndarray) + # (num_outputs, num_frames, H, W, C) or (num_frames, H, W, C) + assert frame_data.ndim in (4, 5), f"Expected 4D or 5D video array, got shape {frame_data.shape}" + assert frame_data.shape[-3] == HEIGHT, f"Expected height {HEIGHT}, got {frame_data.shape[-3]}" + assert frame_data.shape[-2] == WIDTH, f"Expected width {WIDTH}, got {frame_data.shape[-2]}" + assert frame_data.shape[-1] == 3, f"Expected 3 channels (RGB), got {frame_data.shape[-1]}" + + ### Offline helpers def _run_offline_t2i(omni: Omni, params: OmniDiffusionSamplingParams = IMAGE_GEN_SAMPLING_PARAMS): return omni.generate({"prompt": PROMPT}, params) +def _run_offline_t2v(omni: Omni, params: OmniDiffusionSamplingParams = VIDEO_GEN_SAMPLING_PARAMS): + return omni.generate({"prompt": PROMPT}, params) + + def _run_offline_i2i(omni: Omni): return omni.generate( {"prompt": PROMPT, "multi_modal_data": {"image": INPUT_IMAGE}}, @@ -130,6 +168,11 @@ def run_and_validate_image_to_image_request(omni: Omni): _validate_images(_get_offline_images(_run_offline_i2i(omni))) +def run_and_validate_text_to_video_request(omni: Omni): + """Run and validate a text to video request.""" + _validate_video(_run_offline_t2v(omni)) + + def run_and_validate_text_to_image_determinism(omni: Omni): """Checks for determinism; for now we just keep this for TTI.""" _validate_image_gen_determinism( @@ -144,6 +187,25 @@ def run_and_validate_text_to_image_multi_output(omni: Omni): _validate_images(_get_offline_images(_run_offline_t2i(omni, params)), expected_n=2) +def _run_online_t2v( + server: OmniServer, client: OpenAIClientHandler, form_data: dict | None = None +) -> list[DiffusionResponse]: + """Run a text to video request through the server's /v1/videos API.""" + data = dict(form_data or VIDEO_GEN_FORM_DATA) + data.setdefault("prompt", PROMPT) + data.setdefault("model", server.model) + return client.send_video_diffusion_request({"form_data": data}) + + +def _get_online_videos(responses: list[DiffusionResponse]) -> list: + """Extract the videos from a server response.""" + assert len(responses) == 1 + videos = responses[0].videos + assert videos is not None + assert len(videos) > 0 + return videos + + ### Online task runners def run_and_validate_online_text_to_image_request(server: OmniServer, client: OpenAIClientHandler): """Run and validate a text to image request through the server.""" @@ -170,3 +232,8 @@ def run_and_validate_online_text_to_image_multi_output(server: OmniServer, clien _get_online_images(_run_online_t2i(server, client, extra_body=extra_body)), expected_n=2, ) + + +def run_and_validate_online_text_to_video_request(server: OmniServer, client: OpenAIClientHandler): + """Run and validate a text to video request through the server.""" + _get_online_videos(_run_online_t2v(server, client)) diff --git a/tests/model_tests/diffusion/test_alignment.py b/tests/model_tests/diffusion/test_alignment.py index 4bc740b01..9d0a4911a 100644 --- a/tests/model_tests/diffusion/test_alignment.py +++ b/tests/model_tests/diffusion/test_alignment.py @@ -30,7 +30,6 @@ EXCLUDED_MODELS = [ "OvisImagePipeline", "WanPipeline", "WanVACEPipeline", - "LTX2Pipeline", "LTX2ImageToVideoPipeline", "LTX2TwoStagesPipeline", "LTX2ImageToVideoTwoStagesPipeline", diff --git a/tests/model_tests/diffusion/test_common_offline.py b/tests/model_tests/diffusion/test_common_offline.py index f24ed3780..d5cf3aba6 100644 --- a/tests/model_tests/diffusion/test_common_offline.py +++ b/tests/model_tests/diffusion/test_common_offline.py @@ -12,6 +12,7 @@ from tests.model_tests.diffusion.task_runners import ( run_and_validate_text_to_image_determinism, run_and_validate_text_to_image_multi_output, run_and_validate_text_to_image_request, + run_and_validate_text_to_video_request, ) # NOTE : Hardware marks are added dynamically based on test requirements @@ -53,6 +54,8 @@ def test_pipeline_on_supported_tasks( run_and_validate_text_to_image_request(omni) elif task_type == DiffusionTasks.IMAGE_TO_IMAGE: run_and_validate_image_to_image_request(omni) + elif task_type == DiffusionTasks.TEXT_TO_VIDEO: + run_and_validate_text_to_video_request(omni) else: raise ValueError(f"Task type {task_type} is not yet supported") diff --git a/tests/model_tests/diffusion/test_common_online.py b/tests/model_tests/diffusion/test_common_online.py index e4c6be0d4..31509c3b3 100644 --- a/tests/model_tests/diffusion/test_common_online.py +++ b/tests/model_tests/diffusion/test_common_online.py @@ -19,6 +19,7 @@ from tests.model_tests.diffusion.task_runners import ( run_and_validate_online_text_to_image_determinism, run_and_validate_online_text_to_image_multi_output, run_and_validate_online_text_to_image_request, + run_and_validate_online_text_to_video_request, ) # NOTE : Hardware marks are added dynamically based on test requirements @@ -61,6 +62,8 @@ def test_online_on_supported_tasks( run_and_validate_online_text_to_image_request(server, client) elif task_type == DiffusionTasks.IMAGE_TO_IMAGE: run_and_validate_online_image_to_image_request(server, client) + elif task_type == DiffusionTasks.TEXT_TO_VIDEO: + run_and_validate_online_text_to_video_request(server, client) else: raise ValueError(f"Task type {task_type} is not yet supported") diff --git a/tests/model_tests/diffusion/tiny_configs/Flux2KleinPipeline/model_index.json b/tests/model_tests/diffusion/tiny_configs/Flux2KleinPipeline/model_index.json new file mode 100644 index 000000000..554a39bf1 --- /dev/null +++ b/tests/model_tests/diffusion/tiny_configs/Flux2KleinPipeline/model_index.json @@ -0,0 +1,25 @@ +{ + "_class_name": "Flux2KleinPipeline", + "_diffusers_version": "0.37.0.dev0", + "is_distilled": true, + "scheduler": [ + "diffusers", + "FlowMatchEulerDiscreteScheduler" + ], + "text_encoder": [ + "transformers", + "Qwen3ForCausalLM" + ], + "tokenizer": [ + "transformers", + "Qwen2TokenizerFast" + ], + "transformer": [ + "diffusers", + "Flux2Transformer2DModel" + ], + "vae": [ + "diffusers", + "AutoencoderKLFlux2" + ] +} diff --git a/tests/model_tests/diffusion/tiny_configs/Flux2KleinPipeline/text_encoder/config.json b/tests/model_tests/diffusion/tiny_configs/Flux2KleinPipeline/text_encoder/config.json new file mode 100644 index 000000000..004914a7f --- /dev/null +++ b/tests/model_tests/diffusion/tiny_configs/Flux2KleinPipeline/text_encoder/config.json @@ -0,0 +1,60 @@ +{ + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 151643, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 16, + "hidden_act": "silu", + "hidden_size": 32, + "initializer_range": 0.02, + "intermediate_size": 64, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 512, + "max_window_layers": 36, + "model_type": "qwen3", + "num_attention_heads": 2, + "num_hidden_layers": 28, + "num_key_value_heads": 2, + "rms_norm_eps": 1e-06, + "rope_scaling": null, + "rope_theta": 1000000, + "sliding_window": null, + "tie_word_embeddings": true, + "transformers_version": "4.56.1", + "use_cache": true, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/tests/model_tests/diffusion/tiny_configs/Flux2KleinPipeline/transformer/config.json b/tests/model_tests/diffusion/tiny_configs/Flux2KleinPipeline/transformer/config.json new file mode 100644 index 000000000..6ff16e1e7 --- /dev/null +++ b/tests/model_tests/diffusion/tiny_configs/Flux2KleinPipeline/transformer/config.json @@ -0,0 +1,24 @@ +{ + "_class_name": "Flux2Transformer2DModel", + "_diffusers_version": "0.37.0.dev0", + "_name_or_path": "/raid/yiyi/klein-4b-distilled-diffusers/transformer", + "attention_head_dim": 16, + "axes_dims_rope": [ + 4, + 4, + 4, + 4 + ], + "eps": 1e-06, + "guidance_embeds": false, + "in_channels": 64, + "joint_attention_dim": 96, + "mlp_ratio": 3.0, + "num_attention_heads": 2, + "num_layers": 2, + "num_single_layers": 2, + "out_channels": null, + "patch_size": 1, + "rope_theta": 2000, + "timestep_guidance_channels": 32 +} diff --git a/tests/model_tests/diffusion/tiny_configs/Flux2KleinPipeline/vae/config.json b/tests/model_tests/diffusion/tiny_configs/Flux2KleinPipeline/vae/config.json new file mode 100644 index 000000000..6410d25bd --- /dev/null +++ b/tests/model_tests/diffusion/tiny_configs/Flux2KleinPipeline/vae/config.json @@ -0,0 +1,31 @@ +{ + "_class_name": "AutoencoderKLFlux2", + "_diffusers_version": "0.37.0.dev0", + "_name_or_path": "black-forest-labs/FLUX.2-dev", + "act_fn": "silu", + "batch_norm_eps": 0.0001, + "batch_norm_momentum": 0.1, + "block_out_channels": [ + 32 + ], + "down_block_types": [ + "DownEncoderBlock2D" + ], + "force_upcast": true, + "in_channels": 3, + "latent_channels": 16, + "layers_per_block": 1, + "mid_block_add_attention": true, + "norm_num_groups": 16, + "out_channels": 3, + "patch_size": [ + 2, + 2 + ], + "sample_size": 1024, + "up_block_types": [ + "UpDecoderBlock2D" + ], + "use_post_quant_conv": true, + "use_quant_conv": true +} diff --git a/tests/model_tests/diffusion/tiny_configs/LTX2Pipeline/audio_vae/config.json b/tests/model_tests/diffusion/tiny_configs/LTX2Pipeline/audio_vae/config.json new file mode 100644 index 000000000..5ebd2f5e8 --- /dev/null +++ b/tests/model_tests/diffusion/tiny_configs/LTX2Pipeline/audio_vae/config.json @@ -0,0 +1,25 @@ +{ + "_class_name": "AutoencoderKLLTX2Audio", + "_diffusers_version": "0.37.0.dev0", + "attn_resolutions": null, + "base_channels": 128, + "causality_axis": "height", + "ch_mult": [ + 1, + 2, + 4 + ], + "double_z": true, + "dropout": 0.0, + "in_channels": 2, + "is_causal": true, + "latent_channels": 8, + "mel_bins": 64, + "mel_hop_length": 160, + "mid_block_add_attention": false, + "norm_type": "pixel", + "num_res_blocks": 1, + "output_channels": 2, + "resolution": 256, + "sample_rate": 16000 +} diff --git a/tests/model_tests/diffusion/tiny_configs/LTX2Pipeline/connectors/config.json b/tests/model_tests/diffusion/tiny_configs/LTX2Pipeline/connectors/config.json new file mode 100644 index 000000000..c63177c7c --- /dev/null +++ b/tests/model_tests/diffusion/tiny_configs/LTX2Pipeline/connectors/config.json @@ -0,0 +1,21 @@ +{ + "_class_name": "LTX2TextConnectors", + "_diffusers_version": "0.37.0.dev0", + "audio_connector_attention_head_dim": 16, + "audio_connector_num_attention_heads": 2, + "audio_connector_num_layers": 1, + "audio_connector_num_learnable_registers": 4, + "caption_channels": 32, + "causal_temporal_positioning": false, + "connector_rope_base_seq_len": 32, + "rope_double_precision": true, + "rope_theta": 10000.0, + "rope_type": "split", + "text_proj_in_factor": 3, + "video_connector_attention_head_dim": 16, + "video_connector_num_attention_heads": 2, + "video_connector_num_layers": 1, + "video_connector_num_learnable_registers": 4, + "video_hidden_dim": 32, + "audio_hidden_dim": 32 +} diff --git a/tests/model_tests/diffusion/tiny_configs/LTX2Pipeline/model_index.json b/tests/model_tests/diffusion/tiny_configs/LTX2Pipeline/model_index.json new file mode 100644 index 000000000..ef9de5d07 --- /dev/null +++ b/tests/model_tests/diffusion/tiny_configs/LTX2Pipeline/model_index.json @@ -0,0 +1,36 @@ +{ + "_class_name": "LTX2Pipeline", + "_diffusers_version": "0.37.0.dev0", + "audio_vae": [ + "diffusers", + "AutoencoderKLLTX2Audio" + ], + "connectors": [ + "ltx2", + "LTX2TextConnectors" + ], + "scheduler": [ + "diffusers", + "FlowMatchEulerDiscreteScheduler" + ], + "text_encoder": [ + "transformers", + "Gemma3ForConditionalGeneration" + ], + "tokenizer": [ + "transformers", + "GemmaTokenizerFast" + ], + "transformer": [ + "diffusers", + "LTX2VideoTransformer3DModel" + ], + "vae": [ + "diffusers", + "AutoencoderKLLTX2Video" + ], + "vocoder": [ + "ltx2", + "LTX2Vocoder" + ] +} diff --git a/tests/model_tests/diffusion/tiny_configs/LTX2Pipeline/text_encoder/config.json b/tests/model_tests/diffusion/tiny_configs/LTX2Pipeline/text_encoder/config.json new file mode 100644 index 000000000..0d722f0c1 --- /dev/null +++ b/tests/model_tests/diffusion/tiny_configs/LTX2Pipeline/text_encoder/config.json @@ -0,0 +1,68 @@ +{ + "architectures": [ + "Gemma3ForConditionalGeneration" + ], + "boi_token_index": 255999, + "dtype": "float32", + "eoi_token_index": 256000, + "eos_token_id": [ + 1, + 106 + ], + "image_token_index": 262144, + "initializer_range": 0.02, + "mm_tokens_per_image": 256, + "model_type": "gemma3", + "text_config": { + "_sliding_window_pattern": 6, + "attention_bias": false, + "attention_dropout": 0.0, + "attn_logit_softcapping": null, + "cache_implementation": "hybrid", + "dtype": "float32", + "final_logit_softcapping": null, + "head_dim": 16, + "hidden_activation": "gelu_pytorch_tanh", + "hidden_size": 32, + "initializer_range": 0.02, + "intermediate_size": 64, + "layer_types": [ + "sliding_attention", + "full_attention" + ], + "max_position_embeddings": 512, + "model_type": "gemma3_text", + "num_attention_heads": 2, + "num_hidden_layers": 2, + "num_key_value_heads": 2, + "query_pre_attn_scalar": 16, + "rms_norm_eps": 1e-06, + "rope_local_base_freq": 10000, + "rope_scaling": { + "factor": 8.0, + "rope_type": "linear" + }, + "rope_theta": 1000000, + "sliding_window": 1024, + "sliding_window_pattern": 6, + "use_bidirectional_attention": false, + "use_cache": true, + "vocab_size": 262208 + }, + "transformers_version": "4.57.3", + "vision_config": { + "attention_dropout": 0.0, + "dtype": "float32", + "hidden_act": "gelu_pytorch_tanh", + "hidden_size": 32, + "image_size": 896, + "intermediate_size": 64, + "layer_norm_eps": 1e-06, + "model_type": "siglip_vision_model", + "num_attention_heads": 2, + "num_channels": 3, + "num_hidden_layers": 2, + "patch_size": 14, + "vision_use_head": false + } +} diff --git a/tests/model_tests/diffusion/tiny_configs/LTX2Pipeline/transformer/config.json b/tests/model_tests/diffusion/tiny_configs/LTX2Pipeline/transformer/config.json new file mode 100644 index 000000000..577b0fb36 --- /dev/null +++ b/tests/model_tests/diffusion/tiny_configs/LTX2Pipeline/transformer/config.json @@ -0,0 +1,44 @@ +{ + "_class_name": "LTX2VideoTransformer3DModel", + "_diffusers_version": "0.37.0.dev0", + "activation_fn": "gelu-approximate", + "attention_bias": true, + "attention_head_dim": 16, + "attention_out_bias": true, + "audio_attention_head_dim": 16, + "audio_cross_attention_dim": 32, + "audio_hop_length": 160, + "audio_in_channels": 128, + "audio_num_attention_heads": 2, + "audio_out_channels": 128, + "audio_patch_size": 1, + "audio_patch_size_t": 1, + "audio_pos_embed_max_pos": 20, + "audio_sampling_rate": 16000, + "audio_scale_factor": 4, + "base_height": 2048, + "base_width": 2048, + "caption_channels": 32, + "causal_offset": 1, + "cross_attention_dim": 32, + "cross_attn_timestep_scale_multiplier": 1000, + "in_channels": 128, + "norm_elementwise_affine": false, + "norm_eps": 1e-06, + "num_attention_heads": 2, + "num_layers": 2, + "out_channels": 128, + "patch_size": 1, + "patch_size_t": 1, + "pos_embed_max_pos": 20, + "qk_norm": "rms_norm_across_heads", + "rope_double_precision": true, + "rope_theta": 10000.0, + "rope_type": "split", + "timestep_scale_multiplier": 1000, + "vae_scale_factors": [ + 8, + 32, + 32 + ] +} diff --git a/tests/model_tests/diffusion/tiny_configs/LTX2Pipeline/vae/config.json b/tests/model_tests/diffusion/tiny_configs/LTX2Pipeline/vae/config.json new file mode 100644 index 000000000..a672971d4 --- /dev/null +++ b/tests/model_tests/diffusion/tiny_configs/LTX2Pipeline/vae/config.json @@ -0,0 +1,81 @@ +{ + "_class_name": "AutoencoderKLLTX2Video", + "_diffusers_version": "0.37.0.dev0", + "block_out_channels": [ + 256, + 512, + 1024, + 2048 + ], + "decoder_block_out_channels": [ + 256, + 512, + 1024 + ], + "decoder_causal": false, + "decoder_inject_noise": [ + false, + false, + false, + false + ], + "decoder_layers_per_block": [ + 1, + 1, + 1, + 1 + ], + "decoder_spatial_padding_mode": "reflect", + "decoder_spatio_temporal_scaling": [ + true, + true, + true + ], + "down_block_types": [ + "LTX2VideoDownBlock3D", + "LTX2VideoDownBlock3D", + "LTX2VideoDownBlock3D", + "LTX2VideoDownBlock3D" + ], + "downsample_type": [ + "spatial", + "temporal", + "spatiotemporal", + "spatiotemporal" + ], + "encoder_causal": true, + "encoder_spatial_padding_mode": "zeros", + "in_channels": 3, + "latent_channels": 128, + "layers_per_block": [ + 1, + 1, + 1, + 1, + 1 + ], + "out_channels": 3, + "patch_size": 4, + "patch_size_t": 1, + "resnet_norm_eps": 1e-06, + "scaling_factor": 1.0, + "spatial_compression_ratio": 32, + "spatio_temporal_scaling": [ + true, + true, + true, + true + ], + "temporal_compression_ratio": 8, + "timestep_conditioning": false, + "upsample_factor": [ + 2, + 2, + 2 + ], + "upsample_residual": [ + true, + true, + true + ] +} From 7d5376f21534482d383c8bab085f5a2ca801b37f Mon Sep 17 00:00:00 2001 From: Alex Brooks Date: Wed, 22 Jul 2026 19:14:55 -0600 Subject: [PATCH 066/240] [CI/Perf] Optimize Docker Caching (#5133) Signed-off-by: Alex Brooks Signed-off-by: Doug Smith Co-authored-by: dougbtv Co-authored-by: Claude Opus 4.6 --- .buildkite/cuda/pipeline.yml | 5 +-- .buildkite/cuda/rebase-pipeline.yml | 5 +-- .buildkite/cuda/scripts/build-ci-image.sh | 52 +++++++++++++++++++++++ .gitignore | 1 - docker/Dockerfile.ci | 42 ++++++++++++++++-- docker/Dockerfile.ci.dockerignore | 37 ++++++++++++++++ 6 files changed, 130 insertions(+), 12 deletions(-) create mode 100755 .buildkite/cuda/scripts/build-ci-image.sh create mode 100644 docker/Dockerfile.ci.dockerignore diff --git a/.buildkite/cuda/pipeline.yml b/.buildkite/cuda/pipeline.yml index 3b460c1c6..e9441a7b5 100644 --- a/.buildkite/cuda/pipeline.yml +++ b/.buildkite/cuda/pipeline.yml @@ -19,10 +19,7 @@ steps: key: image-build if: __IMAGE_BUILD_IF__ commands: - - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7" - - "docker build --progress=plain --file docker/Dockerfile.ci -t vllm-omni-ci ." - - "docker tag vllm-omni-ci public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT" - - "docker push public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT" + - "bash .buildkite/cuda/scripts/build-ci-image.sh" agents: queue: "cpu_queue_premerge" diff --git a/.buildkite/cuda/rebase-pipeline.yml b/.buildkite/cuda/rebase-pipeline.yml index 514c9cc1e..0cb21edcd 100644 --- a/.buildkite/cuda/rebase-pipeline.yml +++ b/.buildkite/cuda/rebase-pipeline.yml @@ -2,10 +2,7 @@ steps: - label: ":docker: Build image" key: image-build commands: - - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7" - - "docker build --progress=plain --file docker/Dockerfile.ci -t vllm-omni-ci ." - - "docker tag vllm-omni-ci public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT" - - "docker push public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT" + - "bash .buildkite/cuda/scripts/build-ci-image.sh" agents: queue: "cpu_queue_premerge" diff --git a/.buildkite/cuda/scripts/build-ci-image.sh b/.buildkite/cuda/scripts/build-ci-image.sh new file mode 100755 index 000000000..9da69bc8b --- /dev/null +++ b/.buildkite/cuda/scripts/build-ci-image.sh @@ -0,0 +1,52 @@ +#!/bin/bash +set -euo pipefail + +# Build and push the CI Docker image with registry-based layer caching. +# +# Usage: build-ci-image.sh +# +# Expects the following environment variables (set by Buildkite): +# BUILDKITE_COMMIT - commit SHA used as the image tag +ECR_NAMESPACE="public.ecr.aws/q9t5s3a7" +REGISTRY="${ECR_NAMESPACE}/vllm-ci-test-repo" +REGION="us-east-1" +DOCKERFILE="docker/Dockerfile.ci" +BUILDER_NAME="vllm-omni-builder" + +# Ensure that the env vars are actually set, otherwise exit early +if [ -z "${BUILDKITE_COMMIT:-}" ]; then + echo "ERROR: BUILDKITE_COMMIT is not set" + exit 1 +fi +echo "BUILDKITE_COMMIT: ${BUILDKITE_COMMIT}" + +# Authenticate to ECR Public +aws ecr-public get-login-password --region "$REGION" \ + | docker login --username AWS --password-stdin "$ECR_NAMESPACE" + +# Compute the cache tag for the dependencies image as the hash of the contents. +# This is because dependencies don't change often, so most PRs will have the same hash. +# +# The Dockerfile is also included to avoid churn in the cached image based on the vLLM version. +DEP_FILES="pyproject.toml setup.py requirements/*.txt docker/Dockerfile.ci" +CACHE_KEY=$(cat ${DEP_FILES} | sha256sum | cut -c1-16) +CACHE_TAG="deps-cache-${CACHE_KEY}" +echo "Cache key: ${CACHE_TAG}" + +# Set up buildx with docker-container driver; we need to do this +# since cache export is not supported for the default docker driver +# that is running in the CI. +docker buildx inspect $BUILDER_NAME >/dev/null 2>&1 \ +|| docker buildx create --name $BUILDER_NAME --driver docker-container +docker buildx use $BUILDER_NAME + +echo "Building image tag: ${REGISTRY}:${BUILDKITE_COMMIT}" + +# Build + push the image to the registry, using the hash from the dependencies. +# Note that the caching here is layerwise, so we're just making sure we always +# can cache hit on everything but the last copy at the moment. +docker buildx build --push --progress=plain \ + --cache-from "type=registry,ref=${REGISTRY}:${CACHE_TAG}" \ + --cache-to "type=registry,ref=${REGISTRY}:${CACHE_TAG},mode=max,compression=zstd" \ + --file "$DOCKERFILE" \ + -t "${REGISTRY}:${BUILDKITE_COMMIT}" . diff --git a/.gitignore b/.gitignore index 224dac867..3c24e40ae 100644 --- a/.gitignore +++ b/.gitignore @@ -259,7 +259,6 @@ configs/production.yaml configs/development.yaml # Docker -.dockerignore Dockerfile.dev discussion tmp_test diff --git a/docker/Dockerfile.ci b/docker/Dockerfile.ci index f4444f8ac..3d2876747 100644 --- a/docker/Dockerfile.ci +++ b/docker/Dockerfile.ci @@ -3,10 +3,22 @@ # be updated on every rebase to match the new upstream vllm commit/tag. ARG VLLM_BASE_IMAGE=vllm/vllm-openai ARG VLLM_BASE_TAG=v0.25.0 -FROM ${VLLM_BASE_IMAGE}:${VLLM_BASE_TAG} + +# We use registry-based layer caching for the dependencies, +# where the tag is determined based on the hash of the contents +# of files like pyproject.toml, etc. The deps stage should have +# everything except the actual project dir, since that should not +# change much across branches. +FROM ${VLLM_BASE_IMAGE}:${VLLM_BASE_TAG} AS deps ARG APP_DIR=/workspace/vllm-omni WORKDIR ${APP_DIR} -COPY . . + +RUN ln -sf /usr/bin/python3 /usr/bin/python + +# Create a vllm_omni dir so that we can install the python package +# without failing to write the version files; we'll copy over it +# after installing +RUN mkdir -p vllm_omni # Install system dependencies RUN apt-get update && \ @@ -14,9 +26,33 @@ RUN apt-get update && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* +# Copy dependency related things first and install deps/ +# We set the version str to 0.0.0 since it doesn't matter +# for CI +COPY pyproject.toml setup.py ./ +COPY requirements/ requirements/ +ENV SETUPTOOLS_SCM_PRETEND_VERSION=0.0.0 RUN uv pip install --system ".[dev]" +# The final stage just copies the working dir on top of the +# dependencies image. This uses --link so that we can skip +# the image pull in Buildkite when we have a cache hit. +# +# IMPORTANT: Do not make any modifications that depend on the +# file system after the --link copy, otherwise we'd need to pull +# the heavy base image to build it. This is the reason we copy +# link & update the pythonpath to point at the copied code instead +# of running a new pip install on it. +FROM deps +ARG APP_DIR=/workspace/vllm-omni +# Add APP_DIR to the pythonpath, since the entrypoint stub +# that was installed in the earlier step must point at the copied code. +# +# NOTE: this will probably warn about undefined vars, but is expected. +# See: https://github.com/docker/buildx/issues/2751 +ENV PYTHONPATH=${APP_DIR}${PYTHONPATH:+:${PYTHONPATH}} +WORKDIR ${APP_DIR} -RUN ln -sf /usr/bin/python3 /usr/bin/python +COPY --link . . ENTRYPOINT [] diff --git a/docker/Dockerfile.ci.dockerignore b/docker/Dockerfile.ci.dockerignore new file mode 100644 index 000000000..ab137c2b5 --- /dev/null +++ b/docker/Dockerfile.ci.dockerignore @@ -0,0 +1,37 @@ +# NOTE: this is largely taken from vLLM + +# Exclude .git for CI builds — version is set via SETUPTOOLS_SCM_PRETEND_VERSION +.git +.venv +/build +dist + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +.mypy_cache + +# Distribution / packaging +.Python +/build/ +cmake-build-*/ +CMakeUserPresets.json +develop-eggs/ +/dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST +rust/target/ From be667a806747230bbf057cf14b12923eff90d8f2 Mon Sep 17 00:00:00 2001 From: bjf-frz Date: Thu, 23 Jul 2026 10:06:08 +0800 Subject: [PATCH 067/240] [Refact] Refactor diffusion output streaming (#5166) Signed-off-by: bjf-frz Co-authored-by: Hongsheng Liu --- docs/configuration/stage_configs.md | 7 +- .../feature/diffusion_continuous_batching.md | 3 +- .../feature/diffusion_step_execution.md | 2 +- docs/design/module/dit_module.md | 123 +++--- docs/user_guide/diffusion/request_batching.md | 21 +- docs/user_guide/diffusion/step_execution.md | 7 +- tests/diffusion/test_diffusion_engine.py | 112 +++++- .../test_diffusion_engine_cleanup.py | 126 ++++-- .../test_diffusion_engine_metrics.py | 24 +- .../test_diffusion_engine_rpc_routing.py | 25 +- .../diffusion/test_diffusion_model_runner.py | 52 +++ .../test_diffusion_output_formatter.py | 24 -- tests/diffusion/test_diffusion_scheduler.py | 42 +- .../test_inline_stage_diffusion_client.py | 15 +- .../test_multiproc_engine_concurrency.py | 2 +- tests/diffusion/test_stage_diffusion_proc.py | 4 +- .../test_async_omni_qwen_image_generate.py | 3 + .../rlhf_test/test_verl_omni_e2e.py | 5 +- vllm_omni/diffusion/diffusion_engine.py | 364 +++++++++--------- vllm_omni/diffusion/executor/abstract.py | 2 +- .../diffusion/executor/multiproc_executor.py | 10 +- .../inline_stage_diffusion_client.py | 9 +- .../models/dreamzero/pipeline_dreamzero.py | 4 +- vllm_omni/diffusion/output_formatter.py | 13 - vllm_omni/diffusion/output_metadata.py | 10 - vllm_omni/diffusion/stage_diffusion_proc.py | 11 +- .../worker/diffusion_model_runner.py | 2 +- 27 files changed, 628 insertions(+), 394 deletions(-) delete mode 100644 vllm_omni/diffusion/output_metadata.py diff --git a/docs/configuration/stage_configs.md b/docs/configuration/stage_configs.md index e49487b72..15cee746a 100644 --- a/docs/configuration/stage_configs.md +++ b/docs/configuration/stage_configs.md @@ -377,7 +377,12 @@ Default: `"0"` #### `engine_args.max_num_seqs` -The maximum number of sequences for concurrent processing in this stage. For LLM stages, this controls the vLLM scheduler's maximum concurrent sequences. For all stage types, this also controls how many tasks can be batched together in the task processing loop. +The maximum number of sequences for concurrent processing in this stage. For +LLM stages, this controls the vLLM scheduler's maximum concurrent sequences. +For diffusion stages, this controls scheduler wave capacity in both request +batch mode and step batch mode. `max_num_seqs: 1` is the serial/conservative +path; values above `1` allow compatible requests to batch when the selected +diffusion pipeline supports that batching mode. Default: `1` diff --git a/docs/design/feature/diffusion_continuous_batching.md b/docs/design/feature/diffusion_continuous_batching.md index 223b88518..482bd1122 100644 --- a/docs/design/feature/diffusion_continuous_batching.md +++ b/docs/design/feature/diffusion_continuous_batching.md @@ -139,7 +139,8 @@ compatibility gating and runner-side `StepInputBatch` packing. ## Current Limitations -- Experimental feature; use `max_num_seqs=1` for the older conservative path. +- Experimental feature; use `max_num_seqs=1` for the conservative single-request + step path. - Only native pipelines that already support `step_execution=True`. - Only homogeneous batches keyed by `StepBatchSamplingParamsKey` are supported. - `cache_backend`, KV transfer, and other request-mode extras are not wired diff --git a/docs/design/feature/diffusion_step_execution.md b/docs/design/feature/diffusion_step_execution.md index 01936e223..88fddb215 100644 --- a/docs/design/feature/diffusion_step_execution.md +++ b/docs/design/feature/diffusion_step_execution.md @@ -32,7 +32,7 @@ Current engine/runtime limitations: - Continuous batching with `max_num_seqs > 1` is experimental and documented in [Continuous Batching for Step-Wise Diffusion](diffusion_continuous_batching.md). - Keep `max_num_seqs=1` if you want the older conservative behavior. + Keep `max_num_seqs=1` for the conservative single-request step path. - `cache_backend` is not supported in step mode. - Request-mode extras such as KV transfer are not wired into step mode yet. - Unsupported pipelines now fail early during model loading instead of failing on the first request. diff --git a/docs/design/module/dit_module.md b/docs/design/module/dit_module.md index 9634adad3..7494ee7b5 100644 --- a/docs/design/module/dit_module.md +++ b/docs/design/module/dit_module.md @@ -52,7 +52,7 @@ The diffusion module follows a **multi-process, distributed architecture** with ### Responsibilities -The `DiffusionEngine` is the **orchestrator** of the diffusion inference system. It manages the lifecycle of worker processes and coordinates the execution flow. +The `DiffusionEngine` is the **orchestrator** of the diffusion inference system. It owns request admission, scheduler/executor coordination, output-stream delivery, cancellation cleanup, and startup warmup. ### Key Components @@ -60,68 +60,84 @@ The `DiffusionEngine` is the **orchestrator** of the diffusion inference system. ```python class DiffusionEngine: - def __init__(self, od_config: OmniDiffusionConfig): + def __init__( + self, + od_config: OmniDiffusionConfig, + scheduler: SchedulerInterface | None = None, + ): self.od_config = od_config - self.post_process_func = get_diffusion_post_process_func(od_config) - self.pre_process_func = get_diffusion_pre_process_func(od_config) - self._processes: list[mp.Process] = [] - self._make_client() + self._init_process_hooks(od_config) + self.execution_mode = self._resolve_execution_mode(od_config) + self._init_executor(od_config) + self._init_scheduler(od_config, scheduler) + self._init_runtime_state() + self._init_execute_fn() ``` **Key Features**: - **Pre/Post Processing**: Registers model-specific pre-processing and post-processing functions via registry pattern -- **Worker Management**: Launches and manages multiple worker processes (one per GPU) +- **Execution Mode Selection**: Selects one of two execution modes: + - `REQUEST_BATCH`: complete requests are scheduled through request-level execution. `max_num_seqs=1` is the serial request path; values above `1` allow compatible requests to use fused `pipeline.forward(batch)` when the pipeline supports it. + - `STEP_BATCH`: requests are advanced one denoising step at a time through `prepare_encode()`, `denoise_step()`, `step_scheduler()`, and `post_decode()`. `max_num_seqs` controls the maximum number of compatible active requests in one step wave. -- **Process Isolation**: Uses multiprocessing for true parallelism +- **Scheduler Selection**: Uses `StepScheduler` for step-batch execution and `RequestScheduler` for request-batch execution. Tests and custom integrations may inject a `SchedulerInterface` instance explicitly. -#### 1.2 Worker Launch Process +- **Executor Selection**: Builds the configured `DiffusionExecutor`, then routes scheduled work through `execute_batch` or `execute_step` according to the execution mode. `execute_batch` handles the serial `max_num_seqs=1` request path internally. -The engine launches workers using a **spawn** method: +- **Factory-Owned Warmup**: `DiffusionEngine.make_engine()` constructs the selected engine backend and then runs startup warmup through `run_startup_warmup()`. + +#### 1.2 Execution Modes ```python -def _launch_workers(self, broadcast_handle): - # Creates one process per GPU - for i in range(num_gpus): - process = mp.Process( - target=worker_proc.worker_main, - args=(i, od_config, writer, broadcast_handle), - name=f"DiffusionWorker-{i}", - ) - process.start() +class DiffusionExecutionMode(str, Enum): + REQUEST_BATCH = "request_batch" + STEP_BATCH = "step_batch" +``` + +`streaming_output=True` requires step execution. If streaming is enabled without `step_execution=True`, the engine enables step execution so the output stream can receive intermediate chunks or the final step output. + +#### 1.3 Output Stream Lifecycle + +```python +request_id = engine.add_request(request) +async for output in engine.get_output_stream(request_id): + yield output ``` **Design Decisions**: -- **Spawn Method**: Ensures clean state for each worker (no shared memory issues) +- **Unified Stream Semantics**: `step_streaming()` is the public async engine path. Streaming callers forward every yielded output. Non-streaming callers consume the same stream to completion and return only the final output. -- **Pipe Communication**: Uses `mp.Pipe` for initialization handshake +- **Queue Delivery**: Each admitted request gets an `asyncio.Queue[DiffusionOutput]`. The busy loop emits chunks/final outputs into that queue. -- **Device Selection**: Each worker is assigned a specific GPU (`cuda:{rank}`) +- **Cancellation Cleanup**: If a client disconnects or cancels stream consumption, `get_output_stream()` removes only the queue. Scheduler terminal state is still finalized when the scheduler reports the request as finished, so request IDs do not remain active indefinitely. -#### 1.3 Request Processing Flow +#### 1.4 Request Processing Flow ```python -def step(self, requests: list[OmniDiffusionRequest]): +async def step_streaming(self, request: OmniDiffusionRequest): # 1. Pre-process requests - requests = self.pre_process_func(requests) + request = self.pre_process_func(request) - # 2. Send to scheduler and wait for response - output = self.add_req_and_wait_for_response(requests) - - # 3. Post-process results - result = self.post_process_func(output.output) - return result + # 2. Add request and consume the unified output stream + async for output in self.async_add_req_and_stream_response(request): + # 3. Post-process each delivered output + yield self.postprocess_output(request, output, ...) ``` **Flow**: 1. **Pre-processing**: Applies model-specific transformations -2. **Scheduling**: Delegates to scheduler for distribution +2. **Scheduling**: Delegates request state transitions to the selected scheduler -3. **Post-processing**: Converts raw outputs to final format (e.g., PIL images) +3. **Execution**: Calls the selected executor function for request, request-batch, or step execution + +4. **Output Delivery**: Emits intermediate chunks and final outputs through the request output stream + +5. **Post-processing**: Converts raw outputs to final format (e.g., PIL images) --- @@ -915,36 +931,49 @@ def initialize_model_parallel( 1. User Request └─> Omni.generate(prompt) └─> Prepare OmniDiffusionRequest - └─> DiffusionEngine.step(requests) + └─> DiffusionEngine.step_streaming(request) 2. Pre-processing - └─> pre_process_func(requests) + └─> pre_process_func(request) └─> Model-specific transformations 3. Scheduling - └─> scheduler.add_request(request) - └─> scheduler.schedule() - └─> DiffusionEngine submits scheduled request to executor.add_req(req) + └─> scheduler.add_request(request) + └─> scheduler.schedule() + └─> DiffusionEngine selects execute_batch / execute_step 4. Worker Execution └─> WorkerProc._worker_busy_loop() └─> DiffusionWorker.execute_model(req) - └─> Pipeline.forward(req) - ├─> encode_prompt() - ├─> prepare_latents() - ├─> diffuse() [loop] - │ ├─> transformer.forward() [with cache backend hooks] - │ └─> scheduler.step() - └─> vae.decode() + └─> DiffusionModelRunner selects execution mode + ├─> REQUEST_BATCH + │ └─> Pipeline.forward(req_or_batch) + │ ├─> encode_prompt() + │ ├─> prepare_latents() + │ ├─> diffuse() [loop] + │ │ ├─> transformer.forward() [with cache backend hooks] + │ │ └─> scheduler.step() + │ └─> vae.decode() + └─> STEP + ├─> pipeline.prepare_encode(state) [new requests] + ├─> pipeline.denoise_step(input_batch) + ├─> pipeline.step_scheduler(state, noise_pred) + └─> pipeline.post_decode(state) [chunk boundary or final step] 5. Result Collection - └─> Executor returns DiffusionOutput + └─> Executor returns RunnerOutput / BatchRunnerOutput └─> scheduler.update_from_output(...) - └─> DiffusionEngine pops finished request state + ├─> DiffusionEngine emits chunk/final output to request queue + └─> DiffusionEngine finalizes terminal scheduler state 6. Post-processing └─> post_process_func(output) └─> Convert to PIL images / final format + +7. Stream Completion / Cancellation + ├─> Streaming callers receive every yielded output + ├─> Non-streaming callers drain the stream and return the final output + └─> Cancelled streams drop their output queue but still allow scheduler state cleanup ``` --- diff --git a/docs/user_guide/diffusion/request_batching.md b/docs/user_guide/diffusion/request_batching.md index 5cd24802b..35bf71725 100644 --- a/docs/user_guide/diffusion/request_batching.md +++ b/docs/user_guide/diffusion/request_batching.md @@ -15,6 +15,14 @@ error, and abort state. The scheduler decides which requests can run together. ## Enablement +Request-level execution is the default diffusion request mode when +`step_execution=False`. `max_num_seqs` controls the maximum number of compatible +requests in one scheduler wave: + +- `max_num_seqs=1` is the serial request path. +- `max_num_seqs>1` allows fused request-level batching for pipelines that + declare request-batch support. + Increase `max_num_seqs` above `1` to allow the request scheduler to keep more than one compatible request active: @@ -51,11 +59,11 @@ stage_args: ## Compatibility -Only pipelines that declare request-batch support use the fused request-batch -path. The engine validates that the pipeline `forward()` uses the request-batch -contract and returns `list[DiffusionOutput]`. Pipelines that do not support this -contract do not use fused `pipeline.forward(batch)`; scheduled requests are -executed through per-request worker calls. +Only pipelines that declare request-batch support can use the fused +multi-request `pipeline.forward(batch)` path. Pipelines that do not support this +contract can still use request-level execution with `max_num_seqs=1`, which is +the serial request path. Setting `max_num_seqs>1` for a pipeline without +request-batch support fails early during engine initialization. The scheduler batches only compatible requests. Compatibility is based on shape-sensitive and guidance-sensitive sampling fields, including resolution, @@ -69,7 +77,8 @@ step-wise runtime, see [Step Execution](step_execution.md). ## Tuning - `max_num_seqs` caps the number of active compatible requests in one scheduler - wave. + wave. `1` means serial request execution; values above `1` enable fused + request-level batching when the pipeline supports it. - `request_batch_max_wait_ms` is an upper bound on extra admission wait before a new wave starts. Keep it small for latency-sensitive serving; values such as `10` to `50` ms are a practical starting range for bursty HTTP ingress. diff --git a/docs/user_guide/diffusion/step_execution.md b/docs/user_guide/diffusion/step_execution.md index 07184a787..4dfedfa20 100644 --- a/docs/user_guide/diffusion/step_execution.md +++ b/docs/user_guide/diffusion/step_execution.md @@ -39,6 +39,11 @@ vllm serve Qwen/Qwen-Image --omni \ For serving, `--step-execution` enables the step-wise runtime. Continuous batching only becomes relevant when `--max-num-seqs > 1`. +`max_num_seqs` controls the step scheduler capacity: + +- `max_num_seqs=1` runs one request through the step-wise state machine. +- `max_num_seqs>1` allows compatible active requests to share step waves. + ## Supported Pipelines | Pipeline | Example models | Step execution | @@ -51,7 +56,7 @@ batching only becomes relevant when `--max-num-seqs > 1`. the step-wise path can batch compatible requests together. This is experimental. Requests with incompatible sampling parameters are intentionally kept in separate batches, - and `max_num_seqs=1` remains the conservative default. + and `max_num_seqs=1` remains the conservative single-request step path. ## Current Limitations diff --git a/tests/diffusion/test_diffusion_engine.py b/tests/diffusion/test_diffusion_engine.py index 5f9a5a5c0..01c901d60 100644 --- a/tests/diffusion/test_diffusion_engine.py +++ b/tests/diffusion/test_diffusion_engine.py @@ -15,7 +15,11 @@ from pytest_mock import MockerFixture import vllm_omni.diffusion.diffusion_engine as diffusion_engine_module from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig -from vllm_omni.diffusion.diffusion_engine import DiffusionEngine, _move_tensor_tree_to_cpu +from vllm_omni.diffusion.diffusion_engine import ( + DiffusionEngine, + DiffusionExecutionMode, + _move_tensor_tree_to_cpu, +) from vllm_omni.diffusion.request import OmniDiffusionRequest from vllm_omni.diffusion.sched.interface import ( CachedRequestData, @@ -221,7 +225,7 @@ class TestRequestBatchCapability: diffusion_engine_module.supports_request_batch(od_config) registry_load.assert_not_called() - def test_engine_disables_batch_dispatch_for_single_request_pipeline( + def test_engine_uses_request_batch_mode_for_single_request_pipeline( self, monkeypatch: pytest.MonkeyPatch, mocker: MockerFixture, @@ -230,6 +234,7 @@ class TestRequestBatchCapability: model_class_name="SinglePipeline", custom_pipeline_args=None, streaming_output=False, + max_num_seqs=1, ) fake_executor = SimpleNamespace( execute_request=mocker.Mock(return_value="per-request"), @@ -255,15 +260,50 @@ class TestRequestBatchCapability: "_try_load_model_cls", lambda model_class_name: _SingleRequestPipeline, ) - monkeypatch.setattr(DiffusionEngine, "_dummy_run", lambda self: None) engine = DiffusionEngine(od_config) - output = engine.execute_fn(_make_request_mode_sched_output("req-a", "req-b")) + output = engine.execute_fn(_make_request_mode_sched_output("req-a")) + assert engine.execution_mode == DiffusionExecutionMode.REQUEST_BATCH assert engine.supports_request_batch is False - assert output == "per-request" - fake_executor.execute_request.assert_called_once() - fake_executor.execute_batch.assert_not_called() + assert output == "batch" + fake_executor.execute_batch.assert_called_once() + fake_executor.execute_request.assert_not_called() + + def test_engine_rejects_multi_seq_request_batch_for_single_request_pipeline( + self, + monkeypatch: pytest.MonkeyPatch, + mocker: MockerFixture, + ) -> None: + od_config = SimpleNamespace( + model_class_name="SinglePipeline", + custom_pipeline_args=None, + streaming_output=False, + max_num_seqs=2, + ) + fake_executor_cls = mocker.Mock() + + monkeypatch.setattr( + "vllm_omni.diffusion.diffusion_engine.get_diffusion_post_process_func", + lambda *args, **kwargs: None, + ) + monkeypatch.setattr( + "vllm_omni.diffusion.diffusion_engine.get_diffusion_pre_process_func", + lambda *args, **kwargs: None, + ) + monkeypatch.setattr( + "vllm_omni.diffusion.diffusion_engine.DiffusionExecutor.get_class", + lambda *args, **kwargs: fake_executor_cls, + ) + monkeypatch.setattr( + diffusion_engine_module.DiffusionModelRegistry, + "_try_load_model_cls", + lambda model_class_name: _SingleRequestPipeline, + ) + + with pytest.raises(ValueError, match="max_num_seqs=1"): + DiffusionEngine(od_config) + fake_executor_cls.assert_not_called() @pytest.mark.parametrize("request_ids", [("req-a",), ("req-a", "req-b")]) def test_engine_enables_batch_dispatch_for_request_batch_pipeline( @@ -276,6 +316,7 @@ class TestRequestBatchCapability: model_class_name="BatchPipeline", custom_pipeline_args=None, streaming_output=False, + max_num_seqs=2, ) fake_executor = SimpleNamespace( execute_request=mocker.Mock(return_value="per-request"), @@ -301,16 +342,55 @@ class TestRequestBatchCapability: "_try_load_model_cls", lambda model_class_name: _BatchCapablePipeline, ) - monkeypatch.setattr(DiffusionEngine, "_dummy_run", lambda self: None) engine = DiffusionEngine(od_config) output = engine.execute_fn(_make_request_mode_sched_output(*request_ids)) + assert engine.execution_mode == DiffusionExecutionMode.REQUEST_BATCH assert engine.supports_request_batch is True assert output == "batch" fake_executor.execute_batch.assert_called_once() fake_executor.execute_request.assert_not_called() + def test_make_engine_runs_startup_warmup(self, monkeypatch: pytest.MonkeyPatch, mocker: MockerFixture) -> None: + od_config = SimpleNamespace( + model_class_name="SinglePipeline", + custom_pipeline_args=None, + engine_backend="default", + streaming_output=False, + ) + fake_executor = SimpleNamespace( + execute_request=mocker.Mock(), + execute_batch=mocker.Mock(), + execute_step=mocker.Mock(), + ) + fake_executor_cls = mocker.Mock(return_value=fake_executor) + warmup = mocker.Mock() + + monkeypatch.setattr( + "vllm_omni.diffusion.diffusion_engine.get_diffusion_post_process_func", + lambda *args, **kwargs: None, + ) + monkeypatch.setattr( + "vllm_omni.diffusion.diffusion_engine.get_diffusion_pre_process_func", + lambda *args, **kwargs: None, + ) + monkeypatch.setattr( + "vllm_omni.diffusion.diffusion_engine.DiffusionExecutor.get_class", + lambda *args, **kwargs: fake_executor_cls, + ) + monkeypatch.setattr( + diffusion_engine_module.DiffusionModelRegistry, + "_try_load_model_cls", + lambda model_class_name: _SingleRequestPipeline, + ) + monkeypatch.setattr(DiffusionEngine, "run_startup_warmup", warmup) + + engine = DiffusionEngine.make_engine(od_config) + + assert isinstance(engine, DiffusionEngine) + warmup.assert_called_once_with() + class TestRequestBatchAdmission: pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu] @@ -470,11 +550,20 @@ def test_move_tensor_tree_moves_nested_cuda_tensors_to_cpu() -> None: assert moved["items"][1][0] == "keep" +async def _consume_final_output(generator): + final_output = None + async for output in generator: + final_output = output + if final_output is None: + raise RuntimeError("Diffusion execution finished without output.") + return final_output + + @pytest.mark.asyncio -async def test_async_add_req_and_wait_for_response(): +async def test_async_add_req_and_stream_response(): engine = object.__new__(DiffusionEngine) engine.scheduler = MockScheduler() - engine._out_queue = {} + engine._out_streams = {} engine.abort_queue: queue.Queue[str] = queue.Queue() engine._rpc_queue = queue.Queue() engine._rpc_lock = threading.RLock() @@ -485,6 +574,7 @@ async def test_async_add_req_and_wait_for_response(): engine._loop_started = False engine.main_loop = None engine.supports_request_batch = False + engine.execution_mode = DiffusionExecutionMode.REQUEST_BATCH engine._finalize_finished_request = lambda rid, out, err: out.result @@ -512,7 +602,7 @@ async def test_async_add_req_and_wait_for_response(): async def run_task(rid): req = SimpleNamespace(request_id=rid) start = time.time() - res = await engine.async_add_req_and_wait_for_response(req) + res = await _consume_final_output(engine.async_add_req_and_stream_response(req)) return rid, res, time.time() - start task_ids = [f"req_{i}" for i in range(5)] diff --git a/tests/diffusion/test_diffusion_engine_cleanup.py b/tests/diffusion/test_diffusion_engine_cleanup.py index 3cc3d48f7..b7a9a345f 100644 --- a/tests/diffusion/test_diffusion_engine_cleanup.py +++ b/tests/diffusion/test_diffusion_engine_cleanup.py @@ -5,15 +5,17 @@ import asyncio import queue import threading from types import SimpleNamespace -from unittest.mock import Mock +from unittest.mock import Mock, patch import pytest +from vllm_omni.diffusion import diffusion_engine as diffusion_engine_module from vllm_omni.diffusion.data import DiffusionOutput -from vllm_omni.diffusion.diffusion_engine import DiffusionEngine +from vllm_omni.diffusion.diffusion_engine import DiffusionEngine, DiffusionExecutionMode from vllm_omni.diffusion.request import OmniDiffusionRequest from vllm_omni.diffusion.sched import DiffusionRequestStatus, RequestScheduler from vllm_omni.inputs.data import OmniDiffusionSamplingParams +from vllm_omni.outputs import OmniRequestOutput pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion] @@ -33,8 +35,7 @@ def _make_engine() -> DiffusionEngine: engine.executor = SimpleNamespace(shutdown=Mock()) engine._rpc_lock = threading.RLock() engine._cv = threading.Condition(engine._rpc_lock) - engine._out_queue = {} - engine._out_queue_streaming = {} + engine._out_streams = {} engine._closed = False engine._shutdown_complete = False engine.abort_queue = queue.Queue() @@ -44,28 +45,13 @@ def _make_engine() -> DiffusionEngine: return engine -def test_close_completes_pending_async_waiters() -> None: - engine = _make_engine() - event_loop = asyncio.new_event_loop() - try: - future = event_loop.create_future() - engine._out_queue["pending-req"] = future - - engine.close() - - assert future.done() - assert future.result().error == "DiffusionEngine is closed." - finally: - event_loop.close() - - -def test_close_completes_pending_streaming_waiters() -> None: +def test_close_completes_pending_output_streams() -> None: engine = _make_engine() event_loop = asyncio.new_event_loop() try: engine.main_loop = event_loop queue: asyncio.Queue[DiffusionOutput] = asyncio.Queue() - engine._out_queue_streaming["pending-stream"] = queue + engine._out_streams["pending-stream"] = queue engine.close() @@ -76,21 +62,101 @@ def test_close_completes_pending_streaming_waiters() -> None: event_loop.close() -def test_handle_finished_requests_ignores_already_drained_waiter() -> None: +def test_emit_finished_outputs_finalizes_already_drained_waiter() -> None: class RacingOutQueue(dict): - def __contains__(self, key): - return True - - def pop(self, key, default=None): + def get(self, key, default=None): return default engine = _make_engine() - engine._out_queue = RacingOutQueue() - engine._finalize_finished_request = Mock(side_effect=AssertionError("should not finalize drained waiters")) + request_id = engine.scheduler.add_request(_make_request("pending-req")) + engine.scheduler.finish_requests(request_id, DiffusionRequestStatus.FINISHED_ABORTED) + engine._out_streams = RacingOutQueue() - engine._handle_finished_requests({"pending-req"}) + engine._emit_finished_outputs({request_id}) - engine._finalize_finished_request.assert_not_called() + assert engine.scheduler.get_request_state(request_id) is None + + +def test_emit_step_outputs_finalizes_finished_request_without_stream() -> None: + engine = _make_engine() + engine.execution_mode = DiffusionExecutionMode.STEP_BATCH + request_id = engine.scheduler.add_request(_make_request("step-drained")) + engine.scheduler.finish_requests(request_id, DiffusionRequestStatus.FINISHED_ABORTED) + + engine._emit_outputs({request_id}, [request_id], SimpleNamespace(get_request_output=lambda _request_id: None)) + + assert engine.scheduler.get_request_state(request_id) is None + + +def test_init_accepts_custom_scheduler(monkeypatch: pytest.MonkeyPatch) -> None: + od_config = SimpleNamespace( + custom_pipeline_args=None, + model_class_name="CustomSchedulerPipeline", + streaming_output=False, + ) + custom_scheduler = RequestScheduler() + fake_executor = SimpleNamespace( + execute_request=Mock(), + execute_batch=Mock(), + execute_step=Mock(), + ) + + monkeypatch.setattr( + "vllm_omni.diffusion.diffusion_engine.get_diffusion_post_process_func", + lambda *args, **kwargs: None, + ) + monkeypatch.setattr( + "vllm_omni.diffusion.diffusion_engine.get_diffusion_pre_process_func", + lambda *args, **kwargs: None, + ) + monkeypatch.setattr( + "vllm_omni.diffusion.diffusion_engine.DiffusionExecutor.get_class", + lambda *args, **kwargs: Mock(return_value=fake_executor), + ) + monkeypatch.setattr( + "vllm_omni.diffusion.diffusion_engine.supports_request_batch", + lambda *args, **kwargs: False, + ) + + engine = DiffusionEngine(od_config, scheduler=custom_scheduler) + + assert engine.scheduler is custom_scheduler + + +@pytest.mark.asyncio +async def test_step_compatibility_wrapper_returns_final_batch() -> None: + engine = _make_engine() + first = [OmniRequestOutput.from_diffusion(request_id="req", images=[], finished=False)] + final = [OmniRequestOutput.from_diffusion(request_id="req", images=[], finished=True)] + + async def _step_streaming(_request): + yield first + yield final + + engine.step_streaming = _step_streaming # type: ignore[method-assign] + with patch.object(diffusion_engine_module.logger, "warning_once") as warning_once: + output = await engine.step(_make_request("req")) + + assert output is final + warning_once.assert_called_once() + + +@pytest.mark.asyncio +async def test_async_wait_compatibility_wrapper_returns_final_output() -> None: + engine = _make_engine() + first = DiffusionOutput(output="chunk", finished=False) + final = DiffusionOutput(output="final", finished=True) + + async def _stream_response(_request): + yield first + yield final + + engine.async_add_req_and_stream_response = _stream_response # type: ignore[method-assign] + with patch.object(diffusion_engine_module.logger, "warning_once") as warning_once: + output = await engine.async_add_req_and_wait_for_response(_make_request("req")) + + assert output is final + warning_once.assert_called_once() def test_abort_request_id_aborts_scheduler_request() -> None: diff --git a/tests/diffusion/test_diffusion_engine_metrics.py b/tests/diffusion/test_diffusion_engine_metrics.py index 2e8eae242..06ad2077f 100644 --- a/tests/diffusion/test_diffusion_engine_metrics.py +++ b/tests/diffusion/test_diffusion_engine_metrics.py @@ -80,27 +80,23 @@ class TestMetricKeys: "format_diffusion_outputs() — should only use 'preprocess_time_ms'" ) - def test_metric_key_naming_consistency(self) -> None: - """Metric keys should map to the explicit timing fields.""" + def test_timing_metric_key_naming_consistency(self) -> None: + """Timing metrics should be attached by the engine orchestration path.""" source = _read_source(_FORMATTER_PATH) formatter_source = _get_function_source(source, None, "format_diffusion_outputs") + engine_source = _read_source(_ENGINE_PATH) + step_streaming_source = _get_function_source(engine_source, "DiffusionEngine", "step_streaming") lines = formatter_source.split("\n") - found_exec = False - found_total = False for line in lines: if '"diffusion_engine_exec_time_ms"' in line: - found_exec = True - assert "timings.exec_time_s" in line, ( - "diffusion_engine_exec_time_ms should measure executor time only (timings.exec_time_s)" - ) + raise AssertionError("diffusion_engine_exec_time_ms should be attached in step_streaming()") if '"diffusion_engine_total_time_ms"' in line: - found_total = True - assert "timings.total_time_ms" in line, ( - "diffusion_engine_total_time_ms should measure full step time (timings.total_time_ms)" - ) - assert found_exec, "diffusion_engine_exec_time_ms key not found in format_diffusion_outputs()" - assert found_total, "diffusion_engine_total_time_ms key not found in format_diffusion_outputs()" + raise AssertionError("diffusion_engine_total_time_ms should be attached in step_streaming()") + + assert '"diffusion_engine_exec_time_ms": exec_total_time * 1000' in step_streaming_source + assert '"diffusion_engine_total_time_ms": step_total_ms' in step_streaming_source + assert '"postprocess_time_ms": postprocess_time * 1000' in step_streaming_source class TestDummyRunAllocation: diff --git a/tests/diffusion/test_diffusion_engine_rpc_routing.py b/tests/diffusion/test_diffusion_engine_rpc_routing.py index 8c4f08152..dff5ac7bc 100644 --- a/tests/diffusion/test_diffusion_engine_rpc_routing.py +++ b/tests/diffusion/test_diffusion_engine_rpc_routing.py @@ -30,7 +30,7 @@ from typing import Any import pytest from vllm_omni.diffusion.data import DiffusionOutput -from vllm_omni.diffusion.diffusion_engine import DiffusionEngine, _RpcTask +from vllm_omni.diffusion.diffusion_engine import DiffusionEngine, DiffusionExecutionMode, _RpcTask from vllm_omni.diffusion.sched import RequestScheduler from vllm_omni.diffusion.sched.interface import RequestBatchSamplingParamsKey from vllm_omni.diffusion.worker.utils import RunnerOutput @@ -91,7 +91,7 @@ class _ConcurrencyTrackingExecutor: # args[0] is the request-like object for execute_model. req = args[0] if args else None tag = req.request_id if req is not None and hasattr(req, "request_id") else "unknown" - return DiffusionOutput(error=f"result_for_{tag}") + return DiffusionOutput(error=f"result_for_{tag}", finished=True) tag = args[0] if args else method return DiffusionOutput(error=f"rpc_result_for_{tag}") finally: @@ -149,12 +149,12 @@ def _make_engine_with_loop( engine.scheduler = sched engine.step_execution = False engine.supports_request_batch = False + engine.execution_mode = DiffusionExecutionMode.REQUEST_BATCH engine.execute_fn = engine.executor.execute_request engine._rpc_lock = threading.RLock() engine._cv = threading.Condition(engine._rpc_lock) - engine._out_queue = {} - engine._out_queue_streaming = {} + engine._out_streams = {} engine._closed = False engine.abort_queue = queue.Queue() engine._rpc_queue = queue.Queue() @@ -187,6 +187,15 @@ def _stop_engine(engine: DiffusionEngine) -> None: assert not engine.worker_thread.is_alive(), "Busy loop thread did not stop" +async def _consume_final_output(generator): + final_output = None + async for output in generator: + final_output = output + if final_output is None: + raise RuntimeError("Diffusion execution finished without output.") + return final_output + + # ─────────────────────── single-thread invariant ─────────────────────────── @@ -199,7 +208,7 @@ async def test_executor_only_called_from_busy_loop_thread(): busy_tid = engine.worker_thread.ident try: # Per-request path - await engine.async_add_req_and_wait_for_response(_make_request("req1")) + await _consume_final_output(engine.async_add_req_and_stream_response(_make_request("req1"))) # Raw RPC path (sync from a worker thread) result = await asyncio.to_thread(engine.collective_rpc, "ping", args=("a",), unique_reply_rank=0) assert result.error == "rpc_result_for_a" @@ -228,7 +237,7 @@ async def test_executor_calls_never_overlap_under_load(): return await engine.async_collective_rpc("ping", args=(f"x{i}",), unique_reply_rank=0) async def _request(i: int): - return await engine.async_add_req_and_wait_for_response(_make_request(f"r{i}")) + return await _consume_final_output(engine.async_add_req_and_stream_response(_make_request(f"r{i}"))) tasks = [_rpc(i) for i in range(20)] + [_request(i) for i in range(5)] results = await asyncio.gather(*tasks) @@ -510,7 +519,9 @@ async def test_rpc_and_request_results_do_not_swap(): engine = _make_engine_with_loop(loop, rpc_delay=0.02) try: rpc_task = asyncio.create_task(engine.async_collective_rpc("ping", args=("rpc1",), unique_reply_rank=0)) - req_task = asyncio.create_task(engine.async_add_req_and_wait_for_response(_make_request("req1"))) + req_task = asyncio.create_task( + _consume_final_output(engine.async_add_req_and_stream_response(_make_request("req1"))) + ) rpc_res, req_res = await asyncio.gather(rpc_task, req_task) finally: _stop_engine(engine) diff --git a/tests/diffusion/test_diffusion_model_runner.py b/tests/diffusion/test_diffusion_model_runner.py index e82b8f7f0..cf9f0d3b4 100644 --- a/tests/diffusion/test_diffusion_model_runner.py +++ b/tests/diffusion/test_diffusion_model_runner.py @@ -98,6 +98,29 @@ class _ChunkStepPipeline: return output +class _FinalOnlyStepPipeline: + device = torch.device("cpu") + supports_step_execution = True + + def prepare_encode(self, state): + state.prompt_embeds = torch.zeros(1, 1, 1) + state.latents = torch.zeros(1, 1) + state.timesteps = torch.tensor([1.0]) + state.step_index = 0 + return state + + def denoise_step(self, input_batch, states): + del states + return torch.ones_like(input_batch.latents) + + def step_scheduler(self, state, noise_pred): + state.latents = noise_pred + state.step_index += 1 + + def post_decode(self, state): + return DiffusionOutput(output=state.latents.clone()) + + def _make_request(): sampling_params = SimpleNamespace( generator=None, @@ -228,6 +251,35 @@ def test_execute_stepwise_streaming_returns_chunks_at_boundaries(monkeypatch): assert fourth.get_request_output("req").finished is True +@pytest.mark.core_model +@pytest.mark.cpu +def test_execute_stepwise_streaming_decodes_final_only_pipeline(monkeypatch): + """Step streaming still returns a final output when no chunk boundaries exist.""" + runner = _make_runner(cache_backend=None, cache_backend_name=None) + runner.pipeline = _FinalOnlyStepPipeline() + runner.od_config.streaming_output = True + runner.od_config.step_execution = True + req = _make_request() + req.request_id = "req" + req.sampling_params.num_inference_steps = 1 + + monkeypatch.setattr(model_runner_module, "set_forward_context", _noop_forward_context) + monkeypatch.setattr(model_runner_module.current_omni_platform, "reset_peak_memory_stats", lambda: None) + monkeypatch.setattr(model_runner_module.current_omni_platform, "max_memory_reserved", lambda: 0) + monkeypatch.setattr(model_runner_module.current_omni_platform, "max_memory_allocated", lambda: 0) + scheduler_output = SimpleNamespace( + finished_req_ids=set(), + scheduled_new_reqs=[SimpleNamespace(request_id="req", req=req)], + scheduled_cached_reqs=SimpleNamespace(request_ids=[]), + ) + + output = DiffusionModelRunner.execute_stepwise(runner, scheduler_output).get_request_output("req") + + assert output.finished is True + assert output.result is not None + assert torch.equal(output.result.output, torch.ones(1, 1)) + + @pytest.mark.core_model @hardware_test(res={"cuda": "L4"}, num_cards=1) def test_execute_model_skips_cache_summary_without_active_cache_backend(monkeypatch): diff --git a/tests/diffusion/test_diffusion_output_formatter.py b/tests/diffusion/test_diffusion_output_formatter.py index 5524e9686..eea53210c 100644 --- a/tests/diffusion/test_diffusion_output_formatter.py +++ b/tests/diffusion/test_diffusion_output_formatter.py @@ -11,7 +11,6 @@ import torch from vllm_omni.diffusion import output_formatter from vllm_omni.diffusion.data import DiffusionOutput from vllm_omni.diffusion.output_formatter import ( - DiffusionStepTimings, format_diffusion_outputs, format_empty_diffusion_outputs, normalize_diffusion_postprocess_output, @@ -42,15 +41,6 @@ def _config(model_class_name: str = "mock_model") -> SimpleNamespace: return SimpleNamespace(model_class_name=model_class_name) -def _timings() -> DiffusionStepTimings: - return DiffusionStepTimings( - preprocess_time_s=0.01, - exec_time_s=0.02, - postprocess_time_s=0.03, - total_time_ms=60.0, - ) - - def test_formatter_preserves_single_video_audio_actions_and_metadata( monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -80,7 +70,6 @@ def test_formatter_preserves_single_video_audio_actions_and_metadata( ), output_data={"raw": "output"}, postprocess_output=postprocess_output, - timings=_timings(), ) assert len(results) == 1 @@ -102,14 +91,10 @@ def test_formatter_preserves_single_video_audio_actions_and_metadata( assert result.stage_durations == {"execute": 1.25} assert result.peak_memory_mb == 321.0 assert result.metrics == { - "preprocess_time_ms": 10.0, - "diffusion_engine_exec_time_ms": 20.0, - "diffusion_engine_total_time_ms": 60.0, "image_num": 1, "resolution": 512, "width": None, "height": None, - "postprocess_time_ms": 30.0, } @@ -162,7 +147,6 @@ def test_formatter_normalizes_payload_metadata_envelope( diffusion_output=DiffusionOutput(output=None), output_data={"raw": "output"}, postprocess_output=postprocess_output, - timings=_timings(), ) assert result.multimodal_output == { @@ -198,7 +182,6 @@ def test_formatter_maps_trajectory_payload_to_request_output( diffusion_output=DiffusionOutput(output=None), output_data={"raw": "output"}, postprocess_output=postprocess_output, - timings=_timings(), ) assert result.images == ["image-0"] @@ -231,7 +214,6 @@ def test_formatter_preserves_text_envelope_metadata(monkeypatch: pytest.MonkeyPa diffusion_output=DiffusionOutput(output=None), output_data={"payload": {"text": "caption"}}, postprocess_output=postprocess_output, - timings=_timings(), ) assert result.images == [] @@ -269,7 +251,6 @@ def test_formatter_preserves_image_with_reasoning_metadata( diffusion_output=DiffusionOutput(output=None), output_data=None, postprocess_output=postprocess_output, - timings=_timings(), ) assert postprocess_output.primary_key == "image" @@ -298,7 +279,6 @@ def test_formatter_preserves_audio_output_with_model_sample_rate_fallback( diffusion_output=DiffusionOutput(output=["waveform"]), output_data=["waveform"], postprocess_output=postprocess_output, - timings=_timings(), ) assert result.images == [] @@ -330,7 +310,6 @@ def test_formatter_preserves_audio_model_video_audio_and_actions( diffusion_output=DiffusionOutput(output=None), output_data={"raw": "output"}, postprocess_output=postprocess_output, - timings=_timings(), ) assert result.images == ["frame-0"] @@ -365,7 +344,6 @@ def test_formatter_preserves_audio_only_postprocess_dict( diffusion_output=DiffusionOutput(output=None), output_data={"raw": "output"}, postprocess_output=postprocess_output, - timings=_timings(), ) assert result.images == [] @@ -397,7 +375,6 @@ def test_formatter_preserves_single_prompt_multiple_audio_outputs( diffusion_output=DiffusionOutput(output=["waveform-0", "waveform-1"]), output_data=["waveform-0", "waveform-1"], postprocess_output=postprocess_output, - timings=_timings(), ) assert result.images == [] @@ -425,7 +402,6 @@ def test_formatter_preserves_single_prompt_audio_and_action_payloads( diffusion_output=DiffusionOutput(output=None), output_data={"raw": "output"}, postprocess_output=postprocess_output, - timings=_timings(), ) assert len(results) == 1 diff --git a/tests/diffusion/test_diffusion_scheduler.py b/tests/diffusion/test_diffusion_scheduler.py index 68b237c67..43ce7a45b 100644 --- a/tests/diffusion/test_diffusion_scheduler.py +++ b/tests/diffusion/test_diffusion_scheduler.py @@ -11,7 +11,7 @@ import torch from pytest_mock import MockerFixture from vllm_omni.diffusion.data import DiffusionOutput, DiffusionRequestAbortedError -from vllm_omni.diffusion.diffusion_engine import DiffusionEngine +from vllm_omni.diffusion.diffusion_engine import DiffusionEngine, DiffusionExecutionMode from vllm_omni.diffusion.request import OmniDiffusionRequest from vllm_omni.diffusion.sched import ( BaseScheduler, @@ -532,13 +532,11 @@ class TestDiffusionEngine: assert output is runner_output.result engine.execute_fn.assert_called_once() - def test_initializes_injected_scheduler( + def test_initializes_default_request_scheduler( self, monkeypatch: pytest.MonkeyPatch, mocker: MockerFixture, ) -> None: - request = _make_request("init") - scheduler = _StubScheduler(request, DiffusionOutput(output=None)) od_config = SimpleNamespace(model_class_name="mock_model", streaming_output=False) fake_executor_cls = mocker.Mock(return_value=mocker.Mock()) @@ -554,11 +552,11 @@ class TestDiffusionEngine: "vllm_omni.diffusion.diffusion_engine.DiffusionExecutor.get_class", lambda *args, **kwargs: fake_executor_cls, ) - monkeypatch.setattr(DiffusionEngine, "_dummy_run", lambda self: None) - DiffusionEngine(od_config, scheduler=scheduler) + engine = DiffusionEngine(od_config) - assert scheduler.initialized_with is od_config + assert isinstance(engine.scheduler, RequestScheduler) + assert engine.scheduler.max_num_running_reqs == 1 fake_executor_cls.assert_called_once_with(od_config) def test_scheduler_alias_keeps_default_request_scheduler(self) -> None: @@ -573,16 +571,19 @@ class TestDiffusionEngine: assert scheduler.get_request_state(req_id).status == DiffusionRequestStatus.FINISHED_COMPLETED @pytest.mark.asyncio - async def test_step_raises_aborted_error(self, mocker: MockerFixture) -> None: + async def test_step_streaming_raises_aborted_error(self, mocker: MockerFixture) -> None: engine = DiffusionEngine.__new__(DiffusionEngine) engine._check_and_start_background_loop = mocker.AsyncMock() engine.pre_process_func = None - engine.async_add_req_and_wait_for_response = mocker.AsyncMock( - return_value=DiffusionOutput(aborted=True, abort_message="Request req-abort aborted.") - ) + + async def _stream(_request): + yield DiffusionOutput(aborted=True, abort_message="Request req-abort aborted.") + + engine.async_add_req_and_stream_response = mocker.Mock(return_value=_stream(None)) with pytest.raises(DiffusionRequestAbortedError, match="Request req-abort aborted"): - await engine.step(_make_request("req-abort")) + async for _ in engine.step_streaming(_make_request("req-abort")): + pass def test_abort_queue_marks_request_finished_aborted(self) -> None: engine = DiffusionEngine.__new__(DiffusionEngine) @@ -619,12 +620,13 @@ class TestDiffusionEngine: engine.scheduler.initialize(SimpleNamespace()) engine._rpc_lock = threading.RLock() engine._cv = threading.Condition(engine._rpc_lock) - engine._out_queue_streaming = {} + engine._out_streams = {} + engine.execution_mode = DiffusionExecutionMode.STEP_BATCH engine.main_loop = asyncio.get_running_loop() req_id = engine.scheduler.add_request(_make_request("stream-engine")) queue: asyncio.Queue[DiffusionOutput] = asyncio.Queue() - engine._out_queue_streaming[req_id] = queue + engine._out_streams[req_id] = queue sched_output = engine.scheduler.schedule() chunk = RunnerOutput( @@ -634,7 +636,7 @@ class TestDiffusionEngine: result=DiffusionOutput(output="chunk-0", finished=False, chunk_index=0, total_chunks=2), ) finished_req_ids = engine.scheduler.update_from_output(sched_output, chunk) - engine._handle_step_streaming_runner_output(finished_req_ids, sched_output.scheduled_request_ids, chunk) + engine._emit_outputs(finished_req_ids, sched_output.scheduled_request_ids, chunk) notified_chunk = await asyncio.wait_for(queue.get(), timeout=1) assert notified_chunk.output == "chunk-0" @@ -648,7 +650,7 @@ class TestDiffusionEngine: result=DiffusionOutput(output="chunk-1", finished=True, chunk_index=1, total_chunks=2), ) finished_req_ids = engine.scheduler.update_from_output(sched_output, final_chunk) - engine._handle_step_streaming_runner_output(finished_req_ids, sched_output.scheduled_request_ids, final_chunk) + engine._emit_outputs(finished_req_ids, sched_output.scheduled_request_ids, final_chunk) notified_final = await asyncio.wait_for(queue.get(), timeout=1) assert notified_final.output == "chunk-1" @@ -662,15 +664,15 @@ class TestDiffusionEngine: engine.scheduler.initialize(SimpleNamespace()) engine._rpc_lock = threading.RLock() engine._cv = threading.Condition(engine._rpc_lock) - engine._out_queue_streaming = {} + engine._out_streams = {} engine.main_loop = asyncio.get_running_loop() req_id = engine.scheduler.add_request(_make_request("stream-abort")) queue: asyncio.Queue[DiffusionOutput] = asyncio.Queue() - engine._out_queue_streaming[req_id] = queue + engine._out_streams[req_id] = queue engine.scheduler.finish_requests(req_id, DiffusionRequestStatus.FINISHED_ABORTED) - engine._handle_empty_streaming_requests({req_id}) + engine._emit_finished_outputs({req_id}) output = await asyncio.wait_for(queue.get(), timeout=1) assert output.aborted is True @@ -699,9 +701,9 @@ class TestDiffusionEngine: "vllm_omni.diffusion.diffusion_engine.DiffusionExecutor.get_class", lambda *args, **kwargs: fake_executor_cls, ) - monkeypatch.setattr(DiffusionEngine, "_dummy_run", lambda self: None) engine = DiffusionEngine(od_config) + assert engine.execution_mode == DiffusionExecutionMode.STEP_BATCH assert isinstance(engine.scheduler, StepScheduler) assert engine.execute_fn is fake_executor.execute_step fake_executor_cls.assert_called_once_with(od_config) diff --git a/tests/diffusion/test_inline_stage_diffusion_client.py b/tests/diffusion/test_inline_stage_diffusion_client.py index b94f86db0..4981f7f92 100644 --- a/tests/diffusion/test_inline_stage_diffusion_client.py +++ b/tests/diffusion/test_inline_stage_diffusion_client.py @@ -54,9 +54,12 @@ def client(mock_engine): @pytest.mark.asyncio async def test_inline_dispatch_request_success(client, mock_engine): - # Setup mock engine step to return a successful result mock_result = OmniRequestOutput.from_diffusion(request_id="req-1", images=[MagicMock()]) - mock_engine.step.return_value = [mock_result] + + async def _step_streaming(_request): + yield [mock_result] + + mock_engine.step_streaming = _step_streaming sampling_params = OmniDiffusionSamplingParams() await client.add_request_async("req-1", "A test prompt", sampling_params) @@ -70,7 +73,6 @@ async def test_inline_dispatch_request_success(client, mock_engine): assert output is not None assert output.request_id == "req-1" - mock_engine.step.assert_called_once() @pytest.mark.asyncio @@ -104,8 +106,11 @@ async def test_inline_dispatch_request_streaming_success(client, mock_engine): @pytest.mark.asyncio async def test_inline_dispatch_request_error(client, mock_engine): - # Setup mock engine step to raise an exception - mock_engine.step.side_effect = RuntimeError("Engine failure") + async def _step_streaming(_request): + raise RuntimeError("Engine failure") + yield # pragma: no cover + + mock_engine.step_streaming = _step_streaming sampling_params = OmniDiffusionSamplingParams() await client.add_request_async("req-err", "A test prompt", sampling_params) diff --git a/tests/diffusion/test_multiproc_engine_concurrency.py b/tests/diffusion/test_multiproc_engine_concurrency.py index 5f5d90f64..f78f8d44b 100644 --- a/tests/diffusion/test_multiproc_engine_concurrency.py +++ b/tests/diffusion/test_multiproc_engine_concurrency.py @@ -91,7 +91,7 @@ def _make_engine(num_gpus: int = 1): engine._loop_started = False engine._rpc_queue = queue.Queue() engine.abort_queue = queue.Queue() - engine.execute_fn = executor.execute_request + engine.execute_fn = executor.execute_batch return engine, executor, req_q, res_q diff --git a/tests/diffusion/test_stage_diffusion_proc.py b/tests/diffusion/test_stage_diffusion_proc.py index c96aa8ba1..697465097 100644 --- a/tests/diffusion/test_stage_diffusion_proc.py +++ b/tests/diffusion/test_stage_diffusion_proc.py @@ -27,7 +27,7 @@ DELAY_BASE = 0.01 class MockDiffusionEngine: - async def step(self, request): + async def step_streaming(self, request): def simulate_step_delay(height, width, num_inference_steps) -> float: return (height / BASE_HEIGHT) * (width / BASE_WIDTH) * (num_inference_steps / BASE_INFER_STEPS) @@ -37,7 +37,7 @@ class MockDiffusionEngine: ) delay = DELAY_BASE + delay_scale * DELAY_BASE await asyncio.sleep(delay) - return [MockOmniRequestOutput(request_id=request.request_id)] + yield [MockOmniRequestOutput(request_id=request.request_id)] @pytest.mark.asyncio diff --git a/tests/e2e/offline_inference/custom_pipeline/test_async_omni_qwen_image_generate.py b/tests/e2e/offline_inference/custom_pipeline/test_async_omni_qwen_image_generate.py index 0681687fe..02519f90f 100644 --- a/tests/e2e/offline_inference/custom_pipeline/test_async_omni_qwen_image_generate.py +++ b/tests/e2e/offline_inference/custom_pipeline/test_async_omni_qwen_image_generate.py @@ -158,6 +158,7 @@ async def test_async_omni_generate(): custom_pipeline_args={"pipeline_class": CUSTOM_PIPELINE_CLASS}, worker_extension_cls=WORKER_EXTENSION_CLASS, enforce_eager=True, + max_num_seqs=1, ) after.callback(engine.shutdown) @@ -183,6 +184,7 @@ async def test_async_omni_generate_with_logprobs(): custom_pipeline_args={"pipeline_class": CUSTOM_PIPELINE_CLASS}, worker_extension_cls=WORKER_EXTENSION_CLASS, enforce_eager=True, + max_num_seqs=1, ) after.callback(engine.shutdown) @@ -220,6 +222,7 @@ async def test_async_omni_generate_concurrent(): custom_pipeline_args={"pipeline_class": CUSTOM_PIPELINE_CLASS}, worker_extension_cls=WORKER_EXTENSION_CLASS, enforce_eager=True, + max_num_seqs=1, ) after.callback(engine.shutdown) diff --git a/tests/e2e/offline_inference/rlhf_test/test_verl_omni_e2e.py b/tests/e2e/offline_inference/rlhf_test/test_verl_omni_e2e.py index 69952435e..420c9bb68 100644 --- a/tests/e2e/offline_inference/rlhf_test/test_verl_omni_e2e.py +++ b/tests/e2e/offline_inference/rlhf_test/test_verl_omni_e2e.py @@ -283,7 +283,7 @@ class vLLMOmniHttpServerLocal: "worker_extension_cls": WORKER_EXTENSION_CLASS, "trust_remote_code": _cfg_get(self.model_config, "trust_remote_code", True), "max_model_len": _cfg_get(self.config, "max_model_len", 1058), - "max_num_seqs": _cfg_get(self.config, "max_num_seqs", 256), + "max_num_seqs": _cfg_get(self.config, "max_num_seqs", 1), "enable_chunked_prefill": _cfg_get(self.config, "enable_chunked_prefill", False), "max_num_batched_tokens": _cfg_get(self.config, "max_num_batched_tokens", 8192), "enable_prefix_caching": _cfg_get(self.config, "enable_prefix_caching", False), @@ -476,7 +476,8 @@ def init_server(): "pipeline_model_parallel_size": 1, "gpu_memory_utilization": 0.8, "max_num_batched_tokens": 8192, - "max_num_seqs": 256, + # The custom log-prob pipeline only supports serial requests. + "max_num_seqs": 1, "max_model_len": 1058, "dtype": "bfloat16", "load_format": "auto", diff --git a/vllm_omni/diffusion/diffusion_engine.py b/vllm_omni/diffusion/diffusion_engine.py index e8ab578db..4d37843d7 100644 --- a/vllm_omni/diffusion/diffusion_engine.py +++ b/vllm_omni/diffusion/diffusion_engine.py @@ -11,6 +11,7 @@ import threading import time from collections.abc import AsyncGenerator, Iterable from dataclasses import dataclass, field +from enum import Enum from typing import TYPE_CHECKING, Any import numpy as np @@ -33,7 +34,6 @@ from vllm_omni.diffusion.io_support import ( supports_multimodal_input, ) from vllm_omni.diffusion.output_formatter import ( - DiffusionStepTimings, format_diffusion_outputs, format_empty_diffusion_outputs, normalize_diffusion_postprocess_output, @@ -57,6 +57,7 @@ logger = init_logger(__name__) __all__ = [ "DiffusionEngine", + "DiffusionExecutionMode", "_RpcTask", "_move_tensor_tree_to_cpu", "get_dummy_run_num_frames", @@ -106,6 +107,13 @@ def supports_request_batch(od_config: OmniDiffusionConfig) -> bool: return bool(getattr(model_cls, "supports_request_batch", False)) +def _max_num_seqs(od_config: OmniDiffusionConfig) -> int: + try: + return max(1, int(getattr(od_config, "max_num_seqs", 1))) + except (TypeError, ValueError): + return 1 + + def _move_tensor_tree_to_cpu(value: object) -> object: if isinstance(value, torch.Tensor): return value.cpu() if value.device.type != "cpu" else value @@ -130,6 +138,11 @@ class _RpcTask: future: concurrent.futures.Future = field(default_factory=concurrent.futures.Future) +class DiffusionExecutionMode(str, Enum): + REQUEST_BATCH = "request_batch" + STEP_BATCH = "step_batch" + + class DiffusionEngine: """The diffusion engine for vLLM-Omni diffusion models.""" @@ -149,27 +162,66 @@ class DiffusionEngine: """Initialize the diffusion engine. Args: - config: The configuration for the diffusion engine. + od_config: The configuration for the diffusion engine. + scheduler: Optional scheduler override for tests or custom engine + integrations. When omitted, the engine selects a scheduler + from the resolved execution mode. """ self.od_config = od_config + self._init_process_hooks(od_config) + self.execution_mode = self._resolve_execution_mode(od_config) + self._init_executor(od_config) + self._init_scheduler(od_config, scheduler) + self._init_runtime_state() + self._init_execute_fn() + self._log_execution_mode(od_config) + + def _init_process_hooks(self, od_config: OmniDiffusionConfig) -> None: self.post_process_func = get_diffusion_post_process_func(od_config) self.pre_process_func = get_diffusion_pre_process_func(od_config) # Cache whether the model-specific postprocess accepts request-level # sampling params so step() can support both legacy and extended hooks. self._post_process_accepts_sampling_params = _func_accepts_parameter(self.post_process_func, "sampling_params") + def _resolve_execution_mode(self, od_config: OmniDiffusionConfig) -> DiffusionExecutionMode: self.step_execution = bool(getattr(od_config, "step_execution", False)) - if self.od_config.streaming_output and not self.step_execution: + if od_config.streaming_output and not self.step_execution: logger.warning("streaming_output=True requires step_execution=True; enabling step execution.") - self.od_config.step_execution = True + od_config.step_execution = True self.step_execution = True + if self.step_execution: + self.supports_request_batch = False + return DiffusionExecutionMode.STEP_BATCH + + self.supports_request_batch = supports_request_batch(od_config) + if not self.supports_request_batch and _max_num_seqs(od_config) > 1: + raise ValueError( + f"{getattr(od_config, 'model_class_name', None)!r} does not support request-level batching. " + "Use max_num_seqs=1 for serial request execution, or choose a pipeline with " + "supports_request_batch=True." + ) + return DiffusionExecutionMode.REQUEST_BATCH + + def _init_executor(self, od_config: OmniDiffusionConfig) -> None: executor_class = DiffusionExecutor.get_class(od_config) self.executor = executor_class(od_config) - self.scheduler: BaseScheduler = scheduler or (StepScheduler() if self.step_execution else RequestScheduler()) + + def _init_scheduler( + self, + od_config: OmniDiffusionConfig, + scheduler: BaseScheduler | None = None, + ) -> None: + if scheduler is not None: + self.scheduler = scheduler + elif self.execution_mode == DiffusionExecutionMode.STEP_BATCH: + self.scheduler = StepScheduler() + else: + self.scheduler = RequestScheduler() self.scheduler.initialize(od_config) - self.supports_request_batch = False if self.step_execution else supports_request_batch(od_config) + + def _init_runtime_state(self) -> None: self.main_loop: asyncio.AbstractEventLoop | None = None self.stop_event: threading.Event | None = None self.worker_thread: threading.Thread | None = None @@ -181,33 +233,26 @@ class DiffusionEngine: # busy loop rather than via mutual exclusion. self._rpc_lock = threading.RLock() self._cv = threading.Condition(self._rpc_lock) - self._out_queue: dict[str, asyncio.Future] = {} - self._out_queue_streaming: dict[str, asyncio.Queue[DiffusionOutput]] = {} + self._out_streams: dict[str, asyncio.Queue[DiffusionOutput]] = {} self._closed = False self._shutdown_complete = False self.abort_queue: queue.Queue[str] = queue.Queue() self._rpc_queue: queue.Queue[_RpcTask] = queue.Queue() - if self.step_execution: - self.execute_fn = self.executor.execute_step - elif self.supports_request_batch: - self.execute_fn = self.executor.execute_batch - else: - self.execute_fn = self.executor.execute_request - if self.supports_request_batch: + def _init_execute_fn(self) -> None: + if self.execution_mode == DiffusionExecutionMode.STEP_BATCH: + self.execute_fn = self.executor.execute_step + else: + self.execute_fn = self.executor.execute_batch + + def _log_execution_mode(self, od_config: OmniDiffusionConfig) -> None: + if self.execution_mode == DiffusionExecutionMode.REQUEST_BATCH: logger.info( "[RequestBatch] engine init max_num_seqs=%s max_wait_ms=%s", getattr(od_config, "max_num_seqs", None), getattr(od_config, "request_batch_max_wait_ms", None), ) - try: - self._dummy_run() - except Exception as e: - logger.error(f"Dummy run failed: {e}") - self.close() - raise e - async def _check_and_start_background_loop(self): if self._closed: raise RuntimeError("DiffusionEngine is closed.") @@ -227,24 +272,6 @@ class DiffusionEngine: self.worker_thread.start() self._loop_started = True - async def step(self, request: OmniDiffusionRequest) -> list[OmniRequestOutput]: - await self._check_and_start_background_loop() - - diffusion_engine_start_time = time.perf_counter() - - # Apply pre-processing if available - preprocess_time = 0.0 - if self.pre_process_func is not None: - preprocess_start_time = time.perf_counter() - request = self.pre_process_func(request) - preprocess_time = time.perf_counter() - preprocess_start_time - logger.debug("Pre-processing completed in %.4f seconds", preprocess_time) - - exec_start_time = time.perf_counter() - output = await self.async_add_req_and_wait_for_response(request) - exec_total_time = time.perf_counter() - exec_start_time - return self.postprocess_output(request, output, diffusion_engine_start_time, preprocess_time, exec_total_time) - async def step_streaming(self, request: OmniDiffusionRequest) -> AsyncGenerator[list[OmniRequestOutput], None]: await self._check_and_start_background_loop() @@ -261,19 +288,50 @@ class DiffusionEngine: generator = self.async_add_req_and_stream_response(request) async for output in generator: exec_total_time = time.perf_counter() - exec_start_time - yield self.postprocess_output( - request, output, diffusion_engine_start_time, preprocess_time, exec_total_time + postprocess_start_time = time.perf_counter() + formatted_outputs = self.postprocess_output(request, output) + postprocess_time = time.perf_counter() - postprocess_start_time + step_total_ms = (time.perf_counter() - diffusion_engine_start_time) * 1000 + logger.debug( + "DiffusionEngine.step_streaming breakdown: preprocess=%.2f ms, " + "add_req_and_wait=%.2f ms, postprocess=%.2f ms, total=%.2f ms", + preprocess_time * 1000, + exec_total_time * 1000, + postprocess_time * 1000, + step_total_ms, ) + for request_output in formatted_outputs: + request_output.metrics.update( + { + "preprocess_time_ms": preprocess_time * 1000, + "diffusion_engine_exec_time_ms": exec_total_time * 1000, + "diffusion_engine_total_time_ms": step_total_ms, + "postprocess_time_ms": postprocess_time * 1000, + } + ) + yield formatted_outputs + + async def step(self, request: OmniDiffusionRequest) -> list[OmniRequestOutput]: + """Deprecated compatibility wrapper over ``step_streaming()``. + + Use ``step_streaming()`` for new callers. This method drains the + unified output stream and returns only the final output batch, matching + the historical non-streaming ``step()`` behavior. + """ + logger.warning_once( + "DiffusionEngine.step() is deprecated; use step_streaming() and consume the final output batch instead." + ) + final_output: list[OmniRequestOutput] | None = None + async for output in self.step_streaming(request): + final_output = output + return final_output or [] def postprocess_output( self, request: OmniDiffusionRequest, output: DiffusionOutput, - diffusion_engine_start_time: float, - preprocess_time: float, - exec_total_time: float, ) -> list[OmniRequestOutput]: - """Convert a DiffusionOutput to a list of OmniRequestOutput, attaching profiling metrics.""" + """Convert a DiffusionOutput to a list of OmniRequestOutput.""" if output.aborted: raise DiffusionRequestAbortedError(output.abort_message or "Diffusion request aborted.") if output.error: @@ -297,7 +355,6 @@ class DiffusionEngine: if self.od_config.enable_cpu_offload: output_data = _move_tensor_tree_to_cpu(output_data) - postprocess_start_time = time.perf_counter() if self.post_process_func is not None: # Some video pipelines need request-level controls during # postprocess (for example worker-side frame interpolation). @@ -309,18 +366,6 @@ class DiffusionEngine: outputs = output_data postprocess_output = normalize_diffusion_postprocess_output(outputs) - postprocess_time = time.perf_counter() - postprocess_start_time - logger.debug("Post-processing completed in %.4f seconds", postprocess_time) - - step_total_ms = (time.perf_counter() - diffusion_engine_start_time) * 1000 - logger.debug( - "DiffusionEngine.step breakdown: preprocess=%.2f ms, " - "add_req_and_wait=%.2f ms, postprocess=%.2f ms, total=%.2f ms", - preprocess_time * 1000, - exec_total_time * 1000, - postprocess_time * 1000, - step_total_ms, - ) return format_diffusion_outputs( request=request, @@ -328,12 +373,6 @@ class DiffusionEngine: diffusion_output=output, output_data=output_data, postprocess_output=postprocess_output, - timings=DiffusionStepTimings( - preprocess_time_s=preprocess_time, - exec_time_s=exec_total_time, - postprocess_time_s=postprocess_time, - total_time_ms=step_total_ms, - ), ) def _busy_loop(self): @@ -357,16 +396,13 @@ class DiffusionEngine: # Only RPC / abort work pending; loop back to drain it. continue - if self.supports_request_batch: + if self.execution_mode == DiffusionExecutionMode.REQUEST_BATCH: self._wait_for_request_batch_admission_locked() sched_output = self.scheduler.schedule() if sched_output.is_empty: - if self.od_config.streaming_output: - self._handle_empty_streaming_requests(sched_output.finished_req_ids) - else: - self._handle_finished_requests(sched_output.finished_req_ids, None) + self._emit_finished_outputs(sched_output.finished_req_ids, None) continue try: @@ -390,14 +426,7 @@ class DiffusionEngine: self._process_aborts_queue() self._process_rpc_queue() finished_req_ids = self.scheduler.update_from_output(sched_output, runner_output) - if self.od_config.streaming_output: - self._handle_step_streaming_runner_output( - finished_req_ids, - sched_output.scheduled_request_ids, - runner_output, - ) - else: - self._handle_finished_requests(finished_req_ids, runner_output) + self._emit_outputs(finished_req_ids, sched_output.scheduled_request_ids, runner_output) # Engine is stopping: fail any RPCs still queued so callers don't hang. self._fail_pending_rpcs(RuntimeError("DiffusionEngine is shutting down.")) @@ -509,74 +538,62 @@ class DiffusionEngine: if not task.future.done(): task.future.set_exception(exc) - def _handle_finished_requests( + def _emit_finished_outputs( self, finished_ids: set[str], runner_output: BaseRunnerOutput | None = None, missing_result_error: str = "Diffusion execution finished without a final output", - ): + ) -> None: for rid in finished_ids: - with self._cv: - fut = self._out_queue.pop(rid, None) - if fut is None: - continue if runner_output is not None: _output = runner_output.get_request_output(rid) else: _output = None out = self._finalize_finished_request(rid, _output, missing_result_error) - self._complete_future(fut, out) + self._put_output(rid, out) - def _handle_empty_streaming_requests( + def _emit_outputs( self, finished_ids: set[str], - missing_result_error: str = "Diffusion streaming request finished without execution output.", - ) -> None: - """Mirrors `_handle_finished_requests()` in non-streaming mode when used for empty scheduler output.""" - for rid in finished_ids: - out = self._finalize_finished_request(rid, None, missing_result_error=missing_result_error) - self._put_streaming_output_with_cv(rid, out) - - def _handle_step_streaming_runner_output( - self, - finished_req_ids: set[str], scheduled_request_ids: list[str], runner_output: BaseRunnerOutput, ) -> None: - """ - Deliver partial step-execution outputs in streaming mode. + """Emit output chunks for every request through the unified output stream.""" + if self.execution_mode != DiffusionExecutionMode.STEP_BATCH: + self._emit_finished_outputs(finished_ids, runner_output) + return - Step execution returns one ``RunnerOutput`` per scheduled request per - engine tick. Most denoise steps have ``result=None``; chunk boundaries - return a ``DiffusionOutput`` that must be delivered even before the - scheduler marks the request finished. - """ delivered_finished_req_ids: set[str] = set() # finished_ids may have some requests that are not scheduler in this round. # First handle this-round requests. for request_id in scheduled_request_ids: req_output = runner_output.get_request_output(request_id) - if request_id in finished_req_ids: + if request_id in finished_ids: # This entire request is finished (this is the last chunk) out = self._finalize_finished_request( request_id, req_output, - missing_result_error="Diffusion streaming execution finished without a final output.", + missing_result_error="Diffusion step execution finished without a final output.", ) - self._put_streaming_output_with_cv(request_id, out) + self._put_output(request_id, out) delivered_finished_req_ids.add(request_id) elif req_output is not None and req_output.result is not None: - # This is a non-terminal chunk. So it is not in scheduler's finished_req_ids, but still need delivering. - self._put_streaming_output_with_cv(request_id, req_output.result) + # This is a non-terminal chunk. It is not in scheduler's + # finished_ids, but it still belongs on the request stream. + self._put_output(request_id, req_output.result) # Then handle other requests that are finished in this round. - for request_id in finished_req_ids - delivered_finished_req_ids: + for request_id in finished_ids - delivered_finished_req_ids: out = self._finalize_finished_request( request_id, - missing_result_error="Diffusion streaming request finished without execution output.", + missing_result_error="Diffusion step request finished without execution output.", ) - self._put_streaming_output_with_cv(request_id, out) + self._put_output(request_id, out) + + def _has_output_stream(self, request_id: str) -> bool: + with self._cv: + return request_id in self._out_streams @staticmethod def resolve_engine_class(config: OmniDiffusionConfig) -> type[DiffusionEngine]: @@ -622,44 +639,31 @@ class DiffusionEngine: Args: config: The configuration for the diffusion engine. + scheduler: Optional scheduler override. When omitted, the selected + engine chooses the scheduler from its execution mode. Returns: An instance of the resolved ``DiffusionEngine`` (sub)class. """ engine_class = DiffusionEngine.resolve_engine_class(config) - return engine_class(config, scheduler=scheduler) + engine = engine_class(config, scheduler=scheduler) + engine.run_startup_warmup() + return engine def add_request(self, request: OmniDiffusionRequest) -> str: with self._cv: if self._closed: raise RuntimeError("DiffusionEngine is closed.") - if not self.od_config.streaming_output: - fut = self.main_loop.create_future() - request_id = self.scheduler.add_request(request) - self._out_queue[request_id] = fut - else: - queue: asyncio.Queue[DiffusionOutput] = asyncio.Queue() - request_id = self.scheduler.add_request(request) - self._out_queue_streaming[request_id] = queue + queue: asyncio.Queue[DiffusionOutput] = asyncio.Queue() + request_id = self.scheduler.add_request(request) + self._out_streams[request_id] = queue self._cv.notify_all() return request_id - async def get_result(self, request_id: str) -> DiffusionOutput: - fut = self._out_queue.get(request_id) - - if fut is None: - raise RuntimeError(f"Request {request_id} not found in output queue.") - try: - return await fut - except Exception as e: - logger.error(f"Wait for response failed: {e}") - raise - - async def get_streaming_result(self, request_id: str) -> AsyncGenerator[DiffusionOutput, None]: - """Mirrors `get_result()` in non-streaming mode.""" + async def get_output_stream(self, request_id: str) -> AsyncGenerator[DiffusionOutput, None]: with self._cv: - queue = self._out_queue_streaming.get(request_id) + queue = self._out_streams.get(request_id) if queue is None: raise RuntimeError(f"Request {request_id} not found in output queue.") try: @@ -672,22 +676,31 @@ class DiffusionEngine: logger.error(f"Wait for response failed: {e}") raise finally: - # In streaming mode, an output queue is maintained until the terminal chunk is met. - # So unlike the non-streaming mode where output Future is popped in `_handle_finished_requests` (immediately - # after the request is returned), the streaming mode needs to pop the output queue here (one layer above). with self._cv: - if self._out_queue_streaming.get(request_id) is queue: - self._out_queue_streaming.pop(request_id, None) - - async def async_add_req_and_wait_for_response(self, request: OmniDiffusionRequest) -> DiffusionOutput: - # No lock needed: add_request is already protected by self._cv, and - # all executor calls are serialized inside the busy loop. - request_id = self.add_request(request) - return await self.get_result(request_id) + if self._out_streams.get(request_id) is queue: + self._out_streams.pop(request_id, None) def async_add_req_and_stream_response(self, request: OmniDiffusionRequest) -> AsyncGenerator[DiffusionOutput, None]: request_id = self.add_request(request) - return self.get_streaming_result(request_id) + return self.get_output_stream(request_id) + + async def async_add_req_and_wait_for_response(self, request: OmniDiffusionRequest) -> DiffusionOutput: + """Deprecated compatibility wrapper over ``async_add_req_and_stream_response()``. + + Use ``async_add_req_and_stream_response()`` for new callers. This + method drains the unified output stream and returns only the final + ``DiffusionOutput``, matching the historical non-streaming behavior. + """ + logger.warning_once( + "DiffusionEngine.async_add_req_and_wait_for_response() is deprecated; " + "use async_add_req_and_stream_response() and consume the final output instead." + ) + final_output: DiffusionOutput | None = None + async for output in self.async_add_req_and_stream_response(request): + final_output = output + if final_output is None: + raise RuntimeError("Diffusion execution completed without an output.") + return final_output def add_req_and_wait_for_response(self, request: OmniDiffusionRequest) -> DiffusionOutput: with self._rpc_lock: @@ -759,6 +772,14 @@ class DiffusionEngine: if is_start: raise RuntimeError(f"Could not {action} profiler: {e}") from e + def run_startup_warmup(self) -> None: + try: + self._dummy_run() + except Exception as e: + logger.error(f"Dummy run failed: {e}") + self.close() + raise e + def _dummy_run(self): """A dummy run to warm up the model.""" num_inference_steps = 1 @@ -891,8 +912,8 @@ class DiffusionEngine: ) -> Any: """Async variant of :meth:`collective_rpc` for event-loop callers. - Mirrors :meth:`async_add_req_and_wait_for_response`: enqueue a task - keyed by a future and ``await`` the result without blocking the loop. + Enqueue a task keyed by a future and ``await`` the result without + blocking the loop. """ await self._check_and_start_background_loop() task = self._submit_rpc(method, timeout, args, kwargs, unique_reply_rank) @@ -905,51 +926,26 @@ class DiffusionEngine: task.future.cancel() raise TimeoutError(f"RPC call to {method} timed out.") from exc - def _complete_future(self, fut: asyncio.Future, output: DiffusionOutput) -> None: - if fut.done(): - return - - def _set_result() -> None: - if not fut.done(): - fut.set_result(output) - - try: - loop = fut.get_loop() - except AttributeError: - loop = self.main_loop - - try: - running_loop = asyncio.get_running_loop() - except RuntimeError: - running_loop = None - - if loop is not None and loop.is_running() and loop is not running_loop: - loop.call_soon_threadsafe(_set_result) - else: - _set_result() - - def _put_streaming_queue_output( + def _put_queue_output( self, queue: asyncio.Queue[DiffusionOutput], output: DiffusionOutput, ) -> None: - """Append to streaming output queue in a safe event loop. Mirrors `_complete_future()` in non-streaming mode.""" loop = self.main_loop if loop is not None and loop.is_running(): loop.call_soon_threadsafe(queue.put_nowait, output) else: queue.put_nowait(output) - def _put_streaming_output_with_cv(self, request_id: str, output: DiffusionOutput) -> None: + def _put_output(self, request_id: str, output: DiffusionOutput) -> None: with self._cv: - queue = self._out_queue_streaming.get(request_id) + queue = self._out_streams.get(request_id) if queue is None: return - self._put_streaming_queue_output(queue, output) + self._put_queue_output(queue, output) def close(self) -> None: - pending_futures: list[asyncio.Future] = [] - pending_streaming_queues: list[asyncio.Queue[DiffusionOutput]] = [] + pending_streams: list[asyncio.Queue[DiffusionOutput]] = [] with self._cv: if self._closed and self._shutdown_complete: return @@ -957,17 +953,13 @@ class DiffusionEngine: self._closed = True if self.stop_event is not None: self.stop_event.set() - pending_futures = list(self._out_queue.values()) - pending_streaming_queues = list(self._out_queue_streaming.values()) - self._out_queue.clear() - self._out_queue_streaming.clear() + pending_streams = list(self._out_streams.values()) + self._out_streams.clear() self._cv.notify_all() closed_output = DiffusionOutput(error="DiffusionEngine is closed.") - for fut in pending_futures: - self._complete_future(fut, closed_output) - for streaming_queue in pending_streaming_queues: - self._put_streaming_queue_output(streaming_queue, closed_output) + for stream in pending_streams: + self._put_queue_output(stream, closed_output) worker_thread = self.worker_thread if worker_thread is not None: diff --git a/vllm_omni/diffusion/executor/abstract.py b/vllm_omni/diffusion/executor/abstract.py index 17e5c0504..ec6b3486a 100644 --- a/vllm_omni/diffusion/executor/abstract.py +++ b/vllm_omni/diffusion/executor/abstract.py @@ -81,7 +81,7 @@ class DiffusionExecutor(ABC): @abstractmethod def execute_batch(self, scheduler_output: DiffusionSchedulerOutput) -> BaseRunnerOutput: - """Execute request-mode work as a single batched RPC.""" + """Execute request-mode work through the request-batch path.""" pass @abstractmethod diff --git a/vllm_omni/diffusion/executor/multiproc_executor.py b/vllm_omni/diffusion/executor/multiproc_executor.py index f90624ed7..c7ca34153 100644 --- a/vllm_omni/diffusion/executor/multiproc_executor.py +++ b/vllm_omni/diffusion/executor/multiproc_executor.py @@ -356,14 +356,18 @@ class MultiprocDiffusionExecutor(DiffusionExecutor): return BatchRunnerOutput.from_list(runner_outputs) def execute_batch(self, scheduler_output: DiffusionSchedulerOutput) -> BaseRunnerOutput: - """Execute request-mode work through a single batched worker RPC. + """Execute request-mode work through the unified request-batch path. - The worker builds DiffusionRequestBatch from scheduler output and returns - BatchRunnerOutput with one RunnerOutput per scheduled request. + A scheduler wave with one request is the conservative serial case and + uses the single-request worker RPC. Waves with multiple requests use the + fused request-batch RPC and require pipeline request-batch support. """ from vllm_omni.diffusion.worker.utils import BatchRunnerOutput self._ensure_open() + if len(scheduler_output.scheduled_new_reqs) <= 1: + return self.execute_request(scheduler_output) + result = self.collective_rpc( "execute_model_batch", args=(scheduler_output, self.od_config), diff --git a/vllm_omni/diffusion/inline_stage_diffusion_client.py b/vllm_omni/diffusion/inline_stage_diffusion_client.py index 8b3d90f00..4e0922ff5 100644 --- a/vllm_omni/diffusion/inline_stage_diffusion_client.py +++ b/vllm_omni/diffusion/inline_stage_diffusion_client.py @@ -139,8 +139,13 @@ class InlineStageDiffusionClient(StageClientBase): result.request_id = request_id self._output_queue.put_nowait(result) else: - results = await self._engine.step(request) - result = results[0] + # Non-streaming callers share the streaming engine path but + # only publish the final output. + result = None + async for results in self._engine.step_streaming(request): + result = results[0] + if result is None: + raise RuntimeError("Diffusion execution finished without output.") if not result.request_id: result.request_id = request_id self._output_queue.put_nowait(result) diff --git a/vllm_omni/diffusion/models/dreamzero/pipeline_dreamzero.py b/vllm_omni/diffusion/models/dreamzero/pipeline_dreamzero.py index 107638f81..d43693b0d 100644 --- a/vllm_omni/diffusion/models/dreamzero/pipeline_dreamzero.py +++ b/vllm_omni/diffusion/models/dreamzero/pipeline_dreamzero.py @@ -3,7 +3,7 @@ """DreamZero pipeline for vllm-omni. -Entry point for DiffusionEngine.step() -> pipeline.forward(req) +Entry point for DiffusionEngine.step_streaming() -> pipeline.forward(req) """ from __future__ import annotations @@ -1163,7 +1163,7 @@ class DreamZeroPipeline(nn.Module, CFGParallelMixin): @torch.no_grad() def forward(self, req: DiffusionRequestBatch, **kwargs) -> DiffusionOutput: - """Full inference step. Called by DiffusionEngine.step().""" + """Full inference step. Called by DiffusionEngine.step_streaming().""" extra_args = req.sampling_params.extra_args or {} robot_obs = extra_args.get("robot_obs") if robot_obs is None: diff --git a/vllm_omni/diffusion/output_formatter.py b/vllm_omni/diffusion/output_formatter.py index 05b5b440d..b369ed0c3 100644 --- a/vllm_omni/diffusion/output_formatter.py +++ b/vllm_omni/diffusion/output_formatter.py @@ -27,14 +27,6 @@ from vllm_omni.outputs.output_metadata import ( ) -@dataclass(frozen=True) -class DiffusionStepTimings: - preprocess_time_s: float - exec_time_s: float - postprocess_time_s: float - total_time_ms: float - - @dataclass(frozen=True) class DiffusionPostprocessOutput: outputs: DiffusionPayload @@ -134,23 +126,18 @@ def format_diffusion_outputs( diffusion_output: DiffusionOutput, output_data: DiffusionPostprocessRawOutput, postprocess_output: DiffusionPostprocessOutput, - timings: DiffusionStepTimings, ) -> list[OmniRequestOutput]: """Convert a finished diffusion model output into API-facing outputs.""" primary_payload = _primary_payload(postprocess_output) outputs = _ensure_list(primary_payload) metrics = { - "preprocess_time_ms": timings.preprocess_time_s * 1000, - "diffusion_engine_exec_time_ms": timings.exec_time_s * 1000, - "diffusion_engine_total_time_ms": timings.total_time_ms, "image_num": int(request.sampling_params.num_outputs_per_prompt), "resolution": ( int(request.sampling_params.resolution) if request.sampling_params.resolution is not None else None ), "width": request.sampling_params.width, "height": request.sampling_params.height, - "postprocess_time_ms": timings.postprocess_time_s * 1000, } # Only the primary payload determines the response type. Some image models diff --git a/vllm_omni/diffusion/output_metadata.py b/vllm_omni/diffusion/output_metadata.py deleted file mode 100644 index abea8759e..000000000 --- a/vllm_omni/diffusion/output_metadata.py +++ /dev/null @@ -1,10 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Compatibility imports for diffusion output metadata. - -The shared output payload and metadata schema lives in -``vllm_omni.outputs.output_metadata``. Keep this module as a stable import -path for existing diffusion callers. -""" - -from vllm_omni.outputs.output_metadata import * # noqa: F403 diff --git a/vllm_omni/diffusion/stage_diffusion_proc.py b/vllm_omni/diffusion/stage_diffusion_proc.py index 0e00f8eea..f01348c74 100644 --- a/vllm_omni/diffusion/stage_diffusion_proc.py +++ b/vllm_omni/diffusion/stage_diffusion_proc.py @@ -155,7 +155,7 @@ class StageDiffusionProc: sampling_params_dict: dict, kv_sender_info: dict[str, Any] | None = None, ) -> OmniRequestOutput: - """Build a diffusion request and run DiffusionEngine.step().""" + """Build a diffusion request and consume DiffusionEngine.step_streaming() to completion.""" sampling_params = self._reconstruct_sampling_params(sampling_params_dict) request = OmniDiffusionRequest( @@ -165,8 +165,13 @@ class StageDiffusionProc: kv_sender_info=kv_sender_info, ) - results = await self._engine.step(request) - result = results[0] + # Non-streaming callers share the streaming engine path but only + # return the final output. + result = None + async for results in self._engine.step_streaming(request): + result = results[0] + if result is None: + raise RuntimeError("Diffusion execution finished without output.") if not result.request_id: result.request_id = request_id return result diff --git a/vllm_omni/diffusion/worker/diffusion_model_runner.py b/vllm_omni/diffusion/worker/diffusion_model_runner.py index 34a191506..44a873e56 100644 --- a/vllm_omni/diffusion/worker/diffusion_model_runner.py +++ b/vllm_omni/diffusion/worker/diffusion_model_runner.py @@ -714,7 +714,7 @@ class DiffusionModelRunner(OmniConnectorModelRunnerMixin): req, noise_pred[offset : offset + row_num] if noise_pred is not None else None ) if self.od_config.streaming_output: - should_decode = req.chunk_denoise_completed + should_decode = req.chunk_denoise_completed or req.request_denoise_completed else: should_decode = req.denoise_completed From 00bdb5dcd723d467bd4984fdfc5164235c012616 Mon Sep 17 00:00:00 2001 From: LHXuuu Date: Thu, 23 Jul 2026 11:22:51 +0800 Subject: [PATCH 068/240] [Perf][Ming-TTS] Cache speaker embeddings for uploaded reference audio (#5240) Signed-off-by: LHXuuu --- .../openai_api/test_serving_speech.py | 170 ++++++++++++++++-- .../entrypoints/openai/serving_speech.py | 11 +- .../openai/tts_adapters/ming_tts.py | 52 +++++- .../ming_flash_omni/ming_flash_omni_talker.py | 4 +- .../models/ming_tts/audio_prep.py | 9 +- .../models/ming_tts/constants.py | 1 + .../models/ming_tts/ming_tts_llm.py | 3 +- 7 files changed, 213 insertions(+), 37 deletions(-) diff --git a/tests/entrypoints/openai_api/test_serving_speech.py b/tests/entrypoints/openai_api/test_serving_speech.py index 8faca1b78..aab1acfbc 100644 --- a/tests/entrypoints/openai_api/test_serving_speech.py +++ b/tests/entrypoints/openai_api/test_serving_speech.py @@ -45,6 +45,7 @@ from vllm_omni.model_executor.models.fish_speech.prompt_utils import ( FISH_TEXT_ONLY_SYSTEM_PROMPT, build_fish_voice_clone_prompt_ids, ) +from vllm_omni.model_executor.models.ming_tts.constants import SPEAKER_EMBEDDING_DIM from vllm_omni.outputs import OmniRequestOutput pytestmark = [pytest.mark.core_model, pytest.mark.cpu] @@ -975,21 +976,25 @@ class TestTTSMethods: ) def test_upload_ming_voice_embedding_wrong_dims_rejected_without_replacing_existing(self, speech_server): - """Ming embedding uploads must be 192-dim before replacing stored voices.""" + """Ming embedding uploads must match the model dimension before replacement.""" speech_server._tts_model_type = "ming_tts" + invalid_dim = SPEAKER_EMBEDDING_DIM - 1 existing = { "name": "bad_emb_voice", "file_path": "/tmp/voice_samples/bad_emb_voice.safetensors", "mime_type": "application/x-safetensors", "embedding_source": "direct", - "embedding_dim": 192, + "embedding_dim": SPEAKER_EMBEDDING_DIM, } speech_server.uploaded_speakers = {"bad_emb_voice": existing.copy()} - with pytest.raises(ValueError, match="Ming speaker embedding must have 192 dims, got 191"): + with pytest.raises( + ValueError, + match=f"Ming speaker embedding must have {SPEAKER_EMBEDDING_DIM} dims, got {invalid_dim}", + ): asyncio.run( speech_server.upload_voice_embedding( - embedding_json=json.dumps([0.0] * 191), + embedding_json=json.dumps([0.0] * invalid_dim), consent="consent", name="bad_emb_voice", ) @@ -997,6 +1002,59 @@ class TestTTSMethods: assert speech_server.uploaded_speakers["bad_emb_voice"] == existing + def test_upload_ming_audio_voice_defers_speaker_extraction(self, speech_server, mocker: MockerFixture, tmp_path): + """Ming audio uploads stay model-agnostic and defer extraction to use.""" + speech_server._tts_model_type = "ming_tts" + speech_server.uploaded_speakers_dir = tmp_path + speech_server.uploaded_speakers = {} + speech_server.supported_speakers = set() + speech_server._speaker_cache.clear() + + left = np.full(32000, 0.5, dtype=np.float32) + right = np.full(32000, -0.25, dtype=np.float32) + samples = np.stack((left, right), axis=-1) + mocker.patch("soundfile.read", return_value=(samples, 16000)) + mock_extract = mocker.patch.object(speech_server, "_extract_ming_speaker_embeddings_from_ref_audio") + saved: dict[str, object] = {} + + def fake_save_file(tensors, path, metadata=None): + saved["tensors"] = tensors + saved["metadata"] = metadata + Path(path).touch() + + mocker.patch("safetensors.torch.save_file", side_effect=fake_save_file) + audio_file = mocker.MagicMock() + audio_file.filename = "reference.wav" + audio_file.content_type = "audio/wav" + audio_file.file = io.BytesIO(b"fake audio") + audio_file.read = mocker.AsyncMock(return_value=b"fake audio") + + asyncio.run( + speech_server.upload_voice( + audio_file, + consent="consent", + name="ming_audio_voice", + ref_text="Reference transcript.", + ) + ) + + mock_extract.assert_not_called() + tensors = saved["tensors"] + metadata = saved["metadata"] + assert isinstance(tensors, dict) + assert isinstance(metadata, dict) + assert set(tensors) == {"audio"} + assert tensors["audio"].shape == (32000, 2) + assert metadata["embedding_source"] == "audio" + assert "embedding_dim" not in metadata + speaker_info = speech_server.uploaded_speakers["ming_audio_voice"] + cache_key = speech_server._speaker_cache.make_cache_key( + "ming_audio_voice", + model_type="ming_tts", + created_at=speaker_info["created_at"], + ) + assert speech_server._speaker_cache.get(cache_key) is None + def test_base_task_requires_ref_audio_or_speaker_embedding(self, speech_server): """Base task without ref_audio or speaker_embedding is rejected.""" req = OpenAICreateSpeechRequest(input="Hello", task_type="Base") @@ -1591,10 +1649,10 @@ class TestTTSMethods: "file_path": "/tmp/voice_samples/emb_voice.safetensors", "mime_type": "application/x-safetensors", "embedding_source": "direct", - "embedding_dim": 192, + "embedding_dim": SPEAKER_EMBEDDING_DIM, } } - fake_embedding = [0.1] * 192 + fake_embedding = [0.1] * SPEAKER_EMBEDDING_DIM mock_get_emb = mocker.patch.object( speech_server, "_get_uploaded_speaker_embedding", return_value=fake_embedding ) @@ -1614,25 +1672,36 @@ class TestTTSMethods: mock_prompt.assert_called_once_with(req, ref_audio_data=None) prompt_request = mock_prompt.call_args.args[0] assert prompt_request.speaker_embedding == fake_embedding - assert len(prompt_request.speaker_embedding) == 192 + assert len(prompt_request.speaker_embedding) == SPEAKER_EMBEDDING_DIM assert prepared.prompt["additional_information"]["speaker_count"] == 1 assert prepared.model_type == "ming_tts" - def test_ming_adapter_uploaded_audio_path_preserves_ref_text(self, speech_server, mocker: MockerFixture): - """Ming uploaded audio voices still resolve audio and stored ref_text.""" + def test_ming_adapter_uploaded_audio_uses_cached_embedding(self, speech_server, mocker: MockerFixture): + """Cached Ming audio voices skip repeated speaker extraction.""" + speech_server._speaker_cache.clear() speech_server.uploaded_speakers = { "audio_voice": { "name": "audio_voice", - "file_path": "/tmp/voice_samples/audio_voice.wav", + "file_path": "/tmp/voice_samples/audio_voice.safetensors", + "created_at": 123, "mime_type": "audio/wav", + "embedding_source": "audio", "ref_text": "Reference transcript.", } } ref_audio_source = "data:audio/wav;base64,ZmFrZWF1ZGlv" ref_audio_data = ([0.0, 0.1], 16000) - fake_embedding = [0.2] * 192 - mock_get_audio = mocker.patch.object(speech_server, "_get_uploaded_audio_data", return_value=ref_audio_source) - mock_get_emb = mocker.patch.object(speech_server, "_get_uploaded_speaker_embedding") + fake_embedding = [0.2] * SPEAKER_EMBEDDING_DIM + cache_key = speech_server._speaker_cache.make_cache_key( + "audio_voice", + model_type="ming_tts", + created_at=123, + ) + speech_server._speaker_cache.put( + cache_key, + {"speaker_embedding": torch.tensor(fake_embedding)}, + ) + mocker.patch.object(speech_server, "_get_uploaded_audio_data", return_value=ref_audio_source) mocker.patch.object( speech_server, "_resolve_ref_audio", @@ -1641,7 +1710,6 @@ class TestTTSMethods: mock_extract = mocker.patch.object( speech_server, "_extract_ming_speaker_embeddings_from_ref_audio", - return_value=[fake_embedding], ) mock_prompt = mocker.patch.object( speech_server, @@ -1653,14 +1721,80 @@ class TestTTSMethods: req = OpenAICreateSpeechRequest(input="Hello", voice="audio_voice") prepared = asyncio.run(adapter.build(req, [], False)) - mock_get_audio.assert_called_once_with("audio_voice") - mock_get_emb.assert_not_called() - mock_extract.assert_called_once_with([ref_audio_data]) + mock_extract.assert_not_called() mock_prompt.assert_called_once_with(req, ref_audio_data=ref_audio_data) assert req.ref_text == "Reference transcript." - assert req.speaker_embedding == fake_embedding + assert req.speaker_embedding == pytest.approx(fake_embedding) assert prepared.model_type == "ming_tts" + def test_ming_adapter_legacy_audio_only_voice_falls_back_and_caches( + self, + speech_server, + mocker: MockerFixture, + tmp_path, + ): + """Legacy audio-only profiles extract once instead of failing after upgrade.""" + from safetensors.torch import save_file + + speech_server._tts_model_type = "ming_tts" + speech_server.uploaded_speakers_dir = tmp_path + speech_server._speaker_cache.clear() + file_path = tmp_path / "legacy_audio_voice.safetensors" + save_file({"audio": torch.zeros(32000, dtype=torch.float32)}, str(file_path)) + speech_server.uploaded_speakers = { + "legacy_audio_voice": { + "name": "legacy_audio_voice", + "file_path": str(file_path), + "created_at": 456, + "mime_type": "audio/wav", + "embedding_source": "audio", + "ref_text": "Reference transcript.", + } + } + ref_audio_source = "data:audio/wav;base64,ZmFrZWF1ZGlv" + ref_audio_data = ([0.0, 0.1], 16000) + fake_embedding = [0.4] * SPEAKER_EMBEDDING_DIM + mocker.patch.object(speech_server, "_get_uploaded_audio_data", return_value=ref_audio_source) + mocker.patch.object( + speech_server, + "_resolve_ref_audio", + new=mocker.AsyncMock(return_value=ref_audio_data), + ) + mock_extract = mocker.patch.object( + speech_server, + "_extract_ming_speaker_embeddings_from_ref_audio", + return_value=[fake_embedding], + ) + mock_to_thread = mocker.spy(asyncio, "to_thread") + mocker.patch.object( + speech_server, + "_build_ming_dense_prompt", + return_value={"additional_information": {"speaker_count": 1}}, + ) + + adapter = MingTTSAdapter(SpeechServingContext(server=speech_server)) + first_req = OpenAICreateSpeechRequest(input="Hello", voice="legacy_audio_voice") + first_prepared = asyncio.run(adapter.build(first_req, [], False)) + second_req = OpenAICreateSpeechRequest(input="Hello again", voice="legacy_audio_voice") + second_prepared = asyncio.run(adapter.build(second_req, [], False)) + + mock_extract.assert_called_once_with([ref_audio_data]) + mock_to_thread.assert_called_once_with(mock_extract, [ref_audio_data]) + assert first_req.speaker_embedding == pytest.approx(fake_embedding) + assert second_req.speaker_embedding == pytest.approx(fake_embedding) + assert first_req.ref_text == "Reference transcript." + assert second_req.ref_text == "Reference transcript." + assert first_prepared.model_type == "ming_tts" + assert second_prepared.model_type == "ming_tts" + cache_key = speech_server._speaker_cache.make_cache_key( + "legacy_audio_voice", + model_type="ming_tts", + created_at=456, + ) + cached = speech_server._speaker_cache.get(cache_key) + assert cached is not None + torch.testing.assert_close(cached["speaker_embedding"], torch.tensor(fake_embedding)) + # ── regression: full flow from issue #1603 ── def test_regression_1603_speaker_key_with_uploaded_audio_voice(self, speech_server, mocker: MockerFixture): diff --git a/vllm_omni/entrypoints/openai/serving_speech.py b/vllm_omni/entrypoints/openai/serving_speech.py index f07ee7e10..ca3b8c21f 100644 --- a/vllm_omni/entrypoints/openai/serving_speech.py +++ b/vllm_omni/entrypoints/openai/serving_speech.py @@ -67,6 +67,7 @@ from vllm_omni.model_executor.models.ming_flash_omni.prompt_utils import ( from vllm_omni.model_executor.models.ming_flash_omni.prompt_utils import ( create_instruction as ming_create_instruction, ) +from vllm_omni.model_executor.models.ming_tts.constants import SPEAKER_EMBEDDING_DIM from vllm_omni.outputs import OmniRequestOutput from vllm_omni.utils.speaker_cache import ( get_speaker_cache, @@ -1396,8 +1397,8 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): emb_dim = len(embedding) if self._tts_model_type == "ming_tts": - if emb_dim != 192: - raise ValueError(f"Ming speaker embedding must have 192 dims, got {emb_dim}") + if emb_dim != SPEAKER_EMBEDDING_DIM: + raise ValueError(f"Ming speaker embedding must have {SPEAKER_EMBEDDING_DIM} dims, got {emb_dim}") else: dim_err = self._validate_qwen_tts_speaker_embedding_dim(emb_dim) if dim_err is not None: @@ -2148,8 +2149,10 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): waveform = torch.as_tensor(wav_samples, dtype=torch.float32).reshape(1, -1) embedding = extractor.extract_from_waveform(waveform, int(sr)) flat = embedding.detach().reshape(-1).to(torch.float32).cpu() - if int(flat.numel()) != 192: - raise ValueError(f"Ming speaker extractor returned {int(flat.numel())} dims; expected 192") + if int(flat.numel()) != SPEAKER_EMBEDDING_DIM: + raise ValueError( + f"Ming speaker extractor returned {int(flat.numel())} dims; expected {SPEAKER_EMBEDDING_DIM}" + ) embeddings.append(flat.tolist()) return embeddings diff --git a/vllm_omni/entrypoints/openai/tts_adapters/ming_tts.py b/vllm_omni/entrypoints/openai/tts_adapters/ming_tts.py index c736afb51..dff392c28 100644 --- a/vllm_omni/entrypoints/openai/tts_adapters/ming_tts.py +++ b/vllm_omni/entrypoints/openai/tts_adapters/ming_tts.py @@ -1,16 +1,19 @@ # SPDX-License-Identifier: Apache-2.0 """Ming-TTS (dense) serving adapter.""" +import asyncio from typing import TYPE_CHECKING +import torch +from vllm.logger import init_logger + from vllm_omni.entrypoints.openai.tts_adapters import register_tts_adapter from vllm_omni.entrypoints.openai.tts_adapters.base import ARTTSAdapter, PreparedRequest +from vllm_omni.model_executor.models.ming_tts.constants import SPEAKER_EMBEDDING_DIM if TYPE_CHECKING: from vllm_omni.entrypoints.openai.protocol.audio import OpenAICreateSpeechRequest -from vllm.logger import init_logger - logger = init_logger(__name__) @@ -33,6 +36,7 @@ class MingTTSAdapter(ARTTSAdapter): ) -> PreparedRequest: server = self.ctx.server ref_audio_source = request.ref_audio + speaker_cache_key: tuple[str, str, int] | None = None voice_lower = request.voice.lower() if isinstance(request.voice, str) else None if ref_audio_source is None and voice_lower in server.uploaded_speakers: speaker_info = server.uploaded_speakers[voice_lower] @@ -42,6 +46,21 @@ class MingTTSAdapter(ARTTSAdapter): if request.speaker_embedding is None: raise ValueError(f"Speaker embedding for uploaded voice '{request.voice}' is missing") else: + if request.speaker_embedding is None: + speaker_cache_key = server._speaker_cache.make_cache_key( + voice_lower, + model_type=self.name, + created_at=int(speaker_info.get("created_at") or 0), + ) + cached = server._speaker_cache.get(speaker_cache_key) + if cached is not None: + cached_embedding = cached.get("speaker_embedding") + if isinstance(cached_embedding, torch.Tensor): + flat = cached_embedding.detach().reshape(-1).to(torch.float32).cpu() + if int(flat.numel()) == SPEAKER_EMBEDDING_DIM: + request.speaker_embedding = flat.tolist() + speaker_cache_key = None + logger.debug("Speaker cache HIT for Ming-TTS speaker '%s'", voice_lower) ref_audio_source = server._get_uploaded_audio_data(request.voice) if not ref_audio_source: raise ValueError(f"Audio file for uploaded voice '{request.voice}' is missing") @@ -51,12 +70,26 @@ class MingTTSAdapter(ARTTSAdapter): if isinstance(ref_audio_source, list): ref_audio_data = await server._resolve_ref_audio_many(ref_audio_source) if request.speaker_embedding is None: - request.speaker_embedding = server._extract_ming_speaker_embeddings_from_ref_audio(ref_audio_data) + request.speaker_embedding = await asyncio.to_thread( + server._extract_ming_speaker_embeddings_from_ref_audio, + ref_audio_data, + ) elif ref_audio_source is not None and isinstance(ref_audio_source, str): wav_list, sr = await server._resolve_ref_audio(ref_audio_source) ref_audio_data = (wav_list, sr) if request.speaker_embedding is None: - request.speaker_embedding = server._extract_ming_speaker_embeddings_from_ref_audio([ref_audio_data])[0] + embeddings = await asyncio.to_thread( + server._extract_ming_speaker_embeddings_from_ref_audio, + [ref_audio_data], + ) + request.speaker_embedding = embeddings[0] + if speaker_cache_key is not None and request.speaker_embedding is not None: + embedding = torch.as_tensor(request.speaker_embedding, dtype=torch.float32).detach().reshape(-1).cpu() + server._speaker_cache.put( + speaker_cache_key, + {"speaker_embedding": embedding}, + ) + logger.debug("Speaker cache STORE for Ming-TTS speaker '%s'", voice_lower) prompt = server._build_ming_dense_prompt(request, ref_audio_data=ref_audio_data) tts_params = prompt.get("additional_information", {}) # Ming stop-token / max_tokens sampling stays in the orchestrator tail. @@ -73,11 +106,12 @@ class MingTTSAdapter(ARTTSAdapter): if not request.speaker_embedding: return "'speaker_embedding' must be a non-empty list of floats" emb_len = len(request.speaker_embedding) - if emb_len != 192: + if emb_len != SPEAKER_EMBEDDING_DIM: logger.warning( - "speaker_embedding has %d dimensions; Ming dense expects 192. " + "speaker_embedding has %d dimensions; Ming dense expects %d. " "Wrong dimensions will likely fail or degrade output.", emb_len, + SPEAKER_EMBEDDING_DIM, ) voice_lower = request.voice.lower() if isinstance(request.voice, str) else None @@ -127,8 +161,10 @@ class MingTTSAdapter(ARTTSAdapter): ) if embeddings and isinstance(embeddings[0], list): for item in embeddings: - if len(item) != 192: - return "Podcast-style Ming speaker embeddings must each have 192 dimensions" + if len(item) != SPEAKER_EMBEDDING_DIM: + return ( + f"Podcast-style Ming speaker embeddings must each have {SPEAKER_EMBEDDING_DIM} dimensions" + ) if request.instructions and len(request.instructions) > server._max_instructions_length: return f"Instructions too long (max {server._max_instructions_length} characters)" diff --git a/vllm_omni/model_executor/models/ming_flash_omni/ming_flash_omni_talker.py b/vllm_omni/model_executor/models/ming_flash_omni/ming_flash_omni_talker.py index e7b305b1e..8868e9d2a 100644 --- a/vllm_omni/model_executor/models/ming_flash_omni/ming_flash_omni_talker.py +++ b/vllm_omni/model_executor/models/ming_flash_omni/ming_flash_omni_talker.py @@ -27,6 +27,7 @@ from vllm.sequence import IntermediateTensors from vllm_omni.model_executor.custom_process_mixin import CustomProcessMixin from vllm_omni.model_executor.model_loader.weight_utils import download_weights_from_hf_specific from vllm_omni.model_executor.models.common.ming.audio_vae import AudioVAE, AudioVAEConfig +from vllm_omni.model_executor.models.ming_tts.constants import SPEAKER_EMBEDDING_DIM from vllm_omni.model_executor.models.output_templates import OmniOutput from vllm_omni.transformers_utils.configs.ming_flash_omni import MingFlashOmniTalkerConfig @@ -120,8 +121,7 @@ class MingFlashOmniTalkerForConditionalGeneration(nn.Module, CustomProcessMixin) ) self.aggregator = Aggregator(llm_input_dim=self.hidden_size, **config.aggregator) self.stop_head = nn.Linear(self.hidden_size, 2, bias=True) - # CAMPPlus 192-dim -> hidden - self.spk_head = nn.Linear(192, self.hidden_size, bias=True) + self.spk_head = nn.Linear(SPEAKER_EMBEDDING_DIM, self.hidden_size, bias=True) # AudioVAE self.audio_vae, self._vae_weight_source = self._init_audio_vae(config, self.talker_dir, model_path) diff --git a/vllm_omni/model_executor/models/ming_tts/audio_prep.py b/vllm_omni/model_executor/models/ming_tts/audio_prep.py index 378cd222d..055948932 100644 --- a/vllm_omni/model_executor/models/ming_tts/audio_prep.py +++ b/vllm_omni/model_executor/models/ming_tts/audio_prep.py @@ -16,6 +16,7 @@ from .config_ming_tts import ( VAE_PATCH_SIZE, MingTTSConfig, ) +from .constants import SPEAKER_EMBEDDING_DIM def pad_prompt_waveform( @@ -56,7 +57,7 @@ def coerce_prompt_waveform(value: Any) -> torch.Tensor: def coerce_speaker_embeddings(value: Any, *, use_zero_spk_emb: bool = False) -> list[torch.Tensor] | None: if value is None: - return [torch.zeros((192,), dtype=torch.float32)] if use_zero_spk_emb else None + return [torch.zeros((SPEAKER_EMBEDDING_DIM,), dtype=torch.float32)] if use_zero_spk_emb else None if isinstance(value, torch.Tensor): tensor = value.detach() if tensor.ndim == 1: @@ -78,10 +79,10 @@ def coerce_speaker_embeddings(value: Any, *, use_zero_spk_emb: bool = False) -> else: return coerce_speaker_embeddings(torch.as_tensor(value), use_zero_spk_emb=use_zero_spk_emb) if not items: - return [torch.zeros((192,), dtype=torch.float32)] if use_zero_spk_emb else None + return [torch.zeros((SPEAKER_EMBEDDING_DIM,), dtype=torch.float32)] if use_zero_spk_emb else None for item in items: - if int(item.numel()) != 192: - raise ValueError(f"Ming speaker embedding must have 192 dims, got {int(item.numel())}") + if int(item.numel()) != SPEAKER_EMBEDDING_DIM: + raise ValueError(f"Ming speaker embedding must have {SPEAKER_EMBEDDING_DIM} dims, got {int(item.numel())}") return items diff --git a/vllm_omni/model_executor/models/ming_tts/constants.py b/vllm_omni/model_executor/models/ming_tts/constants.py index e0508bda7..3db8ff3b8 100644 --- a/vllm_omni/model_executor/models/ming_tts/constants.py +++ b/vllm_omni/model_executor/models/ming_tts/constants.py @@ -43,6 +43,7 @@ LLM_VOCAB_SIZE = 151936 AGGREGATOR_HIDDEN_SIZE = 1024 VAE_PATCH_SIZE = 4 SAMPLE_RATE = 44100 +SPEAKER_EMBEDDING_DIM = 192 # CAMPPlus output width and speaker projection input width # AudioVAE frame/hop geometry (confirmed) AUDIO_FRAME_HOP = 882 # enc input_dim / hop_size / dec output_dim diff --git a/vllm_omni/model_executor/models/ming_tts/ming_tts_llm.py b/vllm_omni/model_executor/models/ming_tts/ming_tts_llm.py index 39ddd7b9f..909befb3a 100644 --- a/vllm_omni/model_executor/models/ming_tts/ming_tts_llm.py +++ b/vllm_omni/model_executor/models/ming_tts/ming_tts_llm.py @@ -33,6 +33,7 @@ from .config_ming_tts import ( KEY_TEXT_MODE, MingTTSConfig, ) +from .constants import SPEAKER_EMBEDDING_DIM from .flowloss_head import FlowLoss from .patch_emission import ( MING_STOP_REASON_CODES, @@ -91,7 +92,7 @@ class MingLLMModel(nn.Module): **self.ming_config.ditar_config, ) self.stop_head = nn.Linear(self.ming_config.llm_hidden_size, 2, bias=True) - self.spk_head = nn.Linear(192, self.ming_config.llm_hidden_size, bias=True) + self.spk_head = nn.Linear(SPEAKER_EMBEDDING_DIM, self.ming_config.llm_hidden_size, bias=True) self.flowloss.to(dtype=self.fm_dtype) self.linear_proj_audio.to(dtype=self.fm_dtype) self.stop_head.to(dtype=self.fm_dtype) From 96235da4682614bc01ed7c66ba8c42c60a8a4942 Mon Sep 17 00:00:00 2001 From: Jack Chen Date: Thu, 23 Jul 2026 12:37:32 +0800 Subject: [PATCH 069/240] Generalize AR-Diffusion KV session capabilities (#5271) Signed-off-by: Jack47 --- .../ar_diffusion_pipeline_capability.md | 165 +++++ tests/ar_diffusion/test_capability_runner.py | 317 ++++++++++ tests/ar_diffusion/test_kv_cache.py | 84 ++- tests/ar_diffusion/test_kv_orchestration.py | 22 +- tests/ar_diffusion/test_paged_attention.py | 69 ++- tests/ar_diffusion/test_pipeline_bridge.py | 251 +++++--- tests/dreamzero/test_pipeline_state.py | 56 +- .../models/dreamzero/pipeline_dreamzero.py | 235 ++++++-- .../experimental/ar_diffusion/__init__.py | 19 +- .../experimental/ar_diffusion/capability.py | 169 ++++++ vllm_omni/experimental/ar_diffusion/engine.py | 5 +- .../ar_diffusion/kv_cache/__init__.py | 2 +- .../ar_diffusion/kv_cache/config.py | 5 +- .../ar_diffusion/kv_cache/manager.py | 355 +++++++---- .../ar_diffusion/kv_cache/paged.py | 21 +- .../ar_diffusion/kv_cache/paged_attention.py | 39 +- .../ar_diffusion/kv_cache/state.py | 225 ++++--- vllm_omni/experimental/ar_diffusion/runner.py | 569 ++++++++---------- 18 files changed, 1808 insertions(+), 800 deletions(-) create mode 100644 docs/design/ar_diffusion_pipeline_capability.md create mode 100644 tests/ar_diffusion/test_capability_runner.py create mode 100644 vllm_omni/experimental/ar_diffusion/capability.py diff --git a/docs/design/ar_diffusion_pipeline_capability.md b/docs/design/ar_diffusion_pipeline_capability.md new file mode 100644 index 000000000..c9e05c8e8 --- /dev/null +++ b/docs/design/ar_diffusion_pipeline_capability.md @@ -0,0 +1,165 @@ +# AR-Diffusion pipeline capability + +> **Status:** experimental and non-normative. This note documents the current +> `experimental/ar_diffusion` contract. It follows the ownership and lifecycle +> direction in [#5137](https://github.com/vllm-project/vllm-omni/issues/5137), +> but it is not yet part of the stable module-design hierarchy. + +`ARDiffusionEngine` is an opt-in diffusion backend for pipelines that keep +autoregressive attention KV across requests. The engine only selects +`ARDiffusionModelRunner`; the runner owns paged KV pools, session lookup, LRU +eviction, and failure cleanup. + +A pipeline opts in by implementing `SupportsARDiffusionPipeline`. Its immutable +`ARDiffusionKVCacheSpec` reports worker-local KV geometry and policy: + +- self-attention layers, TP-local KV heads, head size, and tokens per latent frame; +- latent frames committed by one model block, sliding window, and attention sink; +- logical KV branches mapped to worker-local storage slots; +- named fixed-length cross-attention KV, model-specific scratch geometry, and + the pipeline's requested session-capacity upper bound. + +The runner binds an `ARDiffusionKVState` only around one `forward()` call. The +pipeline may use it during that context but must not retain it. The session +survives subsequent requests with the same `session_id`. Request reset, explicit +close, LRU eviction, and forward exceptions all release runner-owned KV and call +the pipeline's reset/close lifecycle hook. Cross-attention KV is allocated lazily +per session and logical KV branch, and is released on the same path. A pipeline +publishes one named cache transactionally by yielding exactly one `(k, v)` pair +per layer: + +```python +state.populate_cross_attention("main", "text", project_text_kv_by_layer()) +``` + +The cache becomes readable only after the iterable completes successfully. +Logical branches remain independent even when they share a worker-local +`local_index`; that index only maps lazily executed self-attention and scratch +work to worker-local capacity slots. It does not define cross-attention cache +ownership. + +Requests select the session with `extra_args["session_id"]`; setting `reset=True` +replaces it before the forward, while `close_session=True` releases it after a +successful forward. The runner also exposes `reset_session()` and +`close_session()` for an owning serving layer. + +## Current execution limits + +The experimental runner supports request-mode execution with +`max_num_seqs=1`. It rejects step execution and request-batch execution at load +time, and its inherited batch/step entry points also fail explicitly. Supporting +those paths requires a batch-aware state-binding contract rather than silently +bypassing `bind_ar_diffusion_state()`. + +The current implementation retains one session per model instance. A request +for a different `session_id` closes the resident session through the same LRU +lifecycle used by explicit close and failure cleanup before allocating the new +one. Self-attention and cross-attention memory are therefore sized for one +resident session even if the pipeline declares a larger `session_capacity`. +This prioritizes rollout throughput and prevents block-pool exhaustion before a +count-based eviction can run. Multi-session residency can be added later with a +memory-backed capacity calculation or allocation-pressure eviction. + +Scratch storage is also capability-driven. `frames_per_block` reserves space for +an uncommitted current video block, while `max_scratch_tokens_per_branch` +declares the maximum model-specific tokens, such as action/state registers, that +must coexist with it. `AR_DIFFUSION_KV_SCRATCH_BLOCKS_PER_BRANCH` may increase +that derived minimum for deployment experiments but cannot reduce it. + +## DreamZero adapter + +DreamZero implements the capability directly on `DreamZeroPipeline`. Its adapter +reads DreamZero transformer geometry, describes the positive and negative CFG +branches, binds the state with a context manager, and translates the existing +boolean CFG calls to the explicit `"positive"` and `"negative"` KV branch names. +Its optional `ar_diffusion_warmup_requests()` provider owns the robot observation +schema and the one-frame/four-frame warmup convention; none of those details are +present in the generic runner. + +Code migrating from the former experimental KV state API should use the +mechanical mapping below: + +```python +# Before: state.pos, state.neg, state.get_kv_caches(is_negative, ...) +positive = state.adapter("positive") +negative = state.adapter("negative") +contexts = state.get_kv_caches("negative", seq_len=frame_tokens, commit_current=True) +``` + +Direct experimental `ARDiffusionKVCache` construction also migrates +mechanically: replace `local_branches` with explicit `kv_branches`, +`num_frame_per_block` with `frames_per_block`, and the fixed text/image length +arguments with a named `cross_attention_lengths` mapping plus +`session_capacity`. Normal engine users do not construct this object directly. + +Cross-attention writers migrate from incremental publication: + +```python +# Before +for layer_index, (key, value) in enumerate(project_text_kv_by_layer()): + state.write_cross_attention_kv("text", layer_index, branch, key, value) +state.mark_cross_attention_populated(branch, "text") + +# After +state.populate_cross_attention(branch, "text", project_text_kv_by_layer()) +``` + +The iterable is consumed once and must yield exactly `num_layers` pairs. Failed +or incomplete projection leaves the previous complete cache unchanged, or +leaves the cache absent on its first population. + +## Single-KV-branch pipeline sketch + +A future single-branch causal DMD pipeline can opt in without changes to the +runner or KV core: + +```python +class WorldPipeline: + def ar_diffusion_kv_cache_spec(self): + return ARDiffusionKVCacheSpec( + num_layers=self.transformer.num_layers, + num_kv_heads=self.transformer.tp_local_kv_heads, + head_size=self.transformer.head_size, + tokens_per_frame=self.transformer.tokens_per_latent_frame, + frames_per_block=3, + window_frames=self.transformer.sliding_window_frames, + sink_frames=self.transformer.sink_frames, + kv_branches=(ARDiffusionKVBranchSpec("main", 0),), + cross_attention=(ARDiffusionCrossAttentionKVSpec("text", self.text_length),), + max_scratch_tokens_per_branch=self.transformer.max_register_tokens, + session_capacity=1, + ) + + @contextmanager + def bind_ar_diffusion_state(self, session_id, state): + self.current_ar_state = state + try: + yield + finally: + self.current_ar_state = None + + def reset_ar_diffusion_session(self, session_id): + self.model_states.pop(session_id, None) + + def close_ar_diffusion_session(self, session_id): + self.model_states.pop(session_id, None) +``` + +The model writes/reads named cross-attention KV through the bound state and uses +the `"main"` KV branch for paged self-attention. Pipelines without a warmup +provider are loaded normally and skip AR rollout warmup. + +## Relationship to the multi-stage KV manager RFC + +[#5244](https://github.com/vllm-project/vllm-omni/issues/5244) describes a +future multi-stage KV transfer manager, connector integration, and cache-aware +scheduling. This capability has a narrower scope: it owns model-local paged KV +and session lifecycle inside one AR-Diffusion runner. It does not define +cross-stage transfer, connector behavior, or scheduler-visible cache affinity. + +When the experimental runtime migrates toward #5244, the implementation should +reconcile its current full `KVCacheManager` use with the RFC's proposed +`BlockPool`-oriented DiT manager. The pipeline capability should remain the +model-facing geometry and state-binding boundary where possible, while the +experimental `ARDiffusionKVCache` class itself should not be treated as a stable +public API. diff --git a/tests/ar_diffusion/test_capability_runner.py b/tests/ar_diffusion/test_capability_runner.py new file mode 100644 index 000000000..7b94fe80f --- /dev/null +++ b/tests/ar_diffusion/test_capability_runner.py @@ -0,0 +1,317 @@ +# SPDX-License-Identifier: Apache-2.0 +"""CPU contracts for capability-driven AR-Diffusion sessions.""" + +from __future__ import annotations + +from collections import OrderedDict +from contextlib import contextmanager +from types import SimpleNamespace + +import pytest +import torch + +from vllm_omni.diffusion.worker.diffusion_model_runner import DiffusionModelRunner +from vllm_omni.experimental.ar_diffusion.capability import ( + ARDiffusionCrossAttentionKVSpec, + ARDiffusionKVBranchSpec, + ARDiffusionKVCacheSpec, +) +from vllm_omni.experimental.ar_diffusion.kv_cache import ARDiffusionKVConfig +from vllm_omni.experimental.ar_diffusion.runner import ARDiffusionModelRunner + +BLOCK = 16 +POS = "positive" +NEG = "negative" + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +def lingbot_like_spec(*, capacity: int = 2) -> ARDiffusionKVCacheSpec: + """Single-kv_branch causal DMD: 3 latent frames/block + sink/window + text KV.""" + return ARDiffusionKVCacheSpec( + num_layers=2, + num_kv_heads=4, # already TP-local + head_size=64, + tokens_per_frame=BLOCK, + frames_per_block=3, + window_frames=5, + sink_frames=1, + kv_branches=(ARDiffusionKVBranchSpec("main", 0),), + session_capacity=capacity, + cross_attention=(ARDiffusionCrossAttentionKVSpec("text", 8),), + ) + + +def dreamzero_like_spec(*, capacity: int = 2) -> ARDiffusionKVCacheSpec: + return ARDiffusionKVCacheSpec( + num_layers=2, + num_kv_heads=4, + head_size=64, + tokens_per_frame=BLOCK, + frames_per_block=4, + window_frames=6, + kv_branches=(ARDiffusionKVBranchSpec(POS, 0), ARDiffusionKVBranchSpec(NEG, 1)), + session_capacity=capacity, + cross_attention=(ARDiffusionCrossAttentionKVSpec("text", 8),), + ) + + +class CapablePipeline: + def __init__(self, spec: ARDiffusionKVCacheSpec) -> None: + self.spec = spec + self.bound_state = None + self.binds: list[str] = [] + self.resets: list[str] = [] + self.closes: list[str] = [] + + def ar_diffusion_kv_cache_spec(self) -> ARDiffusionKVCacheSpec: + return self.spec + + @contextmanager + def bind_ar_diffusion_state(self, session_id: str, state): + assert self.bound_state is None + self.bound_state = state + self.binds.append(session_id) + try: + yield + finally: + self.bound_state = None + + def reset_ar_diffusion_session(self, session_id: str) -> None: + self.resets.append(session_id) + + def close_ar_diffusion_session(self, session_id: str) -> None: + self.closes.append(session_id) + + +class WarmupPipeline(CapablePipeline): + def __init__(self, spec: ARDiffusionKVCacheSpec, requests: list[object]) -> None: + super().__init__(spec) + self.requests = requests + + def ar_diffusion_warmup_requests(self, session_id: str): + assert session_id == ARDiffusionModelRunner._WARMUP_SID + return iter(self.requests) + + +class BatchCapablePipeline(CapablePipeline): + supports_request_batch = True + + +def make_runner( + pipeline: object, + *, + available_bytes: int = 1 << 28, + step_execution: bool = False, +) -> ARDiffusionModelRunner: + runner = object.__new__(ARDiffusionModelRunner) + runner.od_config = SimpleNamespace( + max_num_seqs=1, + dtype=torch.float32, + enforce_eager=True, + step_execution=step_execution, + ) + runner.device = torch.device("cpu") + runner.pipeline = pipeline + runner.ar_diffusion_kv_config = ARDiffusionKVConfig(enable=True) + runner.kv_cache = None + runner._ar_diffusion_capability = None + runner._ar_diffusion_kv_cache_spec = None + runner._sessions = OrderedDict() + runner._session_capacity = 0 + runner._perf_e2e_times = [] + runner._preallocate_kv_cache(available_bytes=available_bytes) + return runner + + +def commit_one_frame(runner: ARDiffusionModelRunner, session_id: str, kv_branch: str): + state = runner._get_or_create_session(session_id) + ctx = state.get_kv_caches(kv_branch, seq_len=BLOCK, commit_current=True)[0].forward_ctx + ctx.ensure_video_slots(torch.device("cpu")) + state.commit_paged_context(kv_branch) + return state + + +def test_ar_runner_rejects_pipeline_without_capability(): + runner = object.__new__(ARDiffusionModelRunner) + runner.od_config = SimpleNamespace(max_num_seqs=1) + runner.pipeline = object() + with pytest.raises(TypeError, match="SupportsARDiffusionPipeline"): + runner._preallocate_kv_cache(available_bytes=1 << 20) + + +def test_lingbot_like_single_branch_session_reuse_reset_and_close(): + pipeline = CapablePipeline(lingbot_like_spec()) + runner = make_runner(pipeline) + kv = runner.kv_cache + assert kv is not None + assert kv.num_local_kv_branches == 1 + assert kv.frames_per_block == 3 + assert kv.spec.window_chunks == 5 + assert kv.spec.sink_chunks == 1 + assert kv.cross_attention_lengths == {"text": 8} + + first = commit_one_frame(runner, "s1", "main") + assert runner._get_or_create_session("s1") is first + k = torch.randn(1, 8, 4, 64) + v = torch.randn(1, 8, 4, 64) + first.populate_cross_attention("main", "text", [(k, v)] * first.num_layers) + assert first.get_cross_attention_kv("main", "text")[0]["k"].shape == k.shape + + runner.reset_session("s1") + assert "s1" not in runner._sessions + assert "s1" not in kv._cross_sessions + assert pipeline.resets == ["s1"] + second = runner._get_or_create_session("s1") + assert second is not first + + runner.close_session("s1") + assert "s1" not in runner._sessions + assert pipeline.closes == ["s1"] + + +def test_lingbot_like_sink_survives_sliding_window_eviction(): + runner = make_runner(CapablePipeline(lingbot_like_spec())) + kv = runner.kv_cache + assert kv is not None + + for _ in range(8): + state = commit_one_frame(runner, "s1", "main") + + table = kv.block_table(state.adapter("main")) + assert table[0] != kv.null_block_id + assert table[1] == kv.null_block_id + + ctx = state.get_kv_caches("main", seq_len=BLOCK, commit_current=False)[0].forward_ctx + visible, _ = ctx.video_block_table(torch.device("cpu")) + assert visible[0] == table[0] + assert len(visible) == 6 # sink + recent window, including current scratch + + +def test_dreamzero_like_two_branches_are_independent(): + runner = make_runner(CapablePipeline(dreamzero_like_spec())) + kv = runner.kv_cache + assert kv is not None and kv.num_local_kv_branches == 2 + state = commit_one_frame(runner, "s1", POS) + assert len(kv.window_block_ids(state.adapter(POS))) == 1 + assert kv.window_block_ids(state.adapter(NEG)) == [] + + +def test_lru_eviction_releases_blocks_and_notifies_pipeline(): + pipeline = CapablePipeline(lingbot_like_spec(capacity=8)) + runner = make_runner(pipeline) + kv = runner.kv_cache + assert kv is not None + assert runner._session_capacity == 1 + assert kv.session_capacity == 1 + free_total = kv.manager.block_pool.get_num_free_blocks() + old = commit_one_frame(runner, "old", "main") + k = torch.randn(1, 8, 4, 64) + old.populate_cross_attention("main", "text", [(k, k)] * old.num_layers) + assert kv.manager.block_pool.get_num_free_blocks() < free_total + + runner._get_or_create_session("new") + + assert tuple(runner._sessions) == ("new",) + assert pipeline.closes == ["old"] + assert "old" not in kv._cross_sessions + assert kv.manager.block_pool.get_num_free_blocks() == free_total + + +def test_forward_exception_releases_pending_allocation_and_model_state(monkeypatch): + pipeline = CapablePipeline(lingbot_like_spec()) + runner = make_runner(pipeline) + kv = runner.kv_cache + assert kv is not None + free_total = kv.manager.block_pool.get_num_free_blocks() + + def boom(self, req, kv_prefetch_job=None): + state = pipeline.bound_state + ctx = state.get_kv_caches("main", seq_len=BLOCK, commit_current=True)[0].forward_ctx + ctx.ensure_video_slots(torch.device("cpu")) + raise RuntimeError("layer exploded") + + monkeypatch.setattr(DiffusionModelRunner, "execute_model", boom) + request = SimpleNamespace(sampling_params=SimpleNamespace(extra_args={"session_id": "broken"})) + + with pytest.raises(RuntimeError, match="layer exploded"): + runner.execute_model(request) + + assert pipeline.bound_state is None + assert pipeline.closes == ["broken"] + assert not runner._sessions + assert not kv._adapters + assert kv.manager.block_pool.get_num_free_blocks() == free_total + + +def test_synchronize_exception_uses_forward_cleanup_path(monkeypatch): + pipeline = CapablePipeline(lingbot_like_spec()) + runner = make_runner(pipeline) + kv = runner.kv_cache + assert kv is not None + free_total = kv.manager.block_pool.get_num_free_blocks() + + def return_after_allocation(self, req, kv_prefetch_job=None): + state = pipeline.bound_state + ctx = state.get_kv_caches("main", seq_len=BLOCK, commit_current=True)[0].forward_ctx + ctx.ensure_video_slots(torch.device("cpu")) + return object() + + def synchronize_boom(device): + raise RuntimeError("asynchronous kernel failed") + + monkeypatch.setattr(DiffusionModelRunner, "execute_model", return_after_allocation) + monkeypatch.setattr(torch.accelerator, "synchronize", synchronize_boom) + runner.device = torch.device("cuda") + request = SimpleNamespace(sampling_params=SimpleNamespace(extra_args={"session_id": "broken"})) + + with pytest.raises(RuntimeError, match="asynchronous kernel failed"): + runner.execute_model(request) + + assert pipeline.bound_state is None + assert pipeline.closes == ["broken"] + assert not runner._sessions + assert not kv._adapters + assert not runner._perf_e2e_times + assert kv.manager.block_pool.get_num_free_blocks() == free_total + + +def test_ar_runner_rejects_step_and_request_batch_modes(): + with pytest.raises(ValueError, match="step_execution=True"): + make_runner(CapablePipeline(lingbot_like_spec()), step_execution=True) + with pytest.raises(ValueError, match="request-batch execution"): + make_runner(BatchCapablePipeline(lingbot_like_spec())) + + +def test_ar_runner_defensively_rejects_inherited_batch_and_step_entrypoints(): + runner = object.__new__(ARDiffusionModelRunner) + with pytest.raises(RuntimeError, match="request-batch execution"): + runner.execute_model_batch(None, None) + with pytest.raises(RuntimeError, match="step execution"): + runner.execute_stepwise(None) + + +def test_model_specific_warmup_provider_is_consumed(monkeypatch): + requests = [object(), object()] + pipeline = WarmupPipeline(lingbot_like_spec(), requests) + runner = make_runner(pipeline) + seen: list[object] = [] + monkeypatch.setattr(runner, "execute_model", seen.append) + + runner._warmup_ar_rollout() + + assert seen == requests + assert pipeline.closes == [runner._WARMUP_SID] + + +def test_pipeline_without_warmup_provider_is_safely_skipped(monkeypatch): + pipeline = CapablePipeline(lingbot_like_spec()) + runner = make_runner(pipeline) + execute = SimpleNamespace(called=False) + + def fail_if_called(request): + execute.called = True + + monkeypatch.setattr(runner, "execute_model", fail_if_called) + runner._warmup_ar_rollout() + assert execute.called is False diff --git a/tests/ar_diffusion/test_kv_cache.py b/tests/ar_diffusion/test_kv_cache.py index beb575a0b..9ca0e547d 100644 --- a/tests/ar_diffusion/test_kv_cache.py +++ b/tests/ar_diffusion/test_kv_cache.py @@ -12,6 +12,7 @@ from vllm.v1.kv_cache_interface import KVCacheSpecKind, get_kv_cache_spec_kind from vllm.v1.kv_cache_spec_registry import KVCacheSpecRegistry from vllm.v1.request import RequestStatus +from vllm_omni.experimental.ar_diffusion.capability import ARDiffusionKVBranchSpec from vllm_omni.experimental.ar_diffusion.kv_cache import ( ARDiffusionKVCache, ARDiffusionKVConfig, @@ -194,16 +195,29 @@ def test_cross_attn_pool_deducted_from_self_attn_budget(): block_size=BLOCK, max_model_len=4096, available_bytes=avail, - cross_attn_length=L, + kv_branches=(ARDiffusionKVBranchSpec("positive", 0), ARDiffusionKVBranchSpec("negative", 0)), + session_capacity=1, + cross_attention_lengths={"text": L}, device=torch.device("cpu"), ) cross_bytes = 2 * 2 * L * 4 * 64 * torch.float16.itemsize * 2 # K+V, pos+neg, layers expected = compute_num_blocks(avail - cross_bytes, 0.5, kv.spec.page_size_bytes * 2) assert kv.num_blocks == expected - assert expected > 2 * (2 + 1) + 2 # above the min-blocks floor, so the deduction is what's tested + assert expected > 1 * (2 + 1) + 2 # above the local-slot floor, so the cross deduction is what's tested -def _make_kv(*, local_branches, num_frame_per_block=2, window_chunks=9): +def _make_kv( + *, + local_branches, + num_frame_per_block=2, + window_chunks=9, + max_scratch_tokens_per_branch=0, +): + kv_branches = ( + (ARDiffusionKVBranchSpec("positive", 0), ARDiffusionKVBranchSpec("negative", 0)) + if local_branches == 1 + else (ARDiffusionKVBranchSpec("positive", 0), ARDiffusionKVBranchSpec("negative", 1)) + ) return ARDiffusionKVCache( ARDiffusionKVConfig(enable=True, chunk_size=BLOCK, window_chunks=window_chunks), num_layers=1, @@ -214,41 +228,75 @@ def _make_kv(*, local_branches, num_frame_per_block=2, window_chunks=9): max_model_len=4096, available_bytes=1 << 16, # tiny -> the floor binds device=torch.device("cpu"), - local_branches=local_branches, - num_frame_per_block=num_frame_per_block, + kv_branches=kv_branches, + session_capacity=1, + frames_per_block=num_frame_per_block, + max_scratch_tokens_per_branch=max_scratch_tokens_per_branch, ) def test_pool_floor_is_branch_aware(): - """CFG-parallel rank (one local branch) sizes for one window + in-flight chunk; - a single-process run (both branches) sizes for two. Scratch scales the same way.""" + """CFG-parallel rank (one local kv_branch) sizes for one window + in-flight chunk; + a single-process run (both kv_branches) sizes for two. Scratch scales the same way.""" one = _make_kv(local_branches=1) two = _make_kv(local_branches=2) assert one.managed_num_blocks == 1 * (9 + 2) + 2 # 13 assert two.managed_num_blocks == 2 * (9 + 2) + 2 # 24 - assert one.scratch_num_blocks == one.scratch_blocks_per_branch - assert two.scratch_num_blocks == 2 * two.scratch_blocks_per_branch - assert one.num_blocks_total == 13 + one.scratch_blocks_per_branch + assert one.scratch_num_blocks == one.scratch_blocks_per_kv_branch + assert two.scratch_num_blocks == 2 * two.scratch_blocks_per_kv_branch + assert one.scratch_blocks_per_kv_branch == 2 + assert one.num_blocks_total == 13 + one.scratch_blocks_per_kv_branch + + +def test_scratch_capacity_is_derived_from_declared_geometry(monkeypatch): + monkeypatch.delenv("AR_DIFFUSION_KV_SCRATCH_BLOCKS_PER_BRANCH", raising=False) + kv = _make_kv( + local_branches=1, + num_frame_per_block=6, + max_scratch_tokens_per_branch=BLOCK + 1, + ) + assert kv.scratch_blocks_per_kv_branch == 8 # six video blocks + two auxiliary blocks + + +def test_scratch_env_override_cannot_reduce_declared_minimum(monkeypatch): + monkeypatch.setenv("AR_DIFFUSION_KV_SCRATCH_BLOCKS_PER_BRANCH", "1") + kv = _make_kv( + local_branches=1, + num_frame_per_block=6, + max_scratch_tokens_per_branch=BLOCK + 1, + ) + assert kv.scratch_blocks_per_kv_branch == 8 def test_scratch_maps_to_slot_zero_with_one_local_branch(): - """A CFG-parallel rank runs exactly one branch: whichever CFG side it is, + """A CFG-parallel rank runs exactly one kv_branch: whichever CFG side it is, its scratch lands in the rank's single slot (no dead second slot).""" one = _make_kv(local_branches=1) - assert one.scratch_block_ids(True, 0, 2) == one.scratch_block_ids(False, 0, 2) + assert one.scratch_block_ids("positive", 0, 2) == one.scratch_block_ids("negative", 0, 2) two = _make_kv(local_branches=2) - assert two.scratch_block_ids(True, 0, 2) != two.scratch_block_ids(False, 0, 2) + assert two.scratch_block_ids("positive", 0, 2) != two.scratch_block_ids("negative", 0, 2) def test_scratch_exhaustion_still_raises(): one = _make_kv(local_branches=1) - cap = one.scratch_blocks_per_branch + cap = one.scratch_blocks_per_kv_branch with pytest.raises(RuntimeError, match="scratch blocks exhausted"): - one.scratch_block_ids(False, 0, cap + 1) + one.scratch_block_ids("positive", 0, cap + 1) -def test_invalid_local_branches_rejected(): - with pytest.raises(ValueError, match="local_branches"): - _make_kv(local_branches=3) +def test_non_contiguous_branch_indices_rejected(): + with pytest.raises(ValueError, match="contiguous"): + ARDiffusionKVCache( + ARDiffusionKVConfig(enable=True, chunk_size=BLOCK, window_chunks=2), + num_layers=1, + num_kv_heads=4, + head_size=64, + dtype=torch.float32, + block_size=BLOCK, + max_model_len=4096, + available_bytes=1 << 16, + kv_branches=(ARDiffusionKVBranchSpec("main", 1),), + session_capacity=1, + ) diff --git a/tests/ar_diffusion/test_kv_orchestration.py b/tests/ar_diffusion/test_kv_orchestration.py index 4ba5cb49b..b7c77390c 100644 --- a/tests/ar_diffusion/test_kv_orchestration.py +++ b/tests/ar_diffusion/test_kv_orchestration.py @@ -4,9 +4,18 @@ import pytest import torch +from vllm_omni.experimental.ar_diffusion.capability import ARDiffusionKVBranchSpec from vllm_omni.experimental.ar_diffusion.kv_cache import ARDiffusionKVCache, ARDiffusionKVConfig -DIMS = dict(num_layers=2, num_kv_heads=4, head_size=64, dtype=torch.float16, block_size=16) +DIMS = dict( + num_layers=2, + num_kv_heads=4, + head_size=64, + dtype=torch.float16, + block_size=16, + kv_branches=(ARDiffusionKVBranchSpec("positive", 0), ARDiffusionKVBranchSpec("negative", 1)), + session_capacity=2, +) def make_cache(*, chunk_size=16, window_chunks=2, available_bytes=1 << 24): @@ -67,7 +76,7 @@ def test_num_computed_advances_per_chunk(): def test_state_close_frees_both_branch_blocks(): - """ARDiffusionKVState.close() returns both CFG branches' pool blocks to the pool. + """ARDiffusionKVState.close() returns both CFG kv_branches' pool blocks to the pool. This is the primitive the runner's LRU eviction relies on: when a session is evicted, close() must free the blocks both adapters hold, or session churn @@ -80,12 +89,17 @@ def test_state_close_frees_both_branch_blocks(): pos = kv.begin_request("bde__s") neg = kv.begin_request("bde__s__neg") - state = ARDiffusionKVState(kv, pos, neg, num_layers=kv.num_layers) + state = ARDiffusionKVState( + kv, + "s", + {"positive": pos, "negative": neg}, + num_layers=kv.num_layers, + ) for _ in range(3): for adapter in (pos, neg): kv.allocate_chunk(adapter) kv.commit_chunk(adapter) - # Both branches hold resident blocks now. + # Both kv_branches hold resident blocks now. assert kv.manager.block_pool.get_num_free_blocks() < free_total state.close() diff --git a/tests/ar_diffusion/test_paged_attention.py b/tests/ar_diffusion/test_paged_attention.py index 1836656f9..b5c5e5870 100644 --- a/tests/ar_diffusion/test_paged_attention.py +++ b/tests/ar_diffusion/test_paged_attention.py @@ -11,6 +11,7 @@ from unittest.mock import MagicMock import pytest import torch +from vllm_omni.experimental.ar_diffusion.capability import ARDiffusionKVBranchSpec from vllm_omni.experimental.ar_diffusion.kv_cache import ( ARDiffusionKVCache, ARDiffusionKVConfig, @@ -24,6 +25,8 @@ from vllm_omni.experimental.ar_diffusion.kv_cache.state import ARDiffusionKVStat BLOCK = 16 N_HEADS = 4 HEAD_DIM = 64 +POS = "positive" +NEG = "negative" def make_state(*, num_layers=1, window_chunks=2, dtype=torch.float32, device=torch.device("cpu")): @@ -37,11 +40,15 @@ def make_state(*, num_layers=1, window_chunks=2, dtype=torch.float32, device=tor block_size=BLOCK, max_model_len=4096, available_bytes=1 << 26, + kv_branches=(ARDiffusionKVBranchSpec(POS, 0), ARDiffusionKVBranchSpec(NEG, 1)), + session_capacity=2, + frames_per_block=2, + max_scratch_tokens_per_branch=BLOCK, device=device, ) pos = kv.begin_request("r-pos") neg = kv.begin_request("r-neg") - return kv, ARDiffusionKVState(kv, pos, neg, num_layers=num_layers) + return kv, ARDiffusionKVState(kv, "s1", {POS: pos, NEG: neg}, num_layers=num_layers) def _dense_attention(query: torch.Tensor, key: torch.Tensor, value: torch.Tensor) -> torch.Tensor: @@ -79,51 +86,51 @@ def _commit_video_span( kv: ARDiffusionKVCache, st: ARDiffusionKVState, *, - is_negative: bool, + kv_branch: str, n_chunks: int, dtype: torch.dtype, device: torch.device, ) -> tuple[torch.Tensor, torch.Tensor]: - ctx = st.get_kv_caches(is_negative, seq_len=n_chunks * BLOCK, commit_current=True)[0].forward_ctx + ctx = st.get_kv_caches(kv_branch, seq_len=n_chunks * BLOCK, commit_current=True)[0].forward_ctx ctx.ensure_video_slots(device) k = torch.randn(1, n_chunks * BLOCK, N_HEADS, HEAD_DIM, dtype=dtype, device=device) v = torch.randn(1, n_chunks * BLOCK, N_HEADS, HEAD_DIM, dtype=dtype, device=device) kv._k_pools[0][ctx.current_video_slot_mapping] = k[0] kv._v_pools[0][ctx.current_video_slot_mapping] = v[0] - st.commit_paged_context(is_negative) + st.commit_paged_context(kv_branch) return k, v def test_paged_context_allocates_lazily_and_commits_after_forward(): _, st = make_state() - contexts = st.get_kv_caches(False, seq_len=BLOCK, commit_current=True) + contexts = st.get_kv_caches(POS, seq_len=BLOCK, commit_current=True) ctx = contexts[0].forward_ctx assert isinstance(contexts[0], ARDiffusionPagedLayerContext) - assert st.pos.completed_chunks == 0 + assert st.adapter(POS).completed_chunks == 0 assert ctx.current_video_slot_mapping is None ctx.ensure_video_slots(torch.device("cpu")) - assert st.pos.completed_chunks == 0 + assert st.adapter(POS).completed_chunks == 0 assert len(ctx.current_video_block_ids) == 1 - st.commit_paged_context(False) - assert st.pos.completed_chunks == 1 - assert st._committed[False] == BLOCK + st.commit_paged_context(POS) + assert st.adapter(POS).completed_chunks == 1 + assert st._committed[POS] == BLOCK def test_scratch_video_and_action_blocks_do_not_commit(): kv, st = make_state() - ctx = st.get_kv_caches(False, seq_len=2 * BLOCK, commit_current=False)[0].forward_ctx + ctx = st.get_kv_caches(POS, seq_len=2 * BLOCK, commit_current=False)[0].forward_ctx ctx.ensure_video_slots(torch.device("cpu")) ctx.ensure_action_slots(3, torch.device("cpu")) - assert ctx.current_video_block_ids == kv.scratch_block_ids(False, 0, 2) - assert ctx.action_scratch_block_ids == kv.scratch_block_ids(False, 2, 1) - st.commit_paged_context(False) - assert st.pos.completed_chunks == 0 - assert st._committed[False] == 0 + assert ctx.current_video_block_ids == kv.scratch_block_ids(POS, 0, 2) + assert ctx.action_scratch_block_ids == kv.scratch_block_ids(POS, 2, 1) + st.commit_paged_context(POS) + assert st.adapter(POS).completed_chunks == 0 + assert st._committed[POS] == 0 def test_pipeline_kv_get_paged_path_has_no_gather_backend(): @@ -155,7 +162,7 @@ def test_paged_attention_matches_dense_reference_cpu(history_chunks, action_len, k, v = _commit_video_span( kv, st, - is_negative=False, + kv_branch=POS, n_chunks=history_chunks, dtype=dtype, device=device, @@ -163,7 +170,7 @@ def test_paged_attention_matches_dense_reference_cpu(history_chunks, action_len, history_k_parts.append(k) history_v_parts.append(v) - ctx = st.get_kv_caches(False, seq_len=BLOCK, commit_current=commit_current)[0].forward_ctx + ctx = st.get_kv_caches(POS, seq_len=BLOCK, commit_current=commit_current)[0].forward_ctx ctx.ensure_video_slots(device) current_k = torch.randn(1, BLOCK, N_HEADS, HEAD_DIM, dtype=dtype, device=device) current_v = torch.randn(1, BLOCK, N_HEADS, HEAD_DIM, dtype=dtype, device=device) @@ -212,9 +219,9 @@ def test_paged_attention_matches_dense_reference_cpu(history_chunks, action_len, torch.testing.assert_close(paged, ref, rtol=1e-5, atol=1e-5) - before = st.pos.completed_chunks - st.commit_paged_context(False) - assert st.pos.completed_chunks == before + (1 if commit_current else 0) + before = st.adapter(POS).completed_chunks + st.commit_paged_context(POS) + assert st.adapter(POS).completed_chunks == before + (1 if commit_current else 0) @pytest.mark.skipif(not _cuda_flash_attn_usable(), reason="usable CUDA FlashAttention is required") @@ -231,13 +238,13 @@ def test_paged_attention_matches_dense_reference_gpu(history_chunks, action_len, history_k, history_v = _commit_video_span( kv, st, - is_negative=False, + kv_branch=POS, n_chunks=history_chunks, dtype=dtype, device=device, ) - layer_ctx = st.get_kv_caches(False, seq_len=BLOCK, commit_current=commit_current)[0] + layer_ctx = st.get_kv_caches(POS, seq_len=BLOCK, commit_current=commit_current)[0] ctx = layer_ctx.forward_ctx current_k = torch.randn(1, BLOCK, N_HEADS, HEAD_DIM, dtype=dtype, device=device) current_v = torch.randn(1, BLOCK, N_HEADS, HEAD_DIM, dtype=dtype, device=device) @@ -294,13 +301,13 @@ def test_block_table_padded_to_fixed_width(): device = torch.device("cpu") kv, st = make_state(window_chunks=2) - ctx1 = st.get_kv_caches(False, seq_len=BLOCK, commit_current=True)[0].forward_ctx + ctx1 = st.get_kv_caches(POS, seq_len=BLOCK, commit_current=True)[0].forward_ctx ctx1.prepare(device=device, action_len=0, query_len=BLOCK) - st.commit_paged_context(False) + st.commit_paged_context(POS) - ctx2 = st.get_kv_caches(False, seq_len=BLOCK, commit_current=True)[0].forward_ctx + ctx2 = st.get_kv_caches(POS, seq_len=BLOCK, commit_current=True)[0].forward_ctx ctx2.prepare(device=device, action_len=0, query_len=BLOCK) - st.commit_paged_context(False) + st.commit_paged_context(POS) # 1-block vs 2-block visible history: same table width, same max_seq_len. assert ctx1.block_table.shape == ctx2.block_table.shape @@ -315,7 +322,7 @@ def test_block_table_padded_to_fixed_width(): def test_prepare_is_idempotent_and_layers_share_metadata(): device = torch.device("cpu") _, st = make_state(num_layers=2) - contexts = st.get_kv_caches(False, seq_len=BLOCK, commit_current=False) + contexts = st.get_kv_caches(POS, seq_len=BLOCK, commit_current=False) fctx = contexts[0].forward_ctx fctx.prepare(device=device, action_len=0, query_len=BLOCK) table = fctx.block_table @@ -335,7 +342,7 @@ def test_prepare_is_idempotent_and_layers_share_metadata(): def test_layer_inputs_before_prepare_raises(): _, st = make_state() - layer_ctx = st.get_kv_caches(False, seq_len=BLOCK, commit_current=False)[0] + layer_ctx = st.get_kv_caches(POS, seq_len=BLOCK, commit_current=False)[0] with pytest.raises(RuntimeError, match="before prepare"): layer_ctx.to_layer_inputs() @@ -363,7 +370,7 @@ def test_custom_op_compiles_fullgraph_without_recompile_on_value_change(): kv, st = make_state(num_layers=2, window_chunks=2) def run_one_forward(commit): - contexts = st.get_kv_caches(False, seq_len=BLOCK, commit_current=commit) + contexts = st.get_kv_caches(POS, seq_len=BLOCK, commit_current=commit) fctx = contexts[0].forward_ctx fctx.prepare(device=device, action_len=0, query_len=BLOCK) q = torch.randn(BLOCK, N_HEADS, HEAD_DIM) @@ -374,7 +381,7 @@ def test_custom_op_compiles_fullgraph_without_recompile_on_value_change(): # block-forward code object in production). for layer_ctx in contexts: out = compiled(layer_ctx.to_layer_inputs(), q, k, v) - st.commit_paged_context(False) + st.commit_paged_context(POS) return out torch._dynamo.reset() diff --git a/tests/ar_diffusion/test_pipeline_bridge.py b/tests/ar_diffusion/test_pipeline_bridge.py index 2c4284531..92d490b92 100644 --- a/tests/ar_diffusion/test_pipeline_bridge.py +++ b/tests/ar_diffusion/test_pipeline_bridge.py @@ -4,6 +4,7 @@ import pytest import torch +from vllm_omni.experimental.ar_diffusion.capability import ARDiffusionKVBranchSpec from vllm_omni.experimental.ar_diffusion.kv_cache import ( ARDiffusionKVCache, ARDiffusionKVConfig, @@ -14,9 +15,11 @@ from vllm_omni.experimental.ar_diffusion.kv_cache.state import ARDiffusionKVStat BLOCK = 16 N_HEADS = 4 HEAD_DIM = 64 +POS = "positive" +NEG = "negative" -def make_state(num_layers=1, window_chunks=4, cross_attn_length=0): +def make_state(num_layers=1, window_chunks=4, cross_attn_length=0, shared_local_index=False): cfg = ARDiffusionKVConfig(enable=True, chunk_size=BLOCK, window_chunks=window_chunks) kv = ARDiffusionKVCache( cfg, @@ -27,24 +30,29 @@ def make_state(num_layers=1, window_chunks=4, cross_attn_length=0): block_size=BLOCK, max_model_len=4096, available_bytes=1 << 24, - cross_attn_length=cross_attn_length, + kv_branches=( + ARDiffusionKVBranchSpec(POS, 0), + ARDiffusionKVBranchSpec(NEG, 0 if shared_local_index else 1), + ), + session_capacity=2, + cross_attention_lengths={"text": cross_attn_length} if cross_attn_length else None, device=torch.device("cpu"), ) pos = kv.begin_request("r-pos") neg = kv.begin_request("r-neg") - return kv, ARDiffusionKVState(kv, pos, neg, num_layers=num_layers) + return kv, ARDiffusionKVState(kv, "s1", {POS: pos, NEG: neg}, num_layers=num_layers) -def _prepare_and_commit(st: ARDiffusionKVState, is_negative: bool, n_chunks: int) -> None: - ctx = st.get_kv_caches(is_negative, seq_len=n_chunks * BLOCK, commit_current=True)[0].forward_ctx +def _prepare_and_commit(st: ARDiffusionKVState, kv_branch: str, n_chunks: int) -> None: + ctx = st.get_kv_caches(kv_branch, seq_len=n_chunks * BLOCK, commit_current=True)[0].forward_ctx ctx.ensure_video_slots(torch.device("cpu")) - st.commit_paged_context(is_negative) + st.commit_paged_context(kv_branch) def test_get_returns_paged_layer_contexts_when_nothing_committed(): _, st = make_state(num_layers=2) - for neg in (False, True): - contexts = st.get_kv_caches(neg, seq_len=BLOCK, commit_current=False) + for kv_branch in (POS, NEG): + contexts = st.get_kv_caches(kv_branch, seq_len=BLOCK, commit_current=False) assert len(contexts) == 2 assert all(isinstance(ctx, ARDiffusionPagedLayerContext) for ctx in contexts) assert contexts[0].history_block_ids == [] @@ -52,43 +60,43 @@ def test_get_returns_paged_layer_contexts_when_nothing_committed(): def test_managed_paged_context_commits_only_after_forward(): kv, st = make_state(num_layers=1) - contexts = st.get_kv_caches(False, seq_len=2 * BLOCK, commit_current=True) + contexts = st.get_kv_caches(POS, seq_len=2 * BLOCK, commit_current=True) ctx = contexts[0].forward_ctx - assert st.pos.completed_chunks == 0 - assert kv.window_block_ids(st.pos) == [] + assert st.adapter(POS).completed_chunks == 0 + assert kv.window_block_ids(st.adapter(POS)) == [] ctx.ensure_video_slots(torch.device("cpu")) - assert st.pos.completed_chunks == 0 + assert st.adapter(POS).completed_chunks == 0 assert len(ctx.current_video_block_ids) == 2 - st.commit_paged_context(False) + st.commit_paged_context(POS) - assert st.pos.completed_chunks == 2 - assert st._committed[False] == 2 * BLOCK - assert len(kv.window_block_ids(st.pos)) == 2 + assert st.adapter(POS).completed_chunks == 2 + assert st._committed[POS] == 2 * BLOCK + assert len(kv.window_block_ids(st.adapter(POS))) == 2 def test_scratch_paged_context_does_not_commit(): kv, st = make_state(num_layers=1) - contexts = st.get_kv_caches(False, seq_len=BLOCK, commit_current=False) + contexts = st.get_kv_caches(POS, seq_len=BLOCK, commit_current=False) ctx = contexts[0].forward_ctx ctx.ensure_video_slots(torch.device("cpu")) - assert ctx.current_video_block_ids == kv.scratch_block_ids(False, 0, 1) - st.commit_paged_context(False) + assert ctx.current_video_block_ids == kv.scratch_block_ids(POS, 0, 1) + st.commit_paged_context(POS) - assert st.pos.completed_chunks == 0 - assert st._committed[False] == 0 - assert kv.window_block_ids(st.pos) == [] + assert st.adapter(POS).completed_chunks == 0 + assert st._committed[POS] == 0 + assert kv.window_block_ids(st.adapter(POS)) == [] def test_eviction_bounds_resident_window(): kv, st = make_state(num_layers=1, window_chunks=3) - _prepare_and_commit(st, False, 3) - assert len(kv.window_block_ids(st.pos)) == 3 + _prepare_and_commit(st, POS, 3) + assert len(kv.window_block_ids(st.adapter(POS))) == 3 - _prepare_and_commit(st, False, 2) - ctx = st.get_kv_caches(False, seq_len=BLOCK, commit_current=False)[0].forward_ctx + _prepare_and_commit(st, POS, 2) + ctx = st.get_kv_caches(POS, seq_len=BLOCK, commit_current=False)[0].forward_ctx visible_blocks, video_len = ctx.video_block_table(torch.device("cpu")) assert len(visible_blocks) == 3 assert video_len == 3 * BLOCK @@ -96,53 +104,47 @@ def test_eviction_bounds_resident_window(): def test_branches_are_independent(): kv, st = make_state() - _prepare_and_commit(st, False, 1) - assert len(kv.window_block_ids(st.pos)) == 1 - assert kv.window_block_ids(st.neg) == [] + _prepare_and_commit(st, POS, 1) + assert len(kv.window_block_ids(st.adapter(POS))) == 1 + assert kv.window_block_ids(st.adapter(NEG)) == [] def test_reset_clears_session_window(): kv, st = make_state(num_layers=1, window_chunks=4) - _prepare_and_commit(st, False, 2) - _prepare_and_commit(st, True, 1) - assert st._committed[False] == 2 * BLOCK and st._committed[True] == BLOCK + _prepare_and_commit(st, POS, 2) + _prepare_and_commit(st, NEG, 1) + assert st._committed[POS] == 2 * BLOCK and st._committed[NEG] == BLOCK free_before = kv.manager.block_pool.get_num_free_blocks() st.reset() - assert st._committed == {False: 0, True: 0} - assert st._paged_pending == {False: None, True: None} - assert kv.window_block_ids(st.pos) == [] - assert kv.window_block_ids(st.neg) == [] + assert st._committed == {POS: 0, NEG: 0} + assert st._paged_pending == {POS: None, NEG: None} + assert kv.window_block_ids(st.adapter(POS)) == [] + assert kv.window_block_ids(st.adapter(NEG)) == [] assert kv.manager.block_pool.get_num_free_blocks() > free_before - _prepare_and_commit(st, False, 1) - assert len(kv.window_block_ids(st.pos)) == 1 + _prepare_and_commit(st, POS, 1) + assert len(kv.window_block_ids(st.adapter(POS))) == 1 -def test_get_cross_kv_caches_raises_before_populate(): - """Engine ownership guard: a cross-attn read before _kv_populate_cross must fail loud.""" - _, st = make_state() # cross_attn_length == 0, nothing populated - for neg in (False, True): - with pytest.raises(RuntimeError, match="cross-attn read before"): - st.get_cross_kv_caches(neg) - # Guards the full AND: the populated flag alone is not enough without a pool. - st._cross_text_populated[False] = True - with pytest.raises(RuntimeError, match="cross-attn read before"): - st.get_cross_kv_caches(False) +def test_get_cross_attention_kv_raises_before_populate(): + _, st = make_state(cross_attn_length=8) + for kv_branch in (POS, NEG): + with pytest.raises(RuntimeError, match="before it was populated"): + st.get_cross_attention_kv(kv_branch, "text") -def test_get_cross_kv_caches_returns_pool_dicts_when_populated(): +def test_get_cross_attention_kv_returns_session_pool_dicts_when_populated(): L = 8 kv, st = make_state(num_layers=2, cross_attn_length=L) written = [] - for i in range(2): + for _ in range(2): k = torch.randn(1, L, N_HEADS, HEAD_DIM) v = torch.randn(1, L, N_HEADS, HEAD_DIM) - kv.write_cross_kv(i, False, k, v) written.append((k, v)) - st._cross_text_populated[False] = True + st.populate_cross_attention(POS, "text", written) - out = st.get_cross_kv_caches(False) + out = st.get_cross_attention_kv(POS, "text") assert len(out) == 2 for i, (k, v) in enumerate(written): assert out[i]["is_init"] is True @@ -151,6 +153,99 @@ def test_get_cross_kv_caches_returns_pool_dicts_when_populated(): assert torch.equal(out[i]["v"], v) +def test_window_reset_can_keep_named_cross_attention_cache(): + kv, st = make_state(num_layers=1, cross_attn_length=8) + k = torch.randn(1, 8, N_HEADS, HEAD_DIM) + v = torch.randn(1, 8, N_HEADS, HEAD_DIM) + st.populate_cross_attention(POS, "text", [(k, v)]) + old_adapter = st.adapter(POS) + + st.reset(keep_cross_attention=("text",)) + + assert st.adapter(POS) is not old_adapter + assert torch.equal(st.get_cross_attention_kv(POS, "text")[0]["k"], k) + assert "s1" in kv._cross_sessions + + st.reset() + assert "s1" not in kv._cross_sessions + assert not st.is_cross_attention_populated(POS, "text") + + +def test_cross_attention_uses_logical_branch_when_local_slot_is_shared(): + _, st = make_state(num_layers=2, cross_attn_length=8, shared_local_index=True) + assert st.kv_cache.num_local_kv_branches == 1 + pos_layers = [ + (torch.full((1, 8, N_HEADS, HEAD_DIM), 1.0), torch.full((1, 8, N_HEADS, HEAD_DIM), 2.0)) for _ in range(2) + ] + neg_layers = [ + (torch.full((1, 8, N_HEADS, HEAD_DIM), 3.0), torch.full((1, 8, N_HEADS, HEAD_DIM), 4.0)) for _ in range(2) + ] + + st.populate_cross_attention(POS, "text", pos_layers) + st.populate_cross_attention(NEG, "text", neg_layers) + + for layer in st.get_cross_attention_kv(POS, "text"): + assert torch.count_nonzero(layer["k"] != 1.0) == 0 + assert torch.count_nonzero(layer["v"] != 2.0) == 0 + for layer in st.get_cross_attention_kv(NEG, "text"): + assert torch.count_nonzero(layer["k"] != 3.0) == 0 + assert torch.count_nonzero(layer["v"] != 4.0) == 0 + + +def test_cross_attention_population_is_transactional(): + kv, st = make_state(num_layers=2, cross_attn_length=8) + original = [ + (torch.full((1, 8, N_HEADS, HEAD_DIM), value), torch.full((1, 8, N_HEADS, HEAD_DIM), -value)) + for value in (1.0, 2.0) + ] + replacement = ( + torch.full((1, 8, N_HEADS, HEAD_DIM), 9.0), + torch.full((1, 8, N_HEADS, HEAD_DIM), -9.0), + ) + + def interrupted_population(): + yield replacement + raise RuntimeError("projection failed") + + with pytest.raises(RuntimeError, match="projection failed"): + st.populate_cross_attention(POS, "text", interrupted_population()) + assert not st.is_cross_attention_populated(POS, "text") + assert "s1" not in kv._cross_sessions + + st.populate_cross_attention(POS, "text", original) + + with pytest.raises(RuntimeError, match="projection failed"): + st.populate_cross_attention(POS, "text", interrupted_population()) + out = st.get_cross_attention_kv(POS, "text") + for layer_index, layer in enumerate(out): + assert torch.equal(layer["k"], original[layer_index][0]) + assert torch.equal(layer["v"], original[layer_index][1]) + + for invalid_layers in (original[:1], [*original, replacement]): + with pytest.raises(ValueError, match="expected 2 layers"): + st.populate_cross_attention(POS, "text", invalid_layers) + out = st.get_cross_attention_kv(POS, "text") + for layer_index, layer in enumerate(out): + assert torch.equal(layer["k"], original[layer_index][0]) + assert torch.equal(layer["v"], original[layer_index][1]) + + invalid_shape = (torch.empty(1, 7, N_HEADS, HEAD_DIM), original[0][1]) + with pytest.raises(ValueError, match="expected k/v shape"): + st.populate_cross_attention(POS, "text", [invalid_shape, original[1]]) + out = st.get_cross_attention_kv(POS, "text") + assert torch.equal(out[0]["k"], original[0][0]) + + +def test_closed_state_cannot_repopulate_cross_attention(): + kv, st = make_state(num_layers=1, cross_attn_length=8) + layer = (torch.ones(1, 8, N_HEADS, HEAD_DIM), torch.ones(1, 8, N_HEADS, HEAD_DIM)) + st.close() + + with pytest.raises(RuntimeError, match="is closed"): + st.populate_cross_attention(POS, "text", [layer]) + assert "s1" not in kv._cross_sessions + + def test_kv_create_is_noop_engine_owns_allocation(): from unittest.mock import MagicMock @@ -166,57 +261,17 @@ def test_kv_create_is_noop_engine_owns_allocation(): def test_get_kv_cache_requires_frame_aligned_seqlen(): _, st = make_state() with pytest.raises(AssertionError, match="frame-aligned"): - st.get_kv_caches(False, seq_len=BLOCK + 1, commit_current=True) + st.get_kv_caches(POS, seq_len=BLOCK + 1, commit_current=True) def test_prepare_one_branch_allocates_nothing_for_the_other(): - """CFG-parallel laziness: a rank prepares only the branch it runs.""" + """CFG-parallel laziness: a rank prepares only the kv_branch it runs.""" kv, st = make_state(num_layers=1, window_chunks=4) - pos_ctx = st.get_kv_caches(False, seq_len=BLOCK, commit_current=True)[0].forward_ctx - neg_ctx = st.get_kv_caches(True, seq_len=BLOCK, commit_current=True)[0].forward_ctx + pos_ctx = st.get_kv_caches(POS, seq_len=BLOCK, commit_current=True)[0].forward_ctx + neg_ctx = st.get_kv_caches(NEG, seq_len=BLOCK, commit_current=True)[0].forward_ctx pos_ctx.prepare(device=torch.device("cpu"), action_len=0, query_len=BLOCK) assert pos_ctx._allocated_video assert not neg_ctx._allocated_video - assert kv.window_block_ids(st.neg) == [] - - -def test_failed_forward_tears_down_session(monkeypatch): - """Review follow-up (hsliuustc0106): a forward that dies partway must not - leave allocated-but-uncommitted KV behind — the session is torn down (pool - blocks freed, model-local state dropped) and the error propagates.""" - from types import SimpleNamespace - - from vllm_omni.diffusion.worker.diffusion_model_runner import DiffusionModelRunner - from vllm_omni.experimental.ar_diffusion.runner import ARDiffusionModelRunner - - kv, st = make_state(num_layers=1, window_chunks=4) - _prepare_and_commit(st, False, 2) # session owns pool blocks - free_before_failure = kv.manager.block_pool.get_num_free_blocks() - - runner = object.__new__(ARDiffusionModelRunner) - runner.kv_cache = kv - runner._ar_diffusion_states = __import__("collections").OrderedDict({"s1": st}) - runner._max_ar_diffusion_states = 4 - runner.pipeline = SimpleNamespace(_states={"s1": object()}, _ar_diffusion_kv_state=None) - runner.device = None - runner._perf_e2e_times = [] - - def boom(self, req, kv_prefetch_job=None): - raise RuntimeError("layer 17 exploded") - - monkeypatch.setattr(DiffusionModelRunner, "execute_model", boom) - req = SimpleNamespace( - request_id="r0", - sampling_params=SimpleNamespace(extra_args={"session_id": "s1"}), - ) - - with pytest.raises(RuntimeError, match="layer 17 exploded"): - ARDiffusionModelRunner.execute_model(runner, req) - - assert "s1" not in runner._ar_diffusion_states - assert "s1" not in runner.pipeline._states - assert runner.pipeline._ar_diffusion_kv_state is None - # close() freed the session's resident blocks. - assert kv.manager.block_pool.get_num_free_blocks() > free_before_failure + assert kv.window_block_ids(st.adapter(NEG)) == [] diff --git a/tests/dreamzero/test_pipeline_state.py b/tests/dreamzero/test_pipeline_state.py index f00caf53c..366876ef7 100644 --- a/tests/dreamzero/test_pipeline_state.py +++ b/tests/dreamzero/test_pipeline_state.py @@ -2,6 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from collections import OrderedDict +from types import SimpleNamespace import pytest @@ -14,7 +15,6 @@ pytestmark = [pytest.mark.core_model, pytest.mark.cpu] def _empty_pipeline() -> DreamZeroPipeline: pipeline = DreamZeroPipeline.__new__(DreamZeroPipeline) pipeline._states = OrderedDict() - pipeline._max_session_states = 2 return pipeline @@ -32,17 +32,59 @@ def test_dreamzero_pipeline_state_is_session_keyed() -> None: assert session_b.call_count == 3 -def test_dreamzero_pipeline_state_lru_caps_retained_sessions() -> None: +def test_dreamzero_pipeline_state_follows_runner_lifecycle_notifications() -> None: pipeline = _empty_pipeline() session_a = pipeline._get_or_create_state("session-a") - pipeline._get_or_create_state("session-b") - assert pipeline._get_or_create_state("session-a") is session_a + session_b = pipeline._get_or_create_state("session-b") + pipeline.state = session_b - pipeline._get_or_create_state("session-c") + pipeline.close_ar_diffusion_session("session-a") + assert pipeline.state is session_b + pipeline.reset_ar_diffusion_session("session-b") - assert list(pipeline._states) == ["session-a", "session-c"] - assert "session-b" not in pipeline._states + assert not pipeline._states + assert pipeline.state is None + assert pipeline._get_or_create_state("session-a") is not session_a + assert pipeline._get_or_create_state("session-b") is not session_b + + +def test_close_session_clears_active_state_alias() -> None: + pipeline = _empty_pipeline() + session_a = pipeline._get_or_create_state("session-a") + pipeline.state = session_a + + pipeline.close_ar_diffusion_session("session-a") + + assert not pipeline._states + assert pipeline.state is None + + +def test_dreamzero_warmup_provider_builds_session_scoped_requests() -> None: + provider = SimpleNamespace( + ar_diffusion_kv_cache_spec=lambda: SimpleNamespace(window_frames=5, frames_per_block=4), + od_config=SimpleNamespace( + ar_diffusion_kv_config=None, + model_config={"policy_server_config": {"image_resolution": [8, 16]}}, + ), + _ar_warmup_robot_obs=DreamZeroPipeline._ar_warmup_robot_obs, + ) + + requests = list(DreamZeroPipeline.ar_diffusion_warmup_requests(provider, "warmup-session")) + + assert [request.prompt for request in requests] == ["warmup", "warmup"] + assert all(request.sampling_params.extra_args["session_id"] == "warmup-session" for request in requests) + assert requests[0].sampling_params.extra_args["robot_obs"]["observation/exterior_image_0_left"].shape == ( + 8, + 16, + 3, + ) + assert requests[1].sampling_params.extra_args["robot_obs"]["observation/exterior_image_0_left"].shape == ( + 4, + 8, + 16, + 3, + ) def test_dreamzero_state_owns_no_kv_caches() -> None: diff --git a/vllm_omni/diffusion/models/dreamzero/pipeline_dreamzero.py b/vllm_omni/diffusion/models/dreamzero/pipeline_dreamzero.py index d43693b0d..711b345d2 100644 --- a/vllm_omni/diffusion/models/dreamzero/pipeline_dreamzero.py +++ b/vllm_omni/diffusion/models/dreamzero/pipeline_dreamzero.py @@ -11,10 +11,12 @@ from __future__ import annotations import copy import json import logging +import math import os import re as re_module from collections import OrderedDict from collections.abc import Iterable +from contextlib import contextmanager import numpy as np import torch @@ -52,7 +54,14 @@ from vllm_omni.diffusion.models.dreamzero.utils import ( DEFAULT_SIGMA_SHIFT, ) from vllm_omni.diffusion.models.schedulers.scheduling_flow_unipc_multistep import FlowUniPCMultistepScheduler +from vllm_omni.diffusion.request import OmniDiffusionRequest from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch +from vllm_omni.experimental.ar_diffusion.capability import ( + ARDiffusionCrossAttentionKVSpec, + ARDiffusionKVBranchSpec, + ARDiffusionKVCacheSpec, +) +from vllm_omni.inputs.data import OmniDiffusionSamplingParams logger = logging.getLogger(__name__) MAX_DREAMZERO_SESSIONS = 64 @@ -92,18 +101,128 @@ class DreamZeroPipeline(nn.Module, CFGParallelMixin): """DreamZero world model pipeline. Multi-output: predict_noise() returns (video_pred, action_pred). - CFG: video gets standard CFG, action takes positive branch only. + CFG: video gets standard CFG, action takes the positive KV branch only. - KV is managed by the AR-Diffusion engine: ``self._ar_diffusion_kv_state`` is set by the - runner before ``forward()`` and the pipeline routes all KV access (get / update / - commit / reset) through the pool-backed state. Purely duck-typed (no engine import). + KV is managed by the AR-Diffusion engine through the explicit capability + methods below. The runner binds one session state only for ``forward()``. """ - _ar_diffusion_kv_state = None # set by the runner before each forward + _POSITIVE_BRANCH = "positive" + _NEGATIVE_BRANCH = "negative" + _ar_diffusion_kv_state = None + state: DreamZeroState | None + + def ar_diffusion_kv_cache_spec(self) -> ARDiffusionKVCacheSpec: + """Describe DreamZero's local KV geometry to the generic runner.""" + transformer = self.transformer + frame_tokens = int(transformer.frame_seqlen) + max_attention_tokens = int(transformer.blocks[0].self_attn.max_attention_size) + cfg_world = int(get_classifier_free_guidance_world_size()) + negative_local_index = 0 if cfg_world >= 2 else 1 + cross_attention = [ + ARDiffusionCrossAttentionKVSpec("text", int(transformer.text_len)), + ] + if transformer.model_type == "i2v": + cross_attention.append(ARDiffusionCrossAttentionKVSpec("image", 257)) + return ARDiffusionKVCacheSpec( + num_layers=int(transformer.num_layers), + num_kv_heads=int(transformer.blocks[0].self_attn.tp_num_heads), + head_size=int(transformer.dim // transformer.num_heads), + tokens_per_frame=frame_tokens, + frames_per_block=int(transformer.num_frame_per_block), + window_frames=max_attention_tokens // frame_tokens, + kv_branches=( + ARDiffusionKVBranchSpec(self._POSITIVE_BRANCH, 0), + ARDiffusionKVBranchSpec(self._NEGATIVE_BRANCH, negative_local_index), + ), + session_capacity=MAX_DREAMZERO_SESSIONS, + cross_attention=tuple(cross_attention), + max_scratch_tokens_per_branch=int(transformer.num_action_per_block + transformer.num_state_per_block), + ) + + @contextmanager + def bind_ar_diffusion_state(self, session_id, state): + """Bind runner-owned KV only for the duration of one forward.""" + if self._ar_diffusion_kv_state is not None: + raise RuntimeError("DreamZero AR-Diffusion state is already bound") + if state.session_id != session_id: + raise ValueError(f"DreamZero bound session mismatch: {state.session_id!r} != {session_id!r}") + self._ar_diffusion_kv_state = state + try: + yield + finally: + self._ar_diffusion_kv_state = None + + def reset_ar_diffusion_session(self, session_id: str) -> None: + """Reset DreamZero-owned state after the runner releases session KV.""" + self._drop_ar_diffusion_session_state(session_id) + + def close_ar_diffusion_session(self, session_id: str) -> None: + """Drop DreamZero-owned state after close, eviction, or failed forward.""" + self._drop_ar_diffusion_session_state(session_id) + + def _drop_ar_diffusion_session_state(self, session_id: str) -> None: + """Remove model state and clear the compatibility alias when it points there.""" + removed = self._states.pop(str(session_id or "default"), None) + if removed is not None and getattr(self, "state", None) is removed: + self.state = None + + @staticmethod + def _ar_warmup_robot_obs(height: int, width: int, n_frames: int, session_id: str) -> dict: + image = ( + np.zeros((height, width, 3), dtype=np.uint8) + if n_frames == 1 + else np.zeros((n_frames, height, width, 3), dtype=np.uint8) + ) + return { + "observation/exterior_image_0_left": image, + "observation/exterior_image_1_left": image, + "observation/wrist_image_left": image, + "observation/joint_position": np.zeros(7, dtype=np.float32), + "observation/cartesian_position": np.zeros(6, dtype=np.float32), + "observation/gripper_position": np.zeros(1, dtype=np.float32), + "prompt": "warmup", + "session_id": session_id, + } + + def ar_diffusion_warmup_requests(self, session_id: str) -> Iterable[OmniDiffusionRequest]: + """Yield DreamZero-valid requests covering each resident window shape.""" + spec = self.ar_diffusion_kv_cache_spec() + n_forwards = 1 + math.ceil(max(0, spec.window_frames - 1) / spec.frames_per_block) + raw_kv_config = getattr(self.od_config, "ar_diffusion_kv_config", None) + if raw_kv_config is None and isinstance(self.od_config.model_config, dict): + raw_kv_config = self.od_config.model_config.get("ar_diffusion_kv_config") + capture_reset = ( + bool(raw_kv_config.get("warmup_capture_reset", False)) + if isinstance(raw_kv_config, dict) + else bool(getattr(raw_kv_config, "warmup_capture_reset", False)) + ) + if capture_reset: + n_forwards += 1 + policy_config = self.od_config.model_config.get("policy_server_config", {}) + height, width = (int(value) for value in policy_config.get("image_resolution", [180, 320])) + for index in range(n_forwards): + n_frames = 1 if index == 0 else 4 + robot_obs = self._ar_warmup_robot_obs(height, width, n_frames, session_id) + sampling_params = OmniDiffusionSamplingParams( + extra_args={ + "reset": index == 0, + "session_id": session_id, + "robot_obs": robot_obs, + } + ) + yield OmniDiffusionRequest( + prompt="warmup", + sampling_params=sampling_params, + request_id=f"ardiffusion-warmup-{index}", + ) + + def _ar_branch(self, is_negative: bool) -> str: + return self._NEGATIVE_BRANCH if is_negative else self._POSITIVE_BRANCH def _kv_get(self, state, is_negative, seq_len=None, update_kv_cache=False): return self._ar_diffusion_kv_state.get_kv_caches( - is_negative, + self._ar_branch(is_negative), seq_len=seq_len, commit_current=update_kv_cache, ) @@ -115,11 +234,18 @@ class DreamZeroPipeline(nn.Module, CFGParallelMixin): return def _kv_commit(self, is_negative: bool): - self._ar_diffusion_kv_state.commit_paged_context(is_negative) + self._ar_diffusion_kv_state.commit_paged_context(self._ar_branch(is_negative)) def _kv_get_cross(self, state, is_negative): """Cross-attn cache from the engine pool (text k/v + I2V image k_img/v_img).""" - return self._ar_diffusion_kv_state.get_cross_kv_caches(is_negative) + kv_branch = self._ar_branch(is_negative) + text_caches = self._ar_diffusion_kv_state.get_cross_attention_kv(kv_branch, "text") + if "image" in self._ar_diffusion_kv_state.kv_cache.cross_attention_lengths: + image_caches = self._ar_diffusion_kv_state.get_cross_attention_kv(kv_branch, "image") + for text_cache, image_cache in zip(text_caches, image_caches, strict=True): + text_cache["k_img"] = image_cache["k"] + text_cache["v_img"] = image_cache["v"] + return text_caches def _kv_populate_cross(self, context: torch.Tensor, clip_feature, is_negative: bool) -> None: """Eagerly project cross-attn K/V for all layers into the AR-Diffusion pool. @@ -133,32 +259,41 @@ class DreamZeroPipeline(nn.Module, CFGParallelMixin): after the image is encoded so ``clip_feature`` is available. """ s = self._ar_diffusion_kv_state - need_text = not s._cross_text_populated.get(is_negative, False) + kv_branch = self._ar_branch(is_negative) + need_text = not s.is_cross_attention_populated(kv_branch, "text") need_img = ( clip_feature is not None - and getattr(self.transformer, "model_type", "t2v") == "i2v" - and not s._cross_img_populated.get(is_negative, False) + and self.transformer.model_type == "i2v" + and not s.is_cross_attention_populated(kv_branch, "image") ) if not need_text and not need_img: return projected = self.transformer.text_embedding(context) if need_text else None img_ctx = self.transformer.img_emb(clip_feature) if need_img else None - for i, block in enumerate(self.transformer.blocks): - ca = block.cross_attn - n, d = ca.tp_num_heads, ca.head_dim - k = v = None - if projected is not None: - k = ca.norm_k(ca.k(projected)).unflatten(2, (n, d)) - v = ca.v(projected).unflatten(2, (n, d)) - k_img = v_img = None - if img_ctx is not None: - k_img = ca.norm_k_img(ca.k_img(img_ctx)).unflatten(2, (n, d)) - v_img = ca.v_img(img_ctx).unflatten(2, (n, d)) - s.kv_cache.write_cross_kv(i, is_negative, k, v, k_img, v_img) - if need_text: - s._cross_text_populated[is_negative] = True - if need_img: - s._cross_img_populated[is_negative] = True + if projected is not None: + + def text_layer_kv(): + for block in self.transformer.blocks: + ca = block.cross_attn + n, d = ca.tp_num_heads, ca.head_dim + yield ( + ca.norm_k(ca.k(projected)).unflatten(2, (n, d)), + ca.v(projected).unflatten(2, (n, d)), + ) + + s.populate_cross_attention(kv_branch, "text", text_layer_kv()) + if img_ctx is not None: + + def image_layer_kv(): + for block in self.transformer.blocks: + ca = block.cross_attn + n, d = ca.tp_num_heads, ca.head_dim + yield ( + ca.norm_k_img(ca.k_img(img_ctx)).unflatten(2, (n, d)), + ca.v_img(img_ctx).unflatten(2, (n, d)), + ) + + s.populate_cross_attention(kv_branch, "image", image_layer_kv()) logger.info( "AR-Diffusion CROSS POPULATE [%s]: %d layers, text=%s img=%s", "neg" if is_negative else "pos", @@ -179,7 +314,8 @@ class DreamZeroPipeline(nn.Module, CFGParallelMixin): image half repopulates on the restart forward. """ state.reset(clear_video_latents=clear_video_latents) - self._ar_diffusion_kv_state.reset(keep_cross_text=not clear_video_latents) + keep_cross = ("text",) if not clear_video_latents else () + self._ar_diffusion_kv_state.reset(keep_cross_attention=keep_cross) def __init__(self, *, od_config: OmniDiffusionConfig, prefix: str = "") -> None: """Initialize pipeline components. @@ -298,7 +434,6 @@ class DreamZeroPipeline(nn.Module, CFGParallelMixin): ) self._states: OrderedDict[str, DreamZeroState] = OrderedDict() - self._max_session_states = MAX_DREAMZERO_SESSIONS self.state = self._get_or_create_state("default") # DiT step cache is configured by StepCacheBackend @@ -373,9 +508,6 @@ class DreamZeroPipeline(nn.Module, CFGParallelMixin): if state is None: state = DreamZeroState() self._states[session_key] = state - max_states = getattr(self, "_max_session_states", MAX_DREAMZERO_SESSIONS) - while len(self._states) > max_states: - self._states.popitem(last=False) else: self._states.move_to_end(session_key) return state @@ -536,6 +668,10 @@ class DreamZeroPipeline(nn.Module, CFGParallelMixin): if not torch.cuda.is_available(): return + state = self.state + if state is None: + state = self._get_or_create_state("default") + self.state = state device = next(self.text_encoder.parameters()).device with torch.inference_mode(): try: @@ -547,18 +683,18 @@ class DreamZeroPipeline(nn.Module, CFGParallelMixin): try: image = torch.zeros(1, 1, 3, 180, 320, dtype=torch.bfloat16, device=device) - self._encode_image(image, self.num_frames, 180, 320, state=self.state) + self._encode_image(image, self.num_frames, 180, 320, state=state) except Exception as exc: logger.warning("DreamZero compile warmup (image_encoder) skipped: %s", exc) try: - self.state.reset_vae_encoder_stream() + state.reset_vae_encoder_stream() dummy_video = torch.zeros(1, 3, 1, 180, 320, dtype=torch.bfloat16, device=device) - self._vae_stream_seed(self.state, dummy_video[:, :, :1]) + self._vae_stream_seed(state, dummy_video[:, :, :1]) for _ in range(4): - self._vae_stream_append_frame(self.state, dummy_video[:, :, :1]) + self._vae_stream_append_frame(state, dummy_video[:, :, :1]) self._vae_stream_get_observation_latents( - self.state, + state, self.num_frame_per_block, dtype=torch.bfloat16, ) @@ -1034,7 +1170,7 @@ class DreamZeroPipeline(nn.Module, CFGParallelMixin): state_features = kwargs.get("state_features") embodiment_id = kwargs.get("embodiment_id") - # Shared kwargs for predict_noise (both cond & uncond branches) + # Shared kwargs for predict_noise (both conditional and unconditional branches) common_kwargs = dict( seq_len=seq_len, current_start_frame=state.current_start_frame, @@ -1236,21 +1372,14 @@ class DreamZeroPipeline(nn.Module, CFGParallelMixin): text_tokens = text_inputs["input_ids"].to(device) attention_mask = text_inputs["attention_mask"].to(device) - # Explicit reset from OpenPI serving is carried by `extra_args["reset"]` - # on the next inference request after websocket reset/session switch. - if extra_args.get("reset", False): + # Explicit request reset is handled by ARDiffusionModelRunner before it + # binds this forward. Model-detected prompt/window resets still happen + # here because they depend on tokenized DreamZero state. + reset_reason = state.reset_reason(text_tokens, 0, self.transformer.local_attn_size) + if reset_reason == "session": self._kv_reset(state) - else: - # Auto-reset based on model state (before accumulation). Both a "session" - # reset and an "inference" (window-boundary) reset clear the model-local - # KV window, so route both through _kv_reset to drop the AR-Diffusion pool window - # too and stay parity-exact; "inference" keeps the accumulated video - # latents (reset_inference_state == reset(clear_video_latents=False)). - reset_reason = state.reset_reason(text_tokens, 0, self.transformer.local_attn_size) - if reset_reason == "session": - self._kv_reset(state) - elif reset_reason == "inference": - self._kv_reset(state, clear_video_latents=False) + elif reset_reason == "inference": + self._kv_reset(state, clear_video_latents=False) state.language = text_tokens # Frame accumulation: stitched single frame -> multi-frame video @@ -1280,7 +1409,7 @@ class DreamZeroPipeline(nn.Module, CFGParallelMixin): if state.prompt_embeds is None: state.prompt_embeds = self._encode_text(text_tokens, attention_mask) prompt_embeds = state.prompt_embeds - # Negative prompt for CFG uncond branch (model constant) + # Negative prompt for the unconditional CFG branch (model constant) negative_prompt_embeds = None if self.cfg_scale > 1.0: if self._negative_prompt_embeds_cache is None: diff --git a/vllm_omni/experimental/ar_diffusion/__init__.py b/vllm_omni/experimental/ar_diffusion/__init__.py index b312660ec..b87ac1ccc 100644 --- a/vllm_omni/experimental/ar_diffusion/__init__.py +++ b/vllm_omni/experimental/ar_diffusion/__init__.py @@ -2,11 +2,24 @@ """AR-Diffusion Engine (AR-Diffusion). The AR-Diffusion engine: a ``DiffusionEngine`` subclass that adds engine-level KV -cache management for autoregressive / chunked "world-model" diffusion models -(DreamZero and the AR-DiT family). Selected via +cache management for autoregressive / chunked diffusion models. Selected via ``OmniDiffusionConfig.engine_backend = "ar_diffusion"``. """ +from vllm_omni.experimental.ar_diffusion.capability import ( + ARDiffusionCrossAttentionKVSpec, + ARDiffusionKVBranchSpec, + ARDiffusionKVCacheSpec, + SupportsARDiffusionPipeline, + SupportsARDiffusionWarmup, +) from vllm_omni.experimental.ar_diffusion.engine import ARDiffusionEngine -__all__ = ["ARDiffusionEngine"] +__all__ = [ + "ARDiffusionKVBranchSpec", + "ARDiffusionCrossAttentionKVSpec", + "ARDiffusionEngine", + "ARDiffusionKVCacheSpec", + "SupportsARDiffusionPipeline", + "SupportsARDiffusionWarmup", +] diff --git a/vllm_omni/experimental/ar_diffusion/capability.py b/vllm_omni/experimental/ar_diffusion/capability.py new file mode 100644 index 000000000..b30d10c1e --- /dev/null +++ b/vllm_omni/experimental/ar_diffusion/capability.py @@ -0,0 +1,169 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Typed pipeline contract for the model-neutral AR-Diffusion runtime.""" + +from __future__ import annotations + +from collections.abc import Iterable +from contextlib import AbstractContextManager +from dataclasses import dataclass +from typing import TYPE_CHECKING, Protocol, runtime_checkable + +if TYPE_CHECKING: + from vllm_omni.diffusion.request import OmniDiffusionRequest + from vllm_omni.experimental.ar_diffusion.kv_cache.state import ARDiffusionKVState + + +@dataclass(frozen=True) +class ARDiffusionKVBranchSpec: + """One logical model KV branch and its worker-local storage slot. + + Distinct KV branches may share ``local_index`` when distributed execution + guarantees that at most one of them runs on a worker. For example, two CFG + branches use indices ``0, 1`` without CFG parallelism and may both use index + ``0`` when each worker executes only one branch. + """ + + name: str + local_index: int + + def __post_init__(self) -> None: + if not self.name: + raise ValueError("AR-Diffusion KV branch names must be non-empty") + if self.local_index < 0: + raise ValueError(f"AR-Diffusion KV branch local_index must be non-negative, got {self.local_index}") + + +@dataclass(frozen=True) +class ARDiffusionCrossAttentionKVSpec: + """A fixed-length, per-layer cross-attention KV allocation.""" + + name: str + num_tokens: int + + def __post_init__(self) -> None: + if not self.name: + raise ValueError("AR-Diffusion cross-attention cache names must be non-empty") + if self.num_tokens <= 0: + raise ValueError(f"AR-Diffusion cross-attention num_tokens must be positive, got {self.num_tokens}") + + +@dataclass(frozen=True) +class ARDiffusionKVCacheSpec: + """KV geometry and session policy supplied by an AR-Diffusion pipeline. + + ``tokens_per_frame`` is also the paged KV block size. A model forward may + commit ``frames_per_block`` such blocks at once. ``window_frames`` is the + recent sliding tail and ``sink_frames`` is the separately retained prefix; + both are expressed in the same frame/block unit. + + ``max_scratch_tokens_per_branch`` is the maximum non-video KV (for example, + action/state registers) that must coexist with an uncommitted video block. + ``session_capacity`` is the pipeline's upper bound; a runner may retain + fewer sessions when its current memory and execution policy is stricter. + + The runner owns all pool allocations and session lifetime. The pipeline + owns its non-KV model state and receives reset/close notifications through + :class:`SupportsARDiffusionPipeline`. + """ + + num_layers: int + num_kv_heads: int + head_size: int + tokens_per_frame: int + frames_per_block: int + window_frames: int + kv_branches: tuple[ARDiffusionKVBranchSpec, ...] + session_capacity: int + sink_frames: int = 0 + reset_at_boundary: bool = False + cross_attention: tuple[ARDiffusionCrossAttentionKVSpec, ...] = () + max_model_len: int = 1 << 20 + max_scratch_tokens_per_branch: int = 0 + + def __post_init__(self) -> None: + positive_fields = { + "num_layers": self.num_layers, + "num_kv_heads": self.num_kv_heads, + "head_size": self.head_size, + "tokens_per_frame": self.tokens_per_frame, + "frames_per_block": self.frames_per_block, + "window_frames": self.window_frames, + "session_capacity": self.session_capacity, + "max_model_len": self.max_model_len, + } + for name, value in positive_fields.items(): + if value <= 0: + raise ValueError(f"AR-Diffusion {name} must be positive, got {value}") + if self.sink_frames < 0: + raise ValueError(f"AR-Diffusion sink_frames must be non-negative, got {self.sink_frames}") + if self.max_scratch_tokens_per_branch < 0: + raise ValueError( + "AR-Diffusion max_scratch_tokens_per_branch must be non-negative, " + f"got {self.max_scratch_tokens_per_branch}" + ) + if not self.kv_branches: + raise ValueError("AR-Diffusion requires at least one KV branch") + kv_branch_names = [kv_branch.name for kv_branch in self.kv_branches] + if len(kv_branch_names) != len(set(kv_branch_names)): + raise ValueError(f"AR-Diffusion KV branch names must be unique, got {kv_branch_names}") + local_indices = {kv_branch.local_index for kv_branch in self.kv_branches} + if local_indices != set(range(max(local_indices) + 1)): + raise ValueError( + f"AR-Diffusion KV branch local_index values must be contiguous from zero, got {sorted(local_indices)}" + ) + cross_names = [cache.name for cache in self.cross_attention] + if len(cross_names) != len(set(cross_names)): + raise ValueError(f"AR-Diffusion cross-attention cache names must be unique, got {cross_names}") + + @property + def num_local_kv_branches(self) -> int: + """Number of KV branch storage slots allocated on this worker.""" + return max(kv_branch.local_index for kv_branch in self.kv_branches) + 1 + + @property + def cross_attention_lengths(self) -> dict[str, int]: + return {cache.name: cache.num_tokens for cache in self.cross_attention} + + +@runtime_checkable +class SupportsARDiffusionPipeline(Protocol): + """Required pipeline capability for :class:`ARDiffusionModelRunner`. + + A session begins when the runner first sees its ``session_id`` and persists + across requests until reset, explicit close, LRU eviction, or a failed + forward. ``bind_ar_diffusion_state`` exposes the runner-owned KV state only + for the duration of one request. The pipeline must not retain the state + after the context exits. + """ + + def ar_diffusion_kv_cache_spec(self) -> ARDiffusionKVCacheSpec: + """Return immutable KV geometry and session policy for this worker.""" + ... + + def bind_ar_diffusion_state( + self, + session_id: str, + state: ARDiffusionKVState, + ) -> AbstractContextManager[None]: + """Bind ``state`` to model execution for one request.""" + ... + + def reset_ar_diffusion_session(self, session_id: str) -> None: + """Reset model-owned state for ``session_id`` after KV release.""" + ... + + def close_ar_diffusion_session(self, session_id: str) -> None: + """Drop model-owned state after close, eviction, or failed forward.""" + ... + + +@runtime_checkable +class SupportsARDiffusionWarmup(Protocol): + """Optional provider of model-valid warmup requests. + + Pipelines without this capability are loaded without an AR rollout warmup. + """ + + def ar_diffusion_warmup_requests(self, session_id: str) -> Iterable[OmniDiffusionRequest]: + """Yield requests for compiled shapes, each carrying ``session_id``.""" + ... diff --git a/vllm_omni/experimental/ar_diffusion/engine.py b/vllm_omni/experimental/ar_diffusion/engine.py index 6fb6ee7c9..52ef6c9d0 100644 --- a/vllm_omni/experimental/ar_diffusion/engine.py +++ b/vllm_omni/experimental/ar_diffusion/engine.py @@ -20,8 +20,7 @@ class ARDiffusionEngine(DiffusionEngine): (world models, AR-DiT) that materialize persistent attention KV. It reuses vLLM's paged KV stack (``KVCacheManager`` / ``BlockPool`` / ``BlockTables``) as a library, driven from the engine rather than hand-rolled inside each - model. See ``BDE_doc/diffusion_kv_cache_management.md`` for the design and - ``BDE_doc/dreamzero_kv_phase1_plan.md`` for the rollout. + model. It is selected per model via ``OmniDiffusionConfig.engine_backend = "ar_diffusion"`` (resolved by :meth:`DiffusionEngine.make_engine`), so models that do not opt @@ -32,7 +31,7 @@ class ARDiffusionEngine(DiffusionEngine): actual KV *body* is :class:`~vllm_omni.experimental.ar_diffusion.kv_cache.manager.ARDiffusionKVCache`, owned by :class:`~vllm_omni.experimental.ar_diffusion.runner.ARDiffusionModelRunner`. ``ARDiffusionEngine`` itself is the thin selection / injection seam; it wires the AR-Diffusion executor → worker → runner - so DreamZero's rollout runs against the runner-owned KV cache. + so capable pipelines run against runner-owned KV session state. """ #: Workers of this engine build the AR-Diffusion runner (unless the config diff --git a/vllm_omni/experimental/ar_diffusion/kv_cache/__init__.py b/vllm_omni/experimental/ar_diffusion/kv_cache/__init__.py index ab29f6093..b407d24a3 100644 --- a/vllm_omni/experimental/ar_diffusion/kv_cache/__init__.py +++ b/vllm_omni/experimental/ar_diffusion/kv_cache/__init__.py @@ -3,7 +3,7 @@ Thin glue over vLLM's paged KV stack (``KVCacheManager`` / ``BlockPool`` / ``SlidingWindowManager``), used by the AR-Diffusion Engine to manage KV for -AR-diffusion models. See ``BDE_doc/dreamzero_kv_phase1_plan.md``. +AR-diffusion models. Layout: ``config`` (public knob) · ``paged`` (engine-generic paging mechanics + chunk-window eviction spec) · ``manager`` (the ARDiffusionKVCache orchestrator + its diff --git a/vllm_omni/experimental/ar_diffusion/kv_cache/config.py b/vllm_omni/experimental/ar_diffusion/kv_cache/config.py index 4b3c73775..07be2546e 100644 --- a/vllm_omni/experimental/ar_diffusion/kv_cache/config.py +++ b/vllm_omni/experimental/ar_diffusion/kv_cache/config.py @@ -15,14 +15,13 @@ class ARDiffusionKVConfig: """ enable: bool = False - # Persistent KV tokens materialized per AR chunk. For DreamZero this is - # ``num_frame_per_block * frame_seqlen``. + # Persistent KV tokens materialized per paged cache block. chunk_size: int = 0 # Resident window in chunks. ``None`` means full attention (no eviction). window_chunks: int | None = None # Protected leading chunks (attention sink); never evicted. sink_chunks: int = 0 - # DreamZero-style window reset vs. VGGT-style sliding replace. + # Boundary reset vs. sliding replacement. reset_at_boundary: bool = False # Fraction of free device memory budgeted for the AR-Diffusion KV pool. gpu_memory_fraction: float = 0.1 diff --git a/vllm_omni/experimental/ar_diffusion/kv_cache/manager.py b/vllm_omni/experimental/ar_diffusion/kv_cache/manager.py index 4b3b5d788..8c7d4b2ce 100644 --- a/vllm_omni/experimental/ar_diffusion/kv_cache/manager.py +++ b/vllm_omni/experimental/ar_diffusion/kv_cache/manager.py @@ -1,20 +1,19 @@ # SPDX-License-Identifier: Apache-2.0 -"""ARDiffusionKVCache — the engine-level KV cache orchestrator for one AR-Diffusion model. +"""Engine-level KV cache orchestration for AR-Diffusion models. This is the *body* of AR-Diffusion's KV management: it owns a vLLM ``KVCacheManager`` (a single chunk-window group) and the per-request adapter lifecycle, and exposes the per-chunk operations a rollout needs — allocate, slot mapping, commit, window lookup, free. It lives in the model runner (worker / GPU side), co-located with -the model and the KV tensors; the DreamZero pipeline calls these methods during a -rollout. The main-process ``ARDiffusionEngine`` only selects the engine and is otherwise -thin. +the model and the KV tensors. The main-process ``ARDiffusionEngine`` only +selects the engine and is otherwise thin. """ from __future__ import annotations import inspect import os -from collections.abc import Sequence +from collections.abc import Collection, Iterable, Sequence import torch from vllm.logger import init_logger @@ -27,6 +26,7 @@ from vllm.v1.kv_cache_interface import ( ) from vllm.v1.request import RequestStatus +from vllm_omni.experimental.ar_diffusion.capability import ARDiffusionKVBranchSpec from vllm_omni.experimental.ar_diffusion.kv_cache.config import ARDiffusionKVConfig from vllm_omni.experimental.ar_diffusion.kv_cache.paged import ( ChunkWindowSpec, @@ -48,7 +48,7 @@ class ARDiffusionRequestAdapter: real ``KVCacheManager`` against this adapter so the surface cannot silently drift across vLLM versions. - A AR-Diffusion request advances one *chunk* at a time: ``allocate_slots`` is called + An AR-Diffusion request advances one *chunk* at a time: ``allocate_slots`` is called once per chunk and ``num_computed_tokens`` advances only when a chunk is committed (:meth:`on_chunk_committed`), so the ``T`` denoise steps of a chunk reuse the same slots. @@ -145,7 +145,7 @@ def build_kv_manager( class ARDiffusionKVCache: - """Owns the paged KV pool + per-request lifecycle for a AR-Diffusion model. + """Own the paged KV pool and KV-branch-local storage for one model. Build once per loaded model (dimensions known); then per request: ``begin_request`` → per chunk (``allocate_chunk`` → ``chunk_write_slots`` → @@ -163,11 +163,12 @@ class ARDiffusionKVCache: block_size: int, max_model_len: int, available_bytes: int, - cross_attn_length: int = 0, - cross_attn_img_length: int = 0, + kv_branches: tuple[ARDiffusionKVBranchSpec, ...], + session_capacity: int, + cross_attention_lengths: dict[str, int] | None = None, device: torch.device | None = None, - local_branches: int = 2, - num_frame_per_block: int = 1, + frames_per_block: int = 1, + max_scratch_tokens_per_branch: int = 0, ) -> None: if not config.enable: raise ValueError("ARDiffusionKVCache built with a disabled ARDiffusionKVConfig") @@ -175,77 +176,63 @@ class ARDiffusionKVCache: raise ValueError("Phase 1 requires a bounded window (window_chunks)") if config.chunk_size <= 0: raise ValueError("ARDiffusionKVConfig.chunk_size must be set (> 0)") - if local_branches not in (1, 2): - raise ValueError(f"local_branches must be 1 or 2, got {local_branches}") + if not kv_branches: + raise ValueError("ARDiffusionKVCache requires at least one KV branch") + if session_capacity <= 0: + raise ValueError(f"session_capacity must be positive, got {session_capacity}") + kv_branch_names = [kv_branch.name for kv_branch in kv_branches] + if len(kv_branch_names) != len(set(kv_branch_names)): + raise ValueError(f"ARDiffusionKVCache KV branch names must be unique, got {kv_branch_names}") + local_indices = {kv_branch.local_index for kv_branch in kv_branches} + if local_indices != set(range(max(local_indices) + 1)): + raise ValueError( + "ARDiffusionKVCache KV branch local_index values must be contiguous from zero, " + f"got {sorted(local_indices)}" + ) self.config = config - # How many CFG branches allocate from THIS rank's pool. Under - # CFG-parallel x2 each rank executes exactly one branch (rank0 pos, - # rank1 neg) and the other branch's lazy contexts never allocate, so - # sizing for both would leave ~half the pool as idle capacity. A - # single-process run (TP1 / offline) executes both branches -> 2. - self.local_branches = local_branches - self.num_frame_per_block = max(1, int(num_frame_per_block)) + self.kv_branches = kv_branches + self.session_capacity = session_capacity + self._kv_branch_local_indices = {kv_branch.name: kv_branch.local_index for kv_branch in kv_branches} + self.num_local_kv_branches = max(local_indices) + 1 + if frames_per_block <= 0: + raise ValueError(f"frames_per_block must be positive, got {frames_per_block}") + if max_scratch_tokens_per_branch < 0: + raise ValueError(f"max_scratch_tokens_per_branch must be non-negative, got {max_scratch_tokens_per_branch}") + self.frames_per_block = int(frames_per_block) self.block_size = block_size self.num_layers = num_layers self.num_kv_heads = num_kv_heads self.head_size = head_size self.dtype = dtype - self.cross_attn_length = cross_attn_length + self.cross_attention_lengths = dict(cross_attention_lengths or {}) + invalid_cross = {name: length for name, length in self.cross_attention_lengths.items() if length <= 0} + if invalid_cross: + raise ValueError(f"cross_attention_lengths must be positive, got {invalid_cross}") self.device = device or torch.device("cpu") + self._allocate_tensors = device is not None self._adapters: dict[str, ARDiffusionRequestAdapter] = {} - # -- cross-attention pool (static, one-time-fill) ----------------------- - # Cross-attn KV is computed once from the text encoder and never changes. - # Allocate a per-layer contiguous tensor for each branch: (2, text_len, - # kv_heads, head_dim) where dim-0 = [pos, neg]. Allocated separately - # from the self-attn block pool (static, small — ~400 MiB for - # DreamZero I2V). Both pools draw from the same GPU free memory but - # the cross-attn pool is sized directly rather than via the block pool - # budget (no eviction/chunk lifecycle needed). - self.cross_attn_img_length = cross_attn_img_length - self._cross_k: list[torch.Tensor] = [] - self._cross_v: list[torch.Tensor] = [] - # I2V image-token cross-attn pool. Like the text k/v, the image-token - # k_img/v_img are session-invariant (the conditioning image doesn't change), - # so they are cached once and read every denoise step — see - # WanI2VCrossAttention (#4154 caches these model-side too). Empty for T2V. - self._cross_k_img: list[torch.Tensor] = [] - self._cross_v_img: list[torch.Tensor] = [] + # Named cross-attention K/V is allocated lazily per session and released + # with that session. Reserve its worst-case capacity when sizing the + # paged self-attention pool so the two stores share one memory budget. + self._cross_sessions: dict[ + str, + dict[str, dict[str, tuple[list[torch.Tensor], list[torch.Tensor]]]], + ] = {} - # Bytes the cross-attn pools consume directly (K+V, pos+neg, all layers). - # Deducted from the self-attn paged-pool budget below so the two - # allocations together stay within the GPU free-memory budget. def _cross_pool_bytes(length: int) -> int: - return 2 * 2 * length * num_kv_heads * head_size * dtype.itemsize * num_layers + return int(2 * len(self.kv_branches) * length * num_kv_heads * head_size * dtype.itemsize * num_layers) - cross_total_bytes = 0 - if device is not None and cross_attn_length > 0: - cross_shape = (2, cross_attn_length, num_kv_heads, head_size) - cross_bytes = _cross_pool_bytes(cross_attn_length) - cross_total_bytes += cross_bytes - for _ in range(num_layers): - self._cross_k.append(torch.empty(cross_shape, dtype=dtype, device=device)) - self._cross_v.append(torch.empty(cross_shape, dtype=dtype, device=device)) + cross_bytes_per_session = sum(_cross_pool_bytes(length) for length in self.cross_attention_lengths.values()) + cross_total_bytes = cross_bytes_per_session * session_capacity + if cross_total_bytes: _log.info( - "AR-Diffusion cross-attn pool: %d layers × (%d tok × %d heads × %d) = %.1f MiB", - num_layers, - cross_attn_length, - num_kv_heads, - head_size, - cross_bytes / (1024 * 1024), + "AR-Diffusion cross-attn reservation: %.1f MiB/session × %d sessions = %.1f MiB", + cross_bytes_per_session / (1024 * 1024), + session_capacity, + cross_total_bytes / (1024 * 1024), ) - if cross_attn_img_length > 0: - img_shape = (2, cross_attn_img_length, num_kv_heads, head_size) - cross_total_bytes += _cross_pool_bytes(cross_attn_img_length) - for _ in range(num_layers): - self._cross_k_img.append(torch.empty(img_shape, dtype=dtype, device=device)) - self._cross_v_img.append(torch.empty(img_shape, dtype=dtype, device=device)) - _log.info( - "AR-Diffusion cross-attn IMG pool: %d layers × %d img-tok (I2V)", - num_layers, - cross_attn_img_length, - ) self.spec = ChunkWindowSpec( block_size=block_size, @@ -260,46 +247,51 @@ class ARDiffusionKVCache: ) # Each pool block spans all layers' K/V, so size against the per-layer # page size times the layer count. - # Size the self-attn pool against the memory left after the cross-attn - # pools (allocated above), so cross + self-attn stays within the budget. + # Size the self-attn pool against memory reserved for the maximum number + # of lazily allocated cross-attention sessions. num_blocks = compute_num_blocks( max(0, available_bytes - cross_total_bytes), config.gpu_memory_fraction, self.spec.page_size_bytes * num_layers, ) # Floor: one forward needs the resident window plus the in-flight chunk - # (num_frame_per_block frame-blocks) for every branch THIS rank runs, + # (frames_per_block frame-blocks) for every KV branch THIS rank runs, # with a little eviction-transient headroom. The memory-fraction # heuristic can under-size this once block_size grows — e.g. frame-granular # paging at the true frame_seqlen makes each block larger and the pool # fewer-blocks — so guarantee the minimum the rollout cannot run without, # otherwise allocate_chunk hits an exhausted pool mid-forward. - min_blocks = self.local_branches * (config.window_chunks + self.num_frame_per_block) + 2 + resident_blocks = config.sink_chunks + config.window_chunks + min_blocks = self.num_local_kv_branches * (resident_blocks + self.frames_per_block) + 2 if num_blocks < min_blocks: _log.warning( "AR-Diffusion KV pool: memory-fraction sizing gave %d blocks; raising to the %d-block " - "floor (%d local CFG branch(es) x (window_chunks=%d + num_frame_per_block=%d) + 2 headroom)", + "floor (%d local KV branch(es) x (sink_chunks=%d + window_chunks=%d " + "+ frames_per_block=%d) + 2 headroom)", num_blocks, min_blocks, - self.local_branches, + self.num_local_kv_branches, + config.sink_chunks, config.window_chunks, - self.num_frame_per_block, + self.frames_per_block, ) num_blocks = min_blocks layer_names = [f"ar_diffusion.layer.{i}" for i in range(num_layers)] self.manager = build_kv_manager(self.spec, layer_names, num_blocks, max_model_len) self.managed_num_blocks = num_blocks self.num_blocks = num_blocks - # Scratch blocks are outside KVCacheManager ownership. They hold current - # denoise-step video KV when update_kv_cache=False and action/state KV in - # both modes; they are reused every forward and never committed. - # Default 4: measured high-water is num_frame_per_block video blocks (2) - # + 1 action block per branch per forward, +1 margin. - scratch_per_branch = int(os.environ.get("AR_DIFFUSION_KV_SCRATCH_BLOCKS_PER_BRANCH", "4")) - if scratch_per_branch <= 0: - raise ValueError("AR_DIFFUSION_KV_SCRATCH_BLOCKS_PER_BRANCH must be positive") - self.scratch_blocks_per_branch = scratch_per_branch - self.scratch_num_blocks = self.local_branches * scratch_per_branch + # Scratch blocks are outside KVCacheManager ownership. A non-committing + # forward needs one block per current frame plus space for any + # model-declared action/state tokens that coexist with video KV. + declared_scratch_blocks = (max_scratch_tokens_per_branch + block_size - 1) // block_size + minimum_scratch_blocks = self.frames_per_block + declared_scratch_blocks + override = os.environ.get("AR_DIFFUSION_KV_SCRATCH_BLOCKS_PER_BRANCH") + override_blocks = int(override) if override is not None else 0 + if override_blocks < 0: + raise ValueError("AR_DIFFUSION_KV_SCRATCH_BLOCKS_PER_BRANCH must be non-negative") + scratch_per_kv_branch = max(minimum_scratch_blocks, override_blocks) + self.scratch_blocks_per_kv_branch = scratch_per_kv_branch + self.scratch_num_blocks = self.num_local_kv_branches * scratch_per_kv_branch self.num_blocks_total = self.managed_num_blocks + self.scratch_num_blocks self.null_block_id = self.manager.block_pool.null_block.block_id @@ -322,49 +314,147 @@ class ARDiffusionKVCache: # Cross-attn KV is static once populated — write once (from text encoder), # read many (every denoising step). Not managed through the paged block pool. - def write_cross_kv( + def _kv_branch_index(self, kv_branch: str) -> int: + try: + return self._kv_branch_local_indices[kv_branch] + except KeyError as exc: + expected = tuple(self._kv_branch_local_indices) + raise KeyError(f"Unknown AR-Diffusion KV branch {kv_branch!r}; expected {expected}") from exc + + def _cross_attention_pool( self, - layer_idx: int, - is_negative: bool, - k: torch.Tensor | None, - v: torch.Tensor | None, - k_img: torch.Tensor | None = None, - v_img: torch.Tensor | None = None, + session_id: str, + cache_name: str, + kv_branch: str, + ) -> tuple[list[torch.Tensor], list[torch.Tensor]]: + if cache_name not in self.cross_attention_lengths: + raise KeyError( + f"Unknown AR-Diffusion cross-attention cache {cache_name!r}; " + f"expected {tuple(self.cross_attention_lengths)}" + ) + self._kv_branch_index(kv_branch) + session = self._cross_sessions.get(session_id) + if session is None: + raise RuntimeError( + f"AR-Diffusion cross-attention cache {cache_name!r} for session {session_id!r} " + "was read before it was populated" + ) + pool = session.get(cache_name, {}).get(kv_branch) + if pool is None: + raise RuntimeError( + f"AR-Diffusion cross-attention cache {cache_name!r} for session {session_id!r} " + f"and KV branch {kv_branch!r} was read before it was populated" + ) + return pool + + def is_cross_attention_populated(self, session_id: str, cache_name: str, kv_branch: str) -> bool: + """Whether a complete logical-branch cache has been published.""" + if cache_name not in self.cross_attention_lengths: + raise KeyError( + f"Unknown AR-Diffusion cross-attention cache {cache_name!r}; " + f"expected {tuple(self.cross_attention_lengths)}" + ) + self._kv_branch_index(kv_branch) + session = self._cross_sessions.get(session_id) + return session is not None and kv_branch in session.get(cache_name, {}) + + def populate_cross_attention( + self, + session_id: str, + cache_name: str, + kv_branch: str, + layer_kv: Iterable[tuple[torch.Tensor, torch.Tensor]], ) -> None: - """Write one layer's cross-attn K/V into the pool. + """Atomically populate one logical branch of a named cross-attention cache. - ``k`` / ``v``: ``(B, text_len, tp_num_heads, head_dim)``; only batch-0 - is copied (B=1 for inference). ``None`` skips the text half (window - restarts keep the still-valid text K/V and rewrite only the image half). - The ``is_negative`` flag selects the correct CFG branch slot. ``k_img`` / - ``v_img`` (I2V image tokens, ``(B, 257, ...)``) are written when the - image pool is allocated. + ``layer_kv`` must yield exactly one ``(k, v)`` pair per model layer. + Inputs have shape ``(B, length, local_kv_heads, head_size)``; batch zero + is copied because AR-Diffusion currently supports one sequence per + forward. The new cache is published only after every layer is copied, + so failed projection/copy work cannot expose partially initialized KV. """ - branch = 1 if is_negative else 0 - if k is not None: - self._cross_k[layer_idx][branch].copy_(k[0]) - self._cross_v[layer_idx][branch].copy_(v[0]) - if k_img is not None and self._cross_k_img: - self._cross_k_img[layer_idx][branch].copy_(k_img[0]) - self._cross_v_img[layer_idx][branch].copy_(v_img[0]) + try: + length = self.cross_attention_lengths[cache_name] + except KeyError as exc: + raise KeyError( + f"Unknown AR-Diffusion cross-attention cache {cache_name!r}; " + f"expected {tuple(self.cross_attention_lengths)}" + ) from exc + self._kv_branch_index(kv_branch) + session = self._cross_sessions.get(session_id) + if session is None and len(self._cross_sessions) >= self.session_capacity: + raise RuntimeError( + "AR-Diffusion cross-attention session capacity exhausted; " + "the runner must evict a session before allocating another" + ) + if not self._allocate_tensors: + raise RuntimeError("AR-Diffusion cross-attention tensors require a configured pool device") - def read_cross_kv(self, layer_idx: int, is_negative: bool) -> dict: - """Return a pool-backed cross-attn cache dict for one layer. + shape = (length, self.num_kv_heads, self.head_size) + expected_input_shape = (1, *shape) + k_pool = [torch.empty(shape, dtype=self.dtype, device=self.device) for _ in range(self.num_layers)] + v_pool = [torch.empty(shape, dtype=self.dtype, device=self.device) for _ in range(self.num_layers)] + populated_layers = 0 + for layer_idx, (k, v) in enumerate(layer_kv): + if layer_idx >= self.num_layers: + raise ValueError( + f"AR-Diffusion cross-attention cache {cache_name!r} expected {self.num_layers} layers, " + f"got more than {self.num_layers}" + ) + if not isinstance(k, torch.Tensor) or not isinstance(v, torch.Tensor): + raise ValueError( + f"AR-Diffusion cross-attention cache {cache_name!r} layer {layer_idx} " + f"must yield torch.Tensor k/v, got {type(k).__name__} and {type(v).__name__}" + ) + if tuple(k.shape) != expected_input_shape or tuple(v.shape) != expected_input_shape: + raise ValueError( + f"AR-Diffusion cross-attention cache {cache_name!r} layer {layer_idx} expected " + f"k/v shape {expected_input_shape}, got {tuple(k.shape)} and {tuple(v.shape)}" + ) + k_pool[layer_idx].copy_(k[0]) + v_pool[layer_idx].copy_(v[0]) + populated_layers += 1 + if populated_layers != self.num_layers: + raise ValueError( + f"AR-Diffusion cross-attention cache {cache_name!r} expected {self.num_layers} layers, " + f"got {populated_layers}" + ) - The dict matches the ``{"is_init": True, "k": Tensor, "v": Tensor}`` - convention the cross-attention module expects — it reads from the pool - slice rather than from the lazy-initialised model-local dict. For I2V, - ``k_img`` / ``v_img`` are added so the image-token cache (added by #4154) - reads from the pool too. - """ - branch = 1 if is_negative else 0 - k = self._cross_k[layer_idx][branch].unsqueeze(0) # (1, L, heads, dim) - v = self._cross_v[layer_idx][branch].unsqueeze(0) - cache = {"is_init": True, "k": k, "v": v} - if self._cross_k_img: - cache["k_img"] = self._cross_k_img[layer_idx][branch].unsqueeze(0) - cache["v_img"] = self._cross_v_img[layer_idx][branch].unsqueeze(0) - return cache + if session is None: + session = {} + self._cross_sessions[session_id] = session + session.setdefault(cache_name, {})[kv_branch] = (k_pool, v_pool) + + def read_cross_attention_kv( + self, + session_id: str, + cache_name: str, + layer_idx: int, + kv_branch: str, + ) -> dict[str, torch.Tensor | bool]: + """Return a model-facing K/V dict for one named cross-attention pool.""" + k_pool, v_pool = self._cross_attention_pool(session_id, cache_name, kv_branch) + return { + "is_init": True, + "k": k_pool[layer_idx].unsqueeze(0), + "v": v_pool[layer_idx].unsqueeze(0), + } + + def retain_cross_attention(self, session_id: str, cache_names: Collection[str]) -> None: + """Release named cross-attention caches not retained by an internal reset.""" + session = self._cross_sessions.get(session_id) + if session is None: + return + keep = set(cache_names) + for cache_name in tuple(session): + if cache_name not in keep: + del session[cache_name] + if not session: + self._cross_sessions.pop(session_id, None) + + def release_cross_attention(self, session_id: str) -> None: + """Release every named cross-attention allocation for one session.""" + self._cross_sessions.pop(session_id, None) # -- request lifecycle --------------------------------------------------- @@ -431,20 +521,19 @@ class ARDiffusionKVCache: self.block_size, ) - def scratch_block_ids(self, is_negative: bool, start: int, count: int) -> list[int]: - """Return branch-local scratch block ids outside manager ownership.""" + def scratch_block_ids(self, kv_branch: str, start: int, count: int) -> list[int]: + """Return KV-branch-local scratch block ids outside manager ownership.""" if count < 0 or start < 0: raise ValueError(f"scratch start/count must be non-negative, got start={start}, count={count}") - if start + count > self.scratch_blocks_per_branch: + if start + count > self.scratch_blocks_per_kv_branch: raise RuntimeError( "AR-Diffusion paged attention scratch blocks exhausted: " - f"need [{start}, {start + count}) of {self.scratch_blocks_per_branch}. " - "Increase AR_DIFFUSION_KV_SCRATCH_BLOCKS_PER_BRANCH." + f"need [{start}, {start + count}) of {self.scratch_blocks_per_kv_branch}. " + "Declare max_scratch_tokens_per_branch in the pipeline capability " + "or increase AR_DIFFUSION_KV_SCRATCH_BLOCKS_PER_BRANCH." ) - # With one local branch (CFG-parallel x2) the rank's only branch maps to - # slot 0 whichever CFG side it is; the other branch never allocates here. - branch_offset = self.scratch_blocks_per_branch if (is_negative and self.local_branches == 2) else 0 - base = self.managed_num_blocks + branch_offset + start + kv_branch_offset = self.scratch_blocks_per_kv_branch * self._kv_branch_index(kv_branch) + base = self.managed_num_blocks + kv_branch_offset + start return list(range(base, base + count)) def key_cache(self, layer_idx: int) -> torch.Tensor: @@ -455,13 +544,13 @@ class ARDiffusionKVCache: def window_block_ids(self, adapter: ARDiffusionRequestAdapter) -> list[int]: """Resident (non-null) managed blocks visible to paged attention.""" - return resident_block_ids(self.block_table(adapter), self.null_block_id) + return [int(block_id) for block_id in resident_block_ids(self.block_table(adapter), self.null_block_id)] def commit_chunk(self, adapter: ARDiffusionRequestAdapter) -> None: """Advance the adapter by one chunk after its K/V is written. - This standalone primitive is used by low-level manager tests. In the - DreamZero paged-attention path, :meth:`ARDiffusionKVState.commit_paged_context` + This standalone primitive is used by low-level manager tests. The + paged-attention path uses :meth:`ARDiffusionKVState.commit_paged_context`, advances the adapter only after the forward succeeds. Call once per committed chunk, not per denoise step. """ diff --git a/vllm_omni/experimental/ar_diffusion/kv_cache/paged.py b/vllm_omni/experimental/ar_diffusion/kv_cache/paged.py index a39a66b91..6b7d7ffe5 100644 --- a/vllm_omni/experimental/ar_diffusion/kv_cache/paged.py +++ b/vllm_omni/experimental/ar_diffusion/kv_cache/paged.py @@ -153,7 +153,7 @@ def chunk_window_skipped_tokens( Pure function so the eviction policy is unit-testable without constructing a manager. Two strategies: - - ``reset_at_boundary`` (DreamZero): at each chunk boundary everything past + - ``reset_at_boundary``: at each chunk boundary everything past the sink is dropped. - otherwise (VGGT-style sliding replace): keep the last ``window`` tokens (plus the sink); the skip count snaps down to a chunk boundary so a chunk @@ -184,6 +184,25 @@ class ChunkWindowManager(SlidingWindowManager): reset_at_boundary=spec.reset_at_boundary, ) + def remove_skipped_blocks( + self, + request_id: str, + total_computed_tokens: int, + num_prompt_tokens: int | None = None, + ) -> None: + """Free the middle gap while preserving the leading attention sink.""" + del num_prompt_tokens + num_skipped_tokens = self.get_num_skipped_tokens(total_computed_tokens) + if num_skipped_tokens <= 0: + return + sink_blocks = self.kv_cache_spec.sink_chunks + num_skipped_blocks = num_skipped_tokens // self.block_size + self._remove_blocks_in_range( + request_id, + sink_blocks, + sink_blocks + num_skipped_blocks, + ) + # Register so KVCacheManager resolves ChunkWindowSpec to ChunkWindowManager. # Dispatch walks the spec's MRO, so without explicit registration the subclass diff --git a/vllm_omni/experimental/ar_diffusion/kv_cache/paged_attention.py b/vllm_omni/experimental/ar_diffusion/kv_cache/paged_attention.py index 2038e496d..bcd06b63a 100644 --- a/vllm_omni/experimental/ar_diffusion/kv_cache/paged_attention.py +++ b/vllm_omni/experimental/ar_diffusion/kv_cache/paged_attention.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -"""Paged self-attention helpers for AR-Diffusion DreamZero KV reuse.""" +"""Paged self-attention helpers for AR-Diffusion KV reuse.""" from __future__ import annotations @@ -10,7 +10,7 @@ import torch from vllm_omni.experimental.ar_diffusion.kv_cache.paged import compute_slot_mapping -# Set by ARDiffusionPagedForwardContext.prepare() before each branch forward and +# Set by ARDiffusionPagedForwardContext.prepare() before each KV branch forward and # read by the fused write+attend custom op below. The pools are process-lifetime # allocations owned by one ARDiffusionKVCache per worker process (CFG-parallel / # TP ranks are separate processes), mirroring vLLM's forward-context pattern for @@ -39,7 +39,7 @@ class ARDiffusionPagedLayerInputs(NamedTuple): A NamedTuple of plain tensors + ints so ``torch.compile`` treats every field as a pytree graph input (no object-attribute guards, no recompiles when only - tensor *values* change). All layers of one branch forward share the same + tensor *values* change). All layers of one KV branch forward share the same metadata tensor objects, built once by ``prepare()``. ``layer_idx`` is a 0-dim CPU tensor, NOT a python int: all 40 DiT blocks @@ -62,11 +62,11 @@ class ARDiffusionPagedLayerInputs(NamedTuple): @dataclass class ARDiffusionPagedForwardContext: - """Mutable branch-level state shared by all layer contexts in one forward.""" + """Mutable KV-branch state shared by all layer contexts in one forward.""" kv_cache: Any adapter: Any - is_negative: bool + kv_branch: str history_block_ids: list[int] seq_len: int commit_current: bool @@ -80,7 +80,7 @@ class ARDiffusionPagedForwardContext: _allocated_video: bool = False _committed: bool = False _action_len: int = 0 - # Set once by prepare(); shared by all layers of the branch forward. + # Set once by prepare(); shared by all layers of the KV branch forward. block_table: torch.Tensor | None = None query_start_loc: torch.Tensor | None = None seq_lens: torch.Tensor | None = None @@ -102,7 +102,7 @@ class ARDiffusionPagedForwardContext: return self.seq_len // self.block_size def ensure_video_slots(self, device: torch.device) -> None: - """Allocate/write targets for the current video tokens, once per branch.""" + """Allocate/write targets for the current video tokens, once per KV branch.""" if self._allocated_video: return @@ -116,7 +116,7 @@ class ARDiffusionPagedForwardContext: positions = torch.arange(start, start + self.seq_len, dtype=torch.long) self.current_video_slot_mapping = compute_slot_mapping(table, positions, self.block_size).to(device=device) else: - self.current_video_block_ids = self.kv_cache.scratch_block_ids(self.is_negative, 0, n_blocks) + self.current_video_block_ids = self.kv_cache.scratch_block_ids(self.kv_branch, 0, n_blocks) positions = torch.arange(self.seq_len, dtype=torch.long) self.current_video_slot_mapping = compute_slot_mapping( self.current_video_block_ids, @@ -140,7 +140,7 @@ class ARDiffusionPagedForwardContext: action_blocks = (action_len + self.block_size - 1) // self.block_size scratch_offset = 0 if self.commit_current else len(self.current_video_block_ids) self.action_scratch_block_ids = self.kv_cache.scratch_block_ids( - self.is_negative, + self.kv_branch, scratch_offset, action_blocks, ) @@ -161,8 +161,15 @@ class ARDiffusionPagedForwardContext: ) all_video_blocks = self.history_block_ids + self.current_video_block_ids max_video_blocks = self.max_video_tokens // self.block_size - visible_video_blocks = all_video_blocks[-max_video_blocks:] if max_video_blocks else [] - video_len = min(len(all_video_blocks) * self.block_size, self.max_video_tokens) + sink_blocks = int(self.kv_cache.spec.sink_chunks) + if len(all_video_blocks) <= max_video_blocks: + visible_video_blocks = all_video_blocks + else: + tail_blocks = max_video_blocks - sink_blocks + visible_video_blocks = all_video_blocks[:sink_blocks] + if tail_blocks: + visible_video_blocks += all_video_blocks[-tail_blocks:] + video_len = len(visible_video_blocks) * self.block_size return visible_video_blocks, video_len def build_block_table( @@ -201,9 +208,9 @@ class ARDiffusionPagedForwardContext: return block_table, query_start_loc, seq_lens, self.query_len, max_seq_len def prepare(self, device: torch.device, action_len: int, query_len: int) -> None: - """Host-side, once-per-branch-forward setup (called OUTSIDE torch.compile). + """Host-side, once-per-KV-branch setup (called OUTSIDE torch.compile). - Allocates the current video/action slots (still lazy: only the branch a + Allocates the current video/action slots (still lazy: only the KV branch a CFG-parallel rank actually runs reaches its ``_forward_blocks``), builds the padded block-table metadata ONCE for all layers, and publishes the pool registry for the fused custom op. The compiled per-layer code then @@ -245,7 +252,7 @@ class ARDiffusionPagedForwardContext: @dataclass class ARDiffusionPagedLayerContext: - """Layer-specific handle passed through DreamZero's existing ``kv_cache`` slot.""" + """Layer-specific handle passed through a model's ``kv_cache`` slot.""" is_ar_diffusion_paged_context: ClassVar[bool] = True layer_idx: int @@ -260,8 +267,8 @@ class ARDiffusionPagedLayerContext: return self.forward_ctx.adapter @property - def is_negative(self) -> bool: - return self.forward_ctx.is_negative + def kv_branch(self) -> str: + return self.forward_ctx.kv_branch @property def history_block_ids(self) -> list[int]: diff --git a/vllm_omni/experimental/ar_diffusion/kv_cache/state.py b/vllm_omni/experimental/ar_diffusion/kv_cache/state.py index 510bbe043..d114212bb 100644 --- a/vllm_omni/experimental/ar_diffusion/kv_cache/state.py +++ b/vllm_omni/experimental/ar_diffusion/kv_cache/state.py @@ -1,8 +1,12 @@ # SPDX-License-Identifier: Apache-2.0 -"""DreamZero-facing AR-Diffusion KV bridge backed by paged self-attention.""" +"""Model-facing session state backed by paged AR-Diffusion KV storage.""" from __future__ import annotations +from collections.abc import Collection, Iterable, Mapping +from typing import TYPE_CHECKING + +import torch from vllm.logger import init_logger from vllm_omni.experimental.ar_diffusion.kv_cache.paged_attention import ( @@ -10,67 +14,79 @@ from vllm_omni.experimental.ar_diffusion.kv_cache.paged_attention import ( ARDiffusionPagedLayerContext, ) +if TYPE_CHECKING: + from vllm_omni.experimental.ar_diffusion.kv_cache.manager import ( + ARDiffusionKVCache, + ARDiffusionRequestAdapter, + ) + _log = init_logger(__name__) class ARDiffusionKVState: - """Per-request bridge between a DreamZero rollout and the AR-Diffusion KV pool. + """Runner-owned KV state for one session and any number of KV branches. - The self-attention path is paged-only: ``get_kv_caches`` returns lightweight - per-layer contexts, and each attention layer writes current K/V into the - pool before calling the FlashAttention block-table kernel. No resident - window is gathered into a contiguous tensor. + KV branches are addressed by the names in ``ARDiffusionKVCacheSpec``. Each + branch owns an independent request adapter and therefore independent + resident self-attention blocks. Named cross-attention allocations are also + session-scoped and are released by :meth:`close`. """ - def __init__(self, kv_cache, pos_adapter, neg_adapter, num_layers: int) -> None: - """One state per DreamZero session, holding BOTH CFG branches' adapters. - - ``pos``/``neg`` are the classifier-free-guidance branches: ``pos`` is - the conditional pass (real prompt), ``neg`` the unconditional pass - (negative prompt). Each owns an independent request adapter — and - therefore independent pool blocks — because the two branches produce - different K/V for the same frames. Every ``is_negative`` flag in this - module selects between them (``False`` -> ``pos``, ``True`` -> ``neg``). - """ + def __init__( + self, + kv_cache: ARDiffusionKVCache, + session_id: str, + adapters: Mapping[str, ARDiffusionRequestAdapter], + *, + num_layers: int, + ) -> None: + expected = tuple(kv_branch.name for kv_branch in kv_cache.kv_branches) + if set(adapters) != set(expected): + raise ValueError( + "AR-Diffusion session adapters must match the configured KV branches; " + f"expected {expected}, got {tuple(adapters)}" + ) self.kv_cache = kv_cache - self.pos = pos_adapter - self.neg = neg_adapter + self.session_id = session_id + self.adapters = dict(adapters) self.num_layers = num_layers - self._committed: dict[bool, int] = {False: 0, True: 0} - self._paged_pending: dict[bool, ARDiffusionPagedForwardContext | None] = {False: None, True: None} - # Cross-attn KV is populated once after text encoding (eager), then - # read every denoising step. Tracked per half: the text K/V depend only - # on the session prompt (survive window-boundary resets), while the I2V - # image-token K/V come from the current observation's CLIP features and - # must be re-projected on every window restart. - self._cross_text_populated: dict[bool, bool] = {False: False, True: False} - self._cross_img_populated: dict[bool, bool] = {False: False, True: False} + self._committed: dict[str, int] = dict.fromkeys(expected, 0) + self._paged_pending: dict[str, ARDiffusionPagedForwardContext | None] = dict.fromkeys(expected) + self._closed = False - def _adapter(self, is_negative: bool): - return self.neg if is_negative else self.pos + @property + def kv_branch_names(self) -> tuple[str, ...]: + return tuple(self.adapters) + + def adapter(self, kv_branch: str) -> ARDiffusionRequestAdapter: + """Return the request adapter for one logical KV branch.""" + if self._closed: + raise RuntimeError(f"AR-Diffusion session {self.session_id!r} is closed") + try: + return self.adapters[kv_branch] + except KeyError as exc: + raise KeyError(f"Unknown AR-Diffusion KV branch {kv_branch!r}; expected {self.kv_branch_names}") from exc def get_kv_caches( self, - is_negative: bool, + kv_branch: str, seq_len: int | None = None, commit_current: bool = False, ) -> list[ARDiffusionPagedLayerContext]: if seq_len is None: raise ValueError("AR-Diffusion paged self-attention requires seq_len in get_kv_caches()") - return self.prepare_paged_context(is_negative, seq_len, commit_current) + return self.prepare_paged_context(kv_branch, seq_len, commit_current) def prepare_paged_context( self, - is_negative: bool, + kv_branch: str, seq_len: int, commit_current: bool, ) -> list[ARDiffusionPagedLayerContext]: - """Return per-layer paged attention contexts for one branch forward. + """Return per-layer paged attention contexts for one KV branch forward. - Allocation is lazy: constructing both CFG branches' kwargs must not - allocate manager blocks for a branch that a CFG-parallel rank will not run. - The first self-attn layer that consumes the context allocates the current - video slots, then all layers share that branch-level state. + Allocation is lazy so distributed workers allocate only for KV branches + they actually execute. """ cs = self.kv_cache.spec.chunk_size if int(seq_len) % cs != 0: @@ -78,25 +94,26 @@ class ARDiffusionKVState: f"AR-Diffusion expects frame-aligned seq_len (multiple of chunk_size={cs}), got {seq_len}" ) - pending = self._paged_pending.get(is_negative) + pending = self._paged_pending.get(kv_branch) if pending is not None and pending.commit_current and pending._allocated_video and not pending._committed: raise RuntimeError("AR-Diffusion paged context replaced before its managed current chunk was committed") - adapter = self._adapter(is_negative) - branch = "neg" if is_negative else "pos" + adapter = self.adapter(kv_branch) forward_ctx = ARDiffusionPagedForwardContext( kv_cache=self.kv_cache, adapter=adapter, - is_negative=is_negative, + kv_branch=kv_branch, history_block_ids=self.kv_cache.window_block_ids(adapter), seq_len=int(seq_len), commit_current=bool(commit_current), - max_video_tokens=int(self.kv_cache.spec.sliding_window), + max_video_tokens=int( + self.kv_cache.spec.sliding_window + self.kv_cache.spec.sink_chunks * self.kv_cache.spec.chunk_size + ), ) - self._paged_pending[is_negative] = forward_ctx + self._paged_pending[kv_branch] = forward_ctx _log.debug( - "AR-Diffusion GET [%s] source=paged-attn layers=%d history_blocks=%d seq_len=%d commit_current=%s", - branch, + "AR-Diffusion GET [%s] source=paged-attn layers=%d history_blocks=%d seq_len=%d commit_current=%s", + kv_branch, self.num_layers, len(forward_ctx.history_block_ids), int(seq_len), @@ -104,80 +121,94 @@ class ARDiffusionKVState: ) return [ARDiffusionPagedLayerContext(layer_idx=i, forward_ctx=forward_ctx) for i in range(self.num_layers)] - def commit_paged_context(self, is_negative: bool) -> None: - """Commit the managed current video blocks after a successful forward.""" - ctx = self._paged_pending.get(is_negative) + def commit_paged_context(self, kv_branch: str) -> None: + """Commit managed current blocks after one successful KV branch forward.""" + self.adapter(kv_branch) + ctx = self._paged_pending.get(kv_branch) if ctx is None: return - branch = "neg" if is_negative else "pos" if ctx.commit_current and ctx._allocated_video: n_chunks = ctx.seq_len // self.kv_cache.spec.chunk_size for _ in range(n_chunks): ctx.adapter.on_chunk_committed() - self._committed[is_negative] += ctx.seq_len + self._committed[kv_branch] += ctx.seq_len _log.debug( - "AR-Diffusion COMMIT [%s] paged-attn new_tokens=%d chunks=%d resident=%d/%d", - branch, + "AR-Diffusion COMMIT [%s] new_tokens=%d chunks=%d resident=%d/%d", + kv_branch, ctx.seq_len, n_chunks, len(self.kv_cache.window_block_ids(ctx.adapter)), self.kv_cache.spec.window_chunks, ) ctx.mark_committed() - self._paged_pending[is_negative] = None + self._paged_pending[kv_branch] = None - def get_cross_kv_caches(self, is_negative: bool) -> list[dict]: - """Return pool-backed cross-attn cache dicts. + def is_cross_attention_populated(self, kv_branch: str, cache_name: str) -> bool: + self.adapter(kv_branch) + return self.kv_cache.is_cross_attention_populated(self.session_id, cache_name, kv_branch) - Under AR-Diffusion the cross-attn pool is always populated by ``_kv_populate_cross`` - before the first read, so this must never be reached unpopulated — if it - were, the model would lazily project and write cross KV itself, violating - engine ownership. Fail loud rather than fall back to the model. + def populate_cross_attention( + self, + kv_branch: str, + cache_name: str, + layer_kv: Iterable[tuple[torch.Tensor, torch.Tensor]], + ) -> None: + """Publish one named logical-branch cache after all layers are written. + + The iterable is consumed once in layer order. If projection, validation, + or copying fails, the previous complete cache (if any) remains visible. """ - img_ok = not self.kv_cache._cross_k_img or self._cross_img_populated[is_negative] - if not (self._cross_text_populated[is_negative] and img_ok and self.kv_cache.cross_attn_length > 0): + self.adapter(kv_branch) + self.kv_cache.populate_cross_attention(self.session_id, cache_name, kv_branch, layer_kv) + + def get_cross_attention_kv(self, kv_branch: str, cache_name: str) -> list[dict[str, torch.Tensor | bool]]: + """Return all layers for one populated named cross-attention cache.""" + if not self.is_cross_attention_populated(kv_branch, cache_name): raise RuntimeError( - f"AR-Diffusion cross-attn read before _kv_populate_cross (neg={is_negative}, " - f"cross_attn_length={self.kv_cache.cross_attn_length}, " - f"text={self._cross_text_populated[is_negative]} " - f"img={self._cross_img_populated[is_negative]}) — the engine must own all cross KV" + f"AR-Diffusion cross-attention cache {cache_name!r} for KV branch {kv_branch!r} " + "was read before it was populated" ) - return [self.kv_cache.read_cross_kv(i, is_negative) for i in range(self.num_layers)] + return [ + self.kv_cache.read_cross_attention_kv(self.session_id, cache_name, i, kv_branch) + for i in range(self.num_layers) + ] def close(self) -> None: - """Final teardown: free both branches' resident pool blocks.""" - self.kv_cache.end_request(self.pos) - self.kv_cache.end_request(self.neg) - self._paged_pending = {False: None, True: None} + """Release all self- and cross-attention storage owned by this session.""" + if self._closed: + return + for adapter in self.adapters.values(): + self.kv_cache.end_request(adapter) + self.kv_cache.release_cross_attention(self.session_id) + self._paged_pending = dict.fromkeys(self.adapters) + self._closed = True - def reset(self, *, keep_cross_text: bool = False) -> None: - """Drop this session's KV — mirrors the model-local ``state.reset()``. + def reset(self, *, keep_cross_attention: Collection[str] = ()) -> None: + """Release resident blocks and reopen this session with fresh adapters. - DreamZero resets at the attention-window boundary (``should_reset``); the - model starts a fresh sliding window, so AR-Diffusion must free the resident pool - blocks and start a fresh adapter for each branch. The ``ARDiffusionKVState`` - object (and thus ``pipeline._ar_diffusion_kv_state``) is preserved across the reset - so the runner's session mapping stays valid — only the pool-backed state - is recycled. - - Args: - keep_cross_text: On window ("inference") resets the session prompt is - unchanged, so the text cross-attn K/V in the static pool buffers are - still valid — keep them and only force the image half to repopulate. - Session resets (prompt change / explicit reset) pass ``False``. + ``keep_cross_attention`` names allocations whose conditioning remains + valid across a model-internal window reset. Ordinary session reset + should leave it empty. """ - pos_id, neg_id = self.pos.request_id, self.neg.request_id - self.close() - self.pos = self.kv_cache.begin_request(pos_id) - self.neg = self.kv_cache.begin_request(neg_id) - self._committed = {False: 0, True: 0} - self._paged_pending = {False: None, True: None} - if not keep_cross_text: - self._cross_text_populated = {False: False, True: False} - self._cross_img_populated = {False: False, True: False} + unknown = set(keep_cross_attention) - set(self.kv_cache.cross_attention_lengths) + if unknown: + raise KeyError(f"Unknown AR-Diffusion cross-attention caches to keep: {sorted(unknown)}") + request_ids = {kv_branch: adapter.request_id for kv_branch, adapter in self.adapters.items()} + if keep_cross_attention: + for adapter in self.adapters.values(): + self.kv_cache.end_request(adapter) + self.kv_cache.retain_cross_attention(self.session_id, keep_cross_attention) + else: + self.close() + self.adapters = { + kv_branch: self.kv_cache.begin_request(request_id) for kv_branch, request_id in request_ids.items() + } + self._committed = dict.fromkeys(self.adapters, 0) + self._paged_pending = dict.fromkeys(self.adapters) + self._closed = False _log.info( - "AR-Diffusion RESET [%s/%s] session KV cleared (window boundary; cross text %s)", - pos_id, - neg_id, - "kept" if keep_cross_text else "cleared", + "AR-Diffusion RESET session=%s KV branches=%s kept_cross=%s", + self.session_id, + self.kv_branch_names, + sorted(keep_cross_attention), ) diff --git a/vllm_omni/experimental/ar_diffusion/runner.py b/vllm_omni/experimental/ar_diffusion/runner.py index 8c968f8ce..31a23eaf7 100644 --- a/vllm_omni/experimental/ar_diffusion/runner.py +++ b/vllm_omni/experimental/ar_diffusion/runner.py @@ -1,32 +1,23 @@ # SPDX-License-Identifier: Apache-2.0 -"""ARDiffusionModelRunner — the diffusion model runner for the AR-Diffusion engine. - -Subclasses ``DiffusionModelRunner`` and owns the engine-level KV cache -(:class:`ARDiffusionKVCache`). It brackets a request's rollout with the KV lifecycle -(``begin_request`` / ``end_request``) and exposes the live ``ARDiffusionKVCache`` so the -model pipeline (DreamZero) can drive the per-chunk allocate / slot-mapping / -gather / commit operations during ``pipeline.forward``. - -Selecting the AR-Diffusion engine enables KV management (no env gate); optional KV -parameters come from the deploy yaml's ``model_config["ar_diffusion_kv_config"]``. -""" - from __future__ import annotations import dataclasses -import math import time from collections import OrderedDict -import numpy as np import torch from vllm.logger import init_logger from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig -from vllm_omni.diffusion.models.dreamzero.pipeline_dreamzero import MAX_DREAMZERO_SESSIONS from vllm_omni.diffusion.request import OmniDiffusionRequest -from vllm_omni.diffusion.sched.interface import KVPrefetchJob +from vllm_omni.diffusion.sched.interface import DiffusionSchedulerOutput, KVPrefetchJob from vllm_omni.diffusion.worker.diffusion_model_runner import DiffusionModelRunner +from vllm_omni.diffusion.worker.utils import BatchRunnerOutput +from vllm_omni.experimental.ar_diffusion.capability import ( + ARDiffusionKVCacheSpec, + SupportsARDiffusionPipeline, + SupportsARDiffusionWarmup, +) from vllm_omni.experimental.ar_diffusion.kv_cache.config import ARDiffusionKVConfig from vllm_omni.experimental.ar_diffusion.kv_cache.manager import ARDiffusionKVCache from vllm_omni.experimental.ar_diffusion.kv_cache.state import ARDiffusionKVState @@ -35,14 +26,7 @@ logger = init_logger(__name__) def resolve_ar_diffusion_kv_config(od_config: OmniDiffusionConfig) -> ARDiffusionKVConfig: - """Resolve the AR-Diffusion KV config from the deploy/command arg. - - Selecting the AR-Diffusion engine implies KV management, so the result is always - enabled. Optional overrides (``window_chunks`` / ``gpu_memory_fraction`` / - ``sink_chunks`` / ``reset_at_boundary``) may be supplied via the deploy yaml as - ``model_config["ar_diffusion_kv_config"]``. ``chunk_size`` / ``window_chunks`` are - finalized from the model geometry at load (see ``_preallocate_kv_cache``). - """ + """Resolve optional deployment overrides and enable AR-Diffusion KV.""" raw = getattr(od_config, "ar_diffusion_kv_config", None) if raw is None: model_config = getattr(od_config, "model_config", None) @@ -56,359 +40,280 @@ def resolve_ar_diffusion_kv_config(od_config: OmniDiffusionConfig) -> ARDiffusio class ARDiffusionModelRunner(DiffusionModelRunner): - def __init__(self, vllm_config, od_config: OmniDiffusionConfig, device) -> None: + """Own AR KV pools and session lifetime for a capable diffusion pipeline. + + The runner never inspects model internals. At load time it consumes an + :class:`ARDiffusionKVCacheSpec`; during execution it binds a runner-owned + :class:`ARDiffusionKVState` through the pipeline's context manager. Sessions + persist across requests and end on explicit close, LRU eviction, or failed + forward. A request with ``extra_args["reset"]`` releases the old session + before executing with a fresh one. + """ + + _WARMUP_SID = "__ardiffusion_warmup__" + _MAX_RESIDENT_SESSIONS = 1 + + def __init__(self, vllm_config: object, od_config: OmniDiffusionConfig, device: torch.device) -> None: super().__init__(vllm_config, od_config, device) self.ar_diffusion_kv_config = resolve_ar_diffusion_kv_config(od_config) - # Built after the model is loaded (dimensions known); stays None while KV - # management is disabled. self.kv_cache: ARDiffusionKVCache | None = None - # DreamZero KV is session-scoped (persists across a session's forwards), - # so AR-Diffusion KV state is keyed by session_id and reused, not created per request. - # Bounded by the same LRU cap as DreamZeroPipeline._states: an evicted - # session's ARDiffusionKVState owns pool blocks (two CFG adapters), so without a - # bound, session-id churn in a long-running server would leak pool ownership - # until the KV pool is exhausted. Evicting frees those blocks (state.close()). - self._ar_diffusion_states: OrderedDict[str, ARDiffusionKVState] = OrderedDict() - self._max_ar_diffusion_states = MAX_DREAMZERO_SESSIONS - # Per-request server-side forward E2E times (seconds), recorded in - # execute_model. This is the worker-side compute time — the analog of the - # upstream server's per-request E2E — and excludes the engine<->worker IPC - # that the client's omni.generate wall time includes. Read (and cleared) - # by the ar_diffusion_perf_stats worker RPC for the perf-compare summary. + self._ar_diffusion_capability: SupportsARDiffusionPipeline | None = None + self._ar_diffusion_kv_cache_spec: ARDiffusionKVCacheSpec | None = None + self._sessions: OrderedDict[str, ARDiffusionKVState] = OrderedDict() + self._session_capacity = 0 self._perf_e2e_times: list[float] = [] - def build_kv_cache( - self, - *, - num_layers: int, - num_kv_heads: int, - head_size: int, - dtype, - block_size: int, - max_model_len: int, - available_bytes: int, - cross_attn_length: int = 0, - cross_attn_img_length: int = 0, - local_branches: int = 2, - num_frame_per_block: int = 1, - ) -> None: - """Construct the ARDiffusionKVCache (preallocating GPU pools on ``self.device``). + @staticmethod + def _require_capability(pipeline: object) -> SupportsARDiffusionPipeline: + if not isinstance(pipeline, SupportsARDiffusionPipeline): + raise TypeError( + "ARDiffusionEngine requires the loaded pipeline to implement " + "SupportsARDiffusionPipeline: ar_diffusion_kv_cache_spec(), " + "bind_ar_diffusion_state(), reset_ar_diffusion_session(), and " + "close_ar_diffusion_session(). Select the default DiffusionEngine " + "or add an explicit AR-Diffusion capability adapter to the model." + ) + return pipeline - No-op when KV management is disabled. - """ - if not self.ar_diffusion_kv_config.enable: - return - self.kv_cache = ARDiffusionKVCache( - self.ar_diffusion_kv_config, - num_layers=num_layers, - num_kv_heads=num_kv_heads, - head_size=head_size, - dtype=dtype, - block_size=block_size, - max_model_len=max_model_len, - available_bytes=available_bytes, - cross_attn_length=cross_attn_length, - cross_attn_img_length=cross_attn_img_length, - local_branches=local_branches, - num_frame_per_block=num_frame_per_block, - device=self.device, - ) - logger.info( - "AR-Diffusion KV cache enabled: %d blocks, chunk_size=%d, window_chunks=%s, " - "layers=%d kv_heads=%d head_dim=%d block_size=%d cross_attn_len=%d device=%s", - self.kv_cache.num_blocks, - self.ar_diffusion_kv_config.chunk_size, - self.ar_diffusion_kv_config.window_chunks, - num_layers, - num_kv_heads, - head_size, - block_size, - cross_attn_length, - self.device, - ) - - # -- preallocation at load ------------------------------------------------- - - def load_model(self, *args, **kwargs): + def load_model(self, *args: object, **kwargs: object) -> None: super().load_model(*args, **kwargs) - if self.ar_diffusion_kv_config.enable and self.pipeline is not None: - self._preallocate_kv_cache() - # Pre-capture the CUDA graphs for every window-fill shape at load time - # (only when compiling/cuda-graph is on) so serving is fast from chunk 0. - if not self.od_config.enforce_eager and self.ar_diffusion_kv_config.warmup_cudagraph: - self._warmup_ar_rollout() + if self.pipeline is None: + return + self._preallocate_kv_cache() + if not self.od_config.enforce_eager and self.ar_diffusion_kv_config.warmup_cudagraph: + self._warmup_ar_rollout() - def _infer_frame_seqlen(self) -> int: - """frame_seqlen = (H//8)*(W//8)//4 from the configured image_resolution.""" - mc = getattr(self.od_config, "model_config", None) or {} - psc = (mc.get("policy_server_config") if isinstance(mc, dict) else None) or {} - res = psc.get("image_resolution", [180, 320]) - h, w = int(res[0]), int(res[1]) - return (h // 8) * (w // 8) // 4 + def _available_memory_bytes(self) -> int: + if self.device is None or torch.device(self.device).type != "cuda": + raise RuntimeError("AR-Diffusion KV preallocation currently requires a CUDA device") + return int(torch.cuda.mem_get_info(self.device)[0]) - def _preallocate_kv_cache(self) -> None: - """Build the KV pool once, from the loaded DreamZero transformer geometry.""" - # The paged self-attention path carries one rollout per request: slot - # mappings/adapters are single-sequence and K/V writes address batch - # index 0 only. Reject a multi-sequence config at init instead of - # failing (or silently dropping sequences) mid-forward. + def _preallocate_kv_cache(self, *, available_bytes: int | None = None) -> None: + """Build pools solely from the pipeline capability and runner config.""" + if bool(getattr(self.od_config, "step_execution", False)): + raise ValueError( + "ARDiffusionModelRunner currently supports request-mode execution only; " + "step_execution=True would bypass per-request AR session binding." + ) max_num_seqs = int(getattr(self.od_config, "max_num_seqs", 1) or 1) if max_num_seqs > 1: raise ValueError( "AR-Diffusion paged KV supports max_num_seqs=1 (single-sequence " - f"rollouts); got max_num_seqs={max_num_seqs}. Per-sequence " - "adapters/slot maps are not implemented." + f"rollouts); got max_num_seqs={max_num_seqs}." ) - t = self.pipeline.transformer - num_layers = int(t.num_layers) - num_kv_heads = int(getattr(t.blocks[0].self_attn, "tp_num_heads", t.num_heads)) - head_size = int(t.dim // t.num_heads) - num_frame_per_block = int(t.num_frame_per_block) - local_attn_size = int(t.local_attn_size) - # The loaded transformer is the source of truth for frame_seqlen (it derives - # max_attention_size from it). The deploy-config inference is only a - # cross-check — warn on drift rather than silently sizing chunks wrong. - frame_seqlen = int(getattr(t, "frame_seqlen", 0)) or self._infer_frame_seqlen() - inferred = self._infer_frame_seqlen() - if frame_seqlen != inferred: - logger.warning( - "AR-Diffusion frame_seqlen mismatch: transformer=%d deploy-config=%d; using transformer", - frame_seqlen, - inferred, + capability = self._require_capability(self.pipeline) + if bool(getattr(self.pipeline, "supports_request_batch", False)): + raise ValueError( + "ARDiffusionModelRunner currently supports one request at a time; " + "request-batch execution would bypass per-request AR session binding." + ) + spec = capability.ar_diffusion_kv_cache_spec() + if not isinstance(spec, ARDiffusionKVCacheSpec): + raise TypeError( + f"ar_diffusion_kv_cache_spec() must return ARDiffusionKVCacheSpec, got {type(spec).__name__}" ) - # The model's own attention window, in tokens (= the [-max_attention_size:] - # slice it applies). Read it directly rather than recomputing. - max_attention_size = int(t.blocks[0].self_attn.max_attention_size) - # Cross-attn text length — static pool for the text-encoder output. - # The cached k/v spans the full text sequence (the image tokens - # prepended in _forward_blocks are stripped inside the cross-attn - # forward but are not part of the cached k/v). - cross_attn_length = int(getattr(t, "text_len", 0)) - # I2V image-token cross-attn length: the model splits context[:, :257] as the - # image tokens (the img_emb(clip_feature) output), caching k_img/v_img. T2V: 0. - cross_attn_img_length = 257 if getattr(t, "model_type", "t2v") == "i2v" else 0 - # Frame-granular paging: 1 block = 1 frame = frame_seqlen tokens, so the - # resident window matches max_attention_size exactly (it need not be a whole - # number of num_frame_per_block causal blocks). - chunk_size = frame_seqlen - window_chunks = self.ar_diffusion_kv_config.window_chunks or (max_attention_size // frame_seqlen) - - self.ar_diffusion_kv_config = dataclasses.replace( - self.ar_diffusion_kv_config, chunk_size=chunk_size, window_chunks=window_chunks + config = dataclasses.replace( + self.ar_diffusion_kv_config, + chunk_size=spec.tokens_per_frame, + window_chunks=self.ar_diffusion_kv_config.window_chunks or spec.window_frames, + sink_chunks=self.ar_diffusion_kv_config.sink_chunks or spec.sink_frames, + reset_at_boundary=self.ar_diffusion_kv_config.reset_at_boundary or spec.reset_at_boundary, ) - free_bytes = torch.cuda.mem_get_info(self.device)[0] - # Under CFG-parallel each rank executes exactly ONE branch (rank0 pos, - # rank1 neg; the other branch's lazy contexts never allocate on this - # rank), so its pool only needs one branch's capacity. A single-process - # run (cfg world 1) executes both branches from one pool. - try: - from vllm_omni.diffusion.distributed.parallel_state import ( - get_classifier_free_guidance_world_size, - ) - - cfg_world = int(get_classifier_free_guidance_world_size()) - except Exception: - cfg_world = 1 - local_branches = 1 if cfg_world >= 2 else 2 - logger.info( - "AR-Diffusion preallocating (paged): frame_seqlen=%d num_frame_per_block=%d " - "local_attn_size=%d -> chunk_size=%d window_chunks=%d (window=%d tokens)", - frame_seqlen, - num_frame_per_block, - local_attn_size, - chunk_size, - window_chunks, - window_chunks * chunk_size, - ) - self.build_kv_cache( - num_layers=num_layers, - num_kv_heads=num_kv_heads, - head_size=head_size, + self.ar_diffusion_kv_config = config + self._ar_diffusion_capability = capability + self._ar_diffusion_kv_cache_spec = spec + self._session_capacity = min(spec.session_capacity, self._MAX_RESIDENT_SESSIONS) + self.kv_cache = ARDiffusionKVCache( + config, + num_layers=spec.num_layers, + num_kv_heads=spec.num_kv_heads, + head_size=spec.head_size, dtype=self.od_config.dtype, - block_size=chunk_size, # one pool block per chunk - max_model_len=1 << 20, - available_bytes=free_bytes, - cross_attn_length=cross_attn_length, - cross_attn_img_length=cross_attn_img_length, - local_branches=local_branches, - num_frame_per_block=num_frame_per_block, + block_size=spec.tokens_per_frame, + max_model_len=spec.max_model_len, + available_bytes=self._available_memory_bytes() if available_bytes is None else available_bytes, + kv_branches=spec.kv_branches, + session_capacity=self._session_capacity, + cross_attention_lengths=spec.cross_attention_lengths, + frames_per_block=spec.frames_per_block, + max_scratch_tokens_per_branch=spec.max_scratch_tokens_per_branch, + device=self.device, ) + logger.info( + "AR-Diffusion KV cache: blocks=%d layers=%d local_kv_heads=%d head_size=%d " + "tokens/frame=%d frames/block=%d window=%d sink=%d kv_branches=%s cross=%s " + "resident_capacity=%d requested_capacity=%d", + self.kv_cache.num_blocks, + spec.num_layers, + spec.num_kv_heads, + spec.head_size, + spec.tokens_per_frame, + spec.frames_per_block, + config.window_chunks, + config.sink_chunks, + [(kv_branch.name, kv_branch.local_index) for kv_branch in spec.kv_branches], + spec.cross_attention_lengths, + self._session_capacity, + spec.session_capacity, + ) + + def _new_session_state(self, session_id: str) -> ARDiffusionKVState: + if self.kv_cache is None or self._ar_diffusion_kv_cache_spec is None: + raise RuntimeError("AR-Diffusion session requested before KV cache initialization") + adapters = { + kv_branch.name: self.kv_cache.begin_request(f"ar::{session_id}::{kv_branch.name}") + for kv_branch in self._ar_diffusion_kv_cache_spec.kv_branches + } + return ARDiffusionKVState( + self.kv_cache, + session_id, + adapters, + num_layers=self._ar_diffusion_kv_cache_spec.num_layers, + ) + + def _release_session( + self, + session_id: str, + *, + reset_model: bool, + reason: str, + suppress_errors: bool = False, + ) -> None: + """One release path for reset, close, eviction, and failed forwards.""" + state = self._sessions.pop(session_id, None) + errors: list[Exception] = [] + if state is not None: + try: + state.close() + except Exception as exc: # noqa: BLE001 - notify the pipeline even if pool cleanup fails + errors.append(exc) + capability = self._ar_diffusion_capability + if capability is not None: + try: + if reset_model: + capability.reset_ar_diffusion_session(session_id) + else: + capability.close_ar_diffusion_session(session_id) + except Exception as exc: # noqa: BLE001 - preserve all lifecycle cleanup attempts + errors.append(exc) + logger.debug("AR-Diffusion released session=%s reason=%s", session_id, reason) + if errors: + if suppress_errors: + logger.warning( + "AR-Diffusion session=%s cleanup after %s had %d error(s): %s", + session_id, + reason, + len(errors), + errors, + ) + else: + raise errors[0] + + def reset_session(self, session_id: str) -> None: + """Release KV and notify the pipeline to reset model-owned state.""" + self._release_session(session_id, reset_model=True, reason="reset") + + def close_session(self, session_id: str) -> None: + """Release KV and notify the pipeline to drop model-owned state.""" + self._release_session(session_id, reset_model=False, reason="close") + + def _get_or_create_session(self, session_id: str) -> ARDiffusionKVState: + state = self._sessions.get(session_id) + if state is None: + while len(self._sessions) >= self._session_capacity: + oldest = next(iter(self._sessions)) + self._release_session(oldest, reset_model=False, reason="lru_eviction") + state = self._new_session_state(session_id) + self._sessions[session_id] = state + self._sessions.move_to_end(session_id) + return state + + @staticmethod + def _request_session(req: OmniDiffusionRequest) -> tuple[str, dict]: + extra_args = req.sampling_params.extra_args or {} + return str(extra_args.get("session_id") or "default"), extra_args def execute_model( self, req: OmniDiffusionRequest, kv_prefetch_job: KVPrefetchJob | None = None, ) -> DiffusionOutput: - # KV disabled -> base behavior, unchanged. if self.kv_cache is None: return super().execute_model(req, kv_prefetch_job=kv_prefetch_job) + capability = self._ar_diffusion_capability + if capability is None: + raise RuntimeError("AR-Diffusion capability missing after KV cache initialization") - kv = self.kv_cache - # DreamZero KV is session-scoped (the model state persists across a - # session's forwards), so the AR-Diffusion KV state is keyed by session_id and - # reused — matching how pipeline.forward resolves the model-local state. - extra_args = req.sampling_params.extra_args or {} - session_id = str(extra_args.get("session_id") or "default") - state = self._ar_diffusion_states.get(session_id) - if state is None: - pos = kv.begin_request(f"bde__{session_id}") - neg = kv.begin_request(f"bde__{session_id}__neg") - state = ARDiffusionKVState(kv, pos, neg, num_layers=kv.num_layers) - self._ar_diffusion_states[session_id] = state - # Evict the least-recently-used session(s) past the cap, freeing their - # pool blocks — mirrors DreamZeroPipeline._states so the AR-Diffusion pool can't - # outlive the model-local state map it shadows. - while len(self._ar_diffusion_states) > self._max_ar_diffusion_states: - old_id, old_state = self._ar_diffusion_states.popitem(last=False) - old_state.close() - logger.debug("AR-Diffusion evicted session=%s (LRU); freed pool blocks", old_id) - # Track recency on every access (hit or miss) so the LRU order is correct. - self._ar_diffusion_states.move_to_end(session_id) - logger.debug( - "AR-Diffusion execute_model: req=%s session=%s chunk_size=%d window_chunks=%s num_blocks=%d", - req.request_id, - session_id, - kv.spec.chunk_size, - kv.config.window_chunks, - kv.num_blocks, - ) - self.pipeline._ar_diffusion_kv_state = state - # Time the worker-side forward (true per-request compute E2E). The CUDA - # sync below makes the stop reflect completed GPU work, not just dispatch. - _e2e_t0 = time.perf_counter() + session_id, extra_args = self._request_session(req) + if extra_args.get("reset", False): + self.reset_session(session_id) + state = self._get_or_create_session(session_id) + started = time.perf_counter() try: - out = super().execute_model(req, kv_prefetch_job=kv_prefetch_job) + with capability.bind_ar_diffusion_state(session_id, state): + output = super().execute_model(req, kv_prefetch_job=kv_prefetch_job) + if self.device is not None and torch.device(self.device).type == "cuda": + torch.accelerator.synchronize(self.device) except Exception: - # Transactional containment: a forward that died partway may have - # written some layers' K/V into allocated-but-uncommitted blocks - # and advanced model-local state. Tear the whole session down - # (engine pool blocks freed via close(); model-local state dropped) - # so the next request with this session id starts clean instead of - # tripping the pending-commit guard or reading half-written KV. - broken = self._ar_diffusion_states.pop(session_id, None) - if broken is not None: - broken.close() - states = getattr(self.pipeline, "_states", None) - if states is not None: - states.pop(session_id, None) + self._release_session( + session_id, + reset_model=False, + reason="forward_exception", + suppress_errors=True, + ) logger.warning( - "AR-Diffusion execute_model failed for session=%s; session state torn down " - "(pool blocks freed) — the next request starts a fresh session", + "AR-Diffusion forward failed for session=%s; KV and model state were released", session_id, ) raise - finally: - self.pipeline._ar_diffusion_kv_state = None - if self.device is not None and torch.cuda.is_available(): - torch.accelerator.synchronize(self.device) - self._perf_e2e_times.append(time.perf_counter() - _e2e_t0) - return out + self._perf_e2e_times.append(time.perf_counter() - started) + if extra_args.get("close_session", False): + self.close_session(session_id) + return output - # -- cuda-graph warm-up ---------------------------------------------------- + def execute_model_batch( + self, + scheduler_output: DiffusionSchedulerOutput, + od_config: OmniDiffusionConfig, + ) -> BatchRunnerOutput: + """Reject request batching until batch-aware AR state binding exists.""" + raise RuntimeError( + "ARDiffusionModelRunner does not support request-batch execution; use request mode with max_num_seqs=1." + ) - #: Dedicated session id for the warm-up rollout; cleaned from both state maps. - _WARMUP_SID = "__ardiffusion_warmup__" - - def _synth_robot_obs(self, h: int, w: int, n_frames: int) -> dict: - """Zero-filled roboarena observation matching the client's obs layout. - - Mirrors ``examples/.../client_schedule.make_obs_from_video`` (2 ext + wrist - cameras, 7/6/1 state vectors) so it survives ``_transform_robot_obs`` exactly - like a real request. One frame -> ``(H,W,3)``; chunk -> ``(n_frames,H,W,3)``. - """ - img = np.zeros((h, w, 3), dtype=np.uint8) if n_frames == 1 else np.zeros((n_frames, h, w, 3), dtype=np.uint8) - return { - "observation/exterior_image_0_left": img, - "observation/exterior_image_1_left": img, - "observation/wrist_image_left": img, - "observation/joint_position": np.zeros(7, dtype=np.float32), - "observation/cartesian_position": np.zeros(6, dtype=np.float32), - "observation/gripper_position": np.zeros(1, dtype=np.float32), - "prompt": "warmup", - "session_id": self._WARMUP_SID, - } + def execute_stepwise(self, scheduler_output: DiffusionSchedulerOutput) -> BatchRunnerOutput: + """Reject step execution until step-aware AR state binding exists.""" + raise RuntimeError( + "ARDiffusionModelRunner does not support step execution; use request mode with step_execution=False." + ) def _warmup_ar_rollout(self) -> None: - """Drive a synthetic rollout through one window-fill so every DiT graph is - captured at load (off the serving hot path). Frees the synthetic session and - leaves the KV pool exactly as found. Never raises — falls back to lazy capture. - """ - from vllm_omni.inputs.data import OmniDiffusionSamplingParams - - kv = self.kv_cache - if kv is None: + """Run model-provided warmup requests, or safely skip when absent.""" + if self.kv_cache is None or self.pipeline is None: + return + if not isinstance(self.pipeline, SupportsARDiffusionWarmup): + logger.info("AR-Diffusion pipeline provides no warmup requests; skipping rollout warmup") return sid = self._WARMUP_SID - t = self.pipeline.transformer - nfpb = int(t.num_frame_per_block) - window_chunks = int(kv.config.window_chunks or 0) - if window_chunks <= 0: - return - # Distinct DiT window shapes are walked by one fill: prefill + ceil((W-1)/nfpb) - # chunk forwards. +1 optionally to also capture the reset-cycle forward. - n_forwards = 1 + math.ceil(max(0, window_chunks - 1) / nfpb) - if self.ar_diffusion_kv_config.warmup_capture_reset: - n_forwards += 1 - res = self._image_resolution() - h, w = int(res[0]), int(res[1]) - - free_before = kv.manager.block_pool.get_num_free_blocks() - # The step cache is left active: the DiT input shape is per-chunk (the resident - # window length), and the first denoise step of every chunk always computes, so - # every window shape is captured regardless of how many later steps are skipped. - logger.info( - "AR-Diffusion cudagraph warm-up: up to %d synthetic forwards (window_chunks=%d, nfpb=%d)", - n_forwards, - window_chunks, - nfpb, - ) + free_before = self.kv_cache.manager.block_pool.get_num_free_blocks() try: - for i in range(n_forwards): - n_frames = 1 if i == 0 else 4 # client convention: 1-frame prefill, 4-frame chunks - obs = self._synth_robot_obs(h, w, n_frames) - sp = OmniDiffusionSamplingParams(extra_args={"reset": i == 0, "session_id": sid, "robot_obs": obs}) - req = OmniDiffusionRequest(prompts=["warmup"], sampling_params=sp, request_id=f"ardiffusion-warmup-{i}") - self.execute_model(req) - # Stop once the resident window is full — the remaining shapes are - # already captured (the window caps/resets through the same set). - ar_state = self._ar_diffusion_states.get(sid) - if ( - not self.ar_diffusion_kv_config.warmup_capture_reset - and ar_state is not None - and len(kv.window_block_ids(ar_state.pos)) >= window_chunks - ): - break - except Exception as e: # noqa: BLE001 — warm-up must never break model load - logger.warning("AR-Diffusion cudagraph warm-up failed (%s); using lazy capture.", e) + for request in self.pipeline.ar_diffusion_warmup_requests(sid): + self.execute_model(request) + except Exception as exc: # noqa: BLE001 - warmup must not make model load fail + logger.warning("AR-Diffusion rollout warmup failed (%s); using lazy capture", exc) finally: - # Free the synthetic session from BOTH maps (KV pool + pipeline state). - ar_state = self._ar_diffusion_states.pop(sid, None) - if ar_state is not None: - try: - ar_state.close() # return KV blocks to the BlockPool - except Exception as e: # noqa: BLE001 - logger.warning("AR-Diffusion warm-up: state.close() failed (%s)", e) - try: - self.pipeline._states.pop(sid, None) - except Exception: # noqa: BLE001 - pass - # Drop any per-forward timings the warm-up recorded so serving stats start clean. + self._release_session( + sid, + reset_model=False, + reason="warmup_complete", + suppress_errors=True, + ) self._perf_e2e_times.clear() - free_after = kv.manager.block_pool.get_num_free_blocks() + free_after = self.kv_cache.manager.block_pool.get_num_free_blocks() if free_after != free_before: logger.warning( - "AR-Diffusion warm-up: KV pool not restored (free %d -> %d) — possible leak", + "AR-Diffusion warmup did not restore the KV pool (free %d -> %d)", free_before, free_after, ) - else: - logger.info( - "AR-Diffusion cudagraph warm-up complete; KV pool restored (%d free blocks)", - free_after, - ) - - def _image_resolution(self) -> list[int]: - mc = getattr(self.od_config, "model_config", None) or {} - psc = (mc.get("policy_server_config") if isinstance(mc, dict) else None) or {} - return psc.get("image_resolution", [180, 320]) From b471a5cf236cff2419007549c1867a14774e09a6 Mon Sep 17 00:00:00 2001 From: Diya Peng <50388704+dpeng123@users.noreply.github.com> Date: Wed, 22 Jul 2026 23:53:04 -0700 Subject: [PATCH 070/240] [Diffusion] Add FLUX.2-dev CPU Offload (Layerwise) (#5256) Signed-off-by: dpeng123 --- .../diffusion/cpu_offload_diffusion.md | 1 + docs/user_guide/diffusion_features.md | 2 +- .../models/flux2/test_flux2_transformer_tp.py | 25 +++++++++++++++++++ .../test_flux_2_dev_expansion.py | 13 +++++++++- .../models/flux2/flux2_transformer.py | 1 + 5 files changed, 40 insertions(+), 2 deletions(-) diff --git a/docs/user_guide/diffusion/cpu_offload_diffusion.md b/docs/user_guide/diffusion/cpu_offload_diffusion.md index d6fcd8944..31a2670ba 100644 --- a/docs/user_guide/diffusion/cpu_offload_diffusion.md +++ b/docs/user_guide/diffusion/cpu_offload_diffusion.md @@ -223,6 +223,7 @@ reasoner/generator components inside the model forward pass. | Architecture | Example Models | DiT Class | Model-Level Offload | Layerwise Offload | Blocks Attrs (Layerwise specific) | |--------------|----------------|-----------|---------------------|-------------------|-----------------------------------| +| Flux2Pipeline | `black-forest-labs/FLUX.2-dev` | `Flux2Transformer2DModel` | ✓ | ✓ | `"transformer_blocks"`, `"single_transformer_blocks"` | | LongCatImagePipeline | `meituan-longcat/LongCat-Image` | `LongCatImageTransformer2DModel` | - | ✓ | `"transformer_blocks"`, `"single_transformer_blocks"` | | NextStep11Pipeline | `stepfun-ai/NextStep-1.1` | `NextStepModel` | - | ✓ | `"layers"` | | OvisImagePipeline | `AIDC-AI/Ovis-Image-7B` | `OvisImageTransformer2DModel` | - | ✓ | `"transformer"` | diff --git a/docs/user_guide/diffusion_features.md b/docs/user_guide/diffusion_features.md index e5da3c294..bf714b3a0 100644 --- a/docs/user_guide/diffusion_features.md +++ b/docs/user_guide/diffusion_features.md @@ -117,7 +117,7 @@ The following tables show which models support each feature: | **FLUX.1-schnell** | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ | ❌ | ❌ | ✅ | ❌ | | **FLUX.2-klein** | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ❌ | ❌ | ✅ | ❌ | | **FLUX.1-Kontext-dev** | ❌ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | -| **FLUX.2-dev** | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | +| **FLUX.2-dev** | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | | **GLM-Image** | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | | **Hidream-I1-Full** | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | | **HunyuanImage3** | ❌ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | diff --git a/tests/diffusion/models/flux2/test_flux2_transformer_tp.py b/tests/diffusion/models/flux2/test_flux2_transformer_tp.py index c613bb0b4..c935871a0 100644 --- a/tests/diffusion/models/flux2/test_flux2_transformer_tp.py +++ b/tests/diffusion/models/flux2/test_flux2_transformer_tp.py @@ -253,3 +253,28 @@ class TestFlux2PackedModuleMapping: assert len(loaded_params) == 2, "Should load two weights" assert "x_embedder.weight" in loaded_params assert any("to_qkv" in p for p in loaded_params), "to_q should be mapped to to_qkv" + + +@pytest.mark.core_model +@pytest.mark.cpu +class TestFlux2TransformerLayerwiseOffload: + def test_layerwise_offload_blocks_attrs(self): + assert Flux2Transformer2DModel._layerwise_offload_blocks_attrs == [ + "transformer_blocks", + "single_transformer_blocks", + ] + + def test_get_blocks_from_dit(self): + from vllm_omni.diffusion.offloader.layerwise_backend import LayerWiseOffloadBackend + + # Block discovery only needs the ModuleList structure, not production dims. + model = Flux2Transformer2DModel( + num_layers=2, + num_single_layers=2, + num_attention_heads=2, + attention_head_dim=4, + joint_attention_dim=16, + ) + attr_names, blocks = LayerWiseOffloadBackend.get_blocks_from_dit(model) + assert attr_names == ["transformer_blocks", "single_transformer_blocks"] + assert len(blocks) == 4 diff --git a/tests/e2e/online_serving/test_flux_2_dev_expansion.py b/tests/e2e/online_serving/test_flux_2_dev_expansion.py index 77a7a9271..868a1e3bd 100644 --- a/tests/e2e/online_serving/test_flux_2_dev_expansion.py +++ b/tests/e2e/online_serving/test_flux_2_dev_expansion.py @@ -2,7 +2,8 @@ End-to-end diffusion coverage for FLUX.2-dev in online serving mode. Coverage: -- CPU offload +- CPU offload (model-level) +- Layerwise CPU offload - Ulysses sequence parallelism - Ring sequence parallelism @@ -43,6 +44,16 @@ def _get_flux_2_dev_feature_cases(model: str): id="cpu_offload", marks=SINGLE_CARD_FEATURE_MARKS, ), + pytest.param( + OmniServerParams( + model=model, + server_args=[ + "--enable-layerwise-offload", + ], + ), + id="layerwise_offload", + marks=SINGLE_CARD_FEATURE_MARKS, + ), pytest.param( OmniServerParams( model=model, diff --git a/vllm_omni/diffusion/models/flux2/flux2_transformer.py b/vllm_omni/diffusion/models/flux2/flux2_transformer.py index 52e0a4430..2beb1fadb 100644 --- a/vllm_omni/diffusion/models/flux2/flux2_transformer.py +++ b/vllm_omni/diffusion/models/flux2/flux2_transformer.py @@ -765,6 +765,7 @@ class Flux2Transformer2DModel(nn.Module): ) _repeated_blocks = ["Flux2TransformerBlock", "Flux2SingleTransformerBlock"] + _layerwise_offload_blocks_attrs = ["transformer_blocks", "single_transformer_blocks"] _sp_plan = { "": { "hidden_states": SequenceParallelInput(split_dim=1, expected_dims=3, auto_pad=True), From b55120c93b868e8f8c4c34c8e26cb68c2c638340 Mon Sep 17 00:00:00 2001 From: psv666 <150513104+psv666@users.noreply.github.com> Date: Thu, 23 Jul 2026 16:05:54 +0800 Subject: [PATCH 071/240] [Model][MiniCPM-o 4.5] Resolve Hugging Face model ID to local cache path in TTS (#5301) Signed-off-by: psv666 <2693925048@qq.com> --- .../models/minicpmo_4_5/test_hf_model_path.py | 98 +++++++++++++++++++ .../minicpmo_4_5/minicpmo_4_5_omni_tts.py | 11 ++- 2 files changed, 107 insertions(+), 2 deletions(-) create mode 100644 tests/model_executor/models/minicpmo_4_5/test_hf_model_path.py diff --git a/tests/model_executor/models/minicpmo_4_5/test_hf_model_path.py b/tests/model_executor/models/minicpmo_4_5/test_hf_model_path.py new file mode 100644 index 000000000..dc1599e6b --- /dev/null +++ b/tests/model_executor/models/minicpmo_4_5/test_hf_model_path.py @@ -0,0 +1,98 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Regression test for HF model-ID resolution and path caching in MiniCPM-o 4.5 TTS.""" + +from types import SimpleNamespace + +import pytest +import torch + +from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni_tts import ( + MiniCPMO45OmniTTSForConditionalGeneration, +) + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +def test_hf_model_id_path_caching(mocker, tmp_path): + """Verify HF model-ID path is resolved via download_weights_from_hf_specific + once during _lazy_init_tts, cached in self._model_path, and reused in generate_speech. + """ + resolved_path = str(tmp_path / "resolved_hf_model") + assets_dir = tmp_path / "resolved_hf_model" / "assets" + assets_dir.mkdir(parents=True, exist_ok=True) + ref_audio_file = assets_dir / "HT_ref_audio.wav" + ref_audio_file.write_bytes(b"dummy wav content") + + mock_download = mocker.patch( + "vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni_tts.download_weights_from_hf_specific", + return_value=resolved_path, + ) + + # Mock dynamic class loading + fake_tts_class = mocker.MagicMock() + mocker.patch( + "transformers.dynamic_module_utils.get_class_from_dynamic_module", + return_value=fake_tts_class, + ) + + # Build fake vllm_config with HF model ID + model_id = "openbmb/MiniCPM-o-4_5" + tts_cfg = SimpleNamespace( + audio_bos_token_id=151687, + text_eos_token_id=151692, + num_audio_tokens=6562, + hidden_size=768, + normalize_projected_hidden=True, + top_p=0.8, + top_k=100, + repetition_penalty=1.02, + attn_implementation="sdpa", + ) + hf_config = SimpleNamespace(tts_config=tts_cfg) + vllm_config = SimpleNamespace( + model_config=SimpleNamespace( + model=model_id, + hf_config=hf_config, + ) + ) + + tts_model = MiniCPMO45OmniTTSForConditionalGeneration(vllm_config=vllm_config) + + # 1. Trigger lazy init + tts_model._lazy_init_tts() + + # Verify download_weights_from_hf_specific was called with HF model ID and cached + mock_download.assert_called_once_with(model_id, None, ["*"]) + assert tts_model._model_path == resolved_path + assert tts_model._assets_loaded is True + + # 2. Mock TTS object generate method and audio tokenizer for generate_speech + fake_tts_obj = mocker.MagicMock() + fake_tts_obj.emb_text.weight.device = torch.device("cpu") + fake_tts_obj.emb_text.side_effect = lambda t: torch.zeros(t.shape[0], 768) + fake_tts_obj.projector_semantic.side_effect = lambda h, **kw: torch.zeros(h.shape[0], 768) + fake_tts_obj.config = tts_cfg + fake_tts_obj.audio_bos_token_id = 151687 + + fake_output = SimpleNamespace(new_ids=torch.tensor([[[100]]])) + fake_tts_obj.generate.return_value = fake_output + + tts_model.tts_obj = fake_tts_obj + fake_tokenizer = mocker.MagicMock(return_value=b"dummy_wav_bytes") + tts_model.audio_tokenizer = fake_tokenizer + + # Mock sf.read to return fake numpy audio + mocker.patch( + "soundfile.read", + return_value=(torch.zeros(16000).numpy(), 24000), + ) + + # Call generate_speech + tts_token_ids = torch.tensor([1, 2, 3]) + tts_hidden_states = torch.zeros(3, 768) + waveform = tts_model.generate_speech(tts_token_ids, tts_hidden_states) + + # Verify download_weights_from_hf_specific was NOT called a second time + assert mock_download.call_count == 1 + assert waveform is not None diff --git a/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_tts.py b/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_tts.py index 71681c222..f109eb737 100644 --- a/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_tts.py +++ b/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_tts.py @@ -26,6 +26,9 @@ import torch.nn.functional as F from vllm.config import VllmConfig from vllm.model_executor.models.interfaces import SupportsPP +from vllm_omni.model_executor.model_loader.weight_utils import ( + download_weights_from_hf_specific, +) from vllm_omni.platforms import current_omni_platform # Preserve the established external vocoder on CUDA. Ascend uses the in-tree @@ -105,6 +108,7 @@ class MiniCPMO45OmniTTSForConditionalGeneration(nn.Module, SupportsPP): self.tts = None self.audio_tokenizer = None self._assets_loaded = False + self._model_path = None tts_config = getattr(config, "tts_config", None) if tts_config is not None: @@ -121,7 +125,8 @@ class MiniCPMO45OmniTTSForConditionalGeneration(nn.Module, SupportsPP): if self._assets_loaded or self._tts_config is None: return try: - model_path = self.vllm_config.model_config.model + model_path = download_weights_from_hf_specific(self.vllm_config.model_config.model, None, ["*"]) + self._model_path = model_path if model_path not in sys.path: sys.path.insert(0, model_path) @@ -279,7 +284,9 @@ class MiniCPMO45OmniTTSForConditionalGeneration(nn.Module, SupportsPP): import torchaudio - model_path = self.vllm_config.model_config.model + model_path = self._model_path or download_weights_from_hf_specific( + self.vllm_config.model_config.model, None, ["*"] + ) default_ref = os.path.join(model_path, "assets", "HT_ref_audio.wav") prompt_wav_path = default_ref if os.path.exists(default_ref) else None From 05b794f29b02096eba17801d7e2d99911640085e Mon Sep 17 00:00:00 2001 From: Sy03 <1370724210@qq.com> Date: Thu, 23 Jul 2026 16:45:18 +0800 Subject: [PATCH 072/240] [Full Duplex] Feat: Support Full-Duplex realtime runtime & add MiniCPM-o 4.5 demo (#3907) Signed-off-by: Sy03 <1370724210@qq.com> Signed-off-by: lishunyang12 Signed-off-by: Yueqian Lin Co-authored-by: lishunyang12 Co-authored-by: Yueqian Lin Co-authored-by: StormMelody <22371184@zju.edu.cn> Co-authored-by: Gao Han Co-authored-by: Hongsheng Liu --- examples/online_serving/minicpmo/README.md | 204 +- .../minicpmo/realtime_duplex_demo.py | 399 + .../minicpmo/realtime_web/__init__.py | 5 + .../minicpmo/realtime_web/__main__.py | 4 + .../minicpmo/realtime_web/app/index.html | 79 + .../minicpmo/realtime_web/app/static/app.js | 671 ++ .../realtime_web/app/static/pcm_worklet.js | 15 + .../app/static/playback_worklet.js | 153 + .../realtime_web/app/static/styles.css | 127 + .../minicpmo/realtime_web/server.py | 186 + .../core/sched/test_sched_package_imports.py | 26 + .../test_chunk_transfer_adapter.py | 22 + .../minicpmo_realtime_duplex_scenarios.py | 1673 ++++ .../run_minicpmo_realtime_duplex_demo_pair.py | 388 + ..._minicpmo_realtime_duplex_multi_session.py | 556 ++ ...minicpmo_realtime_duplex_soft_interrupt.py | 467 ++ tests/engine/test_async_omni_engine_input.py | 43 + .../engine/test_async_omni_engine_outputs.py | 549 +- tests/engine/test_correlated_rpc_client.py | 104 + tests/engine/test_duplex_import_boundary.py | 102 + tests/engine/test_membership_controller.py | 40 +- tests/engine/test_multimodal_accumulation.py | 46 + tests/engine/test_orchestrator.py | 1070 +++ .../test_orchestrator_stage_input_bridge.py | 16 +- tests/engine/test_output_processor.py | 203 +- tests/engine/test_rpc_result_router.py | 109 + .../openai/test_duplex_protocol.py | 474 ++ .../openai/test_duplex_session_attachment.py | 387 + .../openai_api/test_duplex_capability.py | 19 + .../openai_api/test_duplex_handler.py | 6890 +++++++++++++++++ .../openai_api/test_serving_video_stream.py | 106 + tests/entrypoints/test_async_omni_duplex.py | 600 ++ .../test_duplex_fence_propagation.py | 123 + tests/entrypoints/test_omni_entrypoints.py | 2 + ...st_minicpmo_realtime_duplex_simple_demo.py | 211 + tests/examples/test_minicpmo_realtime_web.py | 1663 ++++ .../test_minicpmo_realtime_web_server.py | 99 + .../test_minicpmo_realtime_web_static.py | 209 + .../engine/test_duplex_control_client.py | 110 + .../engine/test_duplex_control_plane.py | 1270 +++ .../engine/test_duplex_deploy_config.py | 68 + .../engine/test_duplex_intermediate.py | 14 + tests/fullduplex/engine/test_duplex_lease.py | 224 + .../fullduplex/engine/test_duplex_runtime.py | 480 ++ tests/fullduplex/minicpmo45/__init__.py | 0 .../minicpmo45/test_commit_policy.py | 99 + tests/fullduplex/minicpmo45/test_input.py | 131 + tests/fullduplex/openai/test_duplex_audio.py | 54 + .../openai/test_runtime_adapter_boundary.py | 139 + .../fullduplex/openai/test_websocket_actor.py | 145 + tests/fullduplex/test_client.py | 240 + .../models/minicpmo_4_5/test_hf_model_path.py | 54 +- .../models/minicpmo_4_5/test_llm2tts.py | 145 +- .../models/minicpmo_4_5/test_pipeline.py | 14 + .../test_minicpmo_4_5_omni.py | 1347 ++++ tests/test_config_factory.py | 63 + tests/test_data_entry_keys.py | 24 +- tests/test_omni_request.py | 27 +- tests/utils/test_mm_outputs_partition.py | 27 + tests/worker/test_native_duplex_hooks.py | 2576 ++++++ tests/worker/test_omni_gpu_model_runner.py | 96 +- vllm_omni/config/stage_config.py | 48 + vllm_omni/core/sched/__init__.py | 25 +- vllm_omni/core/sched/omni_ar_scheduler.py | 5 + .../core/sched/omni_generation_scheduler.py | 3 + vllm_omni/core/sched/omni_scheduler_mixin.py | 8 +- vllm_omni/core/sched/output.py | 4 + .../minicpmo_4_5_3gpu_stage1_replicas.yaml | 17 + .../minicpmo_4_5_4gpu_stage1_replicas.yaml | 17 + .../minicpmo_4_5_8x4090_stage1_replicas.yaml | 20 + vllm_omni/deploy/minicpmo_4_5_duplex.yaml | 36 + .../chunk_transfer_adapter.py | 4 + vllm_omni/engine/__init__.py | 8 + vllm_omni/engine/async_engine_utils.py | 160 + vllm_omni/engine/async_omni_engine.py | 521 +- vllm_omni/engine/membership_controller.py | 2 +- vllm_omni/engine/messages.py | 4 + vllm_omni/engine/orchestrator.py | 600 +- vllm_omni/engine/rpc_result_router.py | 161 + vllm_omni/engine/stage_pool.py | 135 +- vllm_omni/engine/stage_runtime.py | 1 - vllm_omni/entrypoints/async_omni.py | 199 + vllm_omni/entrypoints/client_request_state.py | 2 +- vllm_omni/entrypoints/omni_base.py | 11 +- vllm_omni/entrypoints/openai/api_server.py | 51 + .../entrypoints/openai/duplex_capability.py | 35 + vllm_omni/entrypoints/openai/serving_chat.py | 64 +- .../entrypoints/openai/video_stream_base.py | 121 +- vllm_omni/experimental/fullduplex/DESIGN.md | 1029 +++ vllm_omni/experimental/fullduplex/README.md | 94 +- vllm_omni/experimental/fullduplex/client.py | 417 + .../fullduplex/engine/__init__.py | 4 + .../fullduplex/engine/contracts.py | 276 + .../engine/duplex_control_client.py | 228 + .../fullduplex/engine/duplex_control_plane.py | 1063 +++ .../fullduplex/engine/duplex_runtime.py | 108 + .../fullduplex/engine/duplex_session.py | 534 ++ .../fullduplex/engine/intermediate.py | 99 + .../experimental/fullduplex/engine/lease.py | 129 + .../fullduplex/engine/messages.py | 130 + .../fullduplex/minicpmo45/__init__.py | 15 + .../fullduplex/minicpmo45/adapter.py | 304 + .../fullduplex/minicpmo45/compat.py | 43 + .../fullduplex/minicpmo45/data_plane.py | 838 ++ .../fullduplex/minicpmo45/input.py | 384 + .../fullduplex/minicpmo45/policy.py | 140 + .../fullduplex/minicpmo45/runtime.py | 354 + .../fullduplex/minicpmo45/serving_adapter.py | 96 + .../fullduplex/minicpmo45/session.py | 56 + .../fullduplex/minicpmo45/stage0.py | 795 ++ .../experimental/fullduplex/model_executor.py | 106 + .../fullduplex/openai/__init__.py | 6 + .../experimental/fullduplex/openai/audio.py | 185 + .../fullduplex/openai/chat_fallback.py | 252 + .../fullduplex/openai/commit_policy.py | 38 + .../fullduplex/openai/protocol.py | 1336 ++++ .../fullduplex/openai/realtime_input.py | 1233 +++ .../fullduplex/openai/realtime_output.py | 907 +++ .../fullduplex/openai/realtime_session.py | 139 + .../fullduplex/openai/realtime_state.py | 197 + .../fullduplex/openai/runtime_adapter.py | 215 + .../fullduplex/openai/runtime_bridge.py | 1298 ++++ .../experimental/fullduplex/openai/serving.py | 1764 +++++ .../fullduplex/openai/session_attachment.py | 486 ++ .../fullduplex/openai/session_runner.py | 1912 +++++ .../fullduplex/openai/websocket.py | 265 + vllm_omni/experimental/fullduplex/output.py | 36 + .../experimental/fullduplex/request_client.py | 383 + vllm_omni/inputs/data.py | 40 +- vllm_omni/inputs/preprocess.py | 14 + vllm_omni/metrics/stats.py | 21 +- .../models/minicpmo_4_5/minicpmo_4_5_omni.py | 1001 ++- .../minicpmo_4_5/minicpmo_4_5_omni_llm.py | 159 +- .../minicpmo_4_5/minicpmo_4_5_omni_tts.py | 1738 ++++- .../models/minicpmo_4_5/pipeline.py | 5 + .../minicpmo_4_5_omni.py | 621 +- vllm_omni/outputs/multimodal_accumulation.py | 91 + vllm_omni/outputs/output_processor.py | 36 +- vllm_omni/request.py | 6 + vllm_omni/utils/mm_outputs.py | 18 + vllm_omni/worker/gpu_ar_model_runner.py | 50 +- vllm_omni/worker/gpu_model_runner.py | 59 +- 142 files changed, 48389 insertions(+), 1013 deletions(-) create mode 100644 examples/online_serving/minicpmo/realtime_duplex_demo.py create mode 100644 examples/online_serving/minicpmo/realtime_web/__init__.py create mode 100644 examples/online_serving/minicpmo/realtime_web/__main__.py create mode 100644 examples/online_serving/minicpmo/realtime_web/app/index.html create mode 100644 examples/online_serving/minicpmo/realtime_web/app/static/app.js create mode 100644 examples/online_serving/minicpmo/realtime_web/app/static/pcm_worklet.js create mode 100644 examples/online_serving/minicpmo/realtime_web/app/static/playback_worklet.js create mode 100644 examples/online_serving/minicpmo/realtime_web/app/static/styles.css create mode 100644 examples/online_serving/minicpmo/realtime_web/server.py create mode 100644 tests/core/sched/test_sched_package_imports.py create mode 100644 tests/e2e/online_serving/minicpmo_realtime_duplex_scenarios.py create mode 100644 tests/e2e/online_serving/run_minicpmo_realtime_duplex_demo_pair.py create mode 100644 tests/e2e/online_serving/run_minicpmo_realtime_duplex_multi_session.py create mode 100644 tests/e2e/online_serving/run_minicpmo_realtime_duplex_soft_interrupt.py create mode 100644 tests/engine/test_correlated_rpc_client.py create mode 100644 tests/engine/test_duplex_import_boundary.py create mode 100644 tests/engine/test_multimodal_accumulation.py create mode 100644 tests/engine/test_rpc_result_router.py create mode 100644 tests/entrypoints/openai/test_duplex_protocol.py create mode 100644 tests/entrypoints/openai/test_duplex_session_attachment.py create mode 100644 tests/entrypoints/openai_api/test_duplex_capability.py create mode 100644 tests/entrypoints/openai_api/test_duplex_handler.py create mode 100644 tests/entrypoints/test_async_omni_duplex.py create mode 100644 tests/entrypoints/test_duplex_fence_propagation.py create mode 100644 tests/examples/test_minicpmo_realtime_duplex_simple_demo.py create mode 100644 tests/examples/test_minicpmo_realtime_web.py create mode 100644 tests/examples/test_minicpmo_realtime_web_server.py create mode 100644 tests/examples/test_minicpmo_realtime_web_static.py create mode 100644 tests/fullduplex/engine/test_duplex_control_client.py create mode 100644 tests/fullduplex/engine/test_duplex_control_plane.py create mode 100644 tests/fullduplex/engine/test_duplex_deploy_config.py create mode 100644 tests/fullduplex/engine/test_duplex_intermediate.py create mode 100644 tests/fullduplex/engine/test_duplex_lease.py create mode 100644 tests/fullduplex/engine/test_duplex_runtime.py create mode 100644 tests/fullduplex/minicpmo45/__init__.py create mode 100644 tests/fullduplex/minicpmo45/test_commit_policy.py create mode 100644 tests/fullduplex/minicpmo45/test_input.py create mode 100644 tests/fullduplex/openai/test_duplex_audio.py create mode 100644 tests/fullduplex/openai/test_runtime_adapter_boundary.py create mode 100644 tests/fullduplex/openai/test_websocket_actor.py create mode 100644 tests/fullduplex/test_client.py create mode 100644 tests/model_executor/stage_input_processors/test_minicpmo_4_5_omni.py create mode 100644 tests/worker/test_native_duplex_hooks.py create mode 100644 vllm_omni/deploy/minicpmo_4_5_3gpu_stage1_replicas.yaml create mode 100644 vllm_omni/deploy/minicpmo_4_5_4gpu_stage1_replicas.yaml create mode 100644 vllm_omni/deploy/minicpmo_4_5_8x4090_stage1_replicas.yaml create mode 100644 vllm_omni/deploy/minicpmo_4_5_duplex.yaml create mode 100644 vllm_omni/engine/async_engine_utils.py create mode 100644 vllm_omni/engine/rpc_result_router.py create mode 100644 vllm_omni/entrypoints/openai/duplex_capability.py create mode 100644 vllm_omni/experimental/fullduplex/DESIGN.md create mode 100644 vllm_omni/experimental/fullduplex/client.py create mode 100644 vllm_omni/experimental/fullduplex/engine/__init__.py create mode 100644 vllm_omni/experimental/fullduplex/engine/contracts.py create mode 100644 vllm_omni/experimental/fullduplex/engine/duplex_control_client.py create mode 100644 vllm_omni/experimental/fullduplex/engine/duplex_control_plane.py create mode 100644 vllm_omni/experimental/fullduplex/engine/duplex_runtime.py create mode 100644 vllm_omni/experimental/fullduplex/engine/duplex_session.py create mode 100644 vllm_omni/experimental/fullduplex/engine/intermediate.py create mode 100644 vllm_omni/experimental/fullduplex/engine/lease.py create mode 100644 vllm_omni/experimental/fullduplex/engine/messages.py create mode 100644 vllm_omni/experimental/fullduplex/minicpmo45/__init__.py create mode 100644 vllm_omni/experimental/fullduplex/minicpmo45/adapter.py create mode 100644 vllm_omni/experimental/fullduplex/minicpmo45/compat.py create mode 100644 vllm_omni/experimental/fullduplex/minicpmo45/data_plane.py create mode 100644 vllm_omni/experimental/fullduplex/minicpmo45/input.py create mode 100644 vllm_omni/experimental/fullduplex/minicpmo45/policy.py create mode 100644 vllm_omni/experimental/fullduplex/minicpmo45/runtime.py create mode 100644 vllm_omni/experimental/fullduplex/minicpmo45/serving_adapter.py create mode 100644 vllm_omni/experimental/fullduplex/minicpmo45/session.py create mode 100644 vllm_omni/experimental/fullduplex/minicpmo45/stage0.py create mode 100644 vllm_omni/experimental/fullduplex/model_executor.py create mode 100644 vllm_omni/experimental/fullduplex/openai/__init__.py create mode 100644 vllm_omni/experimental/fullduplex/openai/audio.py create mode 100644 vllm_omni/experimental/fullduplex/openai/chat_fallback.py create mode 100644 vllm_omni/experimental/fullduplex/openai/commit_policy.py create mode 100644 vllm_omni/experimental/fullduplex/openai/protocol.py create mode 100644 vllm_omni/experimental/fullduplex/openai/realtime_input.py create mode 100644 vllm_omni/experimental/fullduplex/openai/realtime_output.py create mode 100644 vllm_omni/experimental/fullduplex/openai/realtime_session.py create mode 100644 vllm_omni/experimental/fullduplex/openai/realtime_state.py create mode 100644 vllm_omni/experimental/fullduplex/openai/runtime_adapter.py create mode 100644 vllm_omni/experimental/fullduplex/openai/runtime_bridge.py create mode 100644 vllm_omni/experimental/fullduplex/openai/serving.py create mode 100644 vllm_omni/experimental/fullduplex/openai/session_attachment.py create mode 100644 vllm_omni/experimental/fullduplex/openai/session_runner.py create mode 100644 vllm_omni/experimental/fullduplex/openai/websocket.py create mode 100644 vllm_omni/experimental/fullduplex/output.py create mode 100644 vllm_omni/experimental/fullduplex/request_client.py create mode 100644 vllm_omni/outputs/multimodal_accumulation.py diff --git a/examples/online_serving/minicpmo/README.md b/examples/online_serving/minicpmo/README.md index e1c084224..11f6f8195 100644 --- a/examples/online_serving/minicpmo/README.md +++ b/examples/online_serving/minicpmo/README.md @@ -1,159 +1,139 @@ # MiniCPM-o 4.5: Online serving -OpenAI-compatible `/v1/chat/completions` serving for **MiniCPM-o 4.5**, plus a -Gradio UI and curl / Python clients. +This directory contains MiniCPM-o 4.5 online serving demos for vLLM-Omni. +Inputs can include text, image, audio, or video; outputs are text and optional +24 kHz speech. -Inputs: text / image / audio / video. Outputs: text and optional **24 kHz** speech. +For the experimental native duplex runtime architecture, lifecycle invariants, +capability boundary, and validation scope, see +[`vllm_omni/experimental/fullduplex/DESIGN.md`](../../../vllm_omni/experimental/fullduplex/DESIGN.md). ## Installation -Please refer to [README.md](../../../README.md). Install the talker extra: +Install vLLM-Omni with the MiniCPM-o talker dependencies: ```bash pip install 'vllm-omni[minicpmo]' + +# From a source checkout: +pip install -e '.[minicpmo]' ``` -## Launch the server +The `minicpmo` extra installs `stepaudio2-minicpmo` and its audio dependencies, +including `librosa`. -The deploy config auto-loads via `--omni`; the default -`vllm_omni/deploy/minicpmo_4_5.yaml` targets a single-GPU layout (thinker -and talker + t2w co-located on GPU 0). For other hardware layouts pick -one of the deploy variants below. +## Start the backend server -| deploy config | GPUs | Notes | -|---|---|---| -| `minicpmo_4_5.yaml` (default) | 1 | Thinker and talker+t2w co-located on GPU0. | -| `minicpmo_4_5_2gpu.yaml` | 2 | Thinker on GPU0, talker+t2w on GPU1. | -| `minicpmo_4_5_3gpu.yaml` | 3 | Thinker 2-way TP on GPU0/1, talker+t2w share GPU2. | -| `minicpmo_4_5_8x4090.yaml` | 8 | Full 8x4090 layout. | +Pick a deploy config that matches your GPU layout: -Default (single-GPU): +| config | GPUs | TP | Notes | +|---|---|---|---| +| `minicpmo_4_5.yaml` | 1 | 1 | Thinker and talker+t2w co-located on GPU0. | +| `minicpmo_4_5_2gpu.yaml` | 2 | 1 | Thinker on GPU0, talker+t2w on GPU1. | +| `minicpmo_4_5_3gpu.yaml` | 3 | 2 | Thinker 2-way TP on GPU0/1, talker+t2w share GPU2. | +| `minicpmo_4_5_8x4090.yaml` | 8 | 4 | Thinker 4-way TP on GPU0-3, talker+t2w on GPU4. | +| `minicpmo_4_5_3gpu_stage1_replicas.yaml` | 3 | 1 | Experimental validation profile: Thinker on GPU0, two talker+Token2wav replicas on GPU1/2. | +| `minicpmo_4_5_4gpu_stage1_replicas.yaml` | 4 | 1 | Experimental validation profile: Thinker on GPU0, three talker+Token2wav replicas on GPU1/2/3. | +| `minicpmo_4_5_8x4090_stage1_replicas.yaml` | 8 | 4 | Experimental validation profile: Thinker 4-way TP on GPU0-3, four talker+Token2wav replicas on GPU4-7. | +| `minicpmo_4_5_duplex.yaml` | 2 | 1 | Experimental native duplex profile. | ```bash -vllm serve openbmb/MiniCPM-o-4_5 --omni \ +vllm-omni serve openbmb/MiniCPM-o-4_5 \ + --omni \ + --deploy-config vllm_omni/deploy/minicpmo_4_5.yaml \ --trust-remote-code \ --host 0.0.0.0 --port 8099 ``` -Other layouts: +For local ModelScope checkpoints, replace `openbmb/MiniCPM-o-4_5` with the +checkpoint path. To start the experimental native duplex backend, use +`vllm_omni/deploy/minicpmo_4_5_duplex.yaml`. -```bash -vllm serve openbmb/MiniCPM-o-4_5 --omni \ - --deploy-config vllm_omni/deploy/minicpmo_4_5_3gpu.yaml \ - --trust-remote-code \ - --host 0.0.0.0 --port 8099 -``` +The `*_stage1_replicas.yaml` files exercise composite Stage1 replica routing +and failure recovery. They are validation profiles, not recommended production +entrypoints. -### Stage-based CLI (optional) +## Send chat requests -Stage 0 (thinker + API) and stage 1 (talker) can run in separate processes: - -```bash -# Stage 0 -CUDA_VISIBLE_DEVICES=0 vllm serve openbmb/MiniCPM-o-4_5 --omni \ - --trust-remote-code --port 8099 --stage-id 0 \ - --omni-master-address 127.0.0.1 --omni-master-port 26000 - -# Stage 1 (headless) -CUDA_VISIBLE_DEVICES=1 vllm serve openbmb/MiniCPM-o-4_5 --omni \ - --trust-remote-code --stage-id 1 --headless \ - --omni-master-address 127.0.0.1 --omni-master-port 26000 -``` - -### Per-stage overrides - -```bash -vllm serve openbmb/MiniCPM-o-4_5 --omni --trust-remote-code --port 8099 \ - --stage-overrides '{"0": {"gpu_memory_utilization": 0.65}}' -``` - -## Send multimodal requests - -```bash -cd examples/online_serving/minicpmo -``` - -### curl +From this directory, run the MiniCPM-specific curl or Python clients: ```bash bash run_curl_multimodal_generation.sh text bash run_curl_multimodal_generation.sh use_image -bash run_curl_multimodal_generation.sh use_audio '["text"]' # text-only -``` +bash run_curl_multimodal_generation.sh use_audio '["text"]' -Text + speech smoke test (TTS needs top-level `chat_template_kwargs`): - -```bash -curl http://localhost:8099/v1/chat/completions \ - -H "Content-Type: application/json" \ - -d '{ - "model": "openbmb/MiniCPM-o-4_5", - "messages": [{"role": "user", "content": "Say hello, then introduce vLLM in one sentence."}], - "modalities": ["text", "audio"], - "chat_template_kwargs": {"use_tts_template": true} - }' -``` - -### Python client - -```bash python openai_chat_completion_client_for_multimodal_generation.py \ --query-type use_image \ - --port 8099 \ - --host localhost - -# Text-only (faster; no <|tts_bos|>) -python openai_chat_completion_client_for_multimodal_generation.py \ - --query-type text \ - --modalities text \ - --prompt "Briefly introduce yourself." -``` - -Shared helpers also work if you pass MiniCPM defaults yourself: - -```bash -python ../openai_chat_completion_client_for_multimodal_generation.py \ - --model openbmb/MiniCPM-o-4_5 \ - --query-type text \ + --host localhost \ --port 8099 ``` -(Note: the shared client does **not** set `use_tts_template`; prefer the -MiniCPM-specific client above for speech.) +Speech output requires `chat_template_kwargs.use_tts_template=true`. Put that +field at the request root for curl; the OpenAI Python SDK can merge it from +`extra_body`. -### Gradio demo +## Launch the Gradio demo ```bash -bash run_gradio_demo.sh +bash examples/online_serving/minicpmo/run_gradio_demo.sh -# Or: -python gradio_demo.py \ +# Or run the Python entry point directly: +python examples/online_serving/minicpmo/gradio_demo.py \ --minicpmo45-api-base http://localhost:8099/v1 \ --minicpmo45-model openbmb/MiniCPM-o-4_5 \ --port 7862 ``` -Open `http://:7862`. Uncheck **"Generate speech output (TTS)"** for -text-only responses. +Open `http://:7862` in a browser. -## Modality control +## Run the Realtime duplex CLI demo -| Modalities | Output | -|---|---| -| `["text"]` | Text only (no TTS bos) | -| `["text", "audio"]` / unset | Text + 24 kHz speech | +After the duplex backend is running, stream one WAV through the Realtime +WebSocket endpoint: -Speech requires `chat_template_kwargs.use_tts_template=true` so the chat -template appends `<|tts_bos|>`. For **curl**, put that field at the request -root; nested `extra_body` is ignored. The OpenAI Python SDK may use -`extra_body` because it merges those fields into the root. +```bash +python examples/online_serving/minicpmo/realtime_duplex_demo.py \ + --url ws://localhost:8099/v1/realtime?duplex=1 \ + --model openbmb/MiniCPM-o-4_5 \ + --input-wav /path/to/input_16k_mono_pcm16.wav \ + --ref-audio /path/to/MiniCPM-o-Demo/assets/ref_audio/ref_minicpm_signature.wav \ + --output-dir /tmp/minicpmo_realtime_duplex_demo +``` -## Notes +## Open the experimental browser client -- Stage 1 is capped at `max_num_seqs: 1` in the deploy YAML (talker shares - request-0 audio metadata). -- Output audio is base64 WAV in `message.audio.data` (24 kHz mono). -- Offline counterpart: - [`examples/offline_inference/minicpmo/`](../../offline_inference/minicpmo/) -- Recipe: - [`recipes/OpenBMB/MiniCPM-o-4_5.md`](../../../recipes/OpenBMB/MiniCPM-o-4_5.md) +The browser UI serves the page and proxies the same-origin Realtime WebSocket to +the backend: + +```bash +python -m examples.online_serving.minicpmo.realtime_web \ + --port 7862 \ + --ws-backend ws://127.0.0.1:8099 \ + --ref-audio /path/to/MiniCPM-o-Demo/assets/ref_audio/ref_minicpm_signature.wav +``` + +Open `http://:7862/`. When using a reverse proxy, open the URL mapped to +port `7862`; the browser derives its WebSocket endpoint relative to that URL. + +If the page proxy serves HTTP but does not forward WebSocket upgrades, point the +browser at a separately exposed Realtime endpoint: + +```bash +python -m examples.online_serving.minicpmo.realtime_web \ + --port 7862 \ + --ws-backend ws://127.0.0.1:8099 \ + --public-realtime-url wss://public.example/v1/realtime +``` + +## Validate soft-interrupt behavior + +The soft-interrupt E2E driver defaults to `--validation-mode model-policy`, +which checks lifecycle and streaming invariants for arbitrary input audio. The +stronger `response-required` mode is diagnostic: it requires a purpose-built +two-response WAV, its `--input-sha256`, and an +`--expect-second-response-substring` value. + +## Related examples + +- [Offline MiniCPM-o inference](../../offline_inference/minicpmo/) +- [MiniCPM-o 4.5 recipe](../../../recipes/OpenBMB/MiniCPM-o-4_5.md) diff --git a/examples/online_serving/minicpmo/realtime_duplex_demo.py b/examples/online_serving/minicpmo/realtime_duplex_demo.py new file mode 100644 index 000000000..fcb7042a0 --- /dev/null +++ b/examples/online_serving/minicpmo/realtime_duplex_demo.py @@ -0,0 +1,399 @@ +"""Minimal single-input MiniCPM-o 4.5 Realtime duplex demo. + +Run this after starting the duplex server. Strict lifecycle, overlap, and +multi-session validation lives under ``tests/e2e/online_serving``. +""" + +from __future__ import annotations + +import argparse +import asyncio +import base64 +import json +import sys +import time +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[3] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from vllm_omni.experimental.fullduplex.client import ( # noqa: E402 + PCM16_BYTES_PER_SAMPLE, + PCM16_SAMPLE_RATE, + RealtimeDuplexClient, + RealtimeEventCollector, + build_realtime_url, + read_pcm16_wav, + wait_for, + write_pcm16_wav, +) + + +class _StreamingOutputWriter: + """Persist and report output deltas as the client receives them.""" + + def __init__(self, output_dir: Path) -> None: + self.output_dir = output_dir + self.audio_chunk_dir = output_dir / "audio_chunks" + self.audio_chunk_paths: list[Path] = [] + self.output_dir.mkdir(parents=True, exist_ok=True) + self.audio_chunk_dir.mkdir(parents=True, exist_ok=True) + (self.output_dir / "output.pcm").write_bytes(b"") + + def handle(self, event: dict[str, object]) -> None: + event_type = event.get("type") + if event_type in { + "response.audio_transcript.delta", + "response.output_text.delta", + }: + delta = event.get("delta") + if isinstance(delta, str) and delta: + print(delta, end="", file=sys.stderr, flush=True) + return + if event_type != "response.audio.delta": + return + + delta = event.get("delta") or event.get("audio") + if not isinstance(delta, str) or not delta: + return + try: + pcm16 = base64.b64decode(delta) + except ValueError: + return + if not pcm16: + return + + chunk_index = len(self.audio_chunk_paths) + 1 + chunk_path = self.audio_chunk_dir / f"chunk_{chunk_index:04d}.wav" + sample_rate_hz = event.get("sample_rate_hz") + if not isinstance(sample_rate_hz, int) or sample_rate_hz <= 0: + sample_rate_hz = 24_000 + with (self.output_dir / "output.pcm").open("ab") as output_pcm: + output_pcm.write(pcm16) + write_pcm16_wav(chunk_path, pcm16, sample_rate_hz=sample_rate_hz) + self.audio_chunk_paths.append(chunk_path) + print( + f"\n[audio chunk {chunk_index}: {len(pcm16)} bytes -> {chunk_path}]", + file=sys.stderr, + flush=True, + ) + + +class _StreamingEventCollector(RealtimeEventCollector): + def __init__(self, writer: _StreamingOutputWriter) -> None: + super().__init__() + self._writer = writer + + def add(self, event: dict[str, object], *, received_at_s: float | None = None) -> None: + super().add(event, received_at_s=received_at_s) + self._writer.handle(self.events[-1]) + + +def _input_committed_index( + events: list[dict[str, object]], + after_index: int, +) -> int | None: + for index, event in enumerate(events[max(after_index, 0) :], start=max(after_index, 0)): + if event.get("type") == "input_audio_buffer.committed": + return index + return None + + +def _post_commit_model_decision( + events: list[dict[str, object]], + committed_index: int | None, +) -> str | None: + if committed_index is None: + return None + for event in events[committed_index + 1 :]: + event_type = event.get("type") + if event_type == "response.listen": + return "listen" + if event_type == "response.done": + response = event.get("response") + if not isinstance(response, dict) or response.get("status") != "cancelled": + return "speak" + return None + + +def _latest_model_decision( + events: list[dict[str, object]], + after_index: int, +) -> str | None: + decision: str | None = None + for event in events[max(after_index, 0) :]: + event_type = event.get("type") + if event_type == "response.listen": + decision = "listen" + elif event_type == "response.done": + response = event.get("response") + if not isinstance(response, dict) or response.get("status") != "cancelled": + decision = "speak" + return decision + + +def _chunk_period_ms(events: list[dict[str, object]]) -> int: + for event in reversed(events): + session = event.get("session") + if not isinstance(session, dict): + continue + capabilities = session.get("capabilities") + if not isinstance(capabilities, dict): + continue + chunk_period_ms = capabilities.get("chunk_period_ms") + if isinstance(chunk_period_ms, int) and chunk_period_ms > 0: + return chunk_period_ms + return 1000 + + +def _has_residual_model_unit(pcm16: bytes, *, chunk_period_ms: int) -> bool: + unit_bytes = PCM16_SAMPLE_RATE * PCM16_BYTES_PER_SAMPLE * chunk_period_ms // 1000 + return bool(unit_bytes > 0 and len(pcm16) % unit_bytes) + + +def _response_in_progress(events: list[dict[str, object]]) -> bool: + return sum(event.get("type") == "response.created" for event in events) > sum( + event.get("type") == "response.done" for event in events + ) + + +def _event_count_after( + events: list[dict[str, object]], + event_type: str, + index: int | None, +) -> int: + if index is None: + return 0 + return sum(event.get("type") == event_type for event in events[index + 1 :]) + + +def _ref_audio_data_url(path: str | None) -> str | None: + if path is None: + return None + ref_path = Path(path).expanduser() + return "data:audio/wav;base64," + base64.b64encode(ref_path.read_bytes()).decode("ascii") + + +async def run_demo(args: argparse.Namespace) -> dict[str, object]: + input_pcm16 = read_pcm16_wav(Path(args.input_wav)) + if not input_pcm16: + raise ValueError("input WAV has no audio") + + output_dir = Path(args.output_dir) + stream_writer = _StreamingOutputWriter(output_dir) + + url = build_realtime_url( + args.url, + args.model, + autostart=False if args.ref_audio else None, + session_id=args.session_id, + ) + client = RealtimeDuplexClient(url) + client.events = _StreamingEventCollector(stream_writer) + async with client: + await client.configure( + args.model, + ref_audio=_ref_audio_data_url(args.ref_audio), + session_id=args.session_id, + timeout_s=args.timeout_s, + ) + stream_event_cursor = len(client.events.events) + await client.stream_pcm16( + input_pcm16, + chunk_ms=args.chunk_ms, + realtime=not args.no_realtime_pacing, + ) + commit_event_cursor = len(client.events.events) + stream_decision = _latest_model_decision(client.events.events, stream_event_cursor) + input_has_residual_model_unit = _has_residual_model_unit( + input_pcm16, + chunk_period_ms=_chunk_period_ms(client.events.events), + ) + wait_for_post_commit_decision = False + commit_sent_at_s = time.monotonic() + await client.commit() + wait_error: str | None = None + committed_index: int | None = None + post_commit_decision: str | None = None + try: + await wait_for( + lambda: _input_committed_index(client.events.events, commit_event_cursor) is not None, + timeout_s=args.timeout_s, + label="input_audio_buffer.committed", + ) + committed_index = _input_committed_index(client.events.events, commit_event_cursor) + stream_decision = _latest_model_decision(client.events.events[: committed_index + 1], stream_event_cursor) + wait_for_post_commit_decision = input_has_residual_model_unit or _response_in_progress( + client.events.events[: committed_index + 1] + ) + if wait_for_post_commit_decision: + await wait_for( + lambda: _post_commit_model_decision(client.events.events, committed_index) is not None, + timeout_s=args.timeout_s, + label="post-commit model decision or response drain", + ) + post_commit_decision = _post_commit_model_decision(client.events.events, committed_index) + except TimeoutError as exc: + wait_error = str(exc) + await client.acknowledge_playback() + close_error: str | None = None + try: + await client.close_session(timeout_s=args.timeout_s) + except TimeoutError as exc: + close_error = str(exc) + + audio = client.events.audio_bytes() + first_text_at_s = client.events.first_received_at( + "response.audio_transcript.delta", + "response.output_text.delta", + after_s=commit_sent_at_s, + ) + first_audio_at_s = client.events.first_received_at( + "response.audio.delta", + after_s=commit_sent_at_s, + ) + response_created_at_s = client.events.first_received_at( + "response.created", + after_s=commit_sent_at_s, + ) + response_done_at_s = client.events.first_received_at( + "response.done", + after_s=commit_sent_at_s, + ) + audio_duration_s = len(audio) / (client.events.output_sample_rate_hz * 2) + response_generation_s = ( + response_done_at_s - response_created_at_s + if response_done_at_s is not None and response_created_at_s is not None + else None + ) + transcript_deltas = [ + str(event.get("delta", "")) + for event in client.events.events + if event.get("type") + in { + "response.audio_transcript.delta", + "response.output_text.delta", + } + ] + response_id = client.events.response_ids[0] if client.events.response_ids else None + timing = client.events.timing_summary( + after_s=commit_sent_at_s, + input_committed_at_s=commit_sent_at_s, + response_id=response_id, + ) + errors = client.events.errors() + if wait_error: + errors.append({"type": "client.timeout", "message": wait_error}) + if close_error: + errors.append({"type": "client.timeout", "message": close_error}) + model_decision = post_commit_decision or stream_decision + (output_dir / "events.jsonl").write_text( + "".join(json.dumps(event, ensure_ascii=False) + "\n" for event in client.events.events), + encoding="utf-8", + ) + (output_dir / "output.pcm").write_bytes(audio) + if audio: + write_pcm16_wav( + output_dir / "output.wav", + audio, + sample_rate_hz=client.events.output_sample_rate_hz, + ) + + result = { + "ok": ( + client.events.count("session.created") > 0 + and client.events.count("session.closed") > 0 + and not errors + and model_decision is not None + and (bool(audio) or not args.require_audio) + ), + "model_decision": model_decision, + "post_commit": { + "input_committed_event_index": committed_index, + "decision": post_commit_decision, + "decision_required": wait_for_post_commit_decision, + "input_had_residual_model_unit": input_has_residual_model_unit, + "response_listen_count": _event_count_after( + client.events.events, + "response.listen", + committed_index, + ), + "response_done_count": _event_count_after( + client.events.events, + "response.done", + committed_index, + ), + }, + "audio_bytes": len(audio), + "audio_chunk_count": len(stream_writer.audio_chunk_paths), + "audio_chunk_files": [str(path) for path in stream_writer.audio_chunk_paths], + "output_sample_rate_hz": client.events.output_sample_rate_hz, + "latency": { + "ttft_ms": ( + round((first_text_at_s - commit_sent_at_s) * 1000, 2) if first_text_at_s is not None else None + ), + "ttfp_ms": ( + round((first_audio_at_s - commit_sent_at_s) * 1000, 2) if first_audio_at_s is not None else None + ), + "rtf": ( + round(response_generation_s / audio_duration_s, 4) + if response_generation_s is not None and audio_duration_s > 0 + else None + ), + "response_generation_ms": ( + round(response_generation_s * 1000, 2) if response_generation_s is not None else None + ), + "text_stream_ms": ( + round((response_done_at_s - first_text_at_s) * 1000, 2) + if response_done_at_s is not None and first_text_at_s is not None + else None + ), + "transcript_delta_count": len(transcript_deltas), + "audio_duration_s": round(audio_duration_s, 3), + "measurement_origin": "input_audio_buffer.commit send", + }, + "timing": timing, + "response_ids": client.events.response_ids, + "transcript": "".join(transcript_deltas), + "errors": errors, + "output_dir": str(output_dir), + } + (output_dir / "result.json").write_text( + json.dumps(result, ensure_ascii=False, indent=2), + encoding="utf-8", + ) + return result + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--url", default="ws://localhost:8099/v1/realtime?duplex=1") + parser.add_argument("--model", default="openbmb/MiniCPM-o-4_5") + parser.add_argument("--session-id") + parser.add_argument("--input-wav", required=True) + parser.add_argument( + "--ref-audio", + required=True, + help=( + "Reference WAV for the MiniCPM-o duplex assistant voice. " + "This demo matches the official flow by always providing a reference audio clip." + ), + ) + parser.add_argument("--output-dir", default="/tmp/minicpmo_realtime_duplex_demo") + parser.add_argument("--chunk-ms", type=int, default=200) + parser.add_argument("--timeout-s", type=float, default=60.0) + parser.add_argument("--no-realtime-pacing", action="store_true") + parser.add_argument("--require-audio", action="store_true") + return parser.parse_args() + + +def main() -> None: + result = asyncio.run(run_demo(parse_args())) + print(json.dumps(result, ensure_ascii=False, indent=2)) + if not result["ok"]: + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/examples/online_serving/minicpmo/realtime_web/__init__.py b/examples/online_serving/minicpmo/realtime_web/__init__.py new file mode 100644 index 000000000..9a5de5e18 --- /dev/null +++ b/examples/online_serving/minicpmo/realtime_web/__init__.py @@ -0,0 +1,5 @@ +"""Browser demo for the experimental full-duplex runtime.""" + +from .server import build_app + +__all__ = ["build_app"] diff --git a/examples/online_serving/minicpmo/realtime_web/__main__.py b/examples/online_serving/minicpmo/realtime_web/__main__.py new file mode 100644 index 000000000..f5f6e402d --- /dev/null +++ b/examples/online_serving/minicpmo/realtime_web/__main__.py @@ -0,0 +1,4 @@ +from .server import main + +if __name__ == "__main__": + main() diff --git a/examples/online_serving/minicpmo/realtime_web/app/index.html b/examples/online_serving/minicpmo/realtime_web/app/index.html new file mode 100644 index 000000000..46ab15907 --- /dev/null +++ b/examples/online_serving/minicpmo/realtime_web/app/index.html @@ -0,0 +1,79 @@ + + + + + + + MiniCPM-o 4.5 Voice + + + +
+
+

Experimental full-duplex runtime

+

MiniCPM-o 4.5 Voice

+
+ Offline +
+ +
+
+
+ + + + +
+
+ + +
+
+ Mic +
+
+
+
Model
Idle
+
Playback
Idle
+
Session
00:00
+
+
+ +
+
+
+

Live transcript

+

Conversation

+
+ Model-controlled listen / speak +
+
+

Start a session and speak naturally.

+
+
+ +
+ + Event log + 0 events + +
+ No active connection + +
+

+      
+
+ + + + + diff --git a/examples/online_serving/minicpmo/realtime_web/app/static/app.js b/examples/online_serving/minicpmo/realtime_web/app/static/app.js new file mode 100644 index 000000000..e8a3047c6 --- /dev/null +++ b/examples/online_serving/minicpmo/realtime_web/app/static/app.js @@ -0,0 +1,671 @@ +(() => { + 'use strict'; + + const config = window.FULL_DUPLEX_CONFIG || {}; + const callButton = document.getElementById('callButton'); + const muteButton = document.getElementById('muteButton'); + const cameraButton = document.getElementById('cameraButton'); + const cameraPreview = document.getElementById('cameraPreview'); + const promptPreset = document.getElementById('promptPreset'); + const systemPromptInput = document.getElementById('systemPrompt'); + const connectionState = document.getElementById('connectionState'); + const modelState = document.getElementById('modelState'); + const playbackState = document.getElementById('playbackState'); + const sessionTimer = document.getElementById('sessionTimer'); + const meterFill = document.getElementById('meterFill'); + const conversation = document.getElementById('conversation'); + const emptyConversation = document.getElementById('emptyConversation'); + const eventLog = document.getElementById('eventLog'); + const eventCount = document.getElementById('eventCount'); + const runtimeDetail = document.getElementById('runtimeDetail'); + const clearLogButton = document.getElementById('clearLogButton'); + + const INPUT_RATE = 16000; + const OUTPUT_RATE = 24000; + const SEND_INTERVAL_MS = 200; + const ECHO_GUARD_MS = 300; + const INITIAL_PLAYBACK_BUFFER_MS = 400; + const SESSION_CLOSE_TIMEOUT_MS = 1000; + + // Default prompts mirroring the official MiniCPM-o-Demo presets + // (assets/presets/{omni,audio_duplex}/*.yaml). + const PROMPT_PRESETS = { + omni: 'Streaming Omni Conversation.', + chinese_call: '扮演一个具有以上声音特征的助手。请认真、高质量地回复用户的问题。' + + '请用高自然度的方式和用户聊天。你处于双工模式,可以一边听、一边说。' + + '你是由面壁智能开发的人工智能助手:面壁小钢炮。', + english_call: 'Replicate the tone and style from the input audio. Your task is to be ' + + 'a helpful assistant using this voice pattern. Please answer the user\'s questions ' + + 'seriously and in a high quality. Please chat with the user in a high naturalness ' + + 'style. You are in duplex mode, where you can listen and speak at the same time.', + }; + + let socket = null; + let mediaStream = null; + let captureContext = null; + let captureNode = null; + let playbackContext = null; + let playbackNode = null; + let sendTimer = null; + let clockTimer = null; + let startedAt = 0; + let running = false; + let muted = false; + let assistantActive = false; + let captureRate = INPUT_RATE; + let cameraStream = null; + let cameraTimer = null; + let cameraPendingFrame = null; + const cameraCanvas = document.createElement('canvas'); + let playbackRate = OUTPUT_RATE; + let pendingCapture = []; + let currentResponseId = null; + let responseHasAudio = false; + let logCount = 0; + let liveUserTurn = null; + let liveAssistantTurn = null; + let sessionCloseResolver = null; + + function staticAssetUrl(path) { + const version = String(config.appVersion || '').trim(); + return version ? `${path}?v=${encodeURIComponent(version)}` : path; + } + + if (promptPreset && systemPromptInput) { + promptPreset.addEventListener('change', () => { + const preset = PROMPT_PRESETS[promptPreset.value]; + if (preset !== undefined) systemPromptInput.value = preset; + }); + systemPromptInput.addEventListener('input', () => { + promptPreset.value = 'custom'; + }); + } + + function realtimeUrl() { + const url = new URL(config.realtimePath, window.location.href); + url.protocol = window.location.protocol === 'https:' ? 'wss:' : 'ws:'; + url.searchParams.set('duplex', '1'); + url.searchParams.set('model', config.model || 'openbmb/MiniCPM-o-4_5'); + url.searchParams.set('minicpmo45_native_duplex', '1'); + url.searchParams.set('autostart', '0'); + return url.toString(); + } + + function setConnection(label, kind) { + connectionState.textContent = label; + connectionState.className = `status status-${kind}`; + } + + function setModel(label) { + modelState.textContent = label; + } + + function setPlayback(label) { + playbackState.textContent = label; + } + + function compactEvent(event) { + const fields = []; + const payload = event.event || event; + const responseId = responseIdOf(event); + if (responseId) fields.push(`response=${responseId}`); + if (event.item_id) fields.push(`item=${event.item_id}`); + if (event.code) fields.push(`code=${event.code}`); + if (event.response && event.response.status) fields.push(`status=${event.response.status}`); + if (payload.played_ms !== undefined) fields.push(`played=${payload.played_ms}ms`); + if (payload.committed_ms !== undefined) fields.push(`committed=${payload.committed_ms}ms`); + if (payload.history_committed !== undefined) fields.push(`history=${payload.history_committed}`); + return fields.join(' '); + } + + function appendLog(message, error = false) { + const time = new Date().toLocaleTimeString([], { hour12: false }); + const line = document.createElement('span'); + if (error) line.className = 'log-error'; + line.textContent = `${time} ${message}\n`; + eventLog.appendChild(line); + eventLog.scrollTop = eventLog.scrollHeight; + logCount += 1; + eventCount.textContent = `${logCount} ${logCount === 1 ? 'event' : 'events'}`; + } + + function appendEventLog(event) { + const detail = compactEvent(event); + appendLog(`${event.type || 'unknown'}${detail ? ` ${detail}` : ''}`, event.type === 'error'); + } + + function responseIdOf(event) { + return event.response_id || (event.response && event.response.id) || null; + } + + function ensureTurn(role) { + const existing = role === 'user' ? liveUserTurn : liveAssistantTurn; + if (existing) return existing; + if (emptyConversation) emptyConversation.remove(); + const row = document.createElement('div'); + row.className = `turn turn-${role} turn-live`; + const label = document.createElement('div'); + label.className = 'turn-role'; + label.textContent = role === 'user' ? 'You' : 'Assistant'; + const text = document.createElement('div'); + text.className = 'turn-text'; + row.append(label, text); + conversation.appendChild(row); + conversation.scrollTop = conversation.scrollHeight; + const turn = { row, text, value: '' }; + if (role === 'user') liveUserTurn = turn; + else liveAssistantTurn = turn; + return turn; + } + + function addTranscript(role, delta) { + if (!delta) return; + const turn = ensureTurn(role); + turn.value += delta; + turn.text.textContent = turn.value; + conversation.scrollTop = conversation.scrollHeight; + } + + function finishTranscript(role, finalText = '') { + const turn = role === 'user' ? liveUserTurn : liveAssistantTurn; + if (!turn && !finalText) return; + const current = turn || ensureTurn(role); + if (finalText) { + current.value = finalText; + current.text.textContent = finalText; + } + current.row.classList.remove('turn-live'); + if (role === 'user') liveUserTurn = null; + else liveAssistantTurn = null; + } + + function bytesToBase64(bytes) { + let binary = ''; + const chunkSize = 0x8000; + for (let offset = 0; offset < bytes.length; offset += chunkSize) { + binary += String.fromCharCode(...bytes.subarray(offset, offset + chunkSize)); + } + return btoa(binary); + } + + function int16ToBase64(pcm) { + return bytesToBase64(new Uint8Array(pcm.buffer, pcm.byteOffset, pcm.byteLength)); + } + + function base64ToBytes(encoded) { + const binary = atob(encoded); + const bytes = new Uint8Array(binary.length); + for (let index = 0; index < binary.length; index += 1) bytes[index] = binary.charCodeAt(index); + return bytes; + } + + function resampleInt16(input, sourceRate, targetRate) { + if (sourceRate === targetRate) return input; + const ratio = sourceRate / targetRate; + const output = new Int16Array(Math.floor(input.length / ratio)); + for (let index = 0; index < output.length; index += 1) { + const start = Math.floor(index * ratio); + const end = Math.max(start + 1, Math.min(input.length, Math.floor((index + 1) * ratio))); + let sum = 0; + for (let source = start; source < end; source += 1) sum += input[source]; + output[index] = sum / (end - start); + } + return output; + } + + async function decodeAudioDelta(event) { + const encoded = event.delta || (event.response && event.response.audio); + if (!encoded) return null; + const bytes = base64ToBytes(encoded); + const format = String(event.format || event.audio_format || 'pcm16').toLowerCase(); + const sourceRate = Number(event.sample_rate_hz || event.sample_rate || OUTPUT_RATE); + if (format.includes('f32')) { + const floats = new Float32Array(bytes.buffer, bytes.byteOffset, Math.floor(bytes.byteLength / 4)); + const pcm = new Int16Array(floats.length); + for (let index = 0; index < floats.length; index += 1) { + const sample = Math.max(-1, Math.min(1, floats[index])); + pcm[index] = sample < 0 ? sample * 32768 : sample * 32767; + } + return { pcm, sourceRate }; + } + if (format.includes('wav')) { + const decoded = await playbackContext.decodeAudioData(bytes.buffer.slice(0)); + const channel = decoded.getChannelData(0); + const pcm = new Int16Array(channel.length); + for (let index = 0; index < channel.length; index += 1) { + const sample = Math.max(-1, Math.min(1, channel[index])); + pcm[index] = sample < 0 ? sample * 32768 : sample * 32767; + } + return { pcm, sourceRate: decoded.sampleRate }; + } + return { + pcm: new Int16Array(bytes.buffer, bytes.byteOffset, Math.floor(bytes.byteLength / 2)), + sourceRate, + }; + } + + function updateMeter(pcm) { + let peak = 0; + for (let index = 0; index < pcm.length; index += 8) peak = Math.max(peak, Math.abs(pcm[index])); + meterFill.style.width = `${Math.min(100, (peak / 32768) * 150).toFixed(0)}%`; + } + + function microphoneUploadEnabled() { + return running && !muted; + } + + function flushCapture() { + if (!socket || socket.readyState !== WebSocket.OPEN || pendingCapture.length === 0) return; + if (!microphoneUploadEnabled()) { + pendingCapture = []; + return; + } + const length = pendingCapture.reduce((total, chunk) => total + chunk.length, 0); + const merged = new Int16Array(length); + let offset = 0; + for (const chunk of pendingCapture) { + merged.set(chunk, offset); + offset += chunk.length; + } + pendingCapture = []; + const pcm = resampleInt16(merged, captureRate, INPUT_RATE); + const appendEvent = { + type: 'input_audio_buffer.append', + audio: int16ToBase64(pcm), + format: 'pcm16', + sample_rate_hz: INPUT_RATE, + }; + // Omni duplex: ~1 fps camera frame rides the audio append (official + // MiniCPM-o-Demo contract: one base64 JPEG per ~1 s chunk). + if (cameraPendingFrame) { + appendEvent.video_frames = [cameraPendingFrame]; + cameraPendingFrame = null; + } + socket.send(JSON.stringify(appendEvent)); + } + + function beginAssistant(responseId) { + currentResponseId = responseId || currentResponseId; + responseHasAudio = false; + assistantActive = true; + setModel('Speaking'); + } + + function feedPlayback(decoded, responseId) { + if (!decoded || !decoded.pcm || decoded.pcm.length === 0 || !playbackNode) return; + const pcm = resampleInt16(decoded.pcm, decoded.sourceRate, playbackRate); + responseHasAudio = true; + assistantActive = true; + setPlayback('Buffering'); + playbackNode.port.postMessage({ + type: 'audio', + pcm, + responseId: responseId || currentResponseId, + initialBufferMs: INITIAL_PLAYBACK_BUFFER_MS, + }, [pcm.buffer]); + } + + function requestPlaybackDrain(responseId) { + if (!playbackNode) return; + playbackNode.port.postMessage({ type: 'drain', responseId: responseId || currentResponseId }); + } + + function sendPlaybackAck(responseId, playedMs) { + if (!responseId || !socket || socket.readyState !== WebSocket.OPEN || playedMs <= 0) { + if (!responseId && playedMs > 0) appendLog('playback ack skipped: missing response id', true); + return; + } + socket.send(JSON.stringify({ + type: 'playback.ack', + response_id: responseId, + item_id: `item_${responseId}`, + played_ms: playedMs, + committed_ms: playedMs, + })); + } + + function playbackDrained(message) { + const responseId = message.responseId || currentResponseId; + sendPlaybackAck(responseId, Number(message.playedMs) || 0); + setPlayback('Idle'); + if (message.underrunMs > 0) { + appendLog(`playback underrun ${message.underrunMs} ms`); + } + window.setTimeout(() => { + assistantActive = false; + currentResponseId = null; + responseHasAudio = false; + if (running) setModel('Listening'); + }, ECHO_GUARD_MS); + } + + function handleEvent(event) { + appendEventLog(event); + const responseId = responseIdOf(event); + switch (event.type) { + case 'session.created': + case 'session.updated': + setConnection('Connected', 'online'); + setModel('Listening'); + break; + case 'response.listen': + assistantActive = false; + setModel('Listening'); + break; + case 'response.created': + case 'response.speak': + beginAssistant(responseId); + break; + case 'response.audio.delta': + currentResponseId = responseId || currentResponseId; + assistantActive = true; + setModel('Speaking'); + decodeAudioDelta(event) + .then((decoded) => feedPlayback(decoded, responseId)) + .catch((error) => appendLog(`audio decode failed: ${error.message || error}`, true)); + break; + case 'response.audio.done': + requestPlaybackDrain(responseId); + break; + case 'response.audio_transcript.delta': + addTranscript('assistant', event.delta || ''); + break; + case 'response.audio_transcript.done': + finishTranscript('assistant', event.transcript || ''); + break; + case 'conversation.item.input_audio_transcription.delta': + addTranscript('user', event.delta || ''); + break; + case 'conversation.item.input_audio_transcription.completed': + finishTranscript('user', event.transcript || ''); + break; + case 'response.done': + finishTranscript('assistant'); + if (!responseHasAudio) requestPlaybackDrain(responseId); + break; + case 'playback.acknowledged': + { + const acknowledgement = event.event || event; + runtimeDetail.textContent = `Playback committed ${acknowledgement.committed_ms || 0} ms`; + } + break; + case 'session.closed': + if (sessionCloseResolver) sessionCloseResolver(); + sessionCloseResolver = null; + break; + case 'error': + setConnection('Error', 'error'); + runtimeDetail.textContent = String(event.error || event.code || 'Server error'); + break; + default: + break; + } + } + + async function openPlayback() { + playbackContext = new (window.AudioContext || window.webkitAudioContext)({ sampleRate: OUTPUT_RATE }); + playbackRate = playbackContext.sampleRate; + await playbackContext.audioWorklet.addModule(staticAssetUrl('static/playback_worklet.js')); + playbackNode = new AudioWorkletNode(playbackContext, 'fullduplex-pcm-playback'); + playbackNode.port.onmessage = (message) => { + if (message.data.type === 'playback-started') setPlayback('Playing'); + else if (message.data.type === 'playback-drained') playbackDrained(message.data); + else if (message.data.type === 'playback-underrun') { + runtimeDetail.textContent = `Playback underrun ${message.data.underrunMs || 0} ms`; + } + }; + playbackNode.connect(playbackContext.destination); + await playbackContext.resume(); + } + + async function openCapture() { + mediaStream = await navigator.mediaDevices.getUserMedia({ + audio: { + channelCount: 1, + echoCancellation: true, + noiseSuppression: true, + autoGainControl: true, + sampleRate: { ideal: INPUT_RATE }, + }, + }); + try { + captureContext = new (window.AudioContext || window.webkitAudioContext)({ sampleRate: INPUT_RATE }); + } catch (_error) { + captureContext = new (window.AudioContext || window.webkitAudioContext)(); + } + captureRate = captureContext.sampleRate; + await captureContext.audioWorklet.addModule(staticAssetUrl('static/pcm_worklet.js')); + const source = captureContext.createMediaStreamSource(mediaStream); + captureNode = new AudioWorkletNode(captureContext, 'fullduplex-pcm-capture'); + captureNode.port.onmessage = (message) => { + const pcm = new Int16Array(message.data); + updateMeter(pcm); + if (microphoneUploadEnabled()) pendingCapture.push(pcm); + }; + const silentSink = captureContext.createGain(); + silentSink.gain.value = 0; + source.connect(captureNode); + captureNode.connect(silentSink).connect(captureContext.destination); + await captureContext.resume(); + } + + function openSocket() { + return new Promise((resolve, reject) => { + const url = realtimeUrl(); + socket = new WebSocket(url); + let settled = false; + socket.onopen = () => { + settled = true; + const extraBody = { + auto_response: true, + minicpmo45_native_duplex: true, + }; + const session = { + modalities: ['audio', 'text'], + voice: 'default', + extra_body: extraBody, + }; + // Reference voice for TTS cloning, provided by the server via + // --ref-audio (mirrors the official demo's default ref audio). + if (config.refAudio) session.ref_audio = config.refAudio; + const instructions = systemPromptInput ? systemPromptInput.value.trim() : ''; + if (instructions) session.instructions = instructions; + socket.send(JSON.stringify({ type: 'session.update', session })); + runtimeDetail.textContent = `${captureRate} Hz capture / ${playbackRate} Hz playback`; + appendLog(`websocket open ${url}`); + resolve(); + }; + socket.onmessage = (message) => { + if (typeof message.data !== 'string') return; + try { + handleEvent(JSON.parse(message.data)); + } catch (error) { + appendLog(`invalid server event: ${error.message || error}`, true); + } + }; + socket.onerror = () => { + if (!settled) { + settled = true; + reject(new Error(`WebSocket connection failed: ${url}`)); + } + }; + socket.onclose = (event) => { + appendLog(`websocket closed code=${event.code}`); + if (running) { + setConnection('Disconnected', 'error'); + stopSession(); + } + }; + }); + } + + function formatElapsed(seconds) { + const minutes = Math.floor(seconds / 60); + return `${String(minutes).padStart(2, '0')}:${String(seconds % 60).padStart(2, '0')}`; + } + + function startClock() { + startedAt = Date.now(); + clockTimer = window.setInterval(() => { + sessionTimer.textContent = formatElapsed(Math.floor((Date.now() - startedAt) / 1000)); + }, 1000); + } + + async function startSession() { + if (running) return; + callButton.disabled = true; + setConnection('Connecting', 'connecting'); + runtimeDetail.textContent = 'Requesting microphone access'; + try { + await openPlayback(); + await openCapture(); + await openSocket(); + running = true; + muted = false; + assistantActive = false; + sendTimer = window.setInterval(flushCapture, SEND_INTERVAL_MS); + startClock(); + callButton.textContent = 'End session'; + callButton.classList.add('is-active'); + muteButton.disabled = false; + cameraButton.disabled = false; + setConnection('Connected', 'online'); + setModel('Listening'); + appendLog('session started'); + } catch (error) { + appendLog(`start failed: ${error.message || error}`, true); + setConnection('Error', 'error'); + runtimeDetail.textContent = String(error.message || error); + await stopSession(); + } finally { + callButton.disabled = false; + } + } + + async function startCamera() { + if (cameraStream) return; + cameraStream = await navigator.mediaDevices.getUserMedia({ video: true, audio: false }); + cameraPreview.srcObject = cameraStream; + cameraPreview.style.display = ''; + await cameraPreview.play().catch(() => {}); + // Official omni-duplex cadence: one JPEG (quality 0.7) per ~1 s chunk, + // no client-side resize (the server normalizes at scale_resolution=448). + cameraTimer = window.setInterval(() => { + if (!cameraStream || cameraPreview.videoWidth === 0) return; + cameraCanvas.width = cameraPreview.videoWidth; + cameraCanvas.height = cameraPreview.videoHeight; + cameraCanvas.getContext('2d').drawImage(cameraPreview, 0, 0); + cameraPendingFrame = cameraCanvas.toDataURL('image/jpeg', 0.7).split(',')[1]; + }, 1000); + cameraButton.textContent = 'Camera off'; + cameraButton.classList.add('is-active'); + appendLog('camera on (1 fps omni frames)'); + } + + function stopCamera() { + if (cameraTimer !== null) clearInterval(cameraTimer); + cameraTimer = null; + if (cameraStream) { + for (const track of cameraStream.getTracks()) track.stop(); + } + cameraStream = null; + cameraPendingFrame = null; + cameraPreview.srcObject = null; + cameraPreview.style.display = 'none'; + cameraButton.textContent = 'Camera'; + cameraButton.classList.remove('is-active'); + } + + cameraButton.addEventListener('click', () => { + if (cameraStream) { + stopCamera(); + appendLog('camera off'); + return; + } + startCamera().catch((error) => appendLog(`camera failed: ${error.message || error}`, true)); + }); + + function waitForSessionClosed(targetSocket, timeoutMs) { + if (!targetSocket || targetSocket.readyState !== WebSocket.OPEN) return Promise.resolve(); + return new Promise((resolve) => { + let done = false; + const finish = () => { + if (done) return; + done = true; + if (sessionCloseResolver === finish) sessionCloseResolver = null; + resolve(); + }; + sessionCloseResolver = finish; + window.setTimeout(finish, timeoutMs); + }); + } + + async function stopSession({ terminal = true } = {}) { + running = false; + assistantActive = false; + pendingCapture = []; + if (sendTimer !== null) clearInterval(sendTimer); + if (clockTimer !== null) clearInterval(clockTimer); + sendTimer = null; + clockTimer = null; + if (socket) { + const closingSocket = socket; + socket = null; + closingSocket.onclose = null; + if (terminal && closingSocket.readyState === WebSocket.OPEN) { + const closed = waitForSessionClosed(closingSocket, SESSION_CLOSE_TIMEOUT_MS); + closingSocket.send(JSON.stringify({ type: 'session.close' })); + await closed; + } + closingSocket.close(1000, 'client stop'); + } + if (playbackNode) playbackNode.port.postMessage({ type: 'clear' }); + if (mediaStream) { + for (const track of mediaStream.getTracks()) track.stop(); + } + mediaStream = null; + stopCamera(); + cameraButton.disabled = true; + if (captureContext) await captureContext.close().catch(() => {}); + if (playbackContext) await playbackContext.close().catch(() => {}); + captureContext = null; + captureNode = null; + playbackContext = null; + playbackNode = null; + currentResponseId = null; + responseHasAudio = false; + meterFill.style.width = '0%'; + sessionTimer.textContent = '00:00'; + callButton.textContent = 'Start session'; + callButton.classList.remove('is-active'); + muteButton.textContent = 'Mute'; + muteButton.classList.remove('is-active'); + muteButton.disabled = true; + setConnection('Offline', 'offline'); + setModel('Idle'); + setPlayback('Idle'); + if (runtimeDetail.textContent.startsWith('Playback committed')) return; + if (!runtimeDetail.textContent.startsWith('start failed')) runtimeDetail.textContent = 'No active connection'; + } + + function toggleMute() { + if (!running) return; + muted = !muted; + pendingCapture = []; + muteButton.textContent = muted ? 'Unmute' : 'Mute'; + muteButton.classList.toggle('is-active', muted); + appendLog(muted ? 'microphone muted' : 'microphone unmuted'); + } + + callButton.addEventListener('click', () => { + if (running) stopSession(); + else startSession(); + }); + muteButton.addEventListener('click', toggleMute); + clearLogButton.addEventListener('click', () => { + eventLog.textContent = ''; + logCount = 0; + eventCount.textContent = '0 events'; + }); + window.addEventListener('beforeunload', () => { stopSession({ terminal: false }); }); +})(); diff --git a/examples/online_serving/minicpmo/realtime_web/app/static/pcm_worklet.js b/examples/online_serving/minicpmo/realtime_web/app/static/pcm_worklet.js new file mode 100644 index 000000000..608ac6b01 --- /dev/null +++ b/examples/online_serving/minicpmo/realtime_web/app/static/pcm_worklet.js @@ -0,0 +1,15 @@ +class FullDuplexPcmCapture extends AudioWorkletProcessor { + process(inputs) { + const channel = inputs[0] && inputs[0][0]; + if (!channel || channel.length === 0) return true; + const pcm = new Int16Array(channel.length); + for (let index = 0; index < channel.length; index += 1) { + const sample = Math.max(-1, Math.min(1, channel[index])); + pcm[index] = sample < 0 ? sample * 32768 : sample * 32767; + } + this.port.postMessage(pcm.buffer, [pcm.buffer]); + return true; + } +} + +registerProcessor('fullduplex-pcm-capture', FullDuplexPcmCapture); diff --git a/examples/online_serving/minicpmo/realtime_web/app/static/playback_worklet.js b/examples/online_serving/minicpmo/realtime_web/app/static/playback_worklet.js new file mode 100644 index 000000000..c3216fd7f --- /dev/null +++ b/examples/online_serving/minicpmo/realtime_web/app/static/playback_worklet.js @@ -0,0 +1,153 @@ +class FullDuplexPcmPlayback extends AudioWorkletProcessor { + constructor() { + super(); + this.queue = []; + this.offset = 0; + this.playedFrames = 0; + this.underrunFrames = 0; + this.drain = null; + this.started = false; + this.activeResponseId = null; + this.initialBufferFrames = Math.round(sampleRate * 0.2); + this.bufferWaitFrames = this.initialBufferFrames; + this.rebuffering = false; + this.fadeFrames = Math.max(1, Math.round(sampleRate * 0.005)); + this.fadeInFrames = 0; + this.port.onmessage = (event) => this.handleMessage(event.data || {}); + } + + handleMessage(message) { + if (message.type === 'audio' && message.pcm) { + const wasEmpty = this.queue.length === 0; + if (!this.started && !this.activeResponseId) { + this.activeResponseId = message.responseId || null; + } + if (!this.started && Number.isFinite(message.initialBufferMs)) { + this.initialBufferFrames = Math.max(0, Math.round((sampleRate * message.initialBufferMs) / 1000)); + } + this.queue.push(message.pcm); + if (!this.started && wasEmpty && !this.rebuffering) { + this.bufferWaitFrames = this.initialBufferFrames; + } + } else if (message.type === 'drain') { + this.drain = { responseId: message.responseId || null }; + if (!this.started && this.bufferedFrames() > 0) { + this.bufferWaitFrames = 0; + this.startPlayback(); + } + this.notifyIfDrained(); + } else if (message.type === 'clear') { + this.queue = []; + this.offset = 0; + this.playedFrames = 0; + this.underrunFrames = 0; + this.drain = null; + this.started = false; + this.activeResponseId = null; + this.bufferWaitFrames = this.initialBufferFrames; + this.rebuffering = false; + this.fadeInFrames = 0; + } + } + + bufferedFrames() { + return this.queue.reduce((total, pcm, index) => ( + total + pcm.length - (index === 0 ? this.offset : 0) + ), 0); + } + + startPlayback() { + if (this.started) return; + this.started = true; + this.fadeInFrames = this.fadeFrames; + if (this.playedFrames === 0) { + this.port.postMessage({ type: 'playback-started', responseId: this.activeResponseId }); + } + } + + notifyIfDrained() { + if (!this.drain || this.queue.length > 0) return; + this.port.postMessage({ + type: 'playback-drained', + responseId: this.drain.responseId, + playedMs: Math.round((this.playedFrames * 1000) / sampleRate), + underrunMs: Math.round((this.underrunFrames * 1000) / sampleRate), + }); + this.playedFrames = 0; + this.underrunFrames = 0; + this.drain = null; + this.started = false; + this.activeResponseId = null; + this.bufferWaitFrames = this.initialBufferFrames; + this.rebuffering = false; + this.fadeInFrames = 0; + } + + reportUnderrun() { + this.port.postMessage({ + type: 'playback-underrun', + responseId: this.activeResponseId, + underrunMs: Math.round((this.underrunFrames * 1000) / sampleRate), + }); + } + + process(_inputs, outputs) { + const output = outputs[0][0]; + output.fill(0); + if (!this.started) { + if (this.rebuffering && !this.drain) { + this.underrunFrames += output.length; + this.reportUnderrun(); + } + if (this.queue.length > 0 && this.bufferWaitFrames > 0) { + this.bufferWaitFrames = Math.max(0, this.bufferWaitFrames - output.length); + return true; + } + if (this.queue.length > 0) { + this.startPlayback(); + this.rebuffering = false; + } + } + if (!this.started) { + this.notifyIfDrained(); + return true; + } + let target = 0; + while (target < output.length && this.queue.length > 0) { + const pcm = this.queue[0]; + const count = Math.min(output.length - target, pcm.length - this.offset); + for (let index = 0; index < count; index += 1) { + let sample = pcm[this.offset + index] / 32768; + if (this.fadeInFrames > 0) { + const elapsed = this.fadeFrames - this.fadeInFrames; + sample *= elapsed / this.fadeFrames; + this.fadeInFrames -= 1; + } + output[target + index] = sample; + } + target += count; + this.offset += count; + this.playedFrames += count; + if (this.offset >= pcm.length) { + this.queue.shift(); + this.offset = 0; + } + } + if (target < output.length && !this.drain) { + const fadeCount = Math.min(target, this.fadeFrames); + for (let index = 0; index < fadeCount; index += 1) { + output[target - fadeCount + index] *= (fadeCount - index - 1) / fadeCount; + } + this.underrunFrames += output.length - target; + this.reportUnderrun(); + this.started = false; + this.rebuffering = true; + this.bufferWaitFrames = this.initialBufferFrames; + this.fadeInFrames = 0; + } + this.notifyIfDrained(); + return true; + } +} + +registerProcessor('fullduplex-pcm-playback', FullDuplexPcmPlayback); diff --git a/examples/online_serving/minicpmo/realtime_web/app/static/styles.css b/examples/online_serving/minicpmo/realtime_web/app/static/styles.css new file mode 100644 index 000000000..9b080176b --- /dev/null +++ b/examples/online_serving/minicpmo/realtime_web/app/static/styles.css @@ -0,0 +1,127 @@ +:root { + color-scheme: light; + font-family: Inter, ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; + color: #20252b; + background: #f4f6f7; + font-synthesis: none; +} + +* { box-sizing: border-box; } + +body { margin: 0; min-width: 320px; } + +button, summary { font: inherit; } + +button:focus-visible, summary:focus-visible { + outline: 3px solid rgba(21, 101, 192, .25); + outline-offset: 2px; +} + +.app-header { + min-height: 104px; + padding: 24px clamp(20px, 5vw, 64px); + background: #ffffff; + border-bottom: 1px solid #d9dee3; + display: flex; + align-items: center; + justify-content: space-between; + gap: 20px; +} + +h1, h2, p { margin: 0; } +h1 { font-size: 24px; line-height: 1.2; letter-spacing: 0; } +h2 { font-size: 18px; line-height: 1.3; letter-spacing: 0; } + +.eyebrow { + margin-bottom: 5px; + color: #68717b; + font-size: 11px; + font-weight: 700; + text-transform: uppercase; + letter-spacing: .08em; +} + +main { width: min(1040px, calc(100% - 40px)); margin: 28px auto 48px; } + +.status { + min-width: 88px; + padding: 7px 10px; + border: 1px solid; + border-radius: 6px; + text-align: center; + font-size: 13px; + font-weight: 700; +} + +.status-offline { color: #616971; background: #f0f2f3; border-color: #cbd1d6; } +.status-connecting { color: #8a4b00; background: #fff5df; border-color: #e6b76a; } +.status-online { color: #176a3a; background: #e9f7ef; border-color: #7dc49b; } +.status-error { color: #a12222; background: #fff0f0; border-color: #df8f8f; } + +.call-bar { + min-height: 86px; + padding: 16px 18px; + background: #252b31; + color: #ffffff; + border-radius: 6px; + display: grid; + grid-template-columns: auto minmax(150px, 1fr) auto; + align-items: center; + gap: 24px; +} + +.call-actions { display: flex; gap: 8px; } +button { border-radius: 5px; border: 1px solid transparent; cursor: pointer; } +button:disabled { opacity: .46; cursor: default; } +.primary { padding: 10px 15px; color: #ffffff; background: #16834b; font-weight: 700; } +.primary.is-active { background: #b43636; } +.secondary { padding: 10px 13px; color: #ffffff; background: #39414a; border-color: #59636d; } +.secondary.is-active { color: #2b2525; background: #ffc9c9; border-color: #e58f8f; } + +.meter { min-width: 0; display: flex; align-items: center; gap: 10px; color: #c8cfd6; font-size: 12px; } +.meter-track { height: 8px; flex: 1; overflow: hidden; background: #444d56; border-radius: 4px; } +.meter-fill { width: 0; height: 100%; background: #54bd7d; transition: width 70ms linear; } + +.runtime-status { margin: 0; display: flex; gap: 24px; } +.runtime-status div { min-width: 68px; } +.runtime-status dt { color: #9fa8b1; font-size: 10px; text-transform: uppercase; } +.runtime-status dd { margin: 3px 0 0; font-size: 13px; font-weight: 700; font-variant-numeric: tabular-nums; } + +.conversation-section { margin-top: 22px; padding: 22px 0 10px; } +.section-heading { display: flex; justify-content: space-between; align-items: end; gap: 16px; padding: 0 2px 14px; border-bottom: 1px solid #cfd5da; } +.policy-label { color: #5e6871; font-size: 12px; } +.conversation { min-height: 260px; max-height: 440px; overflow-y: auto; padding: 18px 2px; } +.empty-state { color: #7a838b; font-size: 14px; } + +.turn { display: grid; grid-template-columns: 92px minmax(0, 1fr); gap: 12px; padding: 12px 0; border-bottom: 1px solid #e0e4e7; } +.turn-role { padding-top: 2px; color: #68717b; font-size: 11px; font-weight: 800; text-transform: uppercase; } +.turn-user .turn-role { color: #1565c0; } +.turn-assistant .turn-role { color: #a45117; } +.turn-text { min-width: 0; white-space: pre-wrap; overflow-wrap: anywhere; line-height: 1.55; } +.turn-live .turn-text::after { content: ""; display: inline-block; width: 6px; height: 14px; margin-left: 4px; background: #9098a0; vertical-align: -2px; animation: blink 1s step-end infinite; } +@keyframes blink { 50% { opacity: 0; } } + +.diagnostics { margin-top: 18px; border-top: 1px solid #bfc6cc; border-bottom: 1px solid #bfc6cc; background: #ffffff; } +.diagnostics summary { padding: 13px 14px; display: flex; justify-content: space-between; cursor: pointer; font-weight: 700; list-style: none; } +.diagnostics summary::-webkit-details-marker { display: none; } +.event-count { color: #6f7880; font-size: 12px; font-weight: 500; } +.log-toolbar { min-height: 40px; padding: 7px 12px; border-top: 1px solid #e0e4e7; display: flex; align-items: center; justify-content: space-between; gap: 12px; color: #69727a; font-size: 12px; } +.icon-button { padding: 5px 9px; color: #394149; background: #f4f6f7; border-color: #cdd3d8; } +.event-log { height: 200px; margin: 0; padding: 12px 14px; overflow: auto; background: #171b1f; color: #d9e0e6; font: 11px/1.55 ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; white-space: pre-wrap; } +.log-error { color: #ff9f9f; } + +@media (max-width: 760px) { + .app-header { min-height: 88px; padding: 18px 20px; } + h1 { font-size: 20px; } + main { width: min(100% - 24px, 620px); margin-top: 16px; } + .call-bar { grid-template-columns: 1fr; gap: 15px; } + .call-actions button { flex: 1; } + .runtime-status { justify-content: space-between; gap: 10px; } + .section-heading { align-items: start; flex-direction: column; gap: 7px; } + .conversation { min-height: 230px; } + .turn { grid-template-columns: 72px minmax(0, 1fr); } +} + +@media (prefers-reduced-motion: reduce) { + *, *::before, *::after { animation-duration: .01ms !important; animation-iteration-count: 1 !important; } +} diff --git a/examples/online_serving/minicpmo/realtime_web/server.py b/examples/online_serving/minicpmo/realtime_web/server.py new file mode 100644 index 000000000..ee78fff74 --- /dev/null +++ b/examples/online_serving/minicpmo/realtime_web/server.py @@ -0,0 +1,186 @@ +"""Static host and same-origin Realtime proxy for the full-duplex demo.""" + +from __future__ import annotations + +import argparse +import asyncio +import base64 +import contextlib +import hashlib +import json +import logging +from pathlib import Path +from urllib.parse import urlencode + +import uvicorn +import websockets +from fastapi import FastAPI, WebSocket, WebSocketDisconnect +from fastapi.responses import HTMLResponse, Response +from fastapi.staticfiles import StaticFiles + +logger = logging.getLogger(__name__) +APP_DIR = Path(__file__).parent / "app" +STATIC_DIR = APP_DIR / "static" + + +def _join_ws_url(base: str, path: str, query: str) -> str: + return base.rstrip("/") + path + (("?" + query) if query else "") + + +async def _pump_client_to_backend(client: WebSocket, backend) -> None: + try: + while True: + message = await client.receive() + if message["type"] == "websocket.disconnect": + await backend.close() + return + if message.get("text") is not None: + await backend.send(message["text"]) + elif message.get("bytes") is not None: + await backend.send(message["bytes"]) + except (WebSocketDisconnect, websockets.ConnectionClosed): + with contextlib.suppress(Exception): + await backend.close() + + +async def _pump_backend_to_client(client: WebSocket, backend) -> None: + try: + async for message in backend: + if isinstance(message, bytes): + await client.send_bytes(message) + else: + await client.send_text(message) + except (WebSocketDisconnect, websockets.ConnectionClosed): + return + except RuntimeError as exc: + if "websocket.send" in str(exc) and "websocket.close" in str(exc): + return + raise + + +def _expected_proxy_close(exc: BaseException) -> bool: + if isinstance(exc, (WebSocketDisconnect, websockets.ConnectionClosed, asyncio.CancelledError)): + return True + return isinstance(exc, RuntimeError) and "websocket.send" in str(exc) and "websocket.close" in str(exc) + + +def build_app( + *, + ws_backend: str = "ws://127.0.0.1:8099", + model: str = "openbmb/MiniCPM-o-4_5", + public_realtime_url: str | None = None, + ref_audio: str | None = None, +) -> FastAPI: + app = FastAPI(title="Experimental Full-Duplex Web Demo") + index_path = APP_DIR / "index.html" + app_version_hash = hashlib.sha256() + for asset_path in ( + STATIC_DIR / "app.js", + STATIC_DIR / "pcm_worklet.js", + STATIC_DIR / "playback_worklet.js", + ): + app_version_hash.update(asset_path.read_bytes()) + app_version = app_version_hash.hexdigest()[:12] + + ref_audio_uri: str | None = None + if ref_audio: + ref_path = Path(ref_audio) + if not ref_path.is_file(): + raise SystemExit(f"--ref-audio not found: {ref_audio}") + encoded = base64.b64encode(ref_path.read_bytes()).decode("ascii") + ref_audio_uri = f"data:audio/wav;base64,{encoded}" + + @app.get("/", response_class=HTMLResponse) + def index() -> HTMLResponse: + config = json.dumps( + { + "model": model, + "realtimePath": public_realtime_url or "v1/realtime", + "refAudio": ref_audio_uri, + "appVersion": app_version, + }, + ensure_ascii=True, + ) + html = ( + index_path.read_text(encoding="utf-8") + .replace("__FULL_DUPLEX_CONFIG__", config) + .replace("__FULL_DUPLEX_APP_VERSION__", app_version) + ) + return HTMLResponse(html, headers={"Cache-Control": "no-store"}) + + @app.get("/healthz") + def healthz() -> Response: + return Response(content="ok", media_type="text/plain") + + @app.websocket("/v1/realtime") + async def realtime_proxy(websocket: WebSocket) -> None: + await websocket.accept() + query = urlencode(websocket.query_params.multi_items()) + backend_url = _join_ws_url(ws_backend, "/v1/realtime", query) + logger.info("Proxying Realtime WebSocket to %s", backend_url) + try: + async with websockets.connect( + backend_url, + max_size=64 * 1024 * 1024, + ) as backend: + tasks = { + asyncio.create_task(_pump_client_to_backend(websocket, backend)), + asyncio.create_task(_pump_backend_to_client(websocket, backend)), + } + done, pending = await asyncio.wait( + tasks, + return_when=asyncio.FIRST_COMPLETED, + ) + for task in pending: + task.cancel() + await asyncio.gather(*pending, return_exceptions=True) + for result in await asyncio.gather(*done, return_exceptions=True): + if isinstance(result, BaseException) and not _expected_proxy_close(result): + raise result + except (WebSocketDisconnect, websockets.ConnectionClosed): + return + except Exception: + logger.exception("Realtime WebSocket proxy failed") + with contextlib.suppress(Exception): + await websocket.close(code=1011) + + app.mount("/static", StaticFiles(directory=str(STATIC_DIR)), name="static") + return app + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--host", default="0.0.0.0") + parser.add_argument("--port", type=int, default=7862) + parser.add_argument("--ws-backend", default="ws://127.0.0.1:8099") + parser.add_argument( + "--public-realtime-url", + help="Browser-visible ws:// or wss:// Realtime URL; defaults to the same-origin proxy.", + ) + parser.add_argument("--model", default="openbmb/MiniCPM-o-4_5") + parser.add_argument( + "--ref-audio", + required=True, + help=( + "Reference voice wav for TTS voice cloning, e.g. the " + "official MiniCPM-o-Demo assets/ref_audio/ref_minicpm_signature.wav" + ), + ) + args = parser.parse_args() + + logging.basicConfig(level=logging.INFO) + uvicorn.run( + build_app( + ws_backend=args.ws_backend, + model=args.model, + public_realtime_url=args.public_realtime_url, + ref_audio=args.ref_audio, + ), + host=args.host, + port=args.port, + log_level="info", + ) + + +if __name__ == "__main__": + main() diff --git a/tests/core/sched/test_sched_package_imports.py b/tests/core/sched/test_sched_package_imports.py new file mode 100644 index 000000000..bc4ec9d44 --- /dev/null +++ b/tests/core/sched/test_sched_package_imports.py @@ -0,0 +1,26 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from __future__ import annotations + +import ast +from pathlib import Path + +import pytest + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +def test_importing_scheduler_output_does_not_eagerly_import_scheduler_implementations() -> None: + package_init = Path(__file__).parents[3] / "vllm_omni/core/sched/__init__.py" + module = ast.parse(package_init.read_text()) + eager_modules = { + node.module + for node in ast.walk(module) + if isinstance(node, ast.ImportFrom) + and node.level == 1 + and node.module in {"omni_ar_scheduler", "omni_generation_scheduler"} + } + + assert eager_modules == set() + assert any(isinstance(node, ast.FunctionDef) and node.name == "__getattr__" for node in module.body) diff --git a/tests/distributed/omni_connectors/test_chunk_transfer_adapter.py b/tests/distributed/omni_connectors/test_chunk_transfer_adapter.py index 9c6a86ed3..76d0cb550 100644 --- a/tests/distributed/omni_connectors/test_chunk_transfer_adapter.py +++ b/tests/distributed/omni_connectors/test_chunk_transfer_adapter.py @@ -677,6 +677,28 @@ def test_finish_requests_removes_zombies_from_chunk_waiting_deques(build_adapter assert "req-zombie" not in adapter.finished_requests +def test_finish_requests_releases_active_stream_slot(build_adapter): + adapter, _ = build_adapter(stage_id=1, max_num_seqs=1, active_stream_window=1) + aborted = _req("req-aborted", RequestStatus.RUNNING) + waiting = _req("req-waiting", RequestStatus.WAITING) + waiting_queue = DummyWaitingQueue([waiting]) + running_queue = [] + adapter._active_streams[aborted.request_id] = aborted + adapter._held_non_active.append(aborted) + + adapter.finish_requests( + [aborted.request_id], + RequestStatus.FINISHED_ABORTED, + {aborted.request_id: aborted}, + ) + adapter.process_pending_chunks(waiting_queue, running_queue) + + assert aborted.request_id not in adapter._active_streams + assert [request.request_id for request in adapter._held_non_active] == [] + assert list(adapter._active_streams) == [waiting.request_id] + assert waiting.status == RequestStatus.WAITING_FOR_CHUNK + + def test_restore_queues_skips_requests_missing_from_scheduler_requests(build_adapter): adapter, _ = build_adapter(stage_id=1) zombie = _req("req-zombie", RequestStatus.WAITING_FOR_CHUNK) diff --git a/tests/e2e/online_serving/minicpmo_realtime_duplex_scenarios.py b/tests/e2e/online_serving/minicpmo_realtime_duplex_scenarios.py new file mode 100644 index 000000000..8b521c378 --- /dev/null +++ b/tests/e2e/online_serving/minicpmo_realtime_duplex_scenarios.py @@ -0,0 +1,1673 @@ +"""Strict MiniCPM-o 4.5 Realtime duplex E2E scenario harness. + +This script is intentionally scenario-based instead of a generic chat client. +It has two explicit validation contracts: + +1. ``model-policy`` accepts either a model-owned listen decision or a complete + spoken response for each streamed user turn; +2. ``response-required`` uses a known fixture and requires a complete audio + response with an independent transcript for every requested turn. + +Explicit serving-side barge-in is intentionally not part of this smoke test. +MiniCPM-o native duplex currently exposes model-owned listen/speak switching, +not a separate model-level barge-in contract. + +Run only after a MiniCPM-o 4.5 vLLM-Omni server is up: + + python tests/e2e/online_serving/minicpmo_realtime_duplex_scenarios.py \ + --url ws://localhost:8099/v1/realtime?duplex=1 \ + --model openbmb/MiniCPM-o-4_5 \ + --input-wav input_16k_mono_pcm16.wav \ + --validation-mode model-policy \ + --output-dir /tmp/minicpmo_duplex_demo +""" + +from __future__ import annotations + +import argparse +import array +import asyncio +import base64 +import hashlib +import inspect +import json +import sys +import time +import wave +from dataclasses import dataclass, field +from pathlib import Path + +_REPO_ROOT = Path(__file__).resolve().parents[3] +if str(_REPO_ROOT) not in sys.path: + sys.path.insert(0, str(_REPO_ROOT)) + +try: + import websockets + from websockets.exceptions import ConnectionClosed +except ImportError as exc: # pragma: no cover - demo dependency. + raise SystemExit("Install websockets first: pip install websockets") from exc + +from vllm_omni.experimental.fullduplex.client import ( # noqa: E402 + PCM16_BYTES_PER_SAMPLE, + PCM16_SAMPLE_RATE, + RealtimeEventCollector, + build_realtime_url, + read_pcm16_wav, +) + +_url_with_model = build_realtime_url +_read_wav_pcm16 = read_pcm16_wav + + +def _ref_audio_data_url(path: str | None) -> str | None: + if path is None: + return None + ref_path = Path(path).expanduser() + return "data:audio/wav;base64," + base64.b64encode(ref_path.read_bytes()).decode("ascii") + + +@dataclass +class DemoState: + events: list[dict[str, object]] = field(default_factory=list) + timing_events: RealtimeEventCollector = field(default_factory=RealtimeEventCollector) + audio_deltas: list[bytes] = field(default_factory=list) + response_audio_deltas: dict[str, list[bytes]] = field(default_factory=dict) + response_ids: list[str] = field(default_factory=list) + assistant_item_ids: list[str] = field(default_factory=list) + done_count: int = 0 + cancelled_count: int = 0 + listen_count: int = 0 + model_listen_count: int = 0 + buffering_listen_count: int = 0 + model_speak_event_count: int = 0 + model_speak_delta_count: int = 0 + playback_ack_count: int = 0 + playback_history_committed_count: int = 0 + truncate_count: int = 0 + input_transcription_count: int = 0 + audio_marks_seen: bool = False + overlap_decisions: list[dict[str, object]] = field(default_factory=list) + output_sample_rate_hz: int = 24000 + input_commit_sent_at_s: list[float] = field(default_factory=list) + + def add(self, event: dict[str, object], *, received_at_s: float | None = None) -> None: + self.events.append(event) + self.timing_events.add(event, received_at_s=received_at_s) + event_type = event.get("type") + if event_type == "response.created": + response = event.get("response") + response_id = response.get("id") if isinstance(response, dict) else event.get("response_id") + if isinstance(response_id, str) and response_id not in self.response_ids: + self.response_ids.append(response_id) + elif event_type == "conversation.item.added": + item = event.get("item") + if isinstance(item, dict) and item.get("role") == "assistant": + item_id = item.get("id") + if isinstance(item_id, str) and item_id not in self.assistant_item_ids: + self.assistant_item_ids.append(item_id) + elif event_type == "response.audio.delta": + delta = event.get("delta") or event.get("audio") + if isinstance(delta, str) and delta: + try: + decoded = base64.b64decode(delta) + self.audio_deltas.append(decoded) + response_id = self._event_response_id(event) + if isinstance(response_id, str): + self.response_audio_deltas.setdefault(response_id, []).append(decoded) + except Exception: + pass + metadata = event.get("metadata") + if isinstance(metadata, dict): + if metadata.get("model_speak") is True: + self.model_speak_delta_count += 1 + if isinstance(metadata.get("audio_text_marks"), list): + self.audio_marks_seen = True + sample_rate_hz = event.get("sample_rate_hz") + if isinstance(sample_rate_hz, int) and sample_rate_hz > 0: + self.output_sample_rate_hz = sample_rate_hz + elif event_type == "response.done": + self.done_count += 1 + response = event.get("response") + if isinstance(response, dict) and response.get("status") == "cancelled": + self.cancelled_count += 1 + elif event_type in {"audio.cancelled", "input.cancelled"}: + self.cancelled_count += 1 + elif event_type == "response.listen": + self.listen_count += 1 + response = event.get("response") + metadata = response.get("metadata") if isinstance(response, dict) else None + if isinstance(metadata, dict) and metadata.get("model_listen") is True: + self.model_listen_count += 1 + if isinstance(metadata, dict) and metadata.get("buffering") is True: + self.buffering_listen_count += 1 + elif event_type == "response.speak": + self.model_speak_event_count += 1 + elif event_type == "overlap.decision": + self.overlap_decisions.append(event) + elif event_type == "playback.acknowledged": + self.playback_ack_count += 1 + payload = event.get("event") + if isinstance(payload, dict) and payload.get("history_committed") is True: + self.playback_history_committed_count += 1 + elif event_type == "conversation.item.truncated": + self.truncate_count += 1 + elif event_type == "conversation.item.input_audio_transcription.completed": + self.input_transcription_count += 1 + + def count(self, event_type: str) -> int: + return sum(1 for event in self.events if event.get("type") == event_type) + + def response_timing_summaries(self) -> dict[str, dict[str, object]]: + summaries: dict[str, dict[str, object]] = {} + for response_id in self.response_ids: + created_at_s = next( + ( + received_at_s + for event, received_at_s in zip( + self.timing_events.events, + self.timing_events.event_received_at_s, + strict=True, + ) + if event.get("type") == "response.created" and self._event_response_id(event) == response_id + ), + None, + ) + if created_at_s is None: + continue + input_committed_at_s = next( + ( + committed_at_s + for committed_at_s in reversed(self.input_commit_sent_at_s) + if committed_at_s <= created_at_s + ), + None, + ) + timing = self.timing_events.timing_summary( + after_s=created_at_s, + input_committed_at_s=input_committed_at_s, + response_id=response_id, + ) + timing["measurement_origin"] = { + "response": "response.created client receive", + "input": "input_audio_buffer.commit client send" if input_committed_at_s is not None else None, + } + summaries[response_id] = timing + return summaries + + def first_index(self, event_type: str, predicate=None) -> int | None: + for index, event in enumerate(self.events): + if event.get("type") != event_type: + continue + if predicate is not None and not predicate(event): + continue + return index + return None + + @staticmethod + def _event_response_id(event: dict[str, object]) -> str | None: + response_id = event.get("response_id") + if isinstance(response_id, str) and response_id: + return response_id + response = event.get("response") + if isinstance(response, dict): + response_id = response.get("id") + if isinstance(response_id, str) and response_id: + return response_id + return None + + @staticmethod + def _event_item_id(event: dict[str, object]) -> str | None: + item_id = event.get("item_id") + if isinstance(item_id, str) and item_id: + return item_id + item = event.get("item") + if isinstance(item, dict): + item_id = item.get("id") + if isinstance(item_id, str) and item_id: + return item_id + return None + + def first_response_lifecycle_indices(self) -> dict[str, int]: + response_created_index = self.first_index("response.created") + if response_created_index is None: + return {} + response_id = self._event_response_id(self.events[response_created_index]) + if not response_id: + return {} + item_id = f"item_{response_id}" + indices: dict[str, int] = {"response.created": response_created_index} + for event_type in ( + "conversation.item.added", + "response.output_item.added", + "response.content_part.added", + "response.speak", + "response.audio.delta", + "response.audio.done", + "response.content_part.done", + "response.output_item.done", + "response.done", + ): + index = self.first_index( + event_type, + lambda event, event_type=event_type: ( + self._event_item_id(event) == item_id + if event_type == "conversation.item.added" + else self._event_response_id(event) == response_id + ), + ) + if index is None and event_type not in { + "response.speak", + "response.audio.delta", + "response.audio.done", + }: + return {} + if index is not None: + indices[event_type] = index + return indices + + def event_order_ok(self, *, require_input_commit: bool = True) -> bool: + if not self.events or self.events[0].get("type") != "session.created": + return False + first_commit_index = self.first_index("input_audio_buffer.committed") + first_response_index = self.first_index("response.created") + if first_response_index is None: + return False + if require_input_commit and (first_commit_index is None or first_commit_index > first_response_index): + return False + indices_by_type = self.first_response_lifecycle_indices() + if not indices_by_type: + return False + required_types = [ + "response.created", + "conversation.item.added", + "response.output_item.added", + "response.content_part.added", + "response.content_part.done", + "response.output_item.done", + "response.done", + ] + if any(event_type not in indices_by_type for event_type in required_types): + return False + ordered_types = [ + "response.created", + "conversation.item.added", + "response.output_item.added", + "response.content_part.added", + "response.speak", + "response.audio.delta", + "response.audio.done", + "response.content_part.done", + "response.output_item.done", + "response.done", + ] + if "response.audio.delta" not in indices_by_type: + listen_index = self.first_index( + "response.listen", + lambda event: ( + isinstance(event.get("response"), dict) + and isinstance(event["response"].get("metadata"), dict) + and event["response"]["metadata"].get("model_listen") is True + ), + ) + return ( + listen_index is not None + and indices_by_type["response.created"] < listen_index < indices_by_type["response.done"] + ) + if any(event_type not in indices_by_type for event_type in ordered_types): + return False + indices = [indices_by_type[event_type] for event_type in ordered_types] + return indices == sorted(indices) + + def model_policy_event_order_ok( + self, + *, + expected_turns: int, + require_input_commit: bool = True, + ) -> bool: + if not self.events or self.events[0].get("type") != "session.created": + return False + commit_indices = [ + index for index, event in enumerate(self.events) if event.get("type") == "input_audio_buffer.committed" + ] + decision_indices = [ + index + for index, event in enumerate(self.events) + if event.get("type") == "response.created" + or ( + event.get("type") == "response.listen" + and isinstance(event.get("response"), dict) + and isinstance(event["response"].get("metadata"), dict) + and event["response"]["metadata"].get("model_listen") is True + ) + ] + first_input_index = self.first_index("input_audio_buffer.speech_started") + if first_input_index is None: + first_input_index = commit_indices[0] if commit_indices else None + if first_input_index is None and not require_input_commit: + first_input_index = 0 + if ( + first_input_index is None + or (require_input_commit and len(commit_indices) < expected_turns) + or len(decision_indices) < expected_turns + ): + return False + return decision_indices[0] > first_input_index + + def model_speak_before_audio_ok(self) -> bool: + speak_index = self.first_index("response.speak") + audio_index = self.first_index("response.audio.delta") + return speak_index is not None and audio_index is not None and speak_index < audio_index + + def response_done(self, response_id: str | None) -> bool: + if not response_id: + return False + return any( + event.get("type") == "response.done" and self._event_response_id(event) == response_id + for event in self.events + ) + + def response_audio_delta_count(self, response_id: str | None) -> int: + if not response_id: + return 0 + return sum( + 1 + for event in self.events + if event.get("type") == "response.audio.delta" and self._event_response_id(event) == response_id + ) + + def response_playback_sent_ms(self, response_id: str | None) -> int: + if not response_id: + return 0 + for event in reversed(self.events): + if event.get("type") != "response.done" or self._event_response_id(event) != response_id: + continue + response = event.get("response") + metadata = response.get("metadata") if isinstance(response, dict) else None + playback = metadata.get("playback") if isinstance(metadata, dict) else event.get("playback") + sent_ms = playback.get("sent_ms") if isinstance(playback, dict) else None + if isinstance(sent_ms, int | float): + return max(0, int(sent_ms)) + return 0 + + def response_playback_history_committed(self, response_id: str) -> bool: + item_id = f"item_{response_id}" + for event in self.events: + if event.get("type") != "playback.acknowledged": + continue + payload = event.get("event") + if not isinstance(payload, dict): + continue + if payload.get("item_id") == item_id and payload.get("history_committed") is True: + return True + return False + + def response_transcript_delta(self, response_id: str) -> str: + return "".join( + str(event.get("delta", "")) + for event in self.events + if event.get("type") == "response.audio_transcript.delta" and self._event_response_id(event) == response_id + ) + + def response_transcript_done(self, response_id: str) -> list[str]: + return [ + str(event.get("transcript", "")) + for event in self.events + if event.get("type") == "response.audio_transcript.done" and self._event_response_id(event) == response_id + ] + + def completed_response_ids(self) -> list[str]: + response_ids: list[str] = [] + for event in self.events: + if event.get("type") != "response.done": + continue + response_id = self._event_response_id(event) + if isinstance(response_id, str): + response_ids.append(response_id) + return response_ids + + def stale_audio_delta_count(self) -> int: + cancelled_epochs_by_index: list[tuple[int, int]] = [] + for index, event in enumerate(self.events): + if event.get("type") != "response.done": + continue + response = event.get("response") + if not isinstance(response, dict) or response.get("status") != "cancelled": + continue + metadata = response.get("metadata") + if not isinstance(metadata, dict): + continue + cancelled_epoch = metadata.get("cancelled_epoch") + if isinstance(cancelled_epoch, int): + cancelled_epochs_by_index.append((index, cancelled_epoch)) + if not cancelled_epochs_by_index: + return 0 + stale = 0 + for index, event in enumerate(self.events): + if event.get("type") != "response.audio.delta": + continue + metadata = event.get("metadata") + if not isinstance(metadata, dict): + continue + event_epoch = metadata.get("epoch") + for cancel_index, cancelled_epoch in cancelled_epochs_by_index: + if index > cancel_index and event_epoch == cancelled_epoch: + stale += 1 + break + return stale + + +def _session_update_event(args: argparse.Namespace) -> dict[str, object]: + session_payload: dict[str, object] = { + "model": args.model, + "modalities": ["audio", "text"], + "input_audio_format": "pcm16", + "output_audio_format": args.output_audio_format, + "turn_detection": None, + "overlap_policy": "listen_only", + "overlap_short_ack_ms": args.short_ack_ms, + "playback_commit_policy": "ack_only", + "extra_body": { + "auto_response": True, + "minicpmo45_native_duplex": True, + "force_listen_count": 0, + }, + } + temperature = getattr(args, "temperature", None) + if temperature is None and getattr(args, "validation_mode", None) == "response-required": + temperature = 0.0 + if temperature is not None: + session_payload["temperature"] = temperature + ref_audio = _ref_audio_data_url(getattr(args, "ref_audio", None)) + if ref_audio is not None: + session_payload["ref_audio"] = ref_audio + event: dict[str, object] = { + "type": "session.update", + "session": session_payload, + } + session_id = getattr(args, "session_id", None) + if session_id: + session_payload["session_id"] = session_id + return event + + +def _turn_input_paths(primary: Path, additional: list[str], *, turns: int) -> list[Path]: + turn_count = max(1, turns) + if not additional: + return [primary] * turn_count + if len(additional) != turn_count - 1: + raise ValueError( + f"provide exactly one --turn-input-wav for each turn after the first " + f"(expected {turn_count - 1}, got {len(additional)})" + ) + return [primary, *(Path(path) for path in additional)] + + +def _turn_inputs_are_distinct(paths: list[Path], pcm16_inputs: list[bytes]) -> bool: + if len(paths) != len(pcm16_inputs): + return False + distinct_paths = len({str(path.resolve()) for path in paths}) == len(paths) + distinct_audio = len({hashlib.sha256(pcm16).digest() for pcm16 in pcm16_inputs}) == len(pcm16_inputs) + return distinct_paths and distinct_audio + + +def _turn_durations( + explicit: list[int], + *, + turns: int, + first_turn_ms: int, +) -> list[int | None]: + turn_count = max(1, turns) + if not explicit: + return [first_turn_ms, *([min(first_turn_ms, 1200)] * (turn_count - 1))] + if len(explicit) != turn_count: + raise ValueError( + f"provide exactly one --turn-duration-ms for each turn (expected {turn_count}, got {len(explicit)})" + ) + if any(duration_ms < 0 for duration_ms in explicit): + raise ValueError("--turn-duration-ms values must be non-negative") + return [None if duration_ms == 0 else duration_ms for duration_ms in explicit] + + +def _turn_transcripts(first: str, *, turns: int) -> list[str]: + turn_count = max(1, turns) + transcripts = [first, "继续", "再说一次"][:turn_count] + transcripts.extend(f"turn-{turn_index + 1}" for turn_index in range(len(transcripts), turn_count)) + return transcripts + + +def _write_wav(path: Path, pcm_bytes: bytes, *, sample_rate_hz: int) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with wave.open(str(path), "wb") as wf: + wf.setnchannels(1) + wf.setsampwidth(PCM16_BYTES_PER_SAMPLE) + wf.setframerate(sample_rate_hz) + wf.writeframes(pcm_bytes) + + +def _write_demo_artifacts(state: DemoState, output_dir: Path, *, output_audio_format: str) -> None: + output_dir.mkdir(parents=True, exist_ok=True) + if state.audio_deltas and output_audio_format == "pcm16": + _write_wav( + output_dir / "joined_audio_deltas.wav", + b"".join(state.audio_deltas), + sample_rate_hz=state.output_sample_rate_hz, + ) + elif state.audio_deltas: + (output_dir / "joined_audio_deltas.bin").write_bytes(b"".join(state.audio_deltas)) + for index, response_id in enumerate(state.response_ids, start=1): + response_audio = state.response_audio_deltas.get(response_id, []) + if not response_audio: + continue + payload = b"".join(response_audio) + if output_audio_format == "pcm16": + _write_wav( + output_dir / f"response_{index:02d}.wav", + payload, + sample_rate_hz=state.output_sample_rate_hz, + ) + else: + (output_dir / f"response_{index:02d}.bin").write_bytes(payload) + (output_dir / "events.jsonl").write_text( + "\n".join(json.dumps(event, ensure_ascii=False) for event in state.events) + "\n", + encoding="utf-8", + ) + + +def _pcm16_silence(duration_ms: int) -> bytes: + samples = PCM16_SAMPLE_RATE * max(0, duration_ms) // 1000 + return b"\x00\x00" * samples + + +def _pcm16_slice(pcm16: bytes, duration_ms: int) -> bytes: + byte_count = PCM16_SAMPLE_RATE * PCM16_BYTES_PER_SAMPLE * max(1, duration_ms) // 1000 + return pcm16[: min(len(pcm16), byte_count)] + + +def _pcm16_active_slice(pcm16: bytes, duration_ms: int) -> bytes: + byte_count = PCM16_SAMPLE_RATE * PCM16_BYTES_PER_SAMPLE * max(1, duration_ms) // 1000 + byte_count = min(len(pcm16), max(PCM16_BYTES_PER_SAMPLE, byte_count)) + byte_count -= byte_count % PCM16_BYTES_PER_SAMPLE + if byte_count <= 0: + return _pcm16_slice(pcm16, duration_ms) + step = max(PCM16_SAMPLE_RATE * PCM16_BYTES_PER_SAMPLE * 20 // 1000, PCM16_BYTES_PER_SAMPLE) + step -= step % PCM16_BYTES_PER_SAMPLE + best_offset = 0 + best_energy = -1.0 + for offset in range(0, max(1, len(pcm16) - byte_count + 1), max(PCM16_BYTES_PER_SAMPLE, step)): + chunk = pcm16[offset : offset + byte_count] + samples = array.array("h") + samples.frombytes(chunk) + if not samples: + continue + energy = sum(abs(sample) for sample in samples) / len(samples) + if energy > best_energy: + best_energy = energy + best_offset = offset + return pcm16[best_offset : best_offset + byte_count] + + +def _select_turn_audio(pcm16: bytes, duration_ms: int | None) -> bytes: + if duration_ms is None: + return pcm16 + return _pcm16_active_slice(pcm16, duration_ms) + + +def _canonical_transcript(text: str) -> str: + return "".join(text.split()) + + +def _reuses_previous_turn_tail(previous: str, current: str) -> bool: + if not previous or not current: + return False + if len(previous) >= 4 and previous in current: + return True + max_overlap = min(len(previous), len(current)) + return any(previous.endswith(current[:overlap]) for overlap in range(max_overlap, 2, -1)) + + +def _has_terminal_punctuation(text: str) -> bool: + stripped = text.rstrip("\"'”’))]} ") + return bool(stripped) and (len(stripped) <= 4 or stripped[-1] in "。!?!?…") + + +def _all_audio_responses_have_transcript(state: DemoState, response_ids: list[str]) -> bool: + return all( + state.response_audio_delta_count(response_id) == 0 + or bool(_canonical_transcript(state.response_transcript_delta(response_id))) + for response_id in response_ids + ) + + +def _all_response_playback_history_committed(state: DemoState, response_ids: list[str]) -> bool: + return all( + state.response_playback_history_committed(response_id) + for response_id in response_ids + if state.response_playback_sent_ms(response_id) > 0 + ) + + +def _response_cardinality_ok( + completed_response_ids: list[str], + *, + expected_turns: int, + validation_mode: str, +) -> bool: + if validation_mode == "response-required": + return len(completed_response_ids) == expected_turns + return True + + +def _requires_cross_turn_independence( + *, + validation_mode: str, + distinct_inputs_required: bool, +) -> bool: + return distinct_inputs_required and validation_mode == "response-required" + + +def _input_transcription_ok(count: int, *, transcript_hints_enabled: bool) -> bool: + return count > 0 if transcript_hints_enabled else True + + +def _unexpected_error_events(state: DemoState) -> list[dict[str, object]]: + return [event for event in state.events if event.get("type") == "error"] + + +def _evaluate_response_speak_contract(state: DemoState) -> dict[str, object]: + speak_counts: dict[str, int] = {} + invalid_response_speak_count = 0 + text_bearing_response_speak_count = 0 + + for event in state.events: + if event.get("type") != "response.speak": + continue + response_id = state._event_response_id(event) + if not response_id or response_id not in state.response_ids: + invalid_response_speak_count += 1 + else: + speak_counts[response_id] = speak_counts.get(response_id, 0) + 1 + if "text" in event: + text_bearing_response_speak_count += 1 + + duplicate_response_speak_ids = sorted(response_id for response_id, count in speak_counts.items() if count > 1) + return { + "response_speak_contract_ok": not duplicate_response_speak_ids + and invalid_response_speak_count == 0 + and text_bearing_response_speak_count == 0, + "duplicate_response_speak_ids": duplicate_response_speak_ids, + "invalid_response_speak_count": invalid_response_speak_count, + "text_bearing_response_speak_count": text_bearing_response_speak_count, + } + + +def _evaluate_transcript_integrity( + state: DemoState, + response_ids: list[str], + *, + expected_empty_response_ids: set[str], + require_cross_turn_independence: bool, + require_terminal_punctuation: bool = False, +) -> dict[str, object]: + details: list[dict[str, object]] = [] + transcripts: list[str] = [] + transcript_delta_done_ok = True + empty_turns_ok = True + nonempty_audio_has_transcript_ok = True + terminal_punctuation_ok = True + + for response_id in response_ids: + transcript = state.response_transcript_delta(response_id) + done_transcripts = state.response_transcript_done(response_id) + canonical_transcript = _canonical_transcript(transcript) + response_delta_done = ( + len(done_transcripts) == 1 and _canonical_transcript(done_transcripts[0]) == canonical_transcript + ) or (not done_transcripts and not canonical_transcript) + expected_empty = response_id in expected_empty_response_ids + audio_delta_count = state.response_audio_delta_count(response_id) + response_empty_ok = not expected_empty or (not canonical_transcript and audio_delta_count == 0) + response_audio_has_transcript = expected_empty or audio_delta_count == 0 or bool(canonical_transcript) + response_terminal_punctuation_ok = ( + expected_empty + or not canonical_transcript + or not require_terminal_punctuation + or _has_terminal_punctuation(canonical_transcript) + ) + transcript_delta_done_ok = transcript_delta_done_ok and response_delta_done + empty_turns_ok = empty_turns_ok and response_empty_ok + nonempty_audio_has_transcript_ok = nonempty_audio_has_transcript_ok and response_audio_has_transcript + terminal_punctuation_ok = terminal_punctuation_ok and response_terminal_punctuation_ok + transcripts.append(canonical_transcript) + details.append( + { + "response_id": response_id, + "transcript": transcript, + "delta_done_ok": response_delta_done, + "expected_empty": expected_empty, + "empty_ok": response_empty_ok, + "audio_delta_count": audio_delta_count, + "audio_has_transcript": response_audio_has_transcript, + "terminal_punctuation_ok": response_terminal_punctuation_ok, + } + ) + + cross_turn_independent_ok = True + if require_cross_turn_independence: + for index, current in enumerate(transcripts): + for previous in transcripts[:index]: + if _reuses_previous_turn_tail(previous, current): + cross_turn_independent_ok = False + break + if not cross_turn_independent_ok: + break + + return { + "transcript_delta_done_ok": transcript_delta_done_ok, + "cross_turn_independent_ok": cross_turn_independent_ok, + "empty_turns_ok": empty_turns_ok, + "nonempty_audio_has_transcript_ok": nonempty_audio_has_transcript_ok, + "terminal_punctuation_ok": terminal_punctuation_ok, + "transcript_integrity": details, + } + + +async def _reader(ws, state: DemoState, stop: asyncio.Event) -> None: + try: + while not stop.is_set(): + raw = await ws.recv() + if not isinstance(raw, str): + continue + event = json.loads(raw) + if isinstance(event, dict): + state.add(event) + except ConnectionClosed: + return + + +async def _send_pcm16( + ws, + pcm16: bytes, + *, + chunk_ms: int, + realtime_delay: bool, + hints: dict[str, object] | None = None, + first_chunk_hints: dict[str, object] | None = None, + on_model_unit_ready=None, + frame_b64: str | None = None, +) -> None: + hints = hints or {} + first_chunk_hints = first_chunk_hints or {} + chunk_bytes = max(PCM16_SAMPLE_RATE * PCM16_BYTES_PER_SAMPLE * chunk_ms // 1000, PCM16_BYTES_PER_SAMPLE) + model_unit_bytes = PCM16_SAMPLE_RATE * PCM16_BYTES_PER_SAMPLE + audio_ms = 0 + next_model_unit_bytes = model_unit_bytes + frames_sent = 0 + for offset in range(0, len(pcm16), chunk_bytes): + chunk = pcm16[offset : offset + chunk_bytes] + duration_ms = int(len(chunk) / (PCM16_SAMPLE_RATE * PCM16_BYTES_PER_SAMPLE) * 1000) + audio_ms += duration_ms + chunk_hints = dict(hints) + if offset == 0: + chunk_hints.update(first_chunk_hints) + if frame_b64 is not None and audio_ms > frames_sent * 1000: + # Omni duplex cadence: one camera frame per 1 s of audio. + chunk_hints["video_frames"] = [frame_b64] + frames_sent += 1 + await ws.send( + json.dumps( + { + "type": "input_audio_buffer.append", + "audio": base64.b64encode(chunk).decode("ascii"), + "input_audio_format": "pcm16", + "sample_rate_hz": PCM16_SAMPLE_RATE, + "duration_ms": duration_ms, + "audio_end_ms": audio_ms, + **chunk_hints, + } + ) + ) + while offset + len(chunk) >= next_model_unit_bytes: + next_model_unit_bytes += model_unit_bytes + if on_model_unit_ready is None: + continue + should_continue = on_model_unit_ready() + if inspect.isawaitable(should_continue): + should_continue = await should_continue + if should_continue is False: + return + if realtime_delay: + await asyncio.sleep(duration_ms / 1000) + + +async def _wait_for( + state: DemoState, + predicate, + *, + timeout_s: float, + label: str, +) -> None: + deadline = time.monotonic() + timeout_s + while time.monotonic() < deadline: + if predicate(): + return + await asyncio.sleep(0.02) + raise TimeoutError(f"Timed out waiting for {label}") + + +async def _wait_for_model_policy_outcome( + state: DemoState, + *, + before_created: int, + before_model_listen: int, + timeout_s: float, + listen_settle_s: float, + label: str, +) -> str: + deadline = time.monotonic() + timeout_s + listen_deadline: float | None = None + while time.monotonic() < deadline: + if state.count("response.created") > before_created: + return "speak" + if state.model_listen_count > before_model_listen: + if listen_deadline is None: + listen_deadline = time.monotonic() + max(0.0, listen_settle_s) + if time.monotonic() >= listen_deadline: + return "listen" + await asyncio.sleep(0.02) + raise TimeoutError(f"Timed out waiting for {label}") + + +async def _send_clean_turn( + ws, + state: DemoState, + pcm16: bytes, + *, + transcript: str, + duration_ms: int | None, + chunk_ms: int, + timeout_s: float, + require_audio: bool, + validation_mode: str, + send_transcript_hint: bool = True, + realtime_input: bool = False, + model_policy_settle_s: float = 2.0, + commit_input: bool = True, + frame_b64: str | None = None, +) -> tuple[str | None, str]: + before_created = state.count("response.created") + before_model_listen = state.model_listen_count + + def eligible_response_id() -> str | None: + for response_id in state.response_ids[before_created:]: + if not state.response_done(response_id): + continue + if require_audio and state.response_audio_delta_count(response_id) == 0: + continue + if require_audio and not _canonical_transcript(state.response_transcript_delta(response_id)): + continue + return response_id + return None + + await _send_pcm16( + ws, + _select_turn_audio(pcm16, duration_ms), + chunk_ms=chunk_ms, + realtime_delay=realtime_input, + hints={"transcript": transcript} if send_transcript_hint else {}, + frame_b64=frame_b64, + ) + if commit_input: + state.input_commit_sent_at_s.append(time.monotonic()) + await ws.send(json.dumps({"type": "input_audio_buffer.commit", "final": True})) + if validation_mode == "model-policy": + outcome = await _wait_for_model_policy_outcome( + state, + before_created=before_created, + before_model_listen=before_model_listen, + timeout_s=timeout_s, + listen_settle_s=model_policy_settle_s, + label=f"{transcript} model speak/listen decision", + ) + if outcome == "listen": + return None, "listen" + else: + await _wait_for( + state, + lambda: state.count("response.created") > before_created, + timeout_s=timeout_s, + label=f"{transcript} response.created", + ) + await _wait_for( + state, + lambda: eligible_response_id() is not None, + timeout_s=timeout_s, + label=f"{transcript} completed response", + ) + response_id = ( + eligible_response_id() + if validation_mode == "response-required" + else (state.response_ids[before_created] if len(state.response_ids) > before_created else None) + ) + if require_audio: + await _wait_for( + state, + lambda: state.response_audio_delta_count(response_id) > 0, + timeout_s=timeout_s, + label=f"{transcript} response.audio.delta", + ) + await _wait_for( + state, + lambda: state.response_done(response_id), + timeout_s=timeout_s, + label=f"{transcript} response.done", + ) + if not isinstance(response_id, str): + raise TimeoutError(f"Missing response id for {transcript}") + await _ack_response_playback( + ws, + state, + response_id, + timeout_s=timeout_s, + label=transcript, + ) + return response_id, "speak" + + +async def _ack_response_playback( + ws, + state: DemoState, + response_id: str, + *, + timeout_s: float, + label: str, +) -> None: + played_ms = state.response_playback_sent_ms(response_id) + if played_ms <= 0: + return + before_ack = state.playback_ack_count + await ws.send( + json.dumps( + { + "type": "playback.ack", + "response_id": response_id, + "item_id": f"item_{response_id}", + "played_ms": played_ms, + "committed_ms": played_ms, + } + ) + ) + await _wait_for( + state, + lambda: state.playback_ack_count > before_ack and state.response_playback_history_committed(response_id), + timeout_s=timeout_s, + label=f"{label} playback history committed", + ) + + +async def _ack_all_completed_response_playback( + ws, + state: DemoState, + *, + timeout_s: float, +) -> None: + for response_id in state.completed_response_ids(): + if state.response_playback_sent_ms(response_id) <= 0: + continue + if state.response_playback_history_committed(response_id): + continue + await _ack_response_playback( + ws, + state, + response_id, + timeout_s=timeout_s, + label=response_id, + ) + + +async def _drain_model_policy_responses( + ws, + state: DemoState, + *, + timeout_s: float, + settle_s: float, +) -> None: + deadline = time.monotonic() + timeout_s + settle_s = max(0.0, settle_s) + observed_created_count = state.count("response.created") + quiet_since: float | None = None + + while True: + remaining_s = deadline - time.monotonic() + active_response_ids = [ + response_id for response_id in state.response_ids if not state.response_done(response_id) + ] + if remaining_s <= 0: + raise TimeoutError(f"Timed out draining model-policy responses; active response ids: {active_response_ids}") + + await _ack_all_completed_response_playback( + ws, + state, + timeout_s=remaining_s, + ) + + now = time.monotonic() + created_count = state.count("response.created") + if created_count != observed_created_count: + observed_created_count = created_count + quiet_since = None + + active_response_ids = [ + response_id for response_id in state.response_ids if not state.response_done(response_id) + ] + if active_response_ids: + quiet_since = None + else: + if quiet_since is None: + quiet_since = now + if now - quiet_since >= settle_s: + return + + await asyncio.sleep(min(0.02, max(0.0, deadline - time.monotonic()))) + + +def _event_index_for_response(state: DemoState, event_type: str, response_id: str) -> int | None: + return state.first_index( + event_type, + lambda event: state._event_response_id(event) == response_id, + ) + + +def _nth_event_index(state: DemoState, event_type: str, occurrence: int) -> int | None: + seen = 0 + for index, event in enumerate(state.events): + if event.get("type") != event_type: + continue + seen += 1 + if seen == occurrence: + return index + return None + + +def _continuous_overlap_terminal_is_outcome( + state: DemoState, + *, + validation_mode: str, + model_unit_ready_while_active: bool, + before_created: int, + before_model_listen: int, +) -> bool: + """Accept current-response termination as a model-policy overlap outcome.""" + return ( + validation_mode == "model-policy" + and model_unit_ready_while_active + and state.count("response.created") == before_created + and state.model_listen_count == before_model_listen + ) + + +async def _send_listen_only_overlap_pair( + ws, + state: DemoState, + first_pcm16: bytes, + second_pcm16: bytes, + *, + transcripts: tuple[str, str], + durations_ms: tuple[int | None, int | None], + chunk_ms: int, + timeout_s: float, + send_transcript_hint: bool, + realtime_input: bool, + model_policy_settle_s: float, + validation_mode: str, + silence_ms: int, + frame_b64: str | None = None, +) -> tuple[list[str | None], list[str], bool]: + if not realtime_input: + raise ValueError("listen-only-overlap requires --realtime-input") + + before_first_created = state.count("response.created") + observed_first_created = before_first_created + observed_first_listen = state.model_listen_count + + async def continue_until_first_speak() -> bool: + nonlocal observed_first_created, observed_first_listen + outcome = await _wait_for_model_policy_outcome( + state, + before_created=observed_first_created, + before_model_listen=observed_first_listen, + timeout_s=timeout_s, + listen_settle_s=model_policy_settle_s, + label=f"{transcripts[0]} model-unit speak/listen decision", + ) + observed_first_created = state.count("response.created") + observed_first_listen = state.model_listen_count + return outcome != "speak" + + first_input = _select_turn_audio(first_pcm16, durations_ms[0]) + _pcm16_silence(silence_ms) + await _send_pcm16( + ws, + first_input, + chunk_ms=chunk_ms, + realtime_delay=True, + hints={"transcript": transcripts[0]} if send_transcript_hint else {}, + on_model_unit_ready=continue_until_first_speak, + frame_b64=frame_b64, + ) + state.input_commit_sent_at_s.append(time.monotonic()) + await ws.send(json.dumps({"type": "input_audio_buffer.commit", "final": True})) + await _wait_for( + state, + lambda: state.count("response.created") > before_first_created, + timeout_s=timeout_s, + label=f"{transcripts[0]} response.created", + ) + first_response_id = state.response_ids[before_first_created] + await _wait_for( + state, + lambda: state.response_audio_delta_count(first_response_id) > 0, + timeout_s=timeout_s, + label=f"{transcripts[0]} response.audio.delta", + ) + if state.response_done(first_response_id): + raise RuntimeError("first response completed before overlap input could start") + + before_second_created = state.count("response.created") + before_second_listen = state.model_listen_count + overlap_started_while_active = not state.response_done(first_response_id) + model_unit_ready_while_active = False + + def record_model_unit_ready() -> None: + nonlocal model_unit_ready_while_active + model_unit_ready_while_active = not state.response_done(first_response_id) + + await _send_pcm16( + ws, + _select_turn_audio(second_pcm16, durations_ms[1]), + chunk_ms=chunk_ms, + realtime_delay=True, + hints={"transcript": transcripts[1]} if send_transcript_hint else {}, + on_model_unit_ready=record_model_unit_ready, + frame_b64=frame_b64, + ) + state.input_commit_sent_at_s.append(time.monotonic()) + await ws.send(json.dumps({"type": "input_audio_buffer.commit", "final": True})) + + await _wait_for( + state, + lambda: state.response_done(first_response_id), + timeout_s=timeout_s, + label=f"{transcripts[0]} response.done", + ) + await _ack_response_playback( + ws, + state, + first_response_id, + timeout_s=timeout_s, + label=transcripts[0], + ) + + if _continuous_overlap_terminal_is_outcome( + state, + validation_mode=validation_mode, + model_unit_ready_while_active=model_unit_ready_while_active, + before_created=before_second_created, + before_model_listen=before_second_listen, + ): + second_outcome = "continuous_terminal" + else: + second_outcome = await _wait_for_model_policy_outcome( + state, + before_created=before_second_created, + before_model_listen=before_second_listen, + timeout_s=timeout_s, + listen_settle_s=model_policy_settle_s, + label=f"{transcripts[1]} model speak/listen decision", + ) + second_response_id: str | None = None + if second_outcome == "speak": + second_response_id = state.response_ids[before_second_created] + await _wait_for( + state, + lambda: state.response_done(second_response_id), + timeout_s=timeout_s, + label=f"{transcripts[1]} response.done", + ) + await _ack_response_playback( + ws, + state, + second_response_id, + timeout_s=timeout_s, + label=transcripts[1], + ) + + second_speech_index = _nth_event_index(state, "input_audio_buffer.speech_started", 2) + first_done_index = _event_index_for_response(state, "response.done", first_response_id) + overlap_ok = ( + overlap_started_while_active + and model_unit_ready_while_active + and second_speech_index is not None + and first_done_index is not None + and second_speech_index < first_done_index + ) + return [first_response_id, second_response_id], ["speak", second_outcome], overlap_ok + + +async def run_demo(args: argparse.Namespace) -> dict[str, object]: + demo_frame_b64: str | None = None + if getattr(args, "frame_image", None): + demo_frame_b64 = base64.b64encode(Path(args.frame_image).read_bytes()).decode("ascii") + turn_input_paths = _turn_input_paths( + Path(args.input_wav), + list(getattr(args, "turn_input_wav", []) or []), + turns=args.turns, + ) + turn_pcm16 = [_read_wav_pcm16(path) for path in turn_input_paths] + if any(not pcm16 for pcm16 in turn_pcm16): + raise ValueError("input WAV has no audio") + turn_durations = _turn_durations( + list(getattr(args, "turn_duration_ms", []) or []), + turns=args.turns, + first_turn_ms=args.first_turn_ms, + ) + expected_empty_turns = set(getattr(args, "expect_empty_turn", []) or []) + invalid_empty_turns = sorted( + turn_number for turn_number in expected_empty_turns if turn_number < 1 or turn_number > max(1, args.turns) + ) + if invalid_empty_turns: + raise ValueError( + f"--expect-empty-turn values are 1-based and must refer to an existing turn: {invalid_empty_turns}" + ) + distinct_turn_inputs = _turn_inputs_are_distinct(turn_input_paths, turn_pcm16) + if getattr(args, "require_distinct_inputs", False) and not distinct_turn_inputs: + raise ValueError("--require-distinct-inputs requires a different WAV path and audio payload for every turn") + url = _url_with_model( + args.url, + args.model, + autostart=False if getattr(args, "ref_audio", None) else None, + session_id=getattr(args, "session_id", None), + ) + state = DemoState() + stop = asyncio.Event() + output_dir = Path(args.output_dir) + turn_response_ids: list[str | None] = [] + turn_outcomes: list[str] = [] + validation_mode = getattr(args, "validation_mode", "response-required") + scenario = getattr(args, "scenario", "sequential") + transcript_hints_enabled = not getattr(args, "omit_transcript_hints", False) + realtime_input = getattr(args, "realtime_input", False) + continuous_input = getattr(args, "continuous_input", False) + listen_only_overlap_ok = scenario != "listen-only-overlap" + if scenario == "listen-only-overlap" and args.turns < 2: + raise ValueError("listen-only-overlap requires at least two turns") + + async with websockets.connect(url, max_size=64 * 1024 * 1024) as ws: + reader = asyncio.create_task(_reader(ws, state, stop)) + try: + await ws.send(json.dumps(_session_update_event(args))) + await _wait_for(state, lambda: state.count("session.created") > 0, timeout_s=20, label="session.created") + + turn_transcripts = _turn_transcripts(args.first_turn_transcript, turns=args.turns) + turn_specs = list(zip(turn_transcripts, turn_durations, strict=True)) + if scenario == "listen-only-overlap": + turn_response_ids, turn_outcomes, listen_only_overlap_ok = await _send_listen_only_overlap_pair( + ws, + state, + turn_pcm16[0], + turn_pcm16[1], + transcripts=(turn_specs[0][0], turn_specs[1][0]), + durations_ms=(turn_specs[0][1], turn_specs[1][1]), + chunk_ms=args.chunk_ms, + timeout_s=args.timeout_s, + send_transcript_hint=transcript_hints_enabled, + realtime_input=realtime_input, + model_policy_settle_s=max(0.0, args.model_policy_settle_ms / 1000), + validation_mode=validation_mode, + silence_ms=max(0, args.silence_ms), + frame_b64=demo_frame_b64, + ) + for turn_index in range(2, len(turn_specs)): + transcript, duration_ms = turn_specs[turn_index] + response_id, outcome = await _send_clean_turn( + ws, + state, + turn_pcm16[turn_index], + transcript=transcript, + duration_ms=duration_ms, + chunk_ms=args.chunk_ms, + timeout_s=args.timeout_s, + require_audio=args.require_audio and (turn_index + 1) not in expected_empty_turns, + validation_mode=validation_mode, + send_transcript_hint=transcript_hints_enabled, + realtime_input=realtime_input, + model_policy_settle_s=max(0.0, args.model_policy_settle_ms / 1000), + commit_input=not continuous_input, + frame_b64=demo_frame_b64, + ) + turn_response_ids.append(response_id) + turn_outcomes.append(outcome) + else: + for turn_index, (transcript, duration_ms) in enumerate(turn_specs): + response_id, outcome = await _send_clean_turn( + ws, + state, + turn_pcm16[turn_index], + transcript=transcript, + duration_ms=duration_ms, + chunk_ms=args.chunk_ms, + timeout_s=args.timeout_s, + require_audio=args.require_audio and (turn_index + 1) not in expected_empty_turns, + validation_mode=validation_mode, + send_transcript_hint=transcript_hints_enabled, + realtime_input=realtime_input, + model_policy_settle_s=max(0.0, args.model_policy_settle_ms / 1000), + commit_input=not continuous_input, + frame_b64=demo_frame_b64, + ) + turn_response_ids.append(response_id) + turn_outcomes.append(outcome) + + if validation_mode == "model-policy": + await _drain_model_policy_responses( + ws, + state, + timeout_s=args.timeout_s, + settle_s=max(0.0, args.model_policy_settle_ms / 1000), + ) + await _ack_all_completed_response_playback( + ws, + state, + timeout_s=args.timeout_s, + ) + await ws.send(json.dumps({"type": "session.close"})) + await _wait_for(state, lambda: state.count("session.closed") > 0, timeout_s=20, label="session.closed") + finally: + stop.set() + reader.cancel() + try: + await reader + except asyncio.CancelledError: + pass + _write_demo_artifacts(state, output_dir, output_audio_format=args.output_audio_format) + + overlap_barge_in = any(decision.get("action") == "barge_in" for decision in state.overlap_decisions) + expected_turns = max(1, args.turns) + event_order_ok = ( + state.model_policy_event_order_ok( + expected_turns=expected_turns, + require_input_commit=not continuous_input, + ) + if validation_mode == "model-policy" + else state.event_order_ok(require_input_commit=not continuous_input) + ) + input_transcription_ok = _input_transcription_ok( + state.input_transcription_count, + transcript_hints_enabled=transcript_hints_enabled and not continuous_input, + ) + model_speak_event_ok = state.model_speak_before_audio_ok() + realtime_audio_lifecycle_ok = state.count("response.audio.delta") > 0 and state.count("response.audio.done") > 0 + completed_response_ids = state.completed_response_ids() + observed_turn_response_ids = [response_id for response_id in turn_response_ids if isinstance(response_id, str)] + expected_empty_response_ids = { + response_id + for turn_number in expected_empty_turns + if turn_number <= len(turn_response_ids) + and isinstance((response_id := turn_response_ids[turn_number - 1]), str) + } + transcript_integrity = _evaluate_transcript_integrity( + state, + observed_turn_response_ids, + expected_empty_response_ids=expected_empty_response_ids, + require_cross_turn_independence=_requires_cross_turn_independence( + validation_mode=validation_mode, + distinct_inputs_required=getattr(args, "require_distinct_inputs", False), + ), + require_terminal_punctuation=getattr(args, "require_distinct_inputs", False), + ) + response_speak_contract = _evaluate_response_speak_contract(state) + expected_audio_turns = expected_turns - len(expected_empty_turns) + lifecycle_counts_ok = ( + state.count("response.created") == state.count("response.done") + and len(state.response_ids) == len(completed_response_ids) + and len(completed_response_ids) == len(set(completed_response_ids)) + and state.count("response.audio.done") <= state.count("response.done") + and _response_cardinality_ok( + completed_response_ids, + expected_turns=expected_turns, + validation_mode=validation_mode, + ) + ) + turn_outcomes_ok = len(turn_outcomes) == expected_turns and ( + validation_mode == "model-policy" or all(outcome == "speak" for outcome in turn_outcomes) + ) + clean_turn_audio_ok = all( + state.response_audio_delta_count(response_id) > 0 + for response_id in observed_turn_response_ids + if response_id not in expected_empty_response_ids + ) + playback_history_committed_ok = _all_response_playback_history_committed( + state, + completed_response_ids, + ) + if validation_mode == "response-required": + full_audio_response_ok = ( + len(observed_turn_response_ids) == expected_audio_turns + len(expected_empty_turns) + and clean_turn_audio_ok + and model_speak_event_ok + and state.model_speak_delta_count > 0 + and realtime_audio_lifecycle_ok + and state.audio_marks_seen + ) + else: + full_audio_response_ok = not observed_turn_response_ids or ( + realtime_audio_lifecycle_ok and state.model_speak_delta_count > 0 + ) + stale_audio_delta_count = state.stale_audio_delta_count() + terminal_activity_ok = state.count("response.done") > 0 or ( + validation_mode == "model-policy" and state.model_listen_count > 0 + ) + all_audio_responses_have_transcript = ( + validation_mode != "response-required" + or _all_audio_responses_have_transcript( + state, + completed_response_ids, + ) + ) + unexpected_error_events = _unexpected_error_events(state) + continuous_input_ok = not continuous_input or state.count("input_audio_buffer.committed") == 0 + result = { + "ok": terminal_activity_ok + and state.count("session.closed") > 0 + and state.cancelled_count == 0 + and not overlap_barge_in + and state.truncate_count == 0 + and event_order_ok + and input_transcription_ok + and stale_audio_delta_count == 0 + and lifecycle_counts_ok + and turn_outcomes_ok + and full_audio_response_ok, + "event_counts": { + event_type: state.count(event_type) + for event_type in sorted({str(event.get("type")) for event in state.events}) + }, + "audio_delta_count": len(state.audio_deltas), + "done_count": state.done_count, + "cancelled_count": state.cancelled_count, + "listen_count": state.listen_count, + "model_listen_count": state.model_listen_count, + "buffering_listen_count": state.buffering_listen_count, + "model_speak_event_count": state.model_speak_event_count, + "model_speak_delta_count": state.model_speak_delta_count, + "playback_ack_count": state.playback_ack_count, + "playback_history_committed_count": state.playback_history_committed_count, + "playback_history_committed_ok": playback_history_committed_ok, + "truncate_count": state.truncate_count, + "input_transcription_count": state.input_transcription_count, + "audio_marks_seen": state.audio_marks_seen, + "overlap_decisions": state.overlap_decisions, + "overlap_barge_in": overlap_barge_in, + "event_order_ok": event_order_ok, + "terminal_activity_ok": terminal_activity_ok, + "input_transcription_ok": input_transcription_ok, + "completed_response_ids": completed_response_ids, + "response_timings": state.response_timing_summaries(), + "lifecycle_counts_ok": lifecycle_counts_ok, + "validation_mode": validation_mode, + "scenario": scenario, + "listen_only_overlap_ok": listen_only_overlap_ok, + "transcript_hints_enabled": transcript_hints_enabled, + "realtime_input": realtime_input, + "continuous_input": continuous_input, + "continuous_input_ok": continuous_input_ok, + "input_chunk_ms": args.chunk_ms, + "model_policy_settle_ms": args.model_policy_settle_ms, + "turn_outcomes": turn_outcomes, + "turn_outcomes_ok": turn_outcomes_ok, + "clean_turn_audio_ok": clean_turn_audio_ok, + "full_audio_response_ok": full_audio_response_ok, + "model_speak_event_ok": model_speak_event_ok, + "realtime_audio_lifecycle_ok": realtime_audio_lifecycle_ok, + "stale_audio_delta_count": stale_audio_delta_count, + "all_audio_responses_have_transcript": all_audio_responses_have_transcript, + "distinct_turn_inputs": distinct_turn_inputs, + "error_count": len(unexpected_error_events), + "errors": unexpected_error_events, + **response_speak_contract, + **transcript_integrity, + "turn_inputs": [ + { + "path": str(path), + "source_duration_ms": len(pcm16) * 1000 // (PCM16_SAMPLE_RATE * PCM16_BYTES_PER_SAMPLE), + "requested_duration_ms": duration_ms, + "sent_duration_ms": len(_select_turn_audio(pcm16, duration_ms)) + * 1000 + // (PCM16_SAMPLE_RATE * PCM16_BYTES_PER_SAMPLE), + } + for path, pcm16, duration_ms in zip(turn_input_paths, turn_pcm16, turn_durations, strict=True) + ], + "output_dir": str(output_dir), + } + result["ok"] = bool( + result["ok"] + and transcript_integrity["transcript_delta_done_ok"] + and transcript_integrity["cross_turn_independent_ok"] + and transcript_integrity["empty_turns_ok"] + and transcript_integrity["nonempty_audio_has_transcript_ok"] + and transcript_integrity["terminal_punctuation_ok"] + and all_audio_responses_have_transcript + and playback_history_committed_ok + and listen_only_overlap_ok + and not unexpected_error_events + and response_speak_contract["response_speak_contract_ok"] + and continuous_input_ok + and (distinct_turn_inputs or not getattr(args, "require_distinct_inputs", False)) + ) + return result + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--url", default="ws://localhost:8099/v1/realtime?duplex=1") + parser.add_argument("--model", default="openbmb/MiniCPM-o-4_5") + parser.add_argument("--session-id", help="Use an explicit public session ID, including for close/reopen tests.") + parser.add_argument("--input-wav", required=True) + parser.add_argument("--ref-audio", help="Optional WAV used as the MiniCPM-o voice prompt.") + parser.add_argument( + "--frame-image", + default=None, + help="Optional image file sent as an omni-duplex camera frame (one per 1 s of audio)", + ) + parser.add_argument( + "--turn-input-wav", + action="append", + default=[], + help="WAV for each turn after the first; repeat exactly turns-1 times.", + ) + parser.add_argument("--output-dir", default="/tmp/minicpmo_realtime_duplex_demo") + parser.add_argument( + "--output-audio-format", + default="pcm16", + choices=["pcm16", "wav", "g711_ulaw", "g711_alaw"], + ) + parser.add_argument("--chunk-ms", type=int, default=200) + parser.add_argument( + "--model-policy-settle-ms", + type=int, + default=2000, + help="Wait for a delayed response.created after model-listen and before closing the session.", + ) + parser.add_argument("--first-turn-ms", type=int, default=1400) + parser.add_argument( + "--turn-duration-ms", + action="append", + type=int, + default=[], + help="Audio duration for each turn; repeat turns times. Use 0 to send the complete WAV.", + ) + parser.add_argument("--first-turn-transcript", default="demo input speech") + parser.add_argument( + "--omit-transcript-hints", + action="store_true", + help="Keep local turn labels but do not send transcript hints to the server.", + ) + parser.add_argument( + "--realtime-input", + action="store_true", + help="Pace each input chunk according to its audio duration.", + ) + parser.add_argument( + "--continuous-input", + action="store_true", + help="Model-owned browser mode: stream PCM continuously without input_audio_buffer.commit.", + ) + parser.add_argument( + "--scenario", + choices=["sequential", "listen-only-overlap"], + default="sequential", + help="Run sequential turns or stream turn 2 while turn 1 audio is active without interruption.", + ) + parser.add_argument( + "--validation-mode", + choices=["model-policy", "response-required"], + default="response-required", + help="Allow model listen decisions, or require at least one response for every input turn.", + ) + parser.add_argument( + "--temperature", + type=float, + default=None, + help="Stage0 sampling temperature; response-required defaults to deterministic sampling.", + ) + parser.add_argument( + "--require-distinct-inputs", + action="store_true", + help="Require distinct WAV paths and audio payloads, and reject cross-turn transcript reuse.", + ) + parser.add_argument( + "--expect-empty-turn", + action="append", + type=int, + default=[], + help="1-based turn expected to end without text or audio; repeat for multiple turns.", + ) + parser.add_argument("--short-ack-ms", type=int, default=350) + parser.add_argument("--silence-ms", type=int, default=500) + parser.add_argument("--playback-ack-ms", type=int, default=500) + parser.add_argument("--turns", type=int, default=3) + parser.add_argument("--timeout-s", type=float, default=60.0) + parser.add_argument( + "--require-audio", + action="store_true", + help="Fail if the first native response listens instead of producing audio.", + ) + return parser.parse_args() + + +def main() -> None: + result = asyncio.run(run_demo(parse_args())) + print(json.dumps(result, ensure_ascii=False, indent=2)) + if not result["ok"]: + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/tests/e2e/online_serving/run_minicpmo_realtime_duplex_demo_pair.py b/tests/e2e/online_serving/run_minicpmo_realtime_duplex_demo_pair.py new file mode 100644 index 000000000..f5539968b --- /dev/null +++ b/tests/e2e/online_serving/run_minicpmo_realtime_duplex_demo_pair.py @@ -0,0 +1,388 @@ +"""Run two MiniCPM-o Realtime duplex demo processes concurrently. + +This E2E driver covers the same process boundary users exercise manually: +two independent ``realtime_duplex_demo.py`` invocations, two distinct input +WAVs, and two distinct output directories. +""" + +from __future__ import annotations + +import argparse +import asyncio +import base64 +import hashlib +import io +import json +import sys +import uuid +import wave +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[3] +DEMO_PATH = REPO_ROOT / "examples/online_serving/minicpmo/realtime_duplex_demo.py" +AUDIO_DELTA_EVENTS = {"response.audio.delta", "response.output_audio.delta"} + + +def _canonical_path(path: str) -> Path: + return Path(path).expanduser().resolve(strict=False) + + +def _sha256_file(path: Path) -> str: + hasher = hashlib.sha256() + with path.open("rb") as f: + for chunk in iter(lambda: f.read(1024 * 1024), b""): + hasher.update(chunk) + return hasher.hexdigest() + + +def _validate_pair_args(args: argparse.Namespace) -> dict[str, object]: + input_a = _canonical_path(args.input_wav_a) + input_b = _canonical_path(args.input_wav_b) + output_a = _canonical_path(args.output_dir_a) + output_b = _canonical_path(args.output_dir_b) + if output_a == output_b: + raise ValueError("output directories must be different") + if not input_a.is_file(): + raise ValueError(f"input WAV does not exist: {input_a}") + if not input_b.is_file(): + raise ValueError(f"input WAV does not exist: {input_b}") + sha_a = _sha256_file(input_a) + sha_b = _sha256_file(input_b) + if sha_a == sha_b: + raise ValueError("input WAV files must have different content") + return { + "input_wav_a": str(input_a), + "input_wav_b": str(input_b), + "input_sha256_a": sha_a, + "input_sha256_b": sha_b, + "output_dir_a": str(output_a), + "output_dir_b": str(output_b), + } + + +def _event_response_id(event: dict[str, object]) -> str | None: + response_id = event.get("response_id") + if isinstance(response_id, str): + return response_id + response = event.get("response") + if isinstance(response, dict) and isinstance(response.get("id"), str): + return response["id"] + return None + + +def _read_json(path: Path) -> dict[str, object]: + if not path.is_file(): + return {} + data = json.loads(path.read_text(encoding="utf-8")) + return data if isinstance(data, dict) else {} + + +def _read_events(path: Path) -> list[dict[str, object]]: + if not path.is_file(): + return [] + events: list[dict[str, object]] = [] + for line in path.read_text(encoding="utf-8").splitlines(): + if not line.strip(): + continue + event = json.loads(line) + if isinstance(event, dict): + events.append(event) + return events + + +def _response_ids_from_artifacts(result: dict[str, object], events: list[dict[str, object]]) -> list[str]: + response_ids = result.get("response_ids") + if isinstance(response_ids, list): + ids = [item for item in response_ids if isinstance(item, str)] + if ids: + return ids + seen: list[str] = [] + for event in events: + response_id = _event_response_id(event) + if response_id is not None and response_id not in seen: + seen.append(response_id) + return seen + + +def _audio_delta_duration_ms(event: dict[str, object]) -> float | None: + encoded = event.get("delta") or event.get("audio") + if not isinstance(encoded, str) or not encoded: + return None + try: + raw = base64.b64decode(encoded) + except ValueError: + return None + fmt = str(event.get("format") or event.get("audio_format") or "pcm16").lower() + sample_rate = event.get("sample_rate_hz") or event.get("sample_rate") or 24_000 + if not isinstance(sample_rate, int | float) or sample_rate <= 0: + return None + if "wav" in fmt: + try: + with wave.open(io.BytesIO(raw), "rb") as wav: + return round((wav.getnframes() * 1000.0) / wav.getframerate(), 3) + except (EOFError, wave.Error): + return None + bytes_per_sample = 4 if "f32" in fmt else 2 + return round((len(raw) * 1000.0) / (bytes_per_sample * float(sample_rate)), 3) + + +def _audio_delta_received_at_s(event: dict[str, object]) -> float | None: + raw = event.get("_client_received_at_s") + if isinstance(raw, int | float): + return float(raw) + return None + + +def _session_audio_contract_ok( + *, + audio_delta_count: int, + delta_response_ids: list[str], + response_ids: list[str], + first_audio_delta_before_done: bool, +) -> bool: + if audio_delta_count <= 0 or not response_ids: + return False + if len(delta_response_ids) != audio_delta_count: + return False + delta_id_set = set(delta_response_ids) + if len(delta_id_set) != 1: + return False + return first_audio_delta_before_done and next(iter(delta_id_set)) in set(response_ids) + + +def _summarize_session( + *, + label: str, + input_wav: str, + output_dir: str, + returncode: int, + stdout: str, + stderr: str, +) -> dict[str, object]: + output_path = Path(output_dir) + result = _read_json(output_path / "result.json") + events = _read_events(output_path / "events.jsonl") + audio_delta_indices = [index for index, event in enumerate(events) if event.get("type") in AUDIO_DELTA_EVENTS] + done_indices = [index for index, event in enumerate(events) if event.get("type") == "response.done"] + response_ids = _response_ids_from_artifacts(result, events) + delta_response_ids = [ + response_id + for response_id in (_event_response_id(events[index]) for index in audio_delta_indices) + if response_id is not None + ] + first_audio_delta_before_done = bool(audio_delta_indices and done_indices) and min(audio_delta_indices) < min( + done_indices + ) + audio_delta_events = [events[index] for index in audio_delta_indices] + first_received_at_s = next( + ( + received_at + for received_at in (_audio_delta_received_at_s(event) for event in audio_delta_events) + if received_at is not None + ), + None, + ) + audio_delta_timings = [] + for event in audio_delta_events: + received_at_s = _audio_delta_received_at_s(event) + audio_delta_timings.append( + { + "response_id": _event_response_id(event), + "received_at_s": received_at_s, + "arrival_offset_ms": ( + round((received_at_s - first_received_at_s) * 1000.0, 3) + if received_at_s is not None and first_received_at_s is not None + else None + ), + "audio_duration_ms": _audio_delta_duration_ms(event), + } + ) + all_audio_deltas_same_response = ( + len(delta_response_ids) == len(audio_delta_indices) and len(set(delta_response_ids)) == 1 + ) + response_id_nonempty = bool(response_ids) + response_audio_contract_ok = _session_audio_contract_ok( + audio_delta_count=len(audio_delta_indices), + delta_response_ids=delta_response_ids, + response_ids=response_ids, + first_audio_delta_before_done=first_audio_delta_before_done, + ) + return { + "label": label, + "input_wav": input_wav, + "output_dir": output_dir, + "returncode": returncode, + "ok": returncode == 0 and result.get("ok") is True and response_audio_contract_ok, + "result_ok": result.get("ok"), + "response_ids": response_ids, + "response_id_nonempty": response_id_nonempty, + "audio_bytes": result.get("audio_bytes", 0), + "audio_delta_count": len(audio_delta_indices), + "audio_delta_response_ids": delta_response_ids, + "audio_delta_timings": audio_delta_timings, + "all_audio_deltas_same_response": all_audio_deltas_same_response, + "first_audio_delta_before_done": first_audio_delta_before_done, + "response_audio_contract_ok": response_audio_contract_ok, + "event_count": len(events), + "stdout_tail": stdout[-4000:], + "stderr_tail": stderr[-4000:], + } + + +def _identity_isolation_ok(sessions: list[dict[str, object]]) -> bool: + seen: set[str] = set() + for session in sessions: + ids = {response_id for response_id in session.get("response_ids", []) if isinstance(response_id, str)} + if not ids: + return False + if ids & seen: + return False + seen.update(ids) + return True + + +async def _run_demo_process( + *, + label: str, + args: argparse.Namespace, + input_wav: str, + output_dir: str, +) -> dict[str, object]: + command = [ + sys.executable, + str(DEMO_PATH), + "--url", + args.url, + "--model", + args.model, + "--input-wav", + input_wav, + "--output-dir", + output_dir, + "--chunk-ms", + str(args.chunk_ms), + "--timeout-s", + str(args.timeout_s), + "--session-id", + f"duplex-pair-{label}-{uuid.uuid4().hex}", + ] + if args.no_realtime_pacing: + command.append("--no-realtime-pacing") + if args.require_audio: + command.append("--require-audio") + command.extend(["--ref-audio", str(args.ref_audio)]) + + process = await asyncio.create_subprocess_exec( + *command, + cwd=str(REPO_ROOT), + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + try: + stdout_bytes, stderr_bytes = await asyncio.wait_for( + process.communicate(), + timeout=args.timeout_s + 15.0, + ) + except TimeoutError: + process.kill() + stdout_bytes, stderr_bytes = await process.communicate() + return _summarize_session( + label=label, + input_wav=input_wav, + output_dir=output_dir, + returncode=-1, + stdout=stdout_bytes.decode(errors="replace"), + stderr=(stderr_bytes.decode(errors="replace") + "\nclient timeout"), + ) + return _summarize_session( + label=label, + input_wav=input_wav, + output_dir=output_dir, + returncode=process.returncode if process.returncode is not None else -1, + stdout=stdout_bytes.decode(errors="replace"), + stderr=stderr_bytes.decode(errors="replace"), + ) + + +async def run_pair(args: argparse.Namespace) -> dict[str, object]: + validated = _validate_pair_args(args) + Path(validated["output_dir_a"]).mkdir(parents=True, exist_ok=True) + Path(validated["output_dir_b"]).mkdir(parents=True, exist_ok=True) + + sessions = await asyncio.gather( + _run_demo_process( + label="a", + args=args, + input_wav=str(validated["input_wav_a"]), + output_dir=str(validated["output_dir_a"]), + ), + _run_demo_process( + label="b", + args=args, + input_wav=str(validated["input_wav_b"]), + output_dir=str(validated["output_dir_b"]), + ), + ) + identity_isolation_ok = _identity_isolation_ok(sessions) + min_audio_delta_ok = all( + int(session.get("audio_delta_count", 0)) >= args.min_audio_deltas_per_session for session in sessions + ) + response_audio_contract_ok = all(session.get("response_audio_contract_ok") is True for session in sessions) + result = { + "ok": ( + all(session.get("ok") is True for session in sessions) + and identity_isolation_ok + and min_audio_delta_ok + and response_audio_contract_ok + ), + "input_wavs_distinct": True, + "output_dirs_distinct": True, + "identity_isolation_ok": identity_isolation_ok, + "min_audio_deltas_per_session": args.min_audio_deltas_per_session, + "min_audio_delta_ok": min_audio_delta_ok, + "response_audio_contract_ok": response_audio_contract_ok, + "inputs": { + "a": validated["input_wav_a"], + "b": validated["input_wav_b"], + "sha256_a": validated["input_sha256_a"], + "sha256_b": validated["input_sha256_b"], + }, + "sessions": sessions, + } + summary_output = ( + Path(args.summary_output) if args.summary_output else Path(validated["output_dir_a"]).parent / "summary.json" + ) + summary_output.parent.mkdir(parents=True, exist_ok=True) + summary_output.write_text(json.dumps(result, ensure_ascii=False, indent=2), encoding="utf-8") + return result + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--url", default="ws://127.0.0.1:8099/v1/realtime?duplex=1") + parser.add_argument("--model", default="openbmb/MiniCPM-o-4_5") + parser.add_argument("--input-wav-a", required=True) + parser.add_argument("--input-wav-b", required=True) + parser.add_argument("--ref-audio", required=True) + parser.add_argument("--output-dir-a", required=True) + parser.add_argument("--output-dir-b", required=True) + parser.add_argument("--summary-output") + parser.add_argument("--chunk-ms", type=int, default=200) + parser.add_argument("--timeout-s", type=float, default=120.0) + parser.add_argument("--no-realtime-pacing", action="store_true") + parser.add_argument("--require-audio", action="store_true") + parser.add_argument("--min-audio-deltas-per-session", type=int, default=2) + args = parser.parse_args() + if args.min_audio_deltas_per_session < 0: + parser.error("--min-audio-deltas-per-session must be non-negative") + return args + + +def main() -> None: + result = asyncio.run(run_pair(parse_args())) + print(json.dumps(result, ensure_ascii=False, indent=2)) + raise SystemExit(0 if result["ok"] else 1) + + +if __name__ == "__main__": + main() diff --git a/tests/e2e/online_serving/run_minicpmo_realtime_duplex_multi_session.py b/tests/e2e/online_serving/run_minicpmo_realtime_duplex_multi_session.py new file mode 100644 index 000000000..61a466913 --- /dev/null +++ b/tests/e2e/online_serving/run_minicpmo_realtime_duplex_multi_session.py @@ -0,0 +1,556 @@ +"""Concurrent MiniCPM-o Realtime duplex and resumable-session E2E driver.""" + +from __future__ import annotations + +import argparse +import asyncio +import hashlib +import json +import sys +import uuid +from pathlib import Path +from types import SimpleNamespace +from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit + +import websockets +from websockets.exceptions import ConnectionClosed + +SCRIPT_DIR = Path(__file__).resolve().parent +if str(SCRIPT_DIR) not in sys.path: + sys.path.insert(0, str(SCRIPT_DIR)) + +from minicpmo_realtime_duplex_scenarios import _url_with_model, run_demo # noqa: E402 + + +def _with_resume_mode(url: str) -> str: + parts = urlsplit(url) + query = dict(parse_qsl(parts.query, keep_blank_values=True)) + query["resume"] = "1" + return urlunsplit((parts.scheme, parts.netloc, parts.path, urlencode(query), parts.fragment)) + + +def _response_ids(result: dict[str, object]) -> set[str]: + values = result.get("completed_response_ids") + return {value for value in values if isinstance(value, str)} if isinstance(values, list) else set() + + +def _event_session_id(event: dict[str, object]) -> str | None: + session_id = event.get("session_id") + if isinstance(session_id, str): + return session_id + inner = event.get("event") + if isinstance(inner, dict): + nested_session_id = inner.get("session_id") + if isinstance(nested_session_id, str): + return nested_session_id + return None + + +def _error_code(event: dict[str, object]) -> str | None: + error = event.get("error") + if isinstance(error, dict) and isinstance(error.get("code"), str): + return error["code"] + code = event.get("code") + return code if isinstance(code, str) else None + + +def _validate_identity_isolation(results: list[dict[str, object]]) -> bool: + seen: set[str] = set() + for result in results: + current = _response_ids(result) + if current & seen: + return False + seen.update(current) + return True + + +def _validate_semantic_isolation( + results: list[dict[str, object]], + *, + input_wavs: list[str], + expected_tokens: list[str], +) -> bool: + if not input_wavs: + return True + if len(results) != len(input_wavs): + return False + input_hashes = [hashlib.sha256(Path(path).read_bytes()).digest() for path in input_wavs] + if len(set(input_hashes)) != len(input_hashes): + return False + if not expected_tokens: + return True + if len(expected_tokens) != len(results): + return False + for result, expected_token in zip(results, expected_tokens, strict=True): + details = result.get("transcript_integrity") + transcripts = ( + [str(item.get("transcript", "")) for item in details if isinstance(item, dict)] + if isinstance(details, list) + else [] + ) + if expected_token not in "".join(transcripts): + return False + return True + + +async def _receive_until(ws, event_type: str, *, timeout_s: float) -> tuple[dict[str, object], list[dict[str, object]]]: + async def receive() -> tuple[dict[str, object], list[dict[str, object]]]: + events: list[dict[str, object]] = [] + while True: + raw = await ws.recv() + if not isinstance(raw, str): + continue + event = json.loads(raw) + if not isinstance(event, dict): + continue + events.append(event) + if event.get("type") == event_type: + return event, events + + return await asyncio.wait_for(receive(), timeout=timeout_s) + + +async def _open_admission_session( + args: argparse.Namespace, + session_id: str, +) -> tuple[object, dict[str, object]]: + url = _url_with_model(args.url, args.model, session_id=session_id) + ws = await websockets.connect(url, max_size=64 * 1024 * 1024) + await ws.send( + json.dumps( + { + "type": "session.update", + "session": { + "session_id": session_id, + "model": args.model, + "modalities": ["audio", "text"], + "extra_body": {"minicpmo45_native_duplex": True}, + }, + } + ) + ) + created, _ = await _receive_until(ws, "session.created", timeout_s=args.timeout_s) + return ws, created + + +async def _close_admission_session(ws, *, timeout_s: float) -> None: + await ws.send(json.dumps({"type": "session.close"})) + await _receive_until(ws, "session.closed", timeout_s=timeout_s) + await ws.close() + + +async def _admission_probe(args: argparse.Namespace, *, limit: int) -> dict[str, object]: + if limit < 1: + raise ValueError("admission limit must be positive") + prefix = f"admission-{uuid.uuid4().hex}" + accepted: list[tuple[object, dict[str, object]]] = [] + replacement: tuple[object, dict[str, object]] | None = None + overflow_code = None + try: + for index in range(limit): + accepted.append(await _open_admission_session(args, f"{prefix}-accepted-{index}")) + + overflow_id = f"{prefix}-overflow" + overflow_url = _url_with_model(args.url, args.model, session_id=overflow_id) + async with websockets.connect(overflow_url, max_size=64 * 1024 * 1024) as overflow: + await overflow.send( + json.dumps( + { + "type": "session.update", + "session": { + "session_id": overflow_id, + "model": args.model, + "modalities": ["audio", "text"], + "extra_body": {"minicpmo45_native_duplex": True}, + }, + } + ) + ) + error, _ = await _receive_until(overflow, "error", timeout_s=args.timeout_s) + overflow_code = _error_code(error) + + first_ws, _ = accepted.pop(0) + await _close_admission_session(first_ws, timeout_s=args.timeout_s) + replacement = await _open_admission_session(args, f"{prefix}-replacement") + + first_capabilities = accepted[0][1].get("session") if accepted else replacement[1].get("session") + capabilities = first_capabilities.get("capabilities") if isinstance(first_capabilities, dict) else None + advertised_multi = capabilities.get("supports_multi_session") if isinstance(capabilities, dict) else None + admission_mode = capabilities.get("session_admission_mode") if isinstance(capabilities, dict) else None + return { + "ok": ( + overflow_code == "resource_exhausted" + and replacement[1].get("type") == "session.created" + and admission_mode == "engine_managed" + ), + "configured_limit": limit, + "accepted_before_rejection": limit, + "overflow_error_code": overflow_code, + "replacement_accepted": True, + "advertised_multi_session": advertised_multi, + "session_admission_mode": admission_mode, + } + finally: + cleanup = list(accepted) + if replacement is not None: + cleanup.append(replacement) + for ws, _ in cleanup: + try: + await _close_admission_session(ws, timeout_s=args.timeout_s) + except Exception: + await ws.close() + + +async def _resume_probe( + args: argparse.Namespace, + *, + session_id: str, + expect_expired: bool = False, +) -> dict[str, object]: + url = _url_with_model(args.url, args.model, session_id=session_id) + async with websockets.connect(url, max_size=64 * 1024 * 1024) as first: + await first.send( + json.dumps( + { + "type": "session.update", + "session": { + "model": args.model, + "modalities": ["audio", "text"], + "extra_body": {"minicpmo45_native_duplex": True}, + }, + } + ) + ) + created, first_events = await _receive_until(first, "session.created", timeout_s=args.timeout_s) + token = created.get("resume_token") + incarnation = created.get("incarnation") + generation = created.get("attachment_generation") + if not isinstance(token, str) or not isinstance(incarnation, int): + raise RuntimeError("session.created omitted resumable credentials") + last_seq = max( + ( + event.get("server_event_seq", 0) + for event in first_events + if isinstance(event.get("server_event_seq"), int) + ), + default=0, + ) + + delay_s = args.expire_after_s if expect_expired else args.resume_after_ms / 1000 + if delay_s > 0: + await asyncio.sleep(delay_s) + + resume_url = _with_resume_mode(url) + async with websockets.connect(resume_url, max_size=64 * 1024 * 1024) as second: + await second.send( + json.dumps( + { + "type": "session.resume", + "session_id": session_id, + "incarnation": incarnation, + "resume_token": token, + "last_received_server_event_seq": last_seq, + } + ) + ) + if expect_expired: + error, error_events = await _receive_until(second, "error", timeout_s=args.timeout_s) + error_payload = error.get("error") + code = error_payload.get("code") if isinstance(error_payload, dict) else error.get("code") + return { + "ok": code == "session_resume_expired", + "session_id": session_id, + "expired": True, + "error_code": code, + "event_count": len(error_events), + } + resumed, replay = await _receive_until(second, "session.resumed", timeout_s=args.timeout_s) + rotated = resumed.get("resume_token") + if not isinstance(rotated, str) or rotated == token: + raise RuntimeError("session.resume did not rotate the resume token") + await second.send(json.dumps({"type": "session.heartbeat"})) + heartbeat, heartbeat_events = await _receive_until( + second, + "session.heartbeat_ack", + timeout_s=args.timeout_s, + ) + await second.send(json.dumps({"type": "session.close"})) + closed, close_events = await _receive_until(second, "session.closed", timeout_s=args.timeout_s) + + replay_sequences = [event["server_event_seq"] for event in replay if isinstance(event.get("server_event_seq"), int)] + return { + "ok": ( + resumed.get("session_id") == session_id + and isinstance(generation, int) + and resumed.get("attachment_generation") == generation + 1 + and _event_session_id(heartbeat) == session_id + and _event_session_id(closed) == session_id + and replay_sequences == sorted(replay_sequences) + ), + "session_id": session_id, + "resumed_session_id": resumed.get("session_id"), + "heartbeat_session_id": _event_session_id(heartbeat), + "closed_session_id": _event_session_id(closed), + "initial_attachment_generation": generation, + "resumed_attachment_generation": resumed.get("attachment_generation"), + "replayed_event_count": len(replay_sequences), + "replayed_event_sequences": replay_sequences, + "heartbeat_event_count": len(heartbeat_events), + "close_event_count": len(close_events), + "token_rotated": True, + } + + +async def _takeover_probe( + args: argparse.Namespace, + *, + session_id: str, +) -> dict[str, object]: + url = _url_with_model(args.url, args.model, session_id=session_id) + first = await websockets.connect(url, max_size=64 * 1024 * 1024) + second = None + try: + await first.send( + json.dumps( + { + "type": "session.update", + "session": { + "model": args.model, + "modalities": ["audio", "text"], + "extra_body": {"minicpmo45_native_duplex": True}, + }, + } + ) + ) + created, first_events = await _receive_until(first, "session.created", timeout_s=args.timeout_s) + token = created.get("resume_token") + incarnation = created.get("incarnation") + generation = created.get("attachment_generation") + if not isinstance(token, str) or not isinstance(incarnation, int) or not isinstance(generation, int): + raise RuntimeError("session.created omitted takeover credentials") + last_seq = max( + ( + event.get("server_event_seq", 0) + for event in first_events + if isinstance(event.get("server_event_seq"), int) + ), + default=0, + ) + + second = await websockets.connect(_with_resume_mode(url), max_size=64 * 1024 * 1024) + await second.send( + json.dumps( + { + "type": "session.resume", + "session_id": session_id, + "incarnation": incarnation, + "resume_token": token, + "last_received_server_event_seq": last_seq, + } + ) + ) + resumed, replay_events = await _receive_until(second, "session.resumed", timeout_s=args.timeout_s) + replaced, replaced_events = await _receive_until(first, "session.replaced", timeout_s=args.timeout_s) + await asyncio.wait_for(first.wait_closed(), timeout=args.timeout_s) + + rejected_old_writes = 0 + for _ in range(4): + try: + await first.send(json.dumps({"type": "session.heartbeat"})) + except ConnectionClosed: + rejected_old_writes += 1 + + await second.send(json.dumps({"type": "session.heartbeat"})) + heartbeat, heartbeat_events = await _receive_until( + second, + "session.heartbeat_ack", + timeout_s=args.timeout_s, + ) + await second.send(json.dumps({"type": "session.close"})) + closed, close_events = await _receive_until(second, "session.closed", timeout_s=args.timeout_s) + rotated_token = resumed.get("resume_token") + return { + "ok": ( + resumed.get("session_id") == session_id + and resumed.get("attachment_generation") == generation + 1 + and isinstance(rotated_token, str) + and rotated_token != token + and _event_session_id(replaced) == session_id + and replaced.get("attachment_generation") == generation + and rejected_old_writes == 4 + and _event_session_id(heartbeat) == session_id + and _event_session_id(closed) == session_id + ), + "session_id": session_id, + "initial_attachment_generation": generation, + "resumed_attachment_generation": resumed.get("attachment_generation"), + "replaced_attachment_generation": replaced.get("attachment_generation"), + "token_rotated": isinstance(rotated_token, str) and rotated_token != token, + "old_attachment_closed": True, + "rejected_old_writes": rejected_old_writes, + "replay_event_count": len(replay_events), + "replaced_event_count": len(replaced_events), + "heartbeat_event_count": len(heartbeat_events), + "close_event_count": len(close_events), + } + finally: + if second is not None: + await second.close() + await first.close() + + +def _demo_args(args: argparse.Namespace, index: int) -> SimpleNamespace: + validation_mode = "response-required" if args.response_required else "model-policy" + input_wav = args.session_input_wav[index] if args.session_input_wav else args.input_wav + return SimpleNamespace( + url=args.url, + model=args.model, + session_id=f"multi-{index}-{uuid.uuid4().hex}", + input_wav=input_wav, + ref_audio=args.ref_audio, + turn_input_wav=list(args.turn_input_wav), + output_dir=str(Path(args.output_dir) / f"session_{index:02d}"), + output_audio_format="pcm16", + chunk_ms=args.chunk_ms, + realtime_input=args.realtime_input, + first_turn_ms=args.first_turn_ms, + turn_duration_ms=list(args.turn_duration_ms), + first_turn_transcript=f"session {index} input", + omit_transcript_hints=True, + validation_mode=validation_mode, + temperature=args.temperature, + scenario="sequential", + require_audio=args.response_required, + require_distinct_inputs=False, + expect_empty_turn=[], + short_ack_ms=350, + turns=args.turns, + timeout_s=args.timeout_s, + model_policy_settle_ms=args.model_policy_settle_ms, + ) + + +async def run_multi_session(args: argparse.Namespace) -> dict[str, object]: + if args.sessions < 1: + raise ValueError("--sessions must be positive") + output_dir = Path(args.output_dir) + output_dir.mkdir(parents=True, exist_ok=True) + resume_result = None + if args.disconnect_session_index is not None: + if not 0 <= args.disconnect_session_index < args.sessions: + raise ValueError("--disconnect-session-index is outside the session range") + resume_result = await _resume_probe( + args, + session_id=f"resume-{args.disconnect_session_index}-{uuid.uuid4().hex}", + ) + takeover_result = None + if args.takeover_session_index is not None: + if not 0 <= args.takeover_session_index < args.sessions: + raise ValueError("--takeover-session-index is outside the session range") + takeover_result = await _takeover_probe( + args, + session_id=f"takeover-{args.takeover_session_index}-{uuid.uuid4().hex}", + ) + expiry_result = None + if args.expire_session_index is not None: + if not 0 <= args.expire_session_index < args.sessions: + raise ValueError("--expire-session-index is outside the session range") + expiry_result = await _resume_probe( + args, + session_id=f"expire-{args.expire_session_index}-{uuid.uuid4().hex}", + expect_expired=True, + ) + admission_result = ( + await _admission_probe(args, limit=args.verify_admission_limit) + if args.verify_admission_limit is not None + else None + ) + + session_results = await asyncio.gather( + *(run_demo(_demo_args(args, index)) for index in range(args.sessions)), + return_exceptions=True, + ) + failures = [repr(result) for result in session_results if isinstance(result, BaseException)] + completed = [result for result in session_results if isinstance(result, dict)] + identity_isolation_ok = _validate_identity_isolation(completed) + semantic_isolation_ok = _validate_semantic_isolation( + completed, + input_wavs=list(args.session_input_wav), + expected_tokens=list(args.session_expected_token), + ) + result = { + "ok": ( + not failures + and len(completed) == args.sessions + and all(item.get("ok") is True for item in completed) + and identity_isolation_ok + and semantic_isolation_ok + and (resume_result is None or resume_result.get("ok") is True) + and (takeover_result is None or takeover_result.get("ok") is True) + and (expiry_result is None or expiry_result.get("ok") is True) + and (admission_result is None or admission_result.get("ok") is True) + ), + "session_count": args.sessions, + "identity_isolation_ok": identity_isolation_ok, + "semantic_isolation_ok": semantic_isolation_ok, + "resume": resume_result, + "takeover": takeover_result, + "expiry": expiry_result, + "admission": admission_result, + "failures": failures, + "sessions": completed, + } + (output_dir / "summary.json").write_text( + json.dumps(result, ensure_ascii=False, indent=2), + encoding="utf-8", + ) + return result + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--url", default="ws://127.0.0.1:8113/v1/realtime?duplex=1") + parser.add_argument("--base-url", help="Deprecated alias; /v1/realtime is appended when supplied.") + parser.add_argument("--model", default="openbmb/MiniCPM-o-4_5") + parser.add_argument("--sessions", type=int, default=2) + parser.add_argument("--input-wav", required=True) + parser.add_argument("--ref-audio", help="Optional WAV used as the MiniCPM-o voice prompt for every session.") + parser.add_argument("--session-input-wav", action="append", default=[]) + parser.add_argument("--session-expected-token", action="append", default=[]) + parser.add_argument("--turn-input-wav", action="append", default=[]) + parser.add_argument("--output-dir", default="/tmp/minicpmo_pr3907_multi_session_e2e") + parser.add_argument("--realtime-input", action="store_true") + parser.add_argument("--chunk-ms", type=int, default=200) + parser.add_argument("--turns", type=int, default=1) + parser.add_argument("--first-turn-ms", type=int, default=1400) + parser.add_argument("--turn-duration-ms", type=int, action="append", default=[]) + parser.add_argument("--response-required", action="store_true") + parser.add_argument("--temperature", type=float, default=None) + parser.add_argument("--disconnect-session-index", type=int) + parser.add_argument("--takeover-session-index", type=int) + parser.add_argument("--resume-after-ms", type=int, default=1000) + parser.add_argument("--expire-session-index", type=int) + parser.add_argument("--expire-after-s", type=float, default=6.0) + parser.add_argument("--verify-admission-limit", type=int) + parser.add_argument("--model-policy-settle-ms", type=int, default=600) + parser.add_argument("--timeout-s", type=float, default=120.0) + args = parser.parse_args() + if args.base_url: + args.url = args.base_url.rstrip("/") + "/v1/realtime?duplex=1" + if args.session_input_wav and len(args.session_input_wav) != args.sessions: + parser.error("provide exactly one --session-input-wav per session") + if args.session_expected_token and len(args.session_expected_token) != args.sessions: + parser.error("provide exactly one --session-expected-token per session") + if args.session_expected_token and not args.session_input_wav: + parser.error("--session-expected-token requires --session-input-wav") + return args + + +def main() -> None: + result = asyncio.run(run_multi_session(parse_args())) + print(json.dumps(result, ensure_ascii=False, indent=2)) + raise SystemExit(0 if result["ok"] else 1) + + +if __name__ == "__main__": + main() diff --git a/tests/e2e/online_serving/run_minicpmo_realtime_duplex_soft_interrupt.py b/tests/e2e/online_serving/run_minicpmo_realtime_duplex_soft_interrupt.py new file mode 100644 index 000000000..3eb9405a6 --- /dev/null +++ b/tests/e2e/online_serving/run_minicpmo_realtime_duplex_soft_interrupt.py @@ -0,0 +1,467 @@ +"""Validate MiniCPM-o Realtime duplex soft-interrupt delta streaming. + +This E2E driver runs the public ``realtime_duplex_demo.py`` against a live +duplex backend. Arbitrary audio defaults to the model-policy lifecycle contract. +The stronger response-required mode binds the two-response contract to a known +input checksum and expected second response. +""" + +from __future__ import annotations + +import argparse +import asyncio +import base64 +import hashlib +import io +import json +import sys +import uuid +import wave +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[3] +DEMO_PATH = REPO_ROOT / "examples/online_serving/minicpmo/realtime_duplex_demo.py" +AUDIO_DELTA_EVENTS = {"response.audio.delta", "response.output_audio.delta"} +TRANSCRIPT_DELTA_EVENTS = { + "response.audio_transcript.delta", + "response.output_text.delta", +} + + +def _canonical_path(path: str) -> Path: + return Path(path).expanduser().resolve(strict=False) + + +def _validate_input_sha256(path: Path, expected: str) -> str: + actual = hashlib.sha256(path.read_bytes()).hexdigest() + if actual != expected.lower(): + raise ValueError(f"input WAV SHA256 mismatch: expected {expected.lower()}, got {actual}") + return actual + + +def _read_json(path: Path) -> dict[str, object]: + if not path.is_file(): + return {} + data = json.loads(path.read_text(encoding="utf-8")) + return data if isinstance(data, dict) else {} + + +def _read_events(path: Path) -> list[dict[str, object]]: + if not path.is_file(): + return [] + events: list[dict[str, object]] = [] + for line in path.read_text(encoding="utf-8").splitlines(): + if not line.strip(): + continue + event = json.loads(line) + if isinstance(event, dict): + events.append(event) + return events + + +def _event_response_id(event: dict[str, object]) -> str | None: + response_id = event.get("response_id") + if isinstance(response_id, str): + return response_id + response = event.get("response") + if isinstance(response, dict) and isinstance(response.get("id"), str): + return response["id"] + return None + + +def _event_received_at_s(event: dict[str, object]) -> float | None: + received_at = event.get("_client_received_at_s") + if isinstance(received_at, int | float): + return float(received_at) + return None + + +def _first_event_index(events: list[dict[str, object]], event_type: str) -> int | None: + for index, event in enumerate(events): + if event.get("type") == event_type: + return index + return None + + +def _event_counts(events: list[dict[str, object]]) -> dict[str, int]: + counts: dict[str, int] = {} + for event in events: + event_type = str(event.get("type")) + counts[event_type] = counts.get(event_type, 0) + 1 + return counts + + +def _response_created_ids(events: list[dict[str, object]]) -> list[str]: + response_ids: list[str] = [] + for event in events: + if event.get("type") != "response.created": + continue + response_id = _event_response_id(event) + if response_id and response_id not in response_ids: + response_ids.append(response_id) + return response_ids + + +def _audio_delta_duration_ms(event: dict[str, object]) -> float | None: + encoded = event.get("delta") or event.get("audio") + if not isinstance(encoded, str) or not encoded: + return None + try: + raw = base64.b64decode(encoded) + except ValueError: + return None + fmt = str(event.get("format") or event.get("audio_format") or "pcm16").lower() + sample_rate = event.get("sample_rate_hz") or event.get("sample_rate") or 24_000 + if not isinstance(sample_rate, int | float) or sample_rate <= 0: + return None + if "wav" in fmt: + try: + with wave.open(io.BytesIO(raw), "rb") as wav_file: + return round((wav_file.getnframes() * 1000.0) / wav_file.getframerate(), 3) + except (EOFError, wave.Error): + return None + bytes_per_sample = 4 if "f32" in fmt else 2 + return round((len(raw) * 1000.0) / (bytes_per_sample * float(sample_rate)), 3) + + +def _normalize_text(text: str) -> str: + return "".join(text.split()) + + +def _compact_sequence(events: list[dict[str, object]]) -> list[str]: + sequence: list[str] = [] + for event in events: + event_type = event.get("type") + response_id = _event_response_id(event) + suffix = f" {response_id[-8:]}" if response_id else "" + if event_type == "response.listen": + sequence.append("listen") + elif event_type == "response.created": + sequence.append(f"response.created{suffix}") + elif event_type == "response.speak": + sequence.append(f"speak{suffix}") + elif event_type in AUDIO_DELTA_EVENTS: + sequence.append(f"audio.delta{suffix}") + elif event_type in TRANSCRIPT_DELTA_EVENTS: + delta = str(event.get("delta") or "") + sequence.append(f"text.delta{suffix}> {delta}") + elif event_type == "response.done": + sequence.append(f"response.done{suffix}") + elif event_type == "input_audio_buffer.committed": + sequence.append("input.committed") + return sequence + + +def _response_summary( + events: list[dict[str, object]], + response_id: str, + *, + t0: float | None, +) -> dict[str, object]: + created_indices = [ + index + for index, event in enumerate(events) + if event.get("type") == "response.created" and _event_response_id(event) == response_id + ] + done_indices = [ + index + for index, event in enumerate(events) + if event.get("type") == "response.done" and _event_response_id(event) == response_id + ] + audio_indices = [ + index + for index, event in enumerate(events) + if event.get("type") in AUDIO_DELTA_EVENTS and _event_response_id(event) == response_id + ] + transcript_indices = [ + index + for index, event in enumerate(events) + if event.get("type") in TRANSCRIPT_DELTA_EVENTS and _event_response_id(event) == response_id + ] + done_index = done_indices[0] if done_indices else None + audio_before_done_ok = done_index is not None and all(index < done_index for index in audio_indices) + stale_audio_count = sum(index > done_index for index in audio_indices) if done_index is not None else 0 + + def offset_ms(index: int) -> float | None: + if t0 is None: + return None + received_at = _event_received_at_s(events[index]) + return round((received_at - t0) * 1000.0, 3) if received_at is not None else None + + return { + "response_id": response_id, + "created_index": created_indices[0] if created_indices else None, + "done_indices": done_indices, + "created_offset_ms": offset_ms(created_indices[0]) if created_indices else None, + "done_offset_ms": offset_ms(done_indices[0]) if done_indices else None, + "audio_delta_count": len(audio_indices), + "transcript_delta_count": len(transcript_indices), + "audio_delta_offsets_ms": [offset_ms(index) for index in audio_indices], + "audio_duration_ms": [_audio_delta_duration_ms(events[index]) for index in audio_indices], + "transcript": "".join(str(events[index].get("delta") or "") for index in transcript_indices), + "one_done": len(done_indices) == 1, + "audio_before_done_ok": audio_before_done_ok, + "stale_audio_count": stale_audio_count, + } + + +def summarize_artifacts( + *, + output_dir: Path, + validation_mode: str, + min_responses: int, + min_audio_deltas_per_response: int, + expect_second_response_substring: str | None, +) -> dict[str, object]: + if validation_mode not in {"model-policy", "response-required"}: + raise ValueError(f"unsupported validation mode: {validation_mode}") + result = _read_json(output_dir / "result.json") + events = _read_events(output_dir / "events.jsonl") + response_ids = _response_created_ids(events) + times = [_event_received_at_s(event) for event in events] + clean_times = [received_at for received_at in times if received_at is not None] + t0 = min(clean_times) if clean_times else None + response_summaries = [_response_summary(events, response_id, t0=t0) for response_id in response_ids] + commit_index = _first_event_index(events, "input_audio_buffer.committed") + first_created_index = next( + (summary["created_index"] for summary in response_summaries if isinstance(summary.get("created_index"), int)), + None, + ) + first_done_index = ( + response_summaries[0]["done_indices"][0] + if response_summaries and response_summaries[0]["done_indices"] + else None + ) + second_created_index = ( + response_summaries[1]["created_index"] + if len(response_summaries) >= 2 and isinstance(response_summaries[1].get("created_index"), int) + else None + ) + last_done_index = next( + (summary["done_indices"][0] for summary in reversed(response_summaries) if summary.get("done_indices")), + None, + ) + listen_indices = [index for index, event in enumerate(events) if event.get("type") == "response.listen"] + effective_min_responses = min_responses if validation_mode == "response-required" else 1 + enough_responses = len(response_summaries) >= effective_min_responses + response_lifecycle_ok = enough_responses and all(bool(summary.get("one_done")) for summary in response_summaries) + multi_delta_ok = enough_responses and all( + int(summary.get("audio_delta_count", 0)) >= min_audio_deltas_per_response for summary in response_summaries + ) + response_audio_contract_ok = enough_responses and all( + summary.get("audio_before_done_ok") is True and int(summary.get("stale_audio_count", 0)) == 0 + for summary in response_summaries + ) + response_before_final_commit = ( + first_created_index is not None and commit_index is not None and first_created_index < commit_index + ) + second_response_before_final_commit = ( + second_created_index is not None and commit_index is not None and second_created_index < commit_index + ) + listen_before_first_response = first_created_index is not None and any( + index < first_created_index for index in listen_indices + ) + listen_between_responses = ( + first_done_index is not None + and second_created_index is not None + and any(first_done_index < index < second_created_index for index in listen_indices) + ) + listen_after_last_done = last_done_index is not None and any(index > last_done_index for index in listen_indices) + listen_after_response_before_commit = ( + last_done_index is not None + and commit_index is not None + and any(last_done_index < index < commit_index for index in listen_indices) + ) + final_listen_after_commit = commit_index is not None and any(index > commit_index for index in listen_indices) + transcript = "".join(str(summary.get("transcript") or "") for summary in response_summaries) + second_response_transcript = ( + str(response_summaries[1].get("transcript") or "") if len(response_summaries) >= 2 else "" + ) + second_response_transcript_expectation_ok = not expect_second_response_substring or _normalize_text( + expect_second_response_substring + ) in _normalize_text(second_response_transcript) + error_events = [ + event + for event in events + if event.get("type") == "error" + or str(event.get("type") or "").endswith(".cancelled") + or str(event.get("type") or "").endswith(".truncated") + ] + cancelled_count = sum( + 1 + for event in events + if event.get("type") == "response.done" + and isinstance(event.get("response"), dict) + and event["response"].get("status") == "cancelled" + ) + result_ok = result.get("ok") is True + common_contract_ok = bool( + result_ok + and not error_events + and cancelled_count == 0 + and enough_responses + and response_lifecycle_ok + and multi_delta_ok + and response_audio_contract_ok + and response_before_final_commit + and listen_after_response_before_commit + and final_listen_after_commit + ) + mode_contract_ok = validation_mode == "model-policy" or ( + second_response_before_final_commit + and listen_before_first_response + and listen_between_responses + and listen_after_last_done + and second_response_transcript_expectation_ok + ) + ok = common_contract_ok and mode_contract_ok + return { + "ok": ok, + "result_ok": result_ok, + "validation_mode": validation_mode, + "event_counts": _event_counts(events), + "response_ids": response_ids, + "response_summaries": response_summaries, + "min_responses": min_responses, + "effective_min_responses": effective_min_responses, + "min_audio_deltas_per_response": min_audio_deltas_per_response, + "enough_responses": enough_responses, + "response_lifecycle_ok": response_lifecycle_ok, + "multi_delta_ok": multi_delta_ok, + "response_audio_contract_ok": response_audio_contract_ok, + "response_before_final_commit": response_before_final_commit, + "second_response_before_final_commit": second_response_before_final_commit, + "listen_before_first_response": listen_before_first_response, + "listen_between_responses": listen_between_responses, + "listen_after_last_done": listen_after_last_done, + "listen_after_response_before_commit": listen_after_response_before_commit, + "final_listen_after_commit": final_listen_after_commit, + "transcript": transcript, + "second_response_transcript": second_response_transcript, + "expect_second_response_substring": expect_second_response_substring, + "second_response_transcript_expectation_ok": second_response_transcript_expectation_ok, + "error_count": len(error_events), + "cancelled_count": cancelled_count, + "compact_sequence": _compact_sequence(events), + "output_dir": str(output_dir), + } + + +async def run_soft_interrupt(args: argparse.Namespace) -> dict[str, object]: + input_wav = _canonical_path(args.input_wav) + output_dir = _canonical_path(args.output_dir) + if not input_wav.is_file(): + raise ValueError(f"input WAV does not exist: {input_wav}") + input_sha256 = hashlib.sha256(input_wav.read_bytes()).hexdigest() + if args.input_sha256: + input_sha256 = _validate_input_sha256(input_wav, args.input_sha256) + command = [ + sys.executable, + str(DEMO_PATH), + "--url", + args.url, + "--model", + args.model, + "--input-wav", + str(input_wav), + "--output-dir", + str(output_dir), + "--chunk-ms", + str(args.chunk_ms), + "--timeout-s", + str(args.timeout_s), + "--session-id", + f"duplex-soft-interrupt-{uuid.uuid4().hex}", + ] + command.extend(["--ref-audio", str(_canonical_path(args.ref_audio))]) + if args.require_audio: + command.append("--require-audio") + if args.no_realtime_pacing: + command.append("--no-realtime-pacing") + + process = await asyncio.create_subprocess_exec( + *command, + cwd=str(REPO_ROOT), + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + try: + stdout_bytes, stderr_bytes = await asyncio.wait_for( + process.communicate(), + timeout=args.timeout_s + 30.0, + ) + except TimeoutError: + process.kill() + stdout_bytes, stderr_bytes = await process.communicate() + return { + "ok": False, + "returncode": -1, + "stdout_tail": stdout_bytes.decode(errors="replace")[-4000:], + "stderr_tail": (stderr_bytes.decode(errors="replace") + "\nclient timeout")[-4000:], + "output_dir": str(output_dir), + } + + summary = summarize_artifacts( + output_dir=output_dir, + validation_mode=args.validation_mode, + min_responses=args.min_responses, + min_audio_deltas_per_response=args.min_audio_deltas_per_response, + expect_second_response_substring=args.expect_second_response_substring, + ) + summary.update( + { + "returncode": process.returncode if process.returncode is not None else -1, + "input_wav": str(input_wav), + "input_sha256": input_sha256, + "stdout_tail": stdout_bytes.decode(errors="replace")[-4000:], + "stderr_tail": stderr_bytes.decode(errors="replace")[-4000:], + } + ) + summary["ok"] = bool(summary["ok"] and summary["returncode"] == 0) + summary_output = Path(args.summary_output) if args.summary_output else output_dir / "summary.json" + summary_output.parent.mkdir(parents=True, exist_ok=True) + summary_output.write_text(json.dumps(summary, ensure_ascii=False, indent=2), encoding="utf-8") + return summary + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--url", default="ws://127.0.0.1:8099/v1/realtime?duplex=1") + parser.add_argument("--model", default="openbmb/MiniCPM-o-4_5") + parser.add_argument("--input-wav", required=True) + parser.add_argument("--ref-audio", required=True) + parser.add_argument("--output-dir", required=True) + parser.add_argument("--summary-output") + parser.add_argument("--chunk-ms", type=int, default=200) + parser.add_argument("--timeout-s", type=float, default=120.0) + parser.add_argument("--require-audio", action="store_true") + parser.add_argument("--no-realtime-pacing", action="store_true") + parser.add_argument( + "--validation-mode", + choices=("model-policy", "response-required"), + default="model-policy", + ) + parser.add_argument("--min-responses", type=int, default=2) + parser.add_argument("--min-audio-deltas-per-response", type=int, default=2) + parser.add_argument("--input-sha256") + parser.add_argument("--expect-second-response-substring") + args = parser.parse_args() + if args.validation_mode == "response-required" and args.min_responses < 2: + parser.error("--min-responses must be at least 2") + if args.validation_mode == "response-required" and not args.input_sha256: + parser.error("--input-sha256 is required in response-required mode") + if args.validation_mode == "response-required" and not args.expect_second_response_substring: + parser.error("--expect-second-response-substring is required in response-required mode") + if args.min_audio_deltas_per_response < 1: + parser.error("--min-audio-deltas-per-response must be positive") + return args + + +def main() -> None: + result = asyncio.run(run_soft_interrupt(parse_args())) + print(json.dumps(result, ensure_ascii=False, indent=2)) + raise SystemExit(0 if result["ok"] else 1) + + +if __name__ == "__main__": + main() diff --git a/tests/engine/test_async_omni_engine_input.py b/tests/engine/test_async_omni_engine_input.py index a1ec472f6..91fdc752d 100644 --- a/tests/engine/test_async_omni_engine_input.py +++ b/tests/engine/test_async_omni_engine_input.py @@ -1,4 +1,5 @@ import pytest +import torch from pytest_mock import MockerFixture from vllm.sampling_params import SamplingParams from vllm.v1.engine import EngineCoreRequest @@ -64,6 +65,48 @@ def test_build_add_request_message_preserves_additional_information(mocker: Mock output_processor.add_request.assert_not_called() +def test_build_add_request_message_preserves_model_intermediate_buffer(mocker: MockerFixture): + engine = object.__new__(AsyncOmniEngine) + params = SamplingParams(max_tokens=8) + engine.default_sampling_params_list = [params] + engine.stage_metadata = [StageRuntimeInfo(final_output=False, final_output_type=None, stage_type="llm")] + engine.supported_tasks = ("speech",) + + input_processor = mocker.Mock() + input_processor.process_inputs.return_value = _make_engine_core_request() + engine.input_processor = input_processor + + output_processor = mocker.Mock() + engine.output_processors = [output_processor] + + hidden = torch.arange(6, dtype=torch.float32).reshape(2, 3) + prompt = { + "prompt_token_ids": [1, 1, 1], + "model_intermediate_buffer": { + "ids": {"tts": [11, 12]}, + "hidden_states": {"tts": hidden}, + }, + } + + msg = engine._build_add_request_message( + request_id="req-1", + prompt=prompt, + sampling_params_list=[params], + final_stage_id=0, + arrival_time=0.0, + ) + + request = msg.prompt + assert isinstance(request, OmniEngineCoreRequest) + assert request.additional_information is not None + assert request.additional_information.entries["global_request_id"].list_data == ["req-1"] + assert request.additional_information.entries["omni_final_stage_id"].scalar_data == 0 + assert isinstance(request.model_intermediate_buffer, dict) + info = request.model_intermediate_buffer + assert info["ids"]["tts"] == [11, 12] + assert torch.equal(info["hidden_states"]["tts"], hidden) + + def test_build_add_request_message_with_resumable_streaming(mocker: MockerFixture): engine = object.__new__(AsyncOmniEngine) params = SamplingParams(max_tokens=8) diff --git a/tests/engine/test_async_omni_engine_outputs.py b/tests/engine/test_async_omni_engine_outputs.py index ecf3ee9ee..a1b7a536e 100644 --- a/tests/engine/test_async_omni_engine_outputs.py +++ b/tests/engine/test_async_omni_engine_outputs.py @@ -4,13 +4,33 @@ Focuses on the critical behavior: when the orchestrator thread dies, subsequent attempts to collect output raise RuntimeError. """ +import asyncio import queue +import threading +from concurrent.futures import ThreadPoolExecutor +from types import SimpleNamespace import pytest from pytest_mock import MockerFixture +from vllm_omni.engine.async_engine_utils import weak_shutdown_async_omni_engine from vllm_omni.engine.async_omni_engine import AsyncOmniEngine -from vllm_omni.engine.messages import ErrorMessage, OutputMessage +from vllm_omni.engine.messages import ( + CollectiveRPCResultMessage, + ErrorMessage, + OutputMessage, +) +from vllm_omni.engine.rpc_result_router import CorrelatedRpcClient +from vllm_omni.experimental.fullduplex.engine.duplex_control_client import ( + DuplexControlClient, + DuplexControlRequestError, +) +from vllm_omni.experimental.fullduplex.engine.messages import ( + AppendDuplexInputMessage, + DuplexControlResultMessage, + DuplexFence, + SignalDuplexTurnMessage, +) from vllm_omni.outputs import OmniRequestOutput pytestmark = [pytest.mark.core_model, pytest.mark.cpu] @@ -26,6 +46,33 @@ def _make_engine(output_queue, mocker: MockerFixture, *, thread_alive: bool = Tr return engine +def test_weak_shutdown_closes_rpc_router_before_joining_orchestrator(mocker: MockerFixture): + request_queue = mocker.MagicMock() + output_queue = mocker.MagicMock() + rpc_output_queue = mocker.MagicMock() + router = mocker.MagicMock() + orchestrator_thread = mocker.MagicMock() + orchestrator_thread.is_alive.return_value = True + + def assert_router_closed_before_join(*args, **kwargs): + router.close.assert_called_once_with() + + orchestrator_thread.join.side_effect = assert_router_closed_before_join + + weak_shutdown_async_omni_engine( + orchestrator_thread, + request_queue, + output_queue, + rpc_output_queue, + router, + ) + + request_queue.sync_q.put.assert_called_once() + assert request_queue.sync_q.put.call_args.kwargs["timeout"] > 0 + orchestrator_thread.join.assert_called_once() + assert orchestrator_thread.join.call_args.kwargs["timeout"] > 0 + + def test_try_get_output_raises_after_orchestrator_dies(mocker: MockerFixture): """Draining remaining results then hitting an empty queue with a dead orchestrator must raise RuntimeError so callers know the pipeline is gone.""" @@ -56,18 +103,17 @@ def test_try_get_output_raises_after_orchestrator_dies(mocker: MockerFixture): @pytest.mark.asyncio async def test_try_get_output_async_raises_after_orchestrator_dies(mocker: MockerFixture): """Same scenario as above but for the async variant.""" - mock_queue = mocker.MagicMock() - mock_queue.sync_q.get_nowait.side_effect = [ + raw_queue = queue.Queue() + raw_queue.put_nowait( OutputMessage( request_id="r1", stage_id=0, engine_outputs=OmniRequestOutput(request_id="r1"), finished=False, - ), - queue.Empty, - ] + ) + ) - engine = _make_engine(mock_queue, mocker, thread_alive=True) + engine = _make_engine(SimpleNamespace(sync_q=raw_queue), mocker, thread_alive=True) assert (await engine.try_get_output_async()).request_id == "r1" @@ -102,12 +148,495 @@ async def test_fatal_error_message_surfaces_through_try_get_output_async(mocker: """Async variant of the fatal error message test.""" fatal_msg = ErrorMessage(error="Orchestrator thread crashed", fatal=True) - mock_queue = mocker.MagicMock() - mock_queue.sync_q.get_nowait.return_value = fatal_msg + raw_queue = queue.Queue() + raw_queue.put_nowait(fatal_msg) - engine = _make_engine(mock_queue, mocker, thread_alive=False) + engine = _make_engine(SimpleNamespace(sync_q=raw_queue), mocker, thread_alive=False) msg = await engine.try_get_output_async() assert msg is not None assert msg.type == "error" assert msg.fatal is True + + +def test_output_remains_on_shared_output_path(mocker: MockerFixture): + raw_queue = queue.Queue() + raw_queue.put_nowait( + OutputMessage( + request_id="legacy-duplex-request", + stage_id=0, + engine_outputs=OmniRequestOutput(request_id="legacy-duplex-request"), + finished=False, + ) + ) + engine = _make_engine(SimpleNamespace(sync_q=raw_queue), mocker) + + output = engine.try_get_output(timeout=0.01) + + assert output.request_id == "legacy-duplex-request" + + +def test_duplex_control_client_uses_engine_owned_correlated_transport(): + engine = object.__new__(AsyncOmniEngine) + engine._duplex_control_client = None + engine._correlated_rpc_client = CorrelatedRpcClient(queue.Queue(), queue.Queue()) + + try: + client = engine._get_duplex_control_client() + + assert isinstance(client, DuplexControlClient) + assert client._transport is engine._correlated_rpc_client + assert not hasattr(engine, "_submit_rpc_and_wait") + finally: + engine._correlated_rpc_client.close() + + +def test_duplex_control_api_requires_explicit_fence(): + engine = object.__new__(AsyncOmniEngine) + + with pytest.raises(TypeError, match="fence"): + engine.open_duplex_session("sid") + with pytest.raises(TypeError, match="fence"): + engine.append_duplex_input("sid", mode="append_audio_chunk", payload={}) + with pytest.raises(TypeError, match="fence"): + engine.signal_duplex_turn("sid", event="turn.end") + with pytest.raises(TypeError, match="fence"): + engine.close_duplex_session("sid") + + +def test_open_duplex_session_waits_for_control_ack(mocker: MockerFixture): + request_q = queue.Queue() + rpc_q = queue.Queue() + control_result = DuplexControlResultMessage( + control_id="ctrl-1", + fence=DuplexFence("sid"), + operation="open", + session_id="sid", + ok=False, + stage_results=[{"stage_id": 0, "replica_id": 0, "result": {"supported": False}}], + unsupported_count=1, + error_count=0, + ) + + engine = object.__new__(AsyncOmniEngine) + engine.request_queue = SimpleNamespace(sync_q=request_q) + engine.rpc_output_queue = SimpleNamespace(sync_q=rpc_q) + engine._correlated_rpc_client = CorrelatedRpcClient(request_q, rpc_q) + engine._duplex_control_client = None + mocker.patch("vllm_omni.engine.async_omni_engine.uuid.uuid4", return_value=SimpleNamespace(hex="ctrl-1")) + + with ThreadPoolExecutor(max_workers=1) as executor: + pending = executor.submit(engine.open_duplex_session, "sid", fence=DuplexFence("sid"), timeout=1) + msg = request_q.get(timeout=1) + assert msg.type == "open_duplex_session" + assert msg.control_id == "ctrl-1" + rpc_q.put(control_result) + with pytest.raises(DuplexControlRequestError) as exc_info: + pending.result(timeout=1) + + assert exc_info.value.result["unsupported_count"] == 1 + assert exc_info.value.result["stage_results"][0]["result"]["supported"] is False + engine._correlated_rpc_client.close() + + +def test_signal_duplex_turn_message_carries_next_fence(mocker: MockerFixture): + request_q = queue.Queue() + rpc_q = queue.Queue() + cancelled_fence = DuplexFence("sid-cancel", epoch=4, turn_id=2) + next_fence = DuplexFence("sid-cancel", epoch=5, turn_id=2) + control_result = DuplexControlResultMessage( + control_id="ctrl-signal", + fence=cancelled_fence, + operation="signal", + session_id="sid-cancel", + ok=True, + stage_results=[], + ) + + engine = object.__new__(AsyncOmniEngine) + engine.request_queue = SimpleNamespace(sync_q=request_q) + engine.rpc_output_queue = SimpleNamespace(sync_q=rpc_q) + engine._correlated_rpc_client = CorrelatedRpcClient(request_q, rpc_q) + engine._duplex_control_client = None + mocker.patch("vllm_omni.engine.async_omni_engine.uuid.uuid4", return_value=SimpleNamespace(hex="ctrl-signal")) + + with ThreadPoolExecutor(max_workers=1) as executor: + pending = executor.submit( + engine.signal_duplex_turn, + "sid-cancel", + event="input.cancel", + fence=cancelled_fence, + next_fence=next_fence, + session_config={"temperature": 0.0}, + timeout=1, + ) + msg = request_q.get(timeout=1) + assert isinstance(msg, SignalDuplexTurnMessage) + assert msg.fence == cancelled_fence + assert msg.next_fence == next_fence + assert msg.session_config == {"temperature": 0.0} + rpc_q.put(control_result) + assert pending.result(timeout=1)["ok"] is True + + engine._correlated_rpc_client.close() + + +@pytest.mark.asyncio +async def test_cancelled_async_append_does_not_stop_executor_callable(mocker: MockerFixture): + entered = threading.Event() + release = threading.Event() + completed = threading.Event() + engine = object.__new__(AsyncOmniEngine) + + def blocking_append(*args, **kwargs): + del args, kwargs + entered.set() + release.wait(timeout=1) + completed.set() + return {} + + mocker.patch.object(engine, "append_duplex_input", side_effect=blocking_append) + task = asyncio.create_task( + engine.append_duplex_input_async( + "sid-executor-cancel", + mode="append_audio_chunk", + payload=b"audio", + fence=DuplexFence("sid-executor-cancel"), + ) + ) + assert await asyncio.to_thread(entered.wait, 1) + + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + release.set() + + assert await asyncio.to_thread(completed.wait, 1) + + +def test_duplex_controls_route_out_of_order_without_global_lock(): + request_q = queue.Queue() + rpc_q = queue.Queue() + engine = object.__new__(AsyncOmniEngine) + engine.request_queue = SimpleNamespace(sync_q=request_q) + engine.rpc_output_queue = SimpleNamespace(sync_q=rpc_q) + engine._correlated_rpc_client = CorrelatedRpcClient(request_q, rpc_q) + engine._duplex_control_client = None + first_fence = DuplexFence("sid", epoch=1) + second_fence = DuplexFence("sid", epoch=2) + first = AppendDuplexInputMessage( + control_id="first", + fence=first_fence, + session_id="sid", + expected_epoch=1, + mode="audio", + payload=b"first", + ) + second = AppendDuplexInputMessage( + control_id="second", + fence=second_fence, + session_id="sid", + expected_epoch=2, + mode="audio", + payload=b"second", + ) + + with ThreadPoolExecutor(max_workers=2) as executor: + client = engine._get_duplex_control_client() + first_result = executor.submit(client.execute, first, timeout=1) + second_result = executor.submit(client.execute, second, timeout=1) + submitted = {request_q.get(timeout=1).control_id, request_q.get(timeout=1).control_id} + assert submitted == {"first", "second"} + rpc_q.put( + DuplexControlResultMessage( + control_id="second", + fence=second_fence, + operation="append", + session_id="sid", + ok=True, + stage_results=[], + ) + ) + rpc_q.put( + DuplexControlResultMessage( + control_id="first", + fence=first_fence, + operation="append", + session_id="sid", + ok=True, + stage_results=[], + ) + ) + + assert first_result.result(timeout=1)["fence"] == first_fence + assert second_result.result(timeout=1)["fence"] == second_fence + + assert not hasattr(engine, "_rpc_lock") + engine._correlated_rpc_client.close() + + +def test_terminal_rpc_router_rejects_without_enqueuing_new_control(): + request_q = queue.Queue() + rpc_q = queue.Queue() + engine = object.__new__(AsyncOmniEngine) + engine.request_queue = SimpleNamespace(sync_q=request_q) + engine.rpc_output_queue = SimpleNamespace(sync_q=rpc_q) + engine._correlated_rpc_client = CorrelatedRpcClient(request_q, rpc_q) + engine._duplex_control_client = None + pending_message = AppendDuplexInputMessage( + control_id="pending", + fence=DuplexFence("sid"), + session_id="sid", + mode="audio", + payload=b"audio", + ) + message = AppendDuplexInputMessage( + control_id="after-fatal", + fence=DuplexFence("sid"), + session_id="sid", + mode="audio", + payload=b"audio", + ) + + try: + with ThreadPoolExecutor(max_workers=1) as executor: + pending = executor.submit(engine._get_duplex_control_client().execute, pending_message, timeout=1) + assert request_q.get(timeout=1).control_id == "pending" + rpc_q.put(ErrorMessage(error="orchestrator failed", fatal=True)) + with pytest.raises(RuntimeError, match="orchestrator failed"): + pending.result(timeout=1) + + with pytest.raises(RuntimeError, match="orchestrator failed"): + engine._get_duplex_control_client().execute(message, timeout=1) + + assert request_q.empty() + finally: + engine._correlated_rpc_client.close() + + +def test_collective_and_duplex_results_share_router_without_swapping(mocker: MockerFixture): + request_q = queue.Queue() + rpc_q = queue.Queue() + engine = object.__new__(AsyncOmniEngine) + engine.request_queue = SimpleNamespace(sync_q=request_q) + engine.rpc_output_queue = SimpleNamespace(sync_q=rpc_q) + engine._correlated_rpc_client = CorrelatedRpcClient(request_q, rpc_q) + engine._duplex_control_client = None + fence = DuplexFence("sid") + duplex = AppendDuplexInputMessage( + control_id="duplex-control", + fence=fence, + session_id="sid", + mode="audio", + payload=b"audio", + ) + mocker.patch("vllm_omni.engine.async_omni_engine.uuid.uuid4", return_value=SimpleNamespace(hex="collective-rpc")) + + with ThreadPoolExecutor(max_workers=2) as executor: + duplex_result = executor.submit(engine._get_duplex_control_client().execute, duplex, timeout=1) + collective_result = executor.submit(engine.collective_rpc, "health", timeout=1) + submitted_types = {request_q.get(timeout=1).type, request_q.get(timeout=1).type} + assert submitted_types == {"append_duplex_input", "collective_rpc"} + rpc_q.put( + CollectiveRPCResultMessage( + rpc_id="collective-rpc", + method="health", + stage_ids=[0], + results=["healthy"], + ) + ) + rpc_q.put( + DuplexControlResultMessage( + control_id="duplex-control", + fence=fence, + operation="append", + session_id="sid", + ok=True, + stage_results=[], + ) + ) + + assert collective_result.result(timeout=1) == ["healthy"] + assert duplex_result.result(timeout=1)["fence"] == fence + + engine._correlated_rpc_client.close() + + +def test_collective_rpc_preserves_request_queue_backpressure(mocker: MockerFixture): + class SignallingQueue(queue.Queue): + def __init__(self) -> None: + super().__init__(maxsize=1) + self.put_attempted = threading.Event() + + def put(self, item, block=True, timeout=None): + self.put_attempted.set() + return super().put(item, block=block, timeout=timeout) + + request_q = SignallingQueue() + request_q.put("queue-is-full") + request_q.put_attempted.clear() + rpc_q = queue.Queue() + engine = object.__new__(AsyncOmniEngine) + engine.request_queue = SimpleNamespace(sync_q=request_q) + engine.rpc_output_queue = SimpleNamespace(sync_q=rpc_q) + engine._correlated_rpc_client = CorrelatedRpcClient(request_q, rpc_q) + engine._duplex_control_client = None + mocker.patch("vllm_omni.engine.async_omni_engine.uuid.uuid4", return_value=SimpleNamespace(hex="blocked-rpc")) + + with ThreadPoolExecutor(max_workers=1) as executor: + pending = executor.submit(engine.collective_rpc, "health", timeout=1) + assert request_q.put_attempted.wait(timeout=1) + assert not pending.done() + + assert request_q.get(timeout=1) == "queue-is-full" + assert request_q.get(timeout=1).rpc_id == "blocked-rpc" + rpc_q.put( + CollectiveRPCResultMessage( + rpc_id="blocked-rpc", + method="health", + stage_ids=[0], + results=["healthy"], + ) + ) + assert pending.result(timeout=1) == ["healthy"] + + engine._correlated_rpc_client.close() + + +def test_open_duplex_session_raises_on_stage_control_error(mocker: MockerFixture): + request_q = queue.Queue() + rpc_q = queue.Queue() + control_result = DuplexControlResultMessage( + control_id="ctrl-error", + fence=DuplexFence("sid"), + operation="open", + session_id="sid", + ok=False, + stage_results=[{"stage_id": 0, "replica_id": 0, "result": {"supported": False, "error": "boom"}}], + unsupported_count=1, + error_count=1, + ) + + engine = object.__new__(AsyncOmniEngine) + engine.request_queue = SimpleNamespace(sync_q=request_q) + engine.rpc_output_queue = SimpleNamespace(sync_q=rpc_q) + engine._correlated_rpc_client = CorrelatedRpcClient(request_q, rpc_q) + engine._duplex_control_client = None + mocker.patch("vllm_omni.engine.async_omni_engine.uuid.uuid4", return_value=SimpleNamespace(hex="ctrl-error")) + + with ThreadPoolExecutor(max_workers=1) as executor: + pending = executor.submit(engine.open_duplex_session, "sid", fence=DuplexFence("sid"), timeout=1) + request_q.get(timeout=1) + rpc_q.put(control_result) + with pytest.raises(RuntimeError, match="duplex open failed"): + pending.result(timeout=1) + engine._correlated_rpc_client.close() + + +def test_shutdown_does_not_race_runtime_cleanup_with_live_orchestrator(mocker: MockerFixture): + engine = object.__new__(AsyncOmniEngine) + engine._shutdown_called = False + engine._weak_finalizer = None + engine.request_queue = mocker.MagicMock() + engine.request_queue.sync_q.put.side_effect = queue.Full + engine.output_queue = mocker.MagicMock() + engine.rpc_output_queue = mocker.MagicMock() + engine._correlated_rpc_client = mocker.MagicMock() + engine.orchestrator_thread = mocker.MagicMock() + engine.orchestrator_thread.is_alive.return_value = True + engine._runtime = mocker.MagicMock() + + engine.shutdown() + + engine.request_queue.sync_q.put.assert_called_once() + assert engine.request_queue.sync_q.put.call_args.kwargs["timeout"] > 0 + assert engine.orchestrator_thread.join.call_args_list[0].kwargs["timeout"] > 0 + engine._correlated_rpc_client.close.assert_called_once_with() + engine.request_queue.close.assert_called_once_with() + engine.output_queue.close.assert_called_once_with() + engine.rpc_output_queue.close.assert_called_once_with() + engine._runtime.shutdown.assert_not_called() + + +def test_shutdown_releases_runtime_after_orchestrator_stops(mocker: MockerFixture): + engine = object.__new__(AsyncOmniEngine) + engine._shutdown_called = False + engine._weak_finalizer = None + engine.request_queue = mocker.MagicMock() + engine.output_queue = mocker.MagicMock() + engine.rpc_output_queue = mocker.MagicMock() + engine._correlated_rpc_client = mocker.MagicMock() + engine.orchestrator_thread = mocker.MagicMock() + engine.orchestrator_thread.is_alive.side_effect = [True, False] + engine._runtime = mocker.MagicMock() + + engine.shutdown() + + engine.orchestrator_thread.join.assert_called_once() + engine._runtime.shutdown.assert_called_once_with() + + +def test_shutdown_defers_runtime_release_until_live_orchestrator_stops(mocker: MockerFixture): + orchestrator_stopped = threading.Event() + runtime_released = threading.Event() + + class ControllableOrchestratorThread: + def __init__(self) -> None: + self.join_timeouts: list[float | None] = [] + + def is_alive(self) -> bool: + return not orchestrator_stopped.is_set() + + def join(self, timeout: float | None = None) -> None: + self.join_timeouts.append(timeout) + if timeout is None: + orchestrator_stopped.wait() + + engine = object.__new__(AsyncOmniEngine) + engine._shutdown_called = False + engine._weak_finalizer = None + engine.request_queue = mocker.MagicMock() + engine.output_queue = mocker.MagicMock() + engine.rpc_output_queue = mocker.MagicMock() + engine._correlated_rpc_client = mocker.MagicMock() + engine.orchestrator_thread = ControllableOrchestratorThread() + engine._runtime = mocker.MagicMock() + engine._runtime.shutdown.side_effect = runtime_released.set + + try: + engine.shutdown() + + engine._runtime.shutdown.assert_not_called() + orchestrator_stopped.set() + assert runtime_released.wait(timeout=1) + engine._runtime.shutdown.assert_called_once_with() + assert engine.orchestrator_thread.join_timeouts[0] is not None + assert engine.orchestrator_thread.join_timeouts[-1] is None + finally: + orchestrator_stopped.set() + + +def test_weak_shutdown_is_bounded_when_request_queue_is_full(mocker: MockerFixture): + request_queue = mocker.MagicMock() + request_queue.sync_q.put.side_effect = queue.Full + output_queue = mocker.MagicMock() + rpc_output_queue = mocker.MagicMock() + router = mocker.MagicMock() + orchestrator_thread = mocker.MagicMock() + orchestrator_thread.is_alive.return_value = True + + weak_shutdown_async_omni_engine( + orchestrator_thread, + request_queue, + output_queue, + rpc_output_queue, + router, + ) + + request_queue.sync_q.put.assert_called_once() + assert request_queue.sync_q.put.call_args.kwargs["timeout"] > 0 + orchestrator_thread.join.assert_called_once() + assert orchestrator_thread.join.call_args.kwargs["timeout"] > 0 + request_queue.close.assert_called_once_with() + output_queue.close.assert_called_once_with() + rpc_output_queue.close.assert_called_once_with() diff --git a/tests/engine/test_correlated_rpc_client.py b/tests/engine/test_correlated_rpc_client.py new file mode 100644 index 000000000..2f470a996 --- /dev/null +++ b/tests/engine/test_correlated_rpc_client.py @@ -0,0 +1,104 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from __future__ import annotations + +import queue +from concurrent.futures import ThreadPoolExecutor + +import pytest + +from vllm_omni.engine.messages import ( + CollectiveRPCRequestMessage, + CollectiveRPCResultMessage, + ErrorMessage, +) +from vllm_omni.engine.rpc_result_router import CorrelatedRpcClient + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +def _request(rpc_id: str) -> CollectiveRPCRequestMessage: + return CollectiveRPCRequestMessage( + rpc_id=rpc_id, + method="health", + timeout=1, + args=(), + kwargs={}, + stage_ids=None, + ) + + +def test_correlated_rpc_client_unregisters_timeout_before_late_result() -> None: + request_queue: queue.Queue = queue.Queue() + result_queue: queue.Queue = queue.Queue() + client = CorrelatedRpcClient(request_queue, result_queue) + + try: + with pytest.raises(TimeoutError, match="first timed out"): + client.execute( + ("collective", "first"), + _request("first"), + timeout=0.01, + timeout_message="first timed out", + ) + assert request_queue.get_nowait().rpc_id == "first" + + result_queue.put( + CollectiveRPCResultMessage( + rpc_id="first", + method="health", + stage_ids=[0], + results=["late"], + ) + ) + result_queue.put( + CollectiveRPCResultMessage( + rpc_id="second", + method="health", + stage_ids=[0], + results=["current"], + ) + ) + current = client.execute( + ("collective", "second"), + _request("second"), + timeout=1, + timeout_message="second timed out", + ) + + assert isinstance(current, CollectiveRPCResultMessage) + assert current.results == ["current"] + finally: + client.close() + + +def test_correlated_rpc_client_rejects_after_fatal_without_enqueuing() -> None: + request_queue: queue.Queue = queue.Queue() + result_queue: queue.Queue = queue.Queue() + client = CorrelatedRpcClient(request_queue, result_queue) + + try: + with ThreadPoolExecutor(max_workers=1) as executor: + pending = executor.submit( + client.execute, + ("collective", "pending"), + _request("pending"), + timeout=1, + timeout_message="unexpected timeout", + ) + assert request_queue.get(timeout=1).rpc_id == "pending" + result_queue.put(ErrorMessage(error="orchestrator failed", fatal=True)) + with pytest.raises(RuntimeError, match="orchestrator failed"): + pending.result(timeout=1) + + with pytest.raises(RuntimeError, match="orchestrator failed"): + client.execute( + ("collective", "after-fatal"), + _request("after-fatal"), + timeout=1, + timeout_message="unexpected timeout", + ) + assert request_queue.empty() + finally: + client.close() diff --git a/tests/engine/test_duplex_import_boundary.py b/tests/engine/test_duplex_import_boundary.py new file mode 100644 index 000000000..6eae2e698 --- /dev/null +++ b/tests/engine/test_duplex_import_boundary.py @@ -0,0 +1,102 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from __future__ import annotations + +import subprocess +import sys +from pathlib import Path + +import pytest + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] +REPO_ROOT = Path(__file__).resolve().parents[2] + + +def _assert_isolated_import_succeeds(script: str) -> None: + result = subprocess.run( + [sys.executable, "-c", script], + cwd=REPO_ROOT, + capture_output=True, + text=True, + timeout=60, + check=False, + ) + assert result.returncode == 0, result.stderr + + +def test_stable_engine_imports_do_not_load_experimental_duplex() -> None: + _assert_isolated_import_succeeds(""" +import sys + +import vllm_omni.engine.async_omni_engine +import vllm_omni.engine.orchestrator +import vllm_omni.entrypoints.async_omni + +loaded = sorted( + name + for name in sys.modules + if name == "vllm_omni.experimental.fullduplex" + or name.startswith("vllm_omni.experimental.fullduplex.") +) +if loaded: + raise SystemExit("stable imports loaded experimental duplex: " + ", ".join(loaded)) +""") + + +def test_stable_engine_does_not_expose_duplex_contract_modules() -> None: + _assert_isolated_import_succeeds(""" +import importlib.util + +from vllm_omni.engine import messages + +legacy_modules = ( + "vllm_omni.engine.duplex_contracts", + "vllm_omni.engine.duplex_lease", + "vllm_omni.engine.resumable", +) +present = [name for name in legacy_modules if importlib.util.find_spec(name) is not None] +if present: + raise SystemExit("stable duplex modules still exist: " + ", ".join(present)) + +duplex_exports = sorted(name for name in vars(messages) if name.startswith("Duplex")) +if duplex_exports: + raise SystemExit("stable messages still expose duplex contracts: " + ", ".join(duplex_exports)) +""") + + +def test_stable_outputs_do_not_declare_duplex_decision_field() -> None: + _assert_isolated_import_succeeds(""" +import dataclasses + +from vllm_omni.outputs import OmniRequestOutput + +fields = {field.name for field in dataclasses.fields(OmniRequestOutput)} +if "duplex_output_decision" in fields: + raise SystemExit("stable output declares duplex_output_decision") +""") + + +def test_stable_model_executor_does_not_expose_duplex_helper_module() -> None: + _assert_isolated_import_succeeds(""" +import importlib.util + +if importlib.util.find_spec("vllm_omni.model_executor.duplex") is not None: + raise SystemExit("stable model_executor still exposes duplex helper module") +""") + + +def test_runtime_package_does_not_bundle_the_browser_demo() -> None: + assert not (REPO_ROOT / "vllm_omni" / "experimental" / "fullduplex" / "web").exists() + + +def test_experimental_engine_uses_canonical_contract_module_names() -> None: + engine_dir = REPO_ROOT / "vllm_omni" / "experimental" / "fullduplex" / "engine" + core_dir = REPO_ROOT / "vllm_omni" / "experimental" / "fullduplex" / "core" + + assert (engine_dir / "contracts.py").is_file() + assert (engine_dir / "lease.py").is_file() + assert (engine_dir / "messages.py").is_file() + assert not (engine_dir / "duplex_lease.py").exists() + assert not (engine_dir / "duplex_types.py").exists() + assert not (core_dir / "identity.py").exists() diff --git a/tests/engine/test_membership_controller.py b/tests/engine/test_membership_controller.py index c00a7f646..1b8bf2adb 100644 --- a/tests/engine/test_membership_controller.py +++ b/tests/engine/test_membership_controller.py @@ -11,6 +11,7 @@ import pytest from vllm_omni.distributed.omni_coordinator.messages import ReplicaStatus from vllm_omni.engine.membership_controller import MembershipController +from vllm_omni.engine.stage_pool import StagePool pytestmark = [pytest.mark.core_model, pytest.mark.cpu] @@ -45,10 +46,10 @@ class FakePool: def attach_load_balancer(self, lb): self.lb = lb - def add_client(self, input_addr, client): - self.added.append((input_addr, client)) + def add_client(self, input_addr, client, *, replica_id=None): + self.added.append((input_addr, client, replica_id)) self.clients.append(client) - return len(self.clients) - 1 + return len(self.clients) - 1 if replica_id is None else replica_id def invalidate_addr(self, input_addr): self.invalidated.append(input_addr) @@ -155,4 +156,35 @@ async def test_do_register_offloads_remote_factory(monkeypatch): assert factory_thread_ids assert factory_thread_ids[0] != threading.get_ident() - assert pool.added[0][0] == "tcp://stage-0-replica-1" + assert pool.added[0] == ("tcp://stage-0-replica-1", pool.clients[0], 1) + + +@pytest.mark.asyncio +async def test_unregister_then_register_restores_coordinator_replica_slot(monkeypatch): + class Client: + def __init__(self, input_addr: str) -> None: + self.client_addresses = {"input_address": input_addr} + self.shutdown_calls = 0 + + def shutdown(self) -> None: + self.shutdown_calls += 1 + + old_clients = [Client("tcp://old-0"), Client("tcp://old-1")] + replacement = Client("tcp://new-1") + pool = StagePool(0, old_clients) + + def factory(stage_id: int, replica_id: int): + assert (stage_id, replica_id) == (0, 1) + return replacement + + controller = _controller(monkeypatch, pool, FakeHub(), remote_replica_factory=factory) + await controller.handle_unregister(0, "tcp://old-0") + await controller.handle_unregister(0, "tcp://old-1") + + await controller.handle_register(0, 1) + await controller.drain_tasks(timeout=1) + + assert pool.clients == [None, replacement] + assert pool.get_replica_id_by_addr("tcp://new-1") == 1 + assert pool.get_replica_id_by_addr("tcp://old-1") is None + assert pool.available_replica_ids() == [1] diff --git a/tests/engine/test_multimodal_accumulation.py b/tests/engine/test_multimodal_accumulation.py new file mode 100644 index 000000000..ca449154c --- /dev/null +++ b/tests/engine/test_multimodal_accumulation.py @@ -0,0 +1,46 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import pytest +import torch + +from vllm_omni.outputs.mm_outputs import MultimodalPayload +from vllm_omni.outputs.multimodal_accumulation import ( + drain_delta_payload, + is_non_final_delta_audio_chunk, + replace_snapshot_keys, +) + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +def test_chunk_accumulation_policy_replaces_snapshots_and_drains_delta_state(): + accumulated = MultimodalPayload.from_dict( + { + "audio": torch.tensor([1.0]), + "meta.tts_is_last_chunk": torch.tensor([0]), + "meta.turn_end": torch.tensor([0]), + "meta.stable_request_value": "keep", + } + ) + incoming = MultimodalPayload.from_dict( + { + "audio": torch.tensor([2.0]), + "meta.tts_is_last_chunk": torch.tensor([1]), + "meta.turn_end": torch.tensor([1]), + } + ) + assert accumulated is not None + assert incoming is not None + + replace_snapshot_keys(accumulated, incoming) + merged = accumulated.merged_with(incoming) + + assert not is_non_final_delta_audio_chunk(merged, "audio") + + drain_delta_payload(merged) + + assert "audio" not in merged + assert "meta.tts_is_last_chunk" not in merged + assert "meta.turn_end" not in merged + assert merged.metadata["meta.stable_request_value"] == "keep" diff --git a/tests/engine/test_orchestrator.py b/tests/engine/test_orchestrator.py index 23f8cfaa2..68d9c2c58 100644 --- a/tests/engine/test_orchestrator.py +++ b/tests/engine/test_orchestrator.py @@ -14,12 +14,14 @@ import janus import pytest from vllm.outputs import CompletionOutput, RequestOutput from vllm.sampling_params import SamplingParams +from vllm.v1.engine.exceptions import EngineDeadError from vllm_omni.engine.messages import ( AbortRequestMessage, AddCompanionRequestMessage, CollectiveRPCRequestMessage, CollectiveRPCResultMessage, + ErrorMessage, OutputMessage, ShutdownRequestMessage, StageSubmissionMessage, @@ -31,12 +33,68 @@ from vllm_omni.engine.orchestrator import ( _infer_stage_audio_sample_rate, ) from vllm_omni.engine.stage_pool import StagePool +from vllm_omni.experimental.fullduplex.engine.duplex_control_plane import DuplexControlPlane +from vllm_omni.experimental.fullduplex.engine.duplex_runtime import ( + DuplexInputMode, + DuplexRuntimeCapabilities, + DuplexSessionRuntimeState, + duplex_resource_request_id, +) +from vllm_omni.experimental.fullduplex.engine.messages import ( + AppendDuplexInputMessage, + CloseDuplexSessionMessage, + DuplexFence, + OpenDuplexSessionMessage, + SignalDuplexTurnMessage, +) +from vllm_omni.experimental.fullduplex.minicpmo45.runtime import MiniCPMO45DuplexRuntimeExtension from vllm_omni.inputs.data import OmniDiffusionSamplingParams from vllm_omni.outputs import OmniRequestOutput pytestmark = [pytest.mark.core_model, pytest.mark.cpu] +class FakeRunningCounter: + def __init__(self) -> None: + self.value = 0 + + def increment(self) -> None: + self.value += 1 + + def decrement(self) -> None: + self.value -= 1 + + +@pytest.mark.asyncio +async def test_engine_dead_broadcasts_fatal_to_rpc_waiters(monkeypatch: pytest.MonkeyPatch) -> None: + rpc_queue: asyncio.Queue = asyncio.Queue() + orchestrator = Orchestrator( + request_async_queue=asyncio.Queue(), + output_async_queue=asyncio.Queue(), + rpc_async_queue=rpc_queue, + stage_pools=[], + ) + orchestrator._fatal_error = "stage engine died" + orchestrator._fatal_error_stage_id = 2 + + async def wait_for_requests() -> None: + await asyncio.Event().wait() + + async def fail_outputs() -> None: + raise EngineDeadError("stage engine died") + + monkeypatch.setattr(orchestrator, "_request_handler", wait_for_requests) + monkeypatch.setattr(orchestrator, "_orchestration_output_handler", fail_outputs) + + await orchestrator.run() + + fatal = rpc_queue.get_nowait() + assert isinstance(fatal, ErrorMessage) + assert fatal.fatal is True + assert fatal.error == "stage engine died" + assert fatal.stage_id == 2 + + @dataclass class OrchestratorFixture: orchestrator: Orchestrator @@ -724,6 +782,778 @@ async def test_run_abort(orchestrator_factory) -> None: await _shutdown_orchestrator(orchestrator_fixture) +def _duplex_open_message( + session_id: str, + *, + incarnation: int = 0, + session_config: dict[str, object] | None = None, + runtime_config: dict[str, object] | None = None, +) -> OpenDuplexSessionMessage: + return OpenDuplexSessionMessage( + control_id=f"open-{session_id}", + fence=DuplexFence(session_id, incarnation=incarnation), + session_id=session_id, + capabilities={ + "input_modes": [DuplexInputMode.APPEND_AUDIO_CHUNK.value], + "implementation_level": "model_native_duplex", + }, + session_config=session_config or {}, + runtime_config=runtime_config or {}, + ) + + +async def _handle_duplex(orchestrator: Orchestrator, message: object) -> None: + await orchestrator._require_duplex_control_plane().handle(message) + + +def _duplex_request_state( + orchestrator: Orchestrator, + session: DuplexSessionRuntimeState, + *, + stage_id: int, +) -> OrchestratorRequestState | None: + context = orchestrator._require_duplex_control_plane().ensure_stage_request( + session, + stage_id=stage_id, + ) + if context is None: + return None + return orchestrator.request_states.get(context.request_id) + + +@pytest.mark.asyncio +async def test_duplex_control_plane_keeps_public_and_runtime_config_separate() -> None: + stage0 = FakeStageClient(stage_type="llm", final_output=True) + rpc_q: asyncio.Queue = asyncio.Queue() + orchestrator = Orchestrator( + request_async_queue=asyncio.Queue(), + output_async_queue=asyncio.Queue(), + rpc_async_queue=rpc_q, + stage_pools=_build_stage_pools([[stage0]]), + duplex_runtime_extension=MiniCPMO45DuplexRuntimeExtension(), + enable_duplex_control=True, + ) + open_message = _duplex_open_message( + "sid-config-channels", + incarnation=3, + session_config={ + "instructions": "public instructions", + "extra_body": {"duplex_stage_max_tokens": {"0": 99}}, + }, + runtime_config={ + "duplex_stage_max_tokens": {"0": 3}, + "duplex_stage_sampling_params": {"0": {"stop_token_ids": [151705]}}, + }, + ) + + await _handle_duplex(orchestrator, open_message) + + assert rpc_q.get_nowait().ok is True + session = orchestrator.duplex_sessions.require(open_message.session_id) + request_state = _duplex_request_state(orchestrator, session, stage_id=0) + assert request_state.sampling_params_list[0].max_tokens == 3 + assert request_state.sampling_params_list[0].stop_token_ids == [151705] + bridge = request_state.streaming.bridge_states["duplex"] + assert bridge["incarnation"] == open_message.fence.incarnation + assert bridge["session_config"] == open_message.session_config + assert bridge["runtime_config"] == open_message.runtime_config + + +@pytest.mark.asyncio +async def test_duplex_control_plane_preserves_turn_commit_without_model_extension() -> None: + rpc_q: asyncio.Queue = asyncio.Queue() + orchestrator = Orchestrator( + request_async_queue=asyncio.Queue(), + output_async_queue=asyncio.Queue(), + rpc_async_queue=rpc_q, + stage_pools=[], + duplex_runtime_extension=None, + enable_duplex_control=True, + ) + assert isinstance(orchestrator.duplex_control_plane, DuplexControlPlane) + + message = OpenDuplexSessionMessage( + control_id="open-turn-commit", + fence=DuplexFence("sid-turn-commit"), + session_id="sid-turn-commit", + capabilities={"input_modes": [DuplexInputMode.TURN_COMMIT_ONLY.value]}, + session_config={}, + ) + await _handle_duplex(orchestrator, message) + + result = rpc_q.get_nowait() + assert result.ok is True + assert result.stage_results[0]["result"]["scheduler_request_context"] is False + + +def test_ordinary_orchestrator_bypasses_duplex_control_plane() -> None: + orchestrator = Orchestrator( + request_async_queue=asyncio.Queue(), + output_async_queue=asyncio.Queue(), + rpc_async_queue=asyncio.Queue(), + stage_pools=[], + duplex_runtime_extension=MiniCPMO45DuplexRuntimeExtension(), + ) + + assert orchestrator.duplex_control_plane is None + + +@pytest.mark.asyncio +async def test_duplex_close_cleans_preregistered_request_without_append() -> None: + stage0 = FakeStageClient(stage_type="llm", final_output=True) + rpc_q: asyncio.Queue = asyncio.Queue() + running_counter = FakeRunningCounter() + orchestrator = Orchestrator( + request_async_queue=asyncio.Queue(), + output_async_queue=asyncio.Queue(), + rpc_async_queue=rpc_q, + stage_pools=_build_stage_pools([[stage0]]), + duplex_runtime_extension=MiniCPMO45DuplexRuntimeExtension(), + enable_duplex_control=True, + running_counter=running_counter, + ) + open_message = _duplex_open_message("sid-preregister-close") + + await _handle_duplex(orchestrator, open_message) + open_result = rpc_q.get_nowait() + request_id = open_result.stage_results[0]["result"]["request_id"] + assert request_id in orchestrator.request_states + + await _handle_duplex( + orchestrator, + CloseDuplexSessionMessage( + control_id="close-preregistered", + fence=open_message.fence, + session_id=open_message.session_id, + ), + ) + + assert rpc_q.get_nowait().ok is True + assert request_id not in orchestrator.request_states + assert orchestrator.duplex_sessions.get(open_message.session_id) is None + assert running_counter.value == 0 + + +@pytest.mark.asyncio +async def test_duplex_open_failure_rolls_back_session_and_reserved_request() -> None: + stage0 = FakeStageClient(stage_type="llm", final_output=True) + rpc_q: asyncio.Queue = asyncio.Queue() + orchestrator = Orchestrator( + request_async_queue=asyncio.Queue(), + output_async_queue=asyncio.Queue(), + rpc_async_queue=rpc_q, + stage_pools=_build_stage_pools([[stage0]]), + duplex_runtime_extension=MiniCPMO45DuplexRuntimeExtension(), + enable_duplex_control=True, + ) + open_message = _duplex_open_message( + "sid-open-rollback", + runtime_config={ + "duplex_stage_sampling_params": { + "0": {"stop_token_ids": 123}, + } + }, + ) + + await _handle_duplex(orchestrator, open_message) + + assert rpc_q.get_nowait().ok is False + assert orchestrator.duplex_sessions.get(open_message.session_id) is None + assert orchestrator.request_states == {} + + +@pytest.mark.asyncio +async def test_duplex_running_counter_tracks_only_submitted_request() -> None: + stage0 = FakeStageClient(stage_type="llm", final_output=True) + rpc_q: asyncio.Queue = asyncio.Queue() + running_counter = FakeRunningCounter() + orchestrator = Orchestrator( + request_async_queue=asyncio.Queue(), + output_async_queue=asyncio.Queue(), + rpc_async_queue=rpc_q, + stage_pools=_build_stage_pools([[stage0]]), + duplex_runtime_extension=MiniCPMO45DuplexRuntimeExtension(), + enable_duplex_control=True, + running_counter=running_counter, + ) + open_message = _duplex_open_message("sid-duplex-counter") + await _handle_duplex(orchestrator, open_message) + assert rpc_q.get_nowait().ok is True + assert running_counter.value == 0 + + await _handle_duplex( + orchestrator, + AppendDuplexInputMessage( + control_id="append-duplex-counter", + fence=open_message.fence, + session_id=open_message.session_id, + mode=DuplexInputMode.APPEND_AUDIO_CHUNK.value, + payload={"is_speech": True}, + ), + ) + assert rpc_q.get_nowait().ok is True + assert running_counter.value == 1 + + await _handle_duplex( + orchestrator, + CloseDuplexSessionMessage( + control_id="close-duplex-counter", + fence=open_message.fence, + session_id=open_message.session_id, + ), + ) + assert rpc_q.get_nowait().ok is True + assert running_counter.value == 0 + + +@pytest.mark.asyncio +async def test_duplex_failed_append_does_not_advance_sequence_or_fence() -> None: + class FailingPlanExtension(MiniCPMO45DuplexRuntimeExtension): + def plan_append(self, **kwargs): + raise RuntimeError("planned append failure") + + stage0 = FakeStageClient(stage_type="llm", final_output=True) + rpc_q: asyncio.Queue = asyncio.Queue() + orchestrator = Orchestrator( + request_async_queue=asyncio.Queue(), + output_async_queue=asyncio.Queue(), + rpc_async_queue=rpc_q, + stage_pools=_build_stage_pools([[stage0]]), + duplex_runtime_extension=FailingPlanExtension(), + enable_duplex_control=True, + ) + fence = DuplexFence("sid-append-rollback", turn_id=1) + session = orchestrator.duplex_sessions.open_session( + DuplexFence("sid-append-rollback"), + capabilities=DuplexRuntimeCapabilities( + input_modes={DuplexInputMode.APPEND_AUDIO_CHUNK}, + ), + ) + + await _handle_duplex( + orchestrator, + AppendDuplexInputMessage( + control_id="append-rollback", + fence=fence, + session_id=fence.session_id, + mode=DuplexInputMode.APPEND_AUDIO_CHUNK.value, + payload={"is_speech": True}, + ), + ) + + result = rpc_q.get_nowait() + assert result.ok is False + assert session.fence == DuplexFence("sid-append-rollback") + assert session.input_seq == 0 + assert session.input_turn_seq == 0 + assert stage0.add_request_calls == [] + + +@pytest.mark.asyncio +async def test_duplex_duplicate_append_operation_is_submitted_once() -> None: + stage0 = FakeStageClient(stage_type="llm", final_output=True) + rpc_q: asyncio.Queue = asyncio.Queue() + orchestrator = Orchestrator( + request_async_queue=asyncio.Queue(), + output_async_queue=asyncio.Queue(), + rpc_async_queue=rpc_q, + stage_pools=_build_stage_pools([[stage0]]), + duplex_runtime_extension=MiniCPMO45DuplexRuntimeExtension(), + enable_duplex_control=True, + ) + fence = DuplexFence("sid-idempotent-append") + session = orchestrator.duplex_sessions.open_session( + fence, + capabilities=DuplexRuntimeCapabilities( + input_modes={DuplexInputMode.APPEND_AUDIO_CHUNK}, + ), + ) + + for control_id in ("append-first", "append-timeout-retry"): + await _handle_duplex( + orchestrator, + AppendDuplexInputMessage( + control_id=control_id, + operation_id="physical-input-1", + fence=fence, + session_id=fence.session_id, + mode=DuplexInputMode.APPEND_AUDIO_CHUNK.value, + payload={"is_speech": True}, + ), + ) + + first = rpc_q.get_nowait() + retry = rpc_q.get_nowait() + assert first.ok is True + assert retry.ok is True + assert first.stage_results == retry.stage_results + assert len(stage0.add_request_calls) == 1 + assert session.input_seq == 1 + + +@pytest.mark.asyncio +async def test_duplex_barge_in_aborts_bound_stage_requests_before_releasing_fence() -> None: + stage0 = FakeStageClient(stage_type="llm", final_output=False) + stage1 = FakeStageClient(stage_type="llm", final_output=True) + stage_pools = _build_stage_pools( + [[stage0], [stage1]], + output_processors=[FakeOutputProcessor(), FakeOutputProcessor()], + stage_vllm_configs=[ + SimpleNamespace(model_config=SimpleNamespace(max_model_len=64)), + SimpleNamespace(model_config=SimpleNamespace(max_model_len=64)), + ], + ) + rpc_q: asyncio.Queue = asyncio.Queue() + orchestrator = Orchestrator( + request_async_queue=asyncio.Queue(), + output_async_queue=asyncio.Queue(), + rpc_async_queue=rpc_q, + stage_pools=stage_pools, + duplex_runtime_extension=MiniCPMO45DuplexRuntimeExtension(), + enable_duplex_control=True, + ) + session = orchestrator.duplex_sessions.open_session( + DuplexFence("sid-stage-signal"), + capabilities=DuplexRuntimeCapabilities( + input_modes={DuplexInputMode.APPEND_AUDIO_CHUNK}, + ), + ) + session.bind_stage_request(0, "req-stage0", fence=session.fence) + session.bind_stage_request(1, "req-stage1", fence=session.fence) + assert stage_pools[0].select_replica_id("req-stage0") == 0 + assert stage_pools[1].select_replica_id("req-stage1") == 0 + session.append_input( + mode=DuplexInputMode.APPEND_AUDIO_CHUNK, + fence=session.fence, + ) + await _handle_duplex( + orchestrator, + SignalDuplexTurnMessage( + control_id="ctrl-signal", + fence=session.fence, + next_fence=DuplexFence("sid-stage-signal", epoch=1), + session_id="sid-stage-signal", + event="barge_in", + ), + ) + + assert stage0.abort_calls == [["req-stage0"]] + assert stage1.abort_calls == [["req-stage1"]] + assert stage0.collective_rpc_calls == [] + assert stage1.collective_rpc_calls == [] + assert session.stage_bindings == {} + assert session.fence == DuplexFence("sid-stage-signal", epoch=1) + result = rpc_q.get_nowait() + assert result.ok is True + + +@pytest.mark.asyncio +async def test_duplex_late_barge_in_releases_only_cancelled_fence_bindings() -> None: + stage0 = FakeStageClient(stage_type="llm", final_output=False) + stage1 = FakeStageClient(stage_type="llm", final_output=True) + stage_pools = _build_stage_pools( + [[stage0], [stage1]], + output_processors=[FakeOutputProcessor(), FakeOutputProcessor()], + stage_vllm_configs=[ + SimpleNamespace(model_config=SimpleNamespace(max_model_len=64)), + SimpleNamespace(model_config=SimpleNamespace(max_model_len=64)), + ], + ) + rpc_q: asyncio.Queue = asyncio.Queue() + orchestrator = Orchestrator( + request_async_queue=asyncio.Queue(), + output_async_queue=asyncio.Queue(), + rpc_async_queue=rpc_q, + stage_pools=stage_pools, + enable_duplex_control=True, + ) + cancelled_fence = DuplexFence("sid-late-stage-signal") + session = orchestrator.duplex_sessions.open_session( + cancelled_fence, + capabilities=DuplexRuntimeCapabilities( + input_modes={DuplexInputMode.APPEND_AUDIO_CHUNK}, + ), + ) + session.bind_stage_request(0, "req-stage0", fence=cancelled_fence) + session.bind_stage_request(1, "req-stage1", fence=cancelled_fence) + assert stage_pools[0].select_replica_id("req-stage0") == 0 + assert stage_pools[1].select_replica_id("req-stage1") == 0 + current_fence = DuplexFence("sid-late-stage-signal", epoch=1) + session.accept_fence(current_fence) + + await _handle_duplex( + orchestrator, + SignalDuplexTurnMessage( + control_id="ctrl-late-signal", + fence=cancelled_fence, + next_fence=current_fence, + session_id="sid-late-stage-signal", + event="barge_in", + ), + ) + + assert stage0.abort_calls == [["req-stage0"]] + assert stage1.abort_calls == [["req-stage1"]] + assert session.stage_bindings == {} + assert session.fence == current_fence + result = rpc_q.get_nowait() + assert result.ok is True + + +@pytest.mark.asyncio +async def test_duplex_cancel_without_next_fence_is_rejected_without_releasing_bindings() -> None: + stage0 = FakeStageClient(stage_type="llm", final_output=True) + stage_pools = _build_stage_pools( + [[stage0]], + output_processors=[FakeOutputProcessor()], + stage_vllm_configs=[SimpleNamespace(model_config=SimpleNamespace(max_model_len=64))], + ) + rpc_q: asyncio.Queue = asyncio.Queue() + orchestrator = Orchestrator( + request_async_queue=asyncio.Queue(), + output_async_queue=asyncio.Queue(), + rpc_async_queue=rpc_q, + stage_pools=stage_pools, + enable_duplex_control=True, + ) + fence = DuplexFence("sid-cancel-contract") + session = orchestrator.duplex_sessions.open_session(fence) + session.bind_stage_request(0, "req-live", fence=fence) + + await _handle_duplex( + orchestrator, + SignalDuplexTurnMessage( + control_id="cancel-without-next", + fence=fence, + session_id=fence.session_id, + event="input.cancel", + ), + ) + + result = rpc_q.get_nowait() + assert result.ok is False + assert result.error_count == 1 + assert "next_fence" in result.stage_results[0]["result"]["error"] + assert session.fence == fence + assert session.stage_request_ids() == ["req-live"] + assert stage0.abort_calls == [] + + +@pytest.mark.asyncio +async def test_duplex_session_update_replaces_runtime_config() -> None: + rpc_q: asyncio.Queue = asyncio.Queue() + orchestrator = Orchestrator( + request_async_queue=asyncio.Queue(), + output_async_queue=asyncio.Queue(), + rpc_async_queue=rpc_q, + stage_pools=[], + enable_duplex_control=True, + ) + fence = DuplexFence("sid-update-config") + session = orchestrator.duplex_sessions.open_session( + fence, + session_config={"temperature": 0.7}, + ) + + await _handle_duplex( + orchestrator, + SignalDuplexTurnMessage( + control_id="update-config", + fence=fence, + session_id=fence.session_id, + event="session.update", + session_config={"temperature": 0.0, "instructions": "updated"}, + ), + ) + + result = rpc_q.get_nowait() + assert result.ok is True + assert session.session_config == {"temperature": 0.0, "instructions": "updated"} + + +@pytest.mark.asyncio +async def test_duplex_session_update_refreshes_next_append_sampling_params() -> None: + stage0 = FakeStageClient(stage_type="llm", final_output=True) + stage_pools = _build_stage_pools( + [[stage0]], + output_processors=[FakeOutputProcessor()], + stage_vllm_configs=[SimpleNamespace(model_config=SimpleNamespace(max_model_len=64))], + ) + rpc_q: asyncio.Queue = asyncio.Queue() + orchestrator = Orchestrator( + request_async_queue=asyncio.Queue(), + output_async_queue=asyncio.Queue(), + rpc_async_queue=rpc_q, + stage_pools=stage_pools, + duplex_runtime_extension=MiniCPMO45DuplexRuntimeExtension(), + enable_duplex_control=True, + ) + fence = DuplexFence("sid-update-policy") + session = orchestrator.duplex_sessions.open_session( + fence, + capabilities=DuplexRuntimeCapabilities( + input_modes={DuplexInputMode.APPEND_AUDIO_CHUNK}, + ), + runtime_config={ + "duplex_stage_max_tokens": {"0": 2}, + "duplex_stage_sampling_params": {"0": {"stop_token_ids": [151645]}}, + }, + ) + req_state = _duplex_request_state(orchestrator, session, stage_id=0) + assert req_state is not None + assert req_state.sampling_params_list[0].max_tokens == 2 + + await _handle_duplex( + orchestrator, + SignalDuplexTurnMessage( + control_id="update-policy", + fence=fence, + session_id=fence.session_id, + event="session.update", + runtime_config={ + "duplex_stage_max_tokens": {"0": 7}, + "duplex_stage_sampling_params": {"0": {"stop_token_ids": [151705]}}, + }, + ), + ) + assert rpc_q.get_nowait().ok is True + + await _handle_duplex( + orchestrator, + AppendDuplexInputMessage( + control_id="append-updated-policy", + fence=fence, + session_id=fence.session_id, + mode=DuplexInputMode.APPEND_AUDIO_CHUNK.value, + payload={"is_speech": True}, + ), + ) + + assert rpc_q.get_nowait().ok is True + assert req_state.sampling_params_list[0].max_tokens == 7 + assert req_state.sampling_params_list[0].stop_token_ids == [151705] + submitted_request = stage0.add_request_calls[0][0] + assert submitted_request.sampling_params.stop_token_ids == [151705] + assert submitted_request.sampling_params.max_tokens == 7 + + +@pytest.mark.asyncio +async def test_duplex_invalid_session_update_preserves_previous_config() -> None: + stage0 = FakeStageClient(stage_type="llm", final_output=True) + rpc_q: asyncio.Queue = asyncio.Queue() + orchestrator = Orchestrator( + request_async_queue=asyncio.Queue(), + output_async_queue=asyncio.Queue(), + rpc_async_queue=rpc_q, + stage_pools=_build_stage_pools([[stage0]]), + duplex_runtime_extension=MiniCPMO45DuplexRuntimeExtension(), + enable_duplex_control=True, + ) + fence = DuplexFence("sid-invalid-update") + original_runtime_config = { + "duplex_stage_sampling_params": {"0": {"stop_token_ids": [151645]}}, + } + session = orchestrator.duplex_sessions.open_session( + fence, + runtime_config=original_runtime_config, + ) + + await _handle_duplex( + orchestrator, + SignalDuplexTurnMessage( + control_id="invalid-update", + fence=fence, + session_id=fence.session_id, + event="session.update", + runtime_config={ + "duplex_stage_sampling_params": {"0": {"stop_token_ids": 123}}, + }, + ), + ) + + result = rpc_q.get_nowait() + assert result.ok is False + assert session.runtime_config == original_runtime_config + assert session.config_generation == 0 + + +@pytest.mark.asyncio +async def test_duplex_arbitrary_non_cancel_signal_is_rejected() -> None: + rpc_q: asyncio.Queue = asyncio.Queue() + orchestrator = Orchestrator( + request_async_queue=asyncio.Queue(), + output_async_queue=asyncio.Queue(), + rpc_async_queue=rpc_q, + stage_pools=[], + enable_duplex_control=True, + ) + fence = DuplexFence("sid-unsupported-signal") + orchestrator.duplex_sessions.open_session(fence) + + await _handle_duplex( + orchestrator, + SignalDuplexTurnMessage( + control_id="unsupported-signal", + fence=fence, + session_id=fence.session_id, + event="turn.end", + ), + ) + + result = rpc_q.get_nowait() + assert result.ok is False + assert "unsupported duplex runtime signal" in result.stage_results[0]["result"]["error"] + + +@pytest.mark.asyncio +async def test_duplex_cancel_rejects_late_old_append_and_accepts_next_epoch() -> None: + stage0 = FakeStageClient(stage_type="llm", final_output=True) + stage_pools = _build_stage_pools( + [[stage0]], + output_processors=[FakeOutputProcessor()], + stage_vllm_configs=[SimpleNamespace(model_config=SimpleNamespace(max_model_len=64))], + ) + rpc_q: asyncio.Queue = asyncio.Queue() + orchestrator = Orchestrator( + request_async_queue=asyncio.Queue(), + output_async_queue=asyncio.Queue(), + rpc_async_queue=rpc_q, + stage_pools=stage_pools, + duplex_runtime_extension=MiniCPMO45DuplexRuntimeExtension(), + enable_duplex_control=True, + ) + cancelled_fence = DuplexFence("sid-cancel-late-append") + next_fence = DuplexFence("sid-cancel-late-append", epoch=1) + session = orchestrator.duplex_sessions.open_session( + cancelled_fence, + capabilities=DuplexRuntimeCapabilities( + input_modes={DuplexInputMode.APPEND_AUDIO_CHUNK}, + ), + ) + + await _handle_duplex( + orchestrator, + SignalDuplexTurnMessage( + control_id="cancel-old-fence", + fence=cancelled_fence, + next_fence=next_fence, + session_id=session.session_id, + event="input.cancel", + ), + ) + + cancel_result = rpc_q.get_nowait() + assert cancel_result.ok is True + assert session.fence == next_fence + + await _handle_duplex( + orchestrator, + AppendDuplexInputMessage( + control_id="late-old-append", + fence=cancelled_fence, + session_id=session.session_id, + mode=DuplexInputMode.APPEND_AUDIO_CHUNK.value, + payload={"is_speech": True}, + ), + ) + + stale_result = rpc_q.get_nowait() + assert stale_result.ok is False + assert stage0.add_request_calls == [] + assert session.stage_bindings == {} + + await _handle_duplex( + orchestrator, + AppendDuplexInputMessage( + control_id="next-epoch-append", + fence=next_fence, + session_id=session.session_id, + mode=DuplexInputMode.APPEND_AUDIO_CHUNK.value, + payload={"is_speech": True}, + ), + ) + + next_result = rpc_q.get_nowait() + assert next_result.ok is True + assert len(stage0.add_request_calls) == 1 + assert session.stage_bindings[0].fence == next_fence + + +@pytest.mark.asyncio +async def test_duplex_append_updates_bridge_turn_id_on_long_lived_stage0_request() -> None: + stage0 = FakeStageClient(stage_type="llm", final_output=True) + stage_pools = _build_stage_pools( + [[stage0]], + output_processors=[FakeOutputProcessor()], + stage_vllm_configs=[SimpleNamespace(model_config=SimpleNamespace(max_model_len=64))], + ) + orchestrator = Orchestrator( + request_async_queue=asyncio.Queue(), + output_async_queue=asyncio.Queue(), + rpc_async_queue=asyncio.Queue(), + stage_pools=stage_pools, + duplex_runtime_extension=MiniCPMO45DuplexRuntimeExtension(), + enable_duplex_control=True, + ) + session = orchestrator.duplex_sessions.open_session( + DuplexFence("sid-bridge-turn"), + capabilities=DuplexRuntimeCapabilities( + input_modes={DuplexInputMode.APPEND_AUDIO_CHUNK}, + ), + session_config={ + "voice": "test", + }, + runtime_config={ + "duplex_stage_sampling_params": { + "0": {"stop_token_ids": [151645]}, + }, + }, + ) + + await _handle_duplex( + orchestrator, + AppendDuplexInputMessage( + control_id="append-1", + fence=DuplexFence("sid-bridge-turn"), + session_id="sid-bridge-turn", + mode=DuplexInputMode.APPEND_AUDIO_CHUNK.value, + payload={"is_speech": True}, + ), + ) + req_state1 = _duplex_request_state(orchestrator, session, stage_id=0) + assert req_state1 is not None + duplex_state1 = req_state1.streaming.bridge_states["duplex"] + assert duplex_state1["session_id"] == "sid-bridge-turn" + assert duplex_state1["epoch"] == 0 + assert duplex_state1["turn_id"] == 0 + assert duplex_state1["session_config"] == session.session_config + assert req_state1.sampling_params_list[0].stop_token_ids == [151645] + submitted_request = stage0.add_request_calls[0][0] + assert submitted_request.sampling_params.stop_token_ids == [151645] + assert submitted_request.sampling_params.max_tokens == 1 + + await _handle_duplex( + orchestrator, + AppendDuplexInputMessage( + control_id="append-2", + fence=DuplexFence("sid-bridge-turn", turn_id=1, response_seq=1), + session_id="sid-bridge-turn", + mode=DuplexInputMode.APPEND_AUDIO_CHUNK.value, + payload={"is_speech": True, "new_user_turn": True}, + ), + ) + req_state2 = _duplex_request_state(orchestrator, session, stage_id=0) + assert req_state2 is req_state1 + duplex_state2 = req_state2.streaming.bridge_states["duplex"] + assert duplex_state2["turn_id"] == 1 + assert duplex_state2["epoch"] == 0 + expected_request_id = duplex_resource_request_id(DuplexFence("sid-bridge-turn"), "stage0") + assert [call[0].request_id for call in stage0.add_request_calls] == [ + expected_request_id, + expected_request_id, + ] + + # --------------------------------------------------------------------------- # Multi-replica tests # --------------------------------------------------------------------------- @@ -898,6 +1728,50 @@ async def test_stage_pool_submit_update_reuses_existing_binding() -> None: assert stage0_r1.add_request_calls[1][0].request_id == "req-1" +@pytest.mark.asyncio +async def test_stage_pool_failed_replica_releases_distributed_affinity_and_stops_polling() -> None: + stage0 = FakeStageClient(stage_type="llm", final_output=False) + pool = StagePool( + 0, + [stage0], + output_processor=FakeOutputProcessor(), + stage_vllm_config=SimpleNamespace(model_config=SimpleNamespace(max_model_len=64)), + ) + pool._addr_to_replica_id["tcp://replica-0"] = 0 + pool.bind("distributed-request", "tcp://replica-0") + pool._request_bindings["legacy-request"] = 0 + + affected = pool.mark_replica_unavailable(0) + + assert set(affected) == {"distributed-request", "legacy-request"} + assert pool.get_bound_replica_id("distributed-request") is None + assert pool.get_bound_replica_id("legacy-request") is None + assert await pool.poll_llm_raw_output(0) is None + + +def test_stage_pool_reattached_replica_becomes_available_again() -> None: + failed_client = FakeStageClient(stage_type="llm", final_output=False) + replacement_client = FakeStageClient(stage_type="llm", final_output=False) + pool = StagePool( + 0, + [failed_client], + output_processor=FakeOutputProcessor(), + stage_vllm_config=SimpleNamespace(model_config=SimpleNamespace(max_model_len=64)), + ) + input_addr = "tcp://replica-0" + pool._addr_to_replica_id[input_addr] = 0 + pool.mark_replica_unavailable(0) + removed = pool.remove_client(input_addr) + + replica_id = pool.add_client(input_addr, replacement_client) + + assert removed is failed_client + assert replica_id == 0 + assert len(pool.clients) == 1 + assert pool.clients[0] is replacement_client + assert pool.available_replica_ids() == [0] + + @pytest.mark.asyncio async def test_stage_pool_submit_update_refreshes_output_processor_state() -> None: output_processor = FakeOutputProcessor() @@ -980,6 +1854,73 @@ async def test_handle_streaming_update_passes_prompt_text_to_stage_pool() -> Non assert orchestrator.request_states["req-stream"].streaming.enabled is True +@pytest.mark.asyncio +async def test_resumable_segment_boundary_builds_stage_metrics() -> None: + built_metrics = SimpleNamespace(pipeline_timings={}) + + class RecordingPool: + def __init__(self) -> None: + self.calls: list[list[Any]] = [] + + def build_stage_metrics(self, outputs, **_kwargs): + self.calls.append(outputs) + return built_metrics + + pool = RecordingPool() + orchestrator = object.__new__(Orchestrator) + req_state = OrchestratorRequestState( + request_id="req-stream", + sampling_params_list=[_sampling_params()], + final_stage_id=0, + ) + req_state.streaming.enabled = True + req_state.streaming.segment_finished = True + req_state.stage_submit_ts[0] = time.time() + orchestrator.request_states = {"req-stream": req_state} + orchestrator.stage_pools = [pool] + routed: list[Any] = [] + + async def record_route(_stage_id, _replica_id, _output, _req_state, stage_metrics): + routed.append(stage_metrics) + + orchestrator._route_output = record_route + output = SimpleNamespace(request_id="req-stream", error=None, finished=False) + + await orchestrator._handle_processed_outputs(0, 0, [output]) + + assert pool.calls == [[output]] + assert routed == [built_metrics] + + +def test_stage_pool_metrics_use_resumable_segment_token_count() -> None: + class SegmentMetricsOutputProcessor(FakeOutputProcessor): + def pop_native_text_metrics(self, request_id: str) -> dict[str, Any]: + assert request_id == "req-stream" + return {"num_generation_tokens": 3} + + stage0 = FakeStageClient(stage_type="llm", final_output=False) + pool = StagePool( + 0, + [stage0], + output_processor=SegmentMetricsOutputProcessor(), + stage_vllm_config=SimpleNamespace(model_config=SimpleNamespace(max_model_len=64)), + ) + output = SimpleNamespace( + request_id="req-stream", + outputs=[SimpleNamespace(cumulative_token_ids=list(range(11)))], + ) + + metrics = pool.build_stage_metrics( + [output], + submit_ts=time.time(), + request_timestamp=time.time(), + replica_id=0, + ) + + assert metrics.num_tokens_out == 3 + assert metrics.output_unit_count == 3 + + @pytest.mark.asyncio async def test_stage_pool_submit_initial_rolls_back_output_processor_when_client_submit_fails() -> None: class FailingStageClient(FakeStageClient): @@ -1179,3 +2120,132 @@ def test_orchestrator_does_not_re_introduce_global_stats_throttle() -> None: "raw_outputs.scheduler_stats being non-None — the per-scheduler 1Hz " "throttle in OmniSchedulerMixin.make_stats() is the only gate needed." ) + + +@pytest.mark.asyncio +async def test_duplex_reaper_loop_waits_between_ticks(): + class _Plane: + def __init__(self) -> None: + self.calls = 0 + + async def reap_expired(self) -> int: + self.calls += 1 + return 0 + + orchestrator = object.__new__(Orchestrator) + orchestrator.duplex_control_plane = _Plane() + orchestrator._duplex_reaper_interval_s = 0.01 + orchestrator._shutdown_event = asyncio.Event() + + task = asyncio.create_task(orchestrator._duplex_reaper_loop()) + await asyncio.sleep(0.035) + orchestrator._shutdown_event.set() + await task + + assert 2 <= orchestrator.duplex_control_plane.calls <= 5 + + +@pytest.mark.asyncio +async def test_duplex_reaper_loop_survives_one_cleanup_failure(): + class _Plane: + def __init__(self) -> None: + self.calls = 0 + + async def reap_expired(self) -> int: + self.calls += 1 + if self.calls == 1: + raise RuntimeError("transient cleanup failure") + return 0 + + orchestrator = object.__new__(Orchestrator) + orchestrator.duplex_control_plane = _Plane() + orchestrator._duplex_reaper_interval_s = 0.01 + orchestrator._shutdown_event = asyncio.Event() + + task = asyncio.create_task(orchestrator._duplex_reaper_loop()) + await asyncio.sleep(0.035) + orchestrator._shutdown_event.set() + await task + + assert orchestrator.duplex_control_plane.calls >= 2 + + +@pytest.mark.asyncio +async def test_abort_retry_does_not_repeat_successful_stage_abort(): + class _Pool: + def __init__(self, *, fail_once: bool = False) -> None: + self.bound = {"req-a"} + self.fail_once = fail_once + self.physical_abort_calls = 0 + + async def abort_requests(self, request_ids: list[str]) -> None: + active = self.bound.intersection(request_ids) + if not active: + return + self.physical_abort_calls += 1 + if self.fail_once: + self.fail_once = False + raise RuntimeError("stage abort failed") + + def release_bindings(self, request_ids: list[str]) -> None: + self.bound.difference_update(request_ids) + + first = _Pool() + second = _Pool(fail_once=True) + orchestrator = object.__new__(Orchestrator) + orchestrator.stage_pools = [first, second] + + with pytest.raises(RuntimeError, match="stage abort failed"): + await orchestrator._abort_request_ids(["req-a"]) + + assert first.bound == set() + assert first.physical_abort_calls == 1 + assert second.bound == {"req-a"} + + await orchestrator._abort_request_ids(["req-a"]) + + assert first.physical_abort_calls == 1 + assert second.physical_abort_calls == 2 + assert second.bound == set() + + +@pytest.mark.asyncio +async def test_request_cleanup_failure_is_deferred_to_control_plane(): + class _FailingPool: + async def abort_requests(self, request_ids: list[str]) -> None: + raise RuntimeError("stage abort failed") + + def release_bindings(self, request_ids: list[str]) -> None: + pass + + class _Plane: + def __init__(self) -> None: + self.deferred: list[str] = [] + self.finalized: list[str] = [] + + def close_sessions_for_request_ids(self, request_ids: list[str], **kwargs): + assert kwargs == {"abort": True, "cleanup_in_progress": True} + return {"sid-cleanup": ["req-a"]} + + def defer_request_cleanups(self, session_ids: list[str]) -> None: + self.deferred.extend(session_ids) + + def finalize_closed_sessions(self, session_ids: list[str]) -> None: + self.finalized.extend(session_ids) + + orchestrator = object.__new__(Orchestrator) + orchestrator.stage_pools = [_FailingPool()] + orchestrator.duplex_control_plane = _Plane() + orchestrator._pd_kv_params = {} + orchestrator.request_states = {} + orchestrator._running_counter = None + + with pytest.raises(RuntimeError, match="stage abort failed"): + await orchestrator._cleanup_request_ids( + ["req-a"], + abort=True, + close_duplex_sessions=True, + ) + + assert orchestrator.duplex_control_plane.deferred == ["sid-cleanup"] + assert orchestrator.duplex_control_plane.finalized == [] diff --git a/tests/engine/test_orchestrator_stage_input_bridge.py b/tests/engine/test_orchestrator_stage_input_bridge.py index 27ee11df4..3abb244eb 100644 --- a/tests/engine/test_orchestrator_stage_input_bridge.py +++ b/tests/engine/test_orchestrator_stage_input_bridge.py @@ -38,6 +38,7 @@ class FakeStageClient: self.custom_process_input_func = None self.next_inputs = list(next_inputs or []) self.add_request_calls: list[tuple[Any, ...]] = [] + self.decoded_source_tokens: str | None = None self._engine_core_outputs = queue.Queue() async def add_request_async(self, *args, **_kwargs) -> None: @@ -50,6 +51,9 @@ class FakeStageClient: return SimpleNamespace(outputs=[]) def process_engine_inputs(self, _source_outputs, prompt=None, streaming_context=None): + decoder = getattr(streaming_context, "source_token_decoder", None) + if callable(decoder): + self.decoded_source_tokens = decoder([11, 12], skip_special_tokens=True) return list(self.next_inputs) async def abort_requests_async(self, _request_ids: list[str]) -> None: @@ -66,6 +70,9 @@ class FakeStageClient: class FakeOutputProcessor: + def __init__(self, tokenizer=None) -> None: + self.tokenizer = tokenizer + def add_request(self, *args, **kwargs) -> None: return None @@ -108,6 +115,11 @@ def _request_output(request_id: str) -> RequestOutput: @pytest.mark.asyncio async def test_forward_text_prompt_uses_target_stage_input_processor() -> None: + class SourceTokenizer: + def decode(self, token_ids, *, skip_special_tokens): + assert skip_special_tokens is True + return ":".join(str(token_id) for token_id in token_ids) + stage0 = FakeStageClient(final_output=True) stage1 = FakeStageClient( final_output=True, @@ -117,7 +129,7 @@ async def test_forward_text_prompt_uses_target_stage_input_processor() -> None: StagePool( 0, [stage0], - output_processor=FakeOutputProcessor(), + output_processor=FakeOutputProcessor(tokenizer=SourceTokenizer()), stage_vllm_config=SimpleNamespace(model_config=SimpleNamespace(max_model_len=64)), ), StagePool( @@ -150,6 +162,8 @@ async def test_forward_text_prompt_uses_target_stage_input_processor() -> None: assert input_processor.calls assert input_processor.calls[0]["prompt"] == {"prompt": "hello", "multi_modal_data": {"video": ["frame"]}} + assert stage1.decoded_source_tokens == "11:12" + assert req_state.streaming.source_token_decoder is None assert stage1.add_request_calls submitted_request = stage1.add_request_calls[0][0] assert submitted_request.prompt_token_ids == [101, 102] diff --git a/tests/engine/test_output_processor.py b/tests/engine/test_output_processor.py index 86e71fcbc..aef78e47e 100644 --- a/tests/engine/test_output_processor.py +++ b/tests/engine/test_output_processor.py @@ -2,6 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """Regression tests for OmniRequestState multimodal DELTA drain and consolidation guard.""" +from types import SimpleNamespace from unittest.mock import MagicMock import numpy as np @@ -10,9 +11,11 @@ import torch from vllm.outputs import PoolingRequestOutput from vllm.sampling_params import RequestOutputKind from vllm.v1.engine import FinishReason +from vllm.v1.engine.output_processor import OutputProcessor as VLLMOutputProcessor +from vllm_omni.outputs import output_processor from vllm_omni.outputs.output_modality import OutputModalityNames -from vllm_omni.outputs.output_processor import OmniRequestState +from vllm_omni.outputs.output_processor import MultimodalOutputProcessor, OmniRequestState pytestmark = [pytest.mark.core_model, pytest.mark.cpu] @@ -62,6 +65,65 @@ def test_init_empty_dict(): assert _make_state(RequestOutputKind.DELTA).mm_accumulated == {} +def test_streaming_update_resets_native_text_metrics_for_next_segment(): + state = _make_state(RequestOutputKind.DELTA) + state.native_text_stats.num_generation_tokens = 12 + state.native_text_stats.first_token_ts = 10.0 + state.native_text_stats.last_token_ts = 10.2 + + state.apply_streaming_update( + SimpleNamespace( + final=False, + prompt=None, + prompt_token_ids=[], + arrival_time=20.0, + ) + ) + + assert state.native_text_stats.arrival_time == 20.0 + assert state.native_text_stats.num_generation_tokens == 0 + assert state.native_text_stats.first_token_ts == 0.0 + assert state.native_text_stats.last_token_ts == 0.0 + + +def test_native_text_tpot_snapshot_matches_vllm_finished_metric_definition(): + calculate = getattr(output_processor, "_mean_time_per_output_token_ms", None) + assert calculate is not None + stats = SimpleNamespace( + num_generation_tokens=4, + first_token_ts=10.0, + last_token_ts=10.045, + ) + + assert calculate(stats) == pytest.approx(15.0) + + +def test_native_text_metrics_include_segment_generation_token_count(monkeypatch): + monkeypatch.setattr(VLLMOutputProcessor, "_update_stats_from_output", lambda *args, **kwargs: None) + processor = object.__new__(MultimodalOutputProcessor) + processor._native_text_metrics_by_request = {} + processor.lora_states = {} + state = _make_state(RequestOutputKind.DELTA) + state.is_prefilling = False + iteration_stats = MagicMock() + + def update_native_stats(_output, _timestamp, _was_prefilling, native_stats, *_args): + native_stats.num_generation_tokens = 27 + native_stats.first_token_ts = 10.0 + native_stats.last_token_ts = 10.26 + + iteration_stats.update_from_output.side_effect = update_native_stats + + processor._update_stats_from_output( + state, + SimpleNamespace(), + 10.26, + iteration_stats, + ) + + assert processor.pop_native_text_metrics("r")["num_generation_tokens"] == 27 + + def test_delta_drains_output_modality_per_step(): """DELTA drains the mm_type key (output modality) but preserves hidden-state keys.""" s = _make_state(RequestOutputKind.DELTA) @@ -172,6 +234,145 @@ def test_finish_consolidation_drains_mm_delta(): assert AUDIO not in s.mm_accumulated # drained +def test_delta_drains_audio_chunk_metadata_per_step(): + s = _make_state(RequestOutputKind.DELTA) + segment_text = "你好" + segment_utf8 = torch.tensor(list(segment_text.encode("utf-8")), dtype=torch.uint8) + + s.add_multimodal_tensor( + { + "model_outputs": torch.ones(5), + "meta.duplex_epoch": torch.tensor([7], dtype=torch.int32), + "meta.duplex_turn_id": torch.tensor([1], dtype=torch.int32), + "meta.llm_output_text_utf8": segment_utf8, + "meta.audio_text_total_chars": torch.tensor([len(segment_text)], dtype=torch.int32), + "meta.tts_is_last_chunk": torch.tensor([0], dtype=torch.int32), + }, + mm_type=AUDIO, + ) + result1 = s.make_request_output([1], None, FinishReason.STOP, None) + assert result1 is not None and not isinstance(result1, PoolingRequestOutput) + meta1 = result1.outputs[0].multimodal_output["meta"] + assert bytes(meta1["llm_output_text_utf8"].tolist()).decode("utf-8") == segment_text + + s.add_multimodal_tensor( + { + "model_outputs": torch.ones(3), + "meta.duplex_epoch": torch.tensor([8], dtype=torch.int32), + "meta.duplex_turn_id": torch.tensor([2], dtype=torch.int32), + "meta.llm_output_text_utf8": segment_utf8, + "meta.audio_text_total_chars": torch.tensor([len(segment_text)], dtype=torch.int32), + "meta.tts_is_last_chunk": torch.tensor([1], dtype=torch.int32), + }, + mm_type=AUDIO, + ) + result2 = s.make_request_output([2], None, FinishReason.STOP, None) + + assert result2 is not None and not isinstance(result2, PoolingRequestOutput) + meta2 = result2.outputs[0].multimodal_output["meta"] + assert bytes(meta2["llm_output_text_utf8"].tolist()).decode("utf-8") == segment_text + assert meta2["audio_text_total_chars"].tolist() == [len(segment_text)] + assert meta2["duplex_epoch"].tolist() == [8] + assert meta2["duplex_turn_id"].tolist() == [2] + + +def test_cumulative_audio_replaces_chunk_metadata_per_step(): + s = _make_state(RequestOutputKind.CUMULATIVE) + segment_text = "你好" + segment_utf8 = torch.tensor(list(segment_text.encode("utf-8")), dtype=torch.uint8) + + s.add_multimodal_tensor( + { + "model_outputs": torch.ones(5), + "meta.duplex_epoch": torch.tensor([7], dtype=torch.int32), + "meta.duplex_turn_id": torch.tensor([1], dtype=torch.int32), + "meta.llm_output_text_utf8": segment_utf8, + "meta.audio_text_total_chars": torch.tensor([len(segment_text)], dtype=torch.int32), + "meta.tts_is_last_chunk": torch.tensor([0], dtype=torch.int32), + }, + mm_type=AUDIO, + ) + result1 = s.make_request_output([1], None, None, None) + assert result1 is not None and not isinstance(result1, PoolingRequestOutput) + assert result1.outputs[0].multimodal_output["audio"].shape[0] == 5 + + s.add_multimodal_tensor( + { + "model_outputs": torch.ones(3), + "meta.duplex_epoch": torch.tensor([8], dtype=torch.int32), + "meta.duplex_turn_id": torch.tensor([2], dtype=torch.int32), + "meta.llm_output_text_utf8": segment_utf8, + "meta.audio_text_total_chars": torch.tensor([len(segment_text)], dtype=torch.int32), + "meta.tts_is_last_chunk": torch.tensor([1], dtype=torch.int32), + }, + mm_type=AUDIO, + ) + result2 = s.make_request_output([2], None, None, None) + + assert result2 is not None and not isinstance(result2, PoolingRequestOutput) + output = result2.outputs[0].multimodal_output + assert output["audio"].shape[0] == 8 + meta = output["meta"] + assert bytes(meta["llm_output_text_utf8"].tolist()).decode("utf-8") == segment_text + assert meta["audio_text_total_chars"].tolist() == [len(segment_text)] + assert meta["duplex_epoch"].tolist() == [8] + assert meta["duplex_turn_id"].tolist() == [2] + assert meta["tts_is_last_chunk"].tolist() == [1] + + +def test_delta_audio_non_final_tts_chunk_overrides_spurious_finish(): + s = _make_state(RequestOutputKind.DELTA) + s.add_multimodal_tensor( + { + "model_outputs": torch.ones(5), + "meta.tts_is_last_chunk": torch.tensor([0]), + }, + mm_type=AUDIO, + ) + + result = s.make_request_output([1], None, FinishReason.STOP, None) + + assert result is not None and not isinstance(result, PoolingRequestOutput) + assert result.finished is False + assert result.outputs[0].finish_reason is None + assert result.outputs[0].multimodal_output["meta"]["tts_is_last_chunk"].tolist() == [0] + assert torch.equal(result.outputs[0].multimodal_output[AUDIO], torch.ones(5)) + + +def test_delta_audio_final_tts_chunk_keeps_finish(): + s = _make_state(RequestOutputKind.DELTA) + s.add_multimodal_tensor( + { + "model_outputs": torch.ones(5), + "meta.tts_is_last_chunk": torch.tensor([1]), + }, + mm_type=AUDIO, + ) + + result = s.make_request_output([151645], None, FinishReason.STOP, None) + + assert result is not None and not isinstance(result, PoolingRequestOutput) + assert result.finished is True + assert result.outputs[0].finish_reason == "stop" + + +def test_delta_audio_non_final_tts_chunk_accepts_nested_meta(): + s = _make_state(RequestOutputKind.DELTA) + s.add_multimodal_tensor( + { + AUDIO: torch.ones(5), + "meta": {"tts_is_last_chunk": torch.tensor([0])}, + }, + mm_type=AUDIO, + ) + + result = s.make_request_output([1], None, FinishReason.STOP, None) + + assert result is not None and not isinstance(result, PoolingRequestOutput) + assert result.finished is False + assert result.outputs[0].multimodal_output["meta"]["tts_is_last_chunk"].tolist() == [0] + + @pytest.mark.parametrize("mm_type", [AUDIO, "hidden"]) def test_final_only_consolidates_drainable_keys(mm_type): """FINAL_ONLY never drains per-step, so modality keys and hidden state diff --git a/tests/engine/test_rpc_result_router.py b/tests/engine/test_rpc_result_router.py new file mode 100644 index 000000000..ec983392b --- /dev/null +++ b/tests/engine/test_rpc_result_router.py @@ -0,0 +1,109 @@ +import queue +from typing import Literal + +import pytest + +from vllm_omni.engine.messages import ( + CollectiveRPCResultMessage, + EngineQueueMessage, + ErrorMessage, +) +from vllm_omni.engine.rpc_result_router import RpcResultRouter + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +class GenericCorrelatedResultMessage(EngineQueueMessage, kw_only=True): + type: Literal["generic_correlated_result"] = "generic_correlated_result" + namespace: str + correlation_id: str + + @property + def rpc_correlation_key(self) -> tuple[str, str]: + return (self.namespace, self.correlation_id) + + +def _generic_result(correlation_id: str) -> GenericCorrelatedResultMessage: + return GenericCorrelatedResultMessage( + namespace="plugin", + correlation_id=correlation_id, + ) + + +def test_rpc_result_router_routes_out_of_order_results_by_correlation_id(): + source: queue.Queue = queue.Queue() + router = RpcResultRouter(source) + first = router.register(("plugin", "first")) + second = router.register(("plugin", "second")) + + source.put(_generic_result("second")) + source.put(_generic_result("first")) + + assert first.get(timeout=1).correlation_id == "first" + assert second.get(timeout=1).correlation_id == "second" + router.close() + + +def test_rpc_result_router_drops_only_the_late_result_after_unregister(): + source: queue.Queue = queue.Queue() + router = RpcResultRouter(source) + expired = router.register(("plugin", "expired")) + active = router.register(("collective", "active")) + router.unregister(("plugin", "expired"), expired) + + source.put(_generic_result("expired")) + source.put( + CollectiveRPCResultMessage( + rpc_id="active", + method="health", + stage_ids=[0], + results=["ok"], + ) + ) + + assert active.get(timeout=1).rpc_id == "active" + assert expired.empty() + router.close() + + +def test_rpc_result_router_broadcasts_fatal_errors_to_pending_waiters(): + source: queue.Queue = queue.Queue() + router = RpcResultRouter(source) + plugin = router.register(("plugin", "one")) + collective = router.register(("collective", "two")) + + source.put(ErrorMessage(error="orchestrator failed", fatal=True)) + + assert plugin.get(timeout=1).error == "orchestrator failed" + assert collective.get(timeout=1).error == "orchestrator failed" + with pytest.raises(RuntimeError, match="orchestrator failed"): + router.register(("plugin", "after-failure")) + router.close() + + +def test_rpc_result_router_does_not_broadcast_uncorrelated_nonfatal_errors(): + source: queue.Queue = queue.Queue() + router = RpcResultRouter(source) + waiter = router.register(("plugin", "active")) + + source.put(ErrorMessage(error="request failed", fatal=False, request_id="other")) + source.put(_generic_result("active")) + + assert waiter.get(timeout=1).correlation_id == "active" + router.close() + + +def test_rpc_result_router_close_unblocks_waiters_and_stops_consumer(): + source: queue.Queue = queue.Queue() + router = RpcResultRouter(source) + waiter = router.register(("plugin", "pending")) + + router.close() + + result = waiter.get(timeout=1) + assert isinstance(result, ErrorMessage) + assert result.fatal is True + assert result.error == "RPC result router closed" + assert not router._thread.is_alive() + with pytest.raises(RuntimeError, match="router is closed"): + router.register(("plugin", "after-close")) diff --git a/tests/entrypoints/openai/test_duplex_protocol.py b/tests/entrypoints/openai/test_duplex_protocol.py new file mode 100644 index 000000000..3c3666026 --- /dev/null +++ b/tests/entrypoints/openai/test_duplex_protocol.py @@ -0,0 +1,474 @@ +import asyncio +import json + +import pytest + +from vllm_omni.experimental.fullduplex.openai.protocol import ( + DuplexCapabilities, + DuplexOverlapPolicy, + DuplexSession, + DuplexSessionConfig, + DuplexSessionRegistry, + DuplexTurnController, + DuplexTurnEventType, + DuplexTurnState, + ResponseCreateOptions, +) +from vllm_omni.experimental.fullduplex.openai.realtime_session import ( + NativeRealtimeSessionProtocol, +) +from vllm_omni.experimental.fullduplex.openai.realtime_state import RealtimeStateOwner + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +def test_realtime_state_owner_uses_explicit_bindings_not_dynamic_attribute_proxy(): + assert "__getattr__" not in RealtimeStateOwner.__dict__ + assert "__setattr__" not in RealtimeStateOwner.__dict__ + + +class _ProtocolWebSocket: + def __init__(self, *events: dict[str, object]) -> None: + self._events = asyncio.Queue() + for event in events: + self._events.put_nowait(json.dumps(event)) + + async def receive_text(self) -> str: + return await self._events.get() + + +def test_duplex_session_commits_text_and_audio_as_one_turn(): + registry = DuplexSessionRegistry() + session = registry.create(DuplexSessionConfig(model="test-model")) + + session.append_text("hello") + session.append_audio("YWJj", fmt="wav", sample_rate_hz=16000) + committed = session.commit_user_input() + + assert committed is not None + assert committed.turn_id == 0 + assert committed.input_commit_seq == 1 + assert committed.epoch == 0 + assert len(session.history) == 1 + content = session.history[0]["content"] + assert isinstance(content, list) + assert content[0] == {"type": "text", "text": "hello"} + assert content[1]["type"] == "audio_url" + assert content[1]["audio_url"]["url"] == "data:audio/wav;base64,YWJj" + assert session.turn_state == DuplexTurnState.USER_COMMITTED + + +def test_response_options_apply_to_one_response_without_mutating_session_defaults(): + session = DuplexSession( + session_id="sid-response-options", + config=DuplexSessionConfig(instructions="base", voice="base-voice", max_tokens=64), + ) + session.reserve_response_options( + ResponseCreateOptions( + instructions="one response", + voice="override-voice", + max_tokens=8, + ) + ) + + assert session.config.instructions == "base" + assert session.config.voice == "base-voice" + session.begin_response() + assert session.config.instructions == "base" + assert session.config.voice == "base-voice" + assert session.response_config.instructions == "one response" + assert session.response_config.voice == "override-voice" + assert session.response_config.max_tokens == 8 + + session.end_response() + assert session.config.instructions == "base" + assert session.config.voice == "base-voice" + assert session.config.max_tokens == 64 + + +def test_response_options_cannot_overwrite_an_unconsumed_reservation(): + session = DuplexSession( + session_id="sid-response-options-pending", + config=DuplexSessionConfig(instructions="base"), + ) + session.reserve_response_options(ResponseCreateOptions(instructions="first")) + + with pytest.raises(RuntimeError, match="already reserved"): + session.reserve_response_options(ResponseCreateOptions(instructions="second")) + + session.begin_response() + assert session.response_config.instructions == "first" + + +def test_response_options_cannot_be_reserved_while_response_is_active(): + session = DuplexSession( + session_id="sid-response-options-active", + config=DuplexSessionConfig(instructions="base"), + ) + session.begin_response() + + with pytest.raises(RuntimeError, match="active"): + session.reserve_response_options(ResponseCreateOptions(instructions="too late")) + + +def test_realtime_model_name_does_not_implicitly_enable_native_duplex(): + protocol = NativeRealtimeSessionProtocol({}) + + event = protocol._session_create_from_realtime({"model": "openbmb/MiniCPM-o-4_5"}) + + assert "minicpmo45_native_duplex" not in event["session"]["extra_body"] + + +def test_realtime_explicit_native_duplex_flag_is_preserved(): + protocol = NativeRealtimeSessionProtocol({}) + + event = protocol._session_create_from_realtime( + { + "model": "openbmb/MiniCPM-o-4_5", + "extra_body": {"minicpmo45_native_duplex": True}, + } + ) + + assert event["session"]["extra_body"]["minicpmo45_native_duplex"] is True + + +def test_realtime_explicit_query_native_duplex_flag_is_available_before_autostart(): + protocol = NativeRealtimeSessionProtocol( + { + "model": "openbmb/MiniCPM-o-4_5", + "minicpmo45_native_duplex": "1", + } + ) + + event = json.loads(asyncio.run(protocol.receive_internal_event_text(None))) + + assert event["type"] == "session.create" + assert event["session"]["extra_body"]["minicpmo45_native_duplex"] is True + + +@pytest.mark.asyncio +async def test_realtime_resume_heartbeat_and_event_ack_translate_without_session_autostart(): + protocol = NativeRealtimeSessionProtocol( + { + "model": "openbmb/MiniCPM-o-4_5", + "minicpmo45_native_duplex": "1", + } + ) + + resume = await protocol._to_duplex_event( + { + "type": "session.resume", + "session_id": "sid-resume", + "incarnation": 3, + "resume_token": "opaque-token", + "last_received_server_event_seq": 17, + } + ) + heartbeat = await protocol._to_duplex_event( + { + "type": "session.heartbeat", + "event_id": "event-heartbeat", + } + ) + event_ack = await protocol._to_duplex_event( + { + "type": "session.event_ack", + "server_event_seq": 19, + } + ) + + assert resume == { + "type": "session.resume", + "session_id": "sid-resume", + "incarnation": 3, + "resume_token": "opaque-token", + "last_received_server_event_seq": 17, + } + assert heartbeat == { + "type": "session.heartbeat", + "event_id": "event-heartbeat", + } + assert event_ack == { + "type": "session.event_ack", + "server_event_seq": 19, + } + assert protocol._opened is False + + wire_protocol = NativeRealtimeSessionProtocol({}) + first_wire_event = json.loads( + await wire_protocol.receive_internal_event_text( + _ProtocolWebSocket( + { + "type": "session.resume", + "session_id": "sid-resume", + "incarnation": 3, + "resume_token": "opaque-token", + "last_received_server_event_seq": 17, + } + ) + ) + ) + + assert first_wire_event["type"] == "session.resume" + + +@pytest.mark.asyncio +async def test_realtime_resume_query_suppresses_model_autostart_on_same_url(): + protocol = NativeRealtimeSessionProtocol( + { + "model": "openbmb/MiniCPM-o-4_5", + "minicpmo45_native_duplex": "1", + "resume": "1", + } + ) + resume = { + "type": "session.resume", + "session_id": "sid-resume", + "incarnation": 3, + "resume_token": "opaque-token", + "last_received_server_event_seq": 17, + } + + first_wire_event = json.loads(await protocol.receive_internal_event_text(_ProtocolWebSocket(resume))) + + assert first_wire_event == resume + assert protocol._autostarted_default_session is False + + +def test_realtime_projects_resume_lifecycle_events_without_duplex_prefix(): + protocol = NativeRealtimeSessionProtocol({}) + + created = protocol.encode_outbound_event( + { + "type": "session.created", + "session": {"id": "sid-resume"}, + "incarnation": 3, + "attachment_generation": 1, + "resume_token": "first-token", + } + )[0] + resumed = protocol.encode_outbound_event( + { + "type": "session.resumed", + "session_id": "sid-resume", + "incarnation": 3, + "attachment_generation": 2, + "resume_token": "second-token", + } + )[0] + heartbeat = protocol.encode_outbound_event( + { + "type": "session.heartbeat_ack", + "session_id": "sid-resume", + } + )[0] + replaced = protocol.encode_outbound_event( + { + "type": "session.replaced", + "session_id": "sid-resume", + "attachment_generation": 1, + } + )[0] + resync = protocol.encode_outbound_event( + { + "type": "session.resync_required", + "session_id": "sid-resume", + "reason": "journal_gap", + } + )[0] + + assert created["type"] == "session.created" + assert created["incarnation"] == 3 + assert created["attachment_generation"] == 1 + assert created["resume_token"] == "first-token" + assert resumed["type"] == "session.resumed" + assert resumed["resume_token"] == "second-token" + assert heartbeat["type"] == "session.heartbeat_ack" + assert replaced["type"] == "session.replaced" + assert resync == { + "type": "session.resync_required", + "session_id": "sid-resume", + "reason": "journal_gap", + "event_id": resync["event_id"], + } + + +def test_duplex_session_registry_advances_incarnation_when_id_is_reused(): + registry = DuplexSessionRegistry() + + first = registry.create(session_id="reused-session") + registry.close(first.session_id) + second = registry.create(session_id="reused-session") + + assert first.incarnation == 0 + assert second.incarnation == 1 + + +def test_native_input_commit_does_not_advance_model_turn_identity(): + session = DuplexSession( + session_id="sid-model-owned-turn", + config=DuplexSessionConfig(extra_body={"auto_response": True}), + ) + + first = session.commit_native_audio_input(transcript="first chunk") + second = session.commit_native_audio_input(transcript="second chunk") + + assert session.input_commit_seq == 2 + assert first.input_commit_seq == 1 + assert second.input_commit_seq == 2 + assert first.turn_id == second.turn_id == session.turn_id == 0 + + +def test_duplex_session_owns_response_and_overlap_identity(): + session = DuplexSession(session_id="sid-state-owner", config=DuplexSessionConfig()) + + response_id = session.begin_response() + session.accumulate_overlap_speech(320) + session.accumulate_overlap_speech(180) + session.end_response() + + assert session.active_response_id is None + assert session.last_response_id == response_id + assert session.overlap_speech_ms == 500 + assert session.reset_overlap_speech() == 500 + assert session.overlap_speech_ms == 0 + + +def test_duplex_session_composes_single_owner_ledgers(): + session = DuplexSession(session_id="sid-ledgers", config=DuplexSessionConfig()) + + session.append_text("hello") + session.bind_request("req-1") + response_id = session.begin_response(turn_id=3) + session.mark_audio_sent(duration_ms=240) + + assert not hasattr(session, "input") + assert not hasattr(session, "response") + assert not hasattr(session, "playback_ledger") + assert not hasattr(session, "conversation") + assert session.pending_text == ("hello",) + assert session.active_request_id == "req-1" + assert session.active_response_id == response_id + assert session.active_response_turn_id == 3 + assert session.playback.sent_ms == 240 + + try: + session.playback.sent_ms = 480 + except (AttributeError, TypeError): + pass + else: + raise AssertionError("playback view must be immutable") + + +def test_duplex_barge_in_advances_epoch_and_drops_uncommitted_assistant_text(): + registry = DuplexSessionRegistry() + session = registry.create() + response_id = session.begin_response() + session.bind_request("chatcmpl-duplex-test") + session.append_assistant_text("unplayed answer") + + new_epoch = session.barge_in() + + assert response_id is not None + assert new_epoch == 1 + assert session.epoch == 1 + assert session.active_request_id is None + assert session.active_response_id is None + assert session.assistant_text_buffer == () + assert session.history == () + assert session.turn_state == DuplexTurnState.BARGE_IN + + +def test_duplex_playback_ack_tracks_committed_cursor_separately(): + registry = DuplexSessionRegistry() + session = registry.create() + session.mark_audio_sent(duration_ms=10_000) + + session.acknowledge_playback(played_ms=2_000) + + assert session.playback.generated_ms == 10_000 + assert session.playback.sent_ms == 10_000 + assert session.playback.played_ms == 2_000 + assert session.playback.committed_ms == 2_000 + + +def test_duplex_history_commit_uses_audio_text_alignment_marks(): + registry = DuplexSessionRegistry() + session = registry.create() + session.begin_response() + + session.append_assistant_text("hello ") + session.mark_audio_sent(duration_ms=1_000, text_chars=6) + session.append_assistant_text("world") + session.mark_audio_sent(duration_ms=2_000, text_chars=11) + session.acknowledge_playback(played_ms=1_200, committed_ms=1_200) + + committed = session.end_response(commit_text=True) + + assert committed == {"role": "assistant", "content": "hello w"} + + +def test_turn_controller_accepts_external_signal_source(): + registry = DuplexSessionRegistry() + session = registry.create() + controller = DuplexTurnController() + + event = controller.signal(session, DuplexTurnEventType.USER_STARTED.value) + + assert event["type"] == "turn.event" + assert event["event"] == "user_started" + assert event["turn_state"] == "user_speaking" + assert session.turn_state == DuplexTurnState.USER_SPEAKING + + +def test_duplex_capabilities_do_not_claim_core_kv_or_input_append(): + registry = DuplexSessionRegistry() + session = registry.create() + caps = session.capabilities.as_dict() + + assert caps["implementation_level"] == "serving_session_adapter" + assert caps["supports_kv_lease"] is False + assert caps["supports_input_append"] is False + assert caps["supports_reencode_context"] is True + assert caps["adapter_patterns"] == ["chunk_group_append"] + assert "turn_commit_only" in caps["input_modes"] + assert "client_event" in caps["signal_sources"] + + +def test_minicpmo_native_capabilities_separate_model_state_from_core_kv_lease(): + caps = DuplexCapabilities.minicpmo45_native(max_sessions=2).as_dict() + + assert caps["implementation_level"] == "model_native_duplex" + assert caps["supports_input_append"] is True + assert caps["input_modes"] == ["append_audio_chunk"] + assert caps["adapter_patterns"] == ["scheduler_data_plane"] + assert caps["supports_model_internal_state"] is True + assert caps["requires_model_runner_kv"] is True + assert caps["requires_native_stage_role"] is True + assert caps["supports_kv_lease"] is False + assert caps["supports_core_kv_lease"] is False + assert caps["supports_stage_resumption"] is True + assert caps["supports_scheduler_native_append"] is False + assert caps["supports_core_resumable_request"] is True + assert caps["supports_stage_connector_handoff"] is True + assert caps["supports_audio_truncate"] is True + assert caps["supports_barge_in"] is False + assert caps["target_barge_in_latency_ms"] is None + assert caps["supports_multi_session"] is True + assert caps["supports_multi_session_same_replica"] is True + assert caps["supports_session_lease"] is True + assert caps["supports_session_resume"] is True + assert caps["session_admission_mode"] == "engine_managed" + assert caps["stage_handoff_transport"] == "scheduler_data_plane" + + +def test_minicpmo_native_capabilities_do_not_overclaim_single_session_deployment(): + caps = DuplexCapabilities.minicpmo45_native(max_sessions=1).as_dict() + + assert caps["supports_multi_session"] is False + assert caps["supports_multi_session_same_replica"] is False + + +def test_duplex_overlap_policy_defaults_and_invalid_values_to_listen_only(): + assert DuplexSessionConfig().overlap_policy == DuplexOverlapPolicy.LISTEN_ONLY.value + assert DuplexSessionConfig._normalize_overlap_policy("not-a-policy") == DuplexOverlapPolicy.LISTEN_ONLY.value diff --git a/tests/entrypoints/openai/test_duplex_session_attachment.py b/tests/entrypoints/openai/test_duplex_session_attachment.py new file mode 100644 index 000000000..24a6f5365 --- /dev/null +++ b/tests/entrypoints/openai/test_duplex_session_attachment.py @@ -0,0 +1,387 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from __future__ import annotations + +import asyncio +import base64 +import json + +import pytest + +from vllm_omni.experimental.fullduplex.openai.session_attachment import ( + DuplexEventJournal, + DuplexJournalGapError, + DuplexJournalOverflowError, + DuplexResumeCredential, + DuplexSessionAttachmentRegistry, + InvalidResumeTokenError, + ResumeToken, +) + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +class _Clock: + def __init__(self) -> None: + self.value = 0.0 + + def __call__(self) -> float: + return self.value + + def advance(self, seconds: float) -> None: + self.value += seconds + + +def _decoded_token_bytes(token: ResumeToken) -> bytes: + padding = "=" * (-len(token.plaintext) % 4) + return base64.urlsafe_b64decode(token.plaintext + padding) + + +def test_resume_credential_uses_256_bit_token_digest_and_redacted_repr(monkeypatch) -> None: + token = ResumeToken.generate() + credential = DuplexResumeCredential.from_token(token) + calls = [] + + import vllm_omni.experimental.fullduplex.openai.session_attachment as attachment_module + + original_compare = attachment_module.hmac.compare_digest + + def recording_compare(left, right): + calls.append((left, right)) + return original_compare(left, right) + + monkeypatch.setattr(attachment_module.hmac, "compare_digest", recording_compare) + + assert len(_decoded_token_bytes(token)) == 32 + assert credential.verify(token.plaintext) is True + assert credential.verify("invalid-token") is False + assert len(calls) == 2 + assert token.plaintext not in repr(token) + assert token.plaintext not in repr(credential) + assert not hasattr(credential, "plaintext") + assert len(credential.token_digest) == 32 + + +def test_resume_credential_rotation_revokes_old_token() -> None: + first = ResumeToken.generate() + credential = DuplexResumeCredential.from_token(first) + + second = credential.rotate() + + assert credential.verify(first.plaintext) is False + assert credential.verify(second.plaintext) is True + assert second.plaintext != first.plaintext + + +def test_event_journal_sequences_acknowledges_and_replays_exact_payloads() -> None: + clock = _Clock() + journal = DuplexEventJournal(max_bytes=4096, ttl_s=60.0, clock=clock) + + first = journal.record({"type": "response.audio.delta", "delta": "AAAA"}) + second = journal.record({"type": "response.done", "response_id": "resp-1"}) + + assert first.sequence == 1 + assert second.sequence == 2 + assert first.payload["server_event_seq"] == 1 + assert second.payload["server_event_seq"] == 2 + assert first.encoded_bytes == len( + json.dumps(dict(first.payload), separators=(",", ":"), ensure_ascii=False).encode("utf-8") + ) + assert [entry.sequence for entry in journal.replay_after(0)] == [1, 2] + assert journal.acknowledge(1) == 1 + assert [entry.sequence for entry in journal.replay_after(1)] == [2] + assert journal.acknowledge(1) == 0 + + +def test_event_journal_ttl_pruning_reports_replay_gap() -> None: + clock = _Clock() + journal = DuplexEventJournal(max_bytes=4096, ttl_s=5.0, clock=clock) + journal.record({"type": "one"}) + clock.advance(3.0) + journal.record({"type": "two"}) + clock.advance(3.0) + + assert journal.prune() == 1 + with pytest.raises(DuplexJournalGapError, match="older than retained"): + journal.replay_after(0) + assert [entry.payload["type"] for entry in journal.replay_after(1)] == ["two"] + + +def test_event_journal_overflow_is_explicit_and_does_not_evict_silently() -> None: + clock = _Clock() + journal = DuplexEventJournal(max_bytes=100, ttl_s=60.0, clock=clock) + first = journal.record({"type": "small", "value": "x"}) + + with pytest.raises(DuplexJournalOverflowError, match="byte limit"): + journal.record({"type": "large", "value": "x" * 200}) + + assert journal.overflowed is True + assert journal.retained_bytes == first.encoded_bytes + with pytest.raises(DuplexJournalGapError, match="overflowed"): + journal.replay_after(0) + + +@pytest.mark.asyncio +async def test_registry_resume_rotates_token_replays_and_atomically_replaces_attachment() -> None: + clock = _Clock() + registry = DuplexSessionAttachmentRegistry( + replay_ttl_s=60.0, + replay_max_bytes_per_session=4096, + clock=clock, + ) + sends_a = [] + closes_a = [] + sends_b = [] + closes_b = [] + + async def send_a(payload): + sends_a.append(payload) + + async def close_a(reason): + closes_a.append(reason) + + async def send_b(payload): + sends_b.append(payload) + + async def close_b(reason): + closes_b.append(reason) + + created = await registry.create("sid", incarnation=2, send=send_a, close=close_a) + await registry.send_event("sid", {"type": "event-1"}) + await registry.send_event("sid", {"type": "event-2"}) + + resumed = await registry.resume( + "sid", + incarnation=2, + resume_token=created.resume_token.plaintext, + last_received_server_event_seq=1, + send=send_b, + close=close_b, + ) + + assert resumed.attachment_generation == 2 + assert resumed.resume_token.plaintext != created.resume_token.plaintext + assert [entry.sequence for entry in resumed.replay_entries] == [2] + assert resumed.replaced_attachment is not None + assert resumed.replaced_attachment.generation == 1 + assert await registry.is_current_attachment("sid", created.attachment_generation) is False + assert await registry.is_current_attachment("sid", resumed.attachment_generation) is True + assert await registry.detach("sid", attachment_generation=1) is False + assert await registry.detach("sid", attachment_generation=2) is True + assert [(payload["type"], payload["server_event_seq"]) for payload in sends_a] == [ + ("event-1", 1), + ("event-2", 2), + ] + assert closes_a == [] and sends_b == [] and closes_b == [] + + +@pytest.mark.asyncio +async def test_registry_resume_sends_activation_then_replay_before_new_live_events() -> None: + registry = DuplexSessionAttachmentRegistry( + replay_ttl_s=60.0, + replay_max_bytes_per_session=4096, + ) + wire = [] + activation_started = asyncio.Event() + release_activation = asyncio.Event() + + async def old_send(payload): + del payload + + async def close(reason): + del reason + + async def new_send(payload): + wire.append(payload["type"]) + if payload["type"] == "session.resumed": + activation_started.set() + await release_activation.wait() + + created = await registry.create("sid-order", incarnation=0, send=old_send, close=close) + await registry.detach("sid-order", attachment_generation=1) + await registry.send_event("sid-order", {"type": "replayed"}) + resume_task = asyncio.create_task( + registry.resume( + "sid-order", + incarnation=0, + resume_token=created.resume_token.plaintext, + last_received_server_event_seq=0, + send=new_send, + close=close, + activation_payload_factory=lambda _token, _generation: {"type": "session.resumed"}, + ) + ) + await activation_started.wait() + live_task = asyncio.create_task(registry.send_event("sid-order", {"type": "live"})) + + release_activation.set() + await asyncio.gather(resume_task, live_task) + + assert wire == ["session.resumed", "replayed", "live"] + + +@pytest.mark.asyncio +async def test_registry_resume_delivery_failure_keeps_one_shot_old_token_recovery() -> None: + registry = DuplexSessionAttachmentRegistry( + replay_ttl_s=60.0, + replay_max_bytes_per_session=4096, + ) + + async def send(payload): + del payload + + async def failing_send(payload): + del payload + raise RuntimeError("transport lost before rotated token arrived") + + async def close(reason): + del reason + + created = await registry.create("sid-recovery", incarnation=0, send=send, close=close) + await registry.detach("sid-recovery", attachment_generation=1) + + with pytest.raises(RuntimeError, match="transport lost"): + await registry.resume( + "sid-recovery", + incarnation=0, + resume_token=created.resume_token.plaintext, + last_received_server_event_seq=0, + send=failing_send, + close=close, + activation_payload_factory=lambda token, generation: { + "type": "session.resumed", + "resume_token": token.plaintext, + "attachment_generation": generation, + }, + ) + + recovered = await registry.resume( + "sid-recovery", + incarnation=0, + resume_token=created.resume_token.plaintext, + last_received_server_event_seq=0, + send=send, + close=close, + ) + + assert recovered.attachment_generation == 3 + with pytest.raises(InvalidResumeTokenError): + await registry.resume( + "sid-recovery", + incarnation=0, + resume_token=created.resume_token.plaintext, + last_received_server_event_seq=0, + send=send, + close=close, + ) + + +@pytest.mark.asyncio +async def test_registry_concurrent_resume_allows_exactly_one_rotated_token_winner() -> None: + registry = DuplexSessionAttachmentRegistry( + replay_ttl_s=60.0, + replay_max_bytes_per_session=4096, + ) + + async def send(payload): + del payload + + async def close(reason): + del reason + + created = await registry.create("sid-race", incarnation=0, send=send, close=close) + + async def attempt(): + try: + return await registry.resume( + "sid-race", + incarnation=0, + resume_token=created.resume_token.plaintext, + last_received_server_event_seq=0, + send=send, + close=close, + ) + except InvalidResumeTokenError as exc: + return exc + + results = await asyncio.gather(attempt(), attempt()) + + assert sum(not isinstance(result, Exception) for result in results) == 1 + assert sum(isinstance(result, InvalidResumeTokenError) for result in results) == 1 + + +@pytest.mark.asyncio +async def test_registry_keeps_sessions_journals_and_incarnations_isolated() -> None: + registry = DuplexSessionAttachmentRegistry( + replay_ttl_s=60.0, + replay_max_bytes_per_session=4096, + ) + + async def send(payload): + del payload + + async def close(reason): + del reason + + created_a = await registry.create("sid-a", incarnation=1, send=send, close=close) + created_b = await registry.create("sid-b", incarnation=4, send=send, close=close) + await registry.detach("sid-a", attachment_generation=1) + await registry.detach("sid-b", attachment_generation=1) + await registry.send_event("sid-a", {"type": "a-only"}) + await registry.send_event("sid-b", {"type": "b-only"}) + + resumed_a = await registry.resume( + "sid-a", + incarnation=1, + resume_token=created_a.resume_token.plaintext, + last_received_server_event_seq=0, + send=send, + close=close, + ) + resumed_b = await registry.resume( + "sid-b", + incarnation=4, + resume_token=created_b.resume_token.plaintext, + last_received_server_event_seq=0, + send=send, + close=close, + ) + + assert [entry.payload["type"] for entry in resumed_a.replay_entries] == ["a-only"] + assert [entry.payload["type"] for entry in resumed_b.replay_entries] == ["b-only"] + with pytest.raises(ValueError, match="incarnation mismatch"): + await registry.resume( + "sid-a", + incarnation=2, + resume_token=resumed_a.resume_token.plaintext, + last_received_server_event_seq=1, + send=send, + close=close, + ) + + +@pytest.mark.asyncio +async def test_registry_repr_never_contains_plaintext_tokens() -> None: + registry = DuplexSessionAttachmentRegistry( + replay_ttl_s=60.0, + replay_max_bytes_per_session=4096, + ) + + async def send(payload): + del payload + + async def close(reason): + del reason + + created = await registry.create("sid-repr", incarnation=0, send=send, close=close) + await registry.detach("sid-repr", attachment_generation=1) + entry = await registry.send_event( + "sid-repr", + {"type": "session.resumed", "resume_token": created.resume_token.plaintext}, + ) + + assert entry is not None + assert created.resume_token.plaintext not in repr(created) + assert created.resume_token.plaintext not in repr(entry) + assert created.resume_token.plaintext not in repr(registry) diff --git a/tests/entrypoints/openai_api/test_duplex_capability.py b/tests/entrypoints/openai_api/test_duplex_capability.py new file mode 100644 index 000000000..153563cbb --- /dev/null +++ b/tests/entrypoints/openai_api/test_duplex_capability.py @@ -0,0 +1,19 @@ +from __future__ import annotations + +import ast +from pathlib import Path + +import pytest + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +def test_api_server_does_not_import_experimental_duplex_at_module_load() -> None: + api_server = Path(__file__).parents[3] / "vllm_omni/entrypoints/openai/api_server.py" + module = ast.parse(api_server.read_text()) + + top_level_imports = { + node.module for node in module.body if isinstance(node, ast.ImportFrom) and isinstance(node.module, str) + } + + assert not any(module.startswith("vllm_omni.experimental.fullduplex") for module in top_level_imports) diff --git a/tests/entrypoints/openai_api/test_duplex_handler.py b/tests/entrypoints/openai_api/test_duplex_handler.py new file mode 100644 index 000000000..3ed30e7e8 --- /dev/null +++ b/tests/entrypoints/openai_api/test_duplex_handler.py @@ -0,0 +1,6890 @@ +from __future__ import annotations + +import asyncio +import base64 +import json +import struct +from types import SimpleNamespace +from typing import Any + +import numpy as np +import pytest +from starlette.websockets import WebSocketDisconnect + +from vllm_omni.config.stage_config import DuplexSessionRuntimeConfig +from vllm_omni.experimental.fullduplex.engine.duplex_control_client import DuplexControlRequestError +from vllm_omni.experimental.fullduplex.engine.duplex_runtime import duplex_resource_request_id +from vllm_omni.experimental.fullduplex.engine.lease import DuplexLeaseActivity +from vllm_omni.experimental.fullduplex.engine.messages import DuplexFence, DuplexSessionLifecycleMessage +from vllm_omni.experimental.fullduplex.minicpmo45 import ( + MiniCPMO45NativeDuplexServingAdapter, + MiniCPMO45PcmAppendBuffer, +) +from vllm_omni.experimental.fullduplex.minicpmo45.data_plane import ( + MiniCPMO45DataPlaneContext, + MiniCPMO45DataPlaneSession, +) +from vllm_omni.experimental.fullduplex.minicpmo45.runtime import ( + MiniCPMO45DuplexRuntimeExtension, +) +from vllm_omni.experimental.fullduplex.minicpmo45.session import ( + MiniCPMO45ServingSessionState, +) +from vllm_omni.experimental.fullduplex.openai.protocol import ( + DuplexCapabilities, + DuplexOverlapPolicy, + DuplexPlaybackCommitPolicy, + DuplexSession, + DuplexSessionConfig, + ResponseCreateOptions, +) +from vllm_omni.experimental.fullduplex.openai.realtime_session import NativeRealtimeSessionProtocol +from vllm_omni.experimental.fullduplex.openai.runtime_adapter import ServingRuntimeConfigError +from vllm_omni.experimental.fullduplex.openai.serving import ( + OmniDuplexSessionHandler, + should_enable_duplex_endpoint, +) +from vllm_omni.experimental.fullduplex.openai.websocket import DuplexWebSocketActor +from vllm_omni.experimental.fullduplex.output import attach_duplex_output_decision +from vllm_omni.outputs import OmniRequestOutput + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +class _ModelConfig: + model = "test-model" + + +class FakeEngineClient: + output_modalities = ["text", "audio"] + + def __init__( + self, + *, + fail_open: bool = False, + fail_signal: bool = False, + fail_touch: bool = False, + fail_signal_events: set[str] | None = None, + fail_close: bool = False, + fail_abort: bool = False, + control_result: dict[str, object] | None = None, + open_result: dict[str, object] | None = None, + append_result: dict[str, object] | None = None, + append_results: list[dict[str, object]] | None = None, + collect_outputs: list[list[object]] | None = None, + collect_delay_s: float = 0.0, + signal_result: dict[str, object] | None = None, + close_result: dict[str, object] | None = None, + ) -> None: + self.fail_open = fail_open + self.fail_signal = fail_signal + self.fail_touch = fail_touch + self.fail_signal_events = set(fail_signal_events or ()) + self.fail_close = fail_close + self.fail_abort = fail_abort + self.control_result = control_result + self.open_result = open_result + self.append_result = append_result + self.append_results = list(append_results or []) + self.collect_outputs = list(collect_outputs or []) + self.collect_delay_s = collect_delay_s + self.signal_result = signal_result + self.close_result = close_result + self.opened: list[str] = [] + self.opened_fences: list[DuplexFence | None] = [] + self.appended: list[tuple[str, str, object, bool]] = [] + self.append_operation_ids: list[str | None] = [] + self.appended_fences: list[DuplexFence | None] = [] + self.opened_configs: list[dict[str, object]] = [] + self.opened_runtime_configs: list[dict[str, object]] = [] + self.signals: list[tuple[str, str]] = [] + self.signal_fences: list[DuplexFence | None] = [] + self.signal_next_fences: list[DuplexFence | None] = [] + self.signal_session_configs: list[dict[str, object] | None] = [] + self.signal_runtime_configs: list[dict[str, object] | None] = [] + self.closed: list[tuple[str, str]] = [] + self.touched: list[tuple[str, DuplexLeaseActivity]] = [] + self.touch_fences: list[DuplexFence] = [] + self.resumed: list[tuple[str, int]] = [] + self.resume_fences: list[DuplexFence] = [] + self.aborted: list[str] = [] + self.abort_batches: list[list[str]] = [] + self.internal_abort_batches: list[list[str]] = [] + self.collected: list[tuple[str, int | None]] = [] + self.duplex_lifecycle_events: asyncio.Queue[DuplexSessionLifecycleMessage] = asyncio.Queue() + + async def open_duplex_session_async( + self, + session_id: str, + *, + session_mode: str = "duplex", + capabilities: dict[str, object] | None = None, + session_config: dict[str, object] | None = None, + runtime_config: dict[str, object] | None = None, + timeout: float | None = None, + fence: DuplexFence | None = None, + ) -> None: + del timeout + if self.fail_open: + raise RuntimeError("open failed") + self.opened.append(session_id) + self.opened_fences.append(fence) + self.opened_configs.append(dict(session_config or {})) + self.opened_runtime_configs.append(dict(runtime_config or {})) + return self.open_result if self.open_result is not None else self.control_result + + async def append_duplex_input_async( + self, + session_id: str, + *, + mode: str, + payload: object, + operation_id: str | None = None, + final: bool = False, + timeout: float | None = None, + collect_outputs: bool = True, + fence: DuplexFence | None = None, + ) -> None: + del timeout, collect_outputs + self.appended.append((session_id, mode, payload, final)) + self.append_operation_ids.append(operation_id) + self.appended_fences.append(fence) + if self.append_results: + return self.append_results.pop(0) + return self.append_result if self.append_result is not None else self.control_result + + async def collect_duplex_data_plane_outputs_async( + self, + request_id: str, + *, + response_stage_id: int | None = None, + timeout: float | None = None, + ) -> list[object]: + del timeout + self.collected.append((request_id, response_stage_id)) + if self.collect_delay_s > 0: + await asyncio.sleep(self.collect_delay_s) + if not self.collect_outputs: + return [] + return self.collect_outputs.pop(0) + + async def signal_duplex_turn_async( + self, + session_id: str, + *, + event: str, + timeout: float | None = None, + fence: DuplexFence | None = None, + next_fence: DuplexFence | None = None, + session_config: dict[str, object] | None = None, + runtime_config: dict[str, object] | None = None, + ) -> None: + del timeout + if self.fail_signal or event in self.fail_signal_events: + raise RuntimeError("signal failed") + self.signals.append((session_id, event)) + self.signal_fences.append(fence) + self.signal_next_fences.append(next_fence) + self.signal_session_configs.append(session_config) + self.signal_runtime_configs.append(runtime_config) + return self.signal_result if self.signal_result is not None else self.control_result + + async def close_duplex_session_async( + self, + session_id: str, + *, + reason: str = "client_close", + timeout: float | None = None, + fence: DuplexFence | None = None, + ) -> None: + del timeout + if self.fail_close: + raise RuntimeError("close failed") + self.closed.append((session_id, reason)) + return self.close_result if self.close_result is not None else self.control_result + + async def touch_duplex_session_async( + self, + session_id: str, + *, + fence: DuplexFence, + activity: DuplexLeaseActivity, + timeout: float | None = None, + ) -> dict[str, object]: + del timeout + if self.fail_touch: + raise RuntimeError("touch failed") + self.touched.append((session_id, activity)) + self.touch_fences.append(fence) + return { + "operation": "touch", + "session_id": session_id, + "ok": True, + "stage_results": [ + { + "stage_id": -1, + "replica_id": -1, + "result": {"supported": True, "lease_generation": 0}, + } + ], + } + + async def resume_duplex_session_async( + self, + session_id: str, + *, + fence: DuplexFence, + expected_lease_generation: int, + timeout: float | None = None, + ) -> dict[str, object]: + del timeout + self.resumed.append((session_id, expected_lease_generation)) + self.resume_fences.append(fence) + return { + "operation": "resume", + "session_id": session_id, + "ok": True, + "stage_results": [ + { + "stage_id": -1, + "replica_id": -1, + "result": { + "supported": True, + "lease_generation": expected_lease_generation + 1, + }, + } + ], + } + + async def abort(self, request_ids: list[str]) -> None: + if self.fail_abort: + raise RuntimeError("abort failed") + assert isinstance(request_ids, list) + self.abort_batches.append(list(request_ids)) + self.aborted.extend(request_ids) + + async def _abort_internal_requests(self, request_ids: list[str]) -> None: + if self.fail_abort: + raise RuntimeError("abort failed") + assert isinstance(request_ids, list) + self.internal_abort_batches.append(list(request_ids)) + self.aborted.extend(request_ids) + + +class FakeChatService: + duplex_serving_adapter_path = ( + "vllm_omni.experimental.fullduplex.minicpmo45.serving_adapter.MiniCPMO45ServingRuntimeAdapter" + ) + + def __init__(self, engine_client: FakeEngineClient) -> None: + self.engine_client = engine_client + self.model_config = _ModelConfig() + self.seen_request_ids: list[str] = [] + + async def create_chat_completion(self, request, raw_request=None): + self.seen_request_ids.append(request.request_id) + + async def _gen(): + await asyncio.sleep(999) + yield "data: [DONE]\n\n" + + return _gen() + + def create_audio(self, audio_obj): + return SimpleNamespace(audio_data=f"wav-{int(audio_obj.audio_tensor.shape[0])}") + + +def _test_data_plane() -> MiniCPMO45DataPlaneSession: + def encode_audio( + audio: object, + sample_rate_hz: int, + response_format: str, + speed: float | None, + ) -> str | None: + del sample_rate_hz, response_format, speed + if audio is None: + return None + if hasattr(audio, "numel"): + return f"wav-{int(audio.numel())}" + return f"wav-{int(np.asarray(audio).size)}" + + return MiniCPMO45DataPlaneSession(encode_audio) + + +def _data_plane_context(session: DuplexSession | None = None) -> MiniCPMO45DataPlaneContext: + if session is None: + return MiniCPMO45DataPlaneContext() + return MiniCPMO45DataPlaneContext( + epoch=session.epoch, + turn_id=session.turn_id, + active_response_turn_id=session.active_response_turn_id, + active_response_id=session.active_response_id, + auto_responds=bool(session.config.extra_body.get("auto_response", False)), + response_format=session.config.response_format, + speed=session.config.speed, + modalities=tuple(session.config.modalities), + ) + + +def _project_data_plane( + data_plane: MiniCPMO45DataPlaneSession, + result: object, + *, + session: DuplexSession | None = None, +) -> list[dict[str, object]]: + return list(data_plane.project(result, context=_data_plane_context(session))) + + +class TimedWebSocket: + def __init__(self, *, on_send=None, receive_timeout_s: float = 1.0): + self._q: asyncio.Queue[str | BaseException] = asyncio.Queue() + self.sent: list[dict[str, Any]] = [] + self.accepted = False + self._on_send = on_send + self._receive_timeout_s = receive_timeout_s + self.query_params: dict[str, str] = {} + + async def accept(self): + self.accepted = True + + async def receive_text(self) -> str: + try: + item = await asyncio.wait_for(self._q.get(), timeout=self._receive_timeout_s) + except asyncio.TimeoutError as exc: + raise WebSocketDisconnect(code=1000) from exc + if isinstance(item, BaseException): + raise item + return item + + async def send_json(self, data: dict[str, Any]): + self.sent.append(data) + if self._on_send is not None: + self._on_send(self, data) + + def put(self, payload: dict[str, Any]) -> None: + self._q.put_nowait(json.dumps(payload)) + + async def close(self, code: int = 1000, reason: str | None = None) -> None: + del reason + self._q.put_nowait(WebSocketDisconnect(code=code)) + + def sent_types(self) -> list[str]: + return [m.get("type", "") for m in self.sent] + + +def test_native_realtime_protocol_emits_speak_once_per_response(): + protocol = NativeRealtimeSessionProtocol({}) + response_id = "resp-speak-once" + + projected = protocol.encode_outbound_event( + { + "type": "response.created", + "response_id": response_id, + "modalities": ["audio", "text"], + } + ) + projected.extend( + protocol.encode_outbound_event( + { + "type": "response.speak", + "response_id": response_id, + "text": "hello", + "model_speak": True, + } + ) + ) + projected.extend( + protocol.encode_outbound_event( + { + "type": "response.output_audio.delta", + "response_id": response_id, + "audio": base64.b64encode(b"\x00\x00").decode("ascii"), + "format": "pcm16", + "sample_rate_hz": 24000, + "model_speak": True, + } + ) + ) + + assert [event["type"] for event in projected].count("response.speak") == 1 + speak = next(event for event in projected if event["type"] == "response.speak") + assert "text" not in speak + assert speak["metadata"] == {"model_speak": True} + + +def test_native_realtime_protocol_speak_does_not_expose_internal_event(): + protocol = NativeRealtimeSessionProtocol({}) + + projected = protocol.encode_outbound_event( + { + "type": "response.speak", + "response_id": "resp-curated-speak", + "session_id": "sid-curated-speak", + "epoch": 3, + "text": "transcript belongs on the delta channel", + "model_speak": True, + "data_plane_request_id": "internal-request-id", + "uses_model_runner_scheduler": True, + } + ) + + assert len(projected) == 1 + projected[0].pop("event_id") + assert projected == [ + { + "type": "response.speak", + "response_id": "resp-curated-speak", + "item_id": "item_resp-curated-speak", + "output_index": 0, + "content_index": 0, + "metadata": { + "session_id": "sid-curated-speak", + "epoch": 3, + "model_speak": True, + }, + } + ] + + +@pytest.mark.asyncio +async def test_native_realtime_protocol_drains_internal_conversation_item_control_event(): + ws = TimedWebSocket() + protocol = NativeRealtimeSessionProtocol(ws) # type: ignore[arg-type] + protocol.bind_sender(ws.send_json) + + ws.put({"type": "session.update", "model": "test-model", "session_id": "rt-delete"}) + session_create = json.loads(await protocol.receive_internal_event_text(ws)) + assert session_create["type"] == "session.create" + + ws.put( + { + "type": "conversation.item.create", + "item": { + "id": "item-a", + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": "hello"}], + }, + } + ) + assert json.loads(await protocol.receive_internal_event_text(ws)) == { + "type": "turn.signal", + "event": "conversation.item.create", + "payload": { + "item": { + "id": "item-a", + "object": "realtime.item", + "type": "message", + "status": "completed", + "role": "user", + "content": [{"type": "input_text", "text": "hello"}], + } + }, + } + + ws.put({"type": "conversation.item.delete", "item_id": "item-a"}) + translated = json.loads(await protocol.receive_internal_event_text(ws)) + + assert translated == { + "type": "turn.signal", + "event": "conversation.item.delete", + "payload": {"item_id": "item-a"}, + } + + +@pytest.mark.asyncio +async def test_native_realtime_protocol_conversation_item_create_commits_user_text(): + ws = TimedWebSocket() + protocol = NativeRealtimeSessionProtocol(ws) # type: ignore[arg-type] + protocol.bind_sender(ws.send_json) + + ws.put({"type": "session.update", "model": "test-model", "session_id": "rt-item-create"}) + session_create = json.loads(await protocol.receive_internal_event_text(ws)) + assert session_create["type"] == "session.create" + + ws.put( + { + "type": "conversation.item.create", + "item": { + "id": "item-user-text", + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": "hello"}], + }, + } + ) + + translated = json.loads(await protocol.receive_internal_event_text(ws)) + + assert translated == { + "type": "turn.signal", + "event": "conversation.item.create", + "payload": { + "item": { + "id": "item-user-text", + "object": "realtime.item", + "type": "message", + "status": "completed", + "role": "user", + "content": [{"type": "input_text", "text": "hello"}], + } + }, + } + + +@pytest.mark.asyncio +async def test_native_realtime_protocol_audio_commit_requires_non_empty_buffer(): + ws = TimedWebSocket() + protocol = NativeRealtimeSessionProtocol(ws) # type: ignore[arg-type] + protocol.bind_sender(ws.send_json) + + ws.put({"type": "session.update", "model": "test-model", "session_id": "rt-audio-commit"}) + session_create = json.loads(await protocol.receive_internal_event_text(ws)) + assert session_create["type"] == "session.create" + + ws.put({"type": "input_audio_buffer.commit", "final": True}) + with pytest.raises(WebSocketDisconnect): + await protocol.receive_internal_event_text(ws) + + assert ws.sent[-1]["type"] == "error" + assert ws.sent[-1]["error"]["code"] == "input_audio_buffer_empty" + + +@pytest.mark.asyncio +async def test_native_realtime_protocol_audio_commit_does_not_auto_create_response(): + ws = TimedWebSocket() + protocol = NativeRealtimeSessionProtocol(ws) # type: ignore[arg-type] + protocol.bind_sender(ws.send_json) + + ws.put({"type": "session.update", "model": "test-model", "session_id": "rt-audio-commit-full"}) + session_create = json.loads(await protocol.receive_internal_event_text(ws)) + assert session_create["type"] == "session.create" + + pcm = struct.pack("<2h", 1024, -1024) + ws.put({"type": "input_audio_buffer.append", "audio": base64.b64encode(pcm).decode("ascii"), "format": "pcm16"}) + append_event = json.loads(await protocol.receive_internal_event_text(ws)) + assert append_event["type"] == "input_audio_buffer.append" + + ws.put({"type": "input_audio_buffer.commit", "final": True}) + commit_event = json.loads(await protocol.receive_internal_event_text(ws)) + + assert commit_event["type"] == "input_audio_buffer.commit" + assert commit_event["response_create"] is False + + +@pytest.mark.asyncio +async def test_native_realtime_protocol_rejects_invalid_sample_rate_without_closing(): + ws = TimedWebSocket() + protocol = NativeRealtimeSessionProtocol(ws) # type: ignore[arg-type] + protocol.bind_sender(ws.send_json) + + ws.put({"type": "session.update", "model": "test-model", "session_id": "rt-invalid-rate"}) + session_create = json.loads(await protocol.receive_internal_event_text(ws)) + assert session_create["type"] == "session.create" + + ws.put( + { + "type": "input_audio_buffer.append", + "audio": _pcm16_b64(1), + "format": "pcm16", + "sample_rate_hz": 192_001, + } + ) + ws.put({"type": "input_audio_buffer.clear"}) + translated = json.loads(await protocol.receive_internal_event_text(ws)) + + assert translated == {"type": "input_audio_buffer.clear", "reason": "input_audio_buffer.clear"} + error = next(event for event in ws.sent if event.get("type") == "error") + assert error["error"]["code"] == "bad_event" + assert error["error"]["param"] == "sample_rate_hz" + + +@pytest.mark.asyncio +async def test_native_realtime_protocol_audio_clear_is_not_barge_in_cancel(): + ws = TimedWebSocket() + protocol = NativeRealtimeSessionProtocol(ws) # type: ignore[arg-type] + protocol.bind_sender(ws.send_json) + + ws.put({"type": "session.update", "model": "test-model", "session_id": "rt-audio-clear"}) + session_create = json.loads(await protocol.receive_internal_event_text(ws)) + assert session_create["type"] == "session.create" + + ws.put({"type": "input_audio_buffer.clear"}) + clear_event = json.loads(await protocol.receive_internal_event_text(ws)) + + assert clear_event == {"type": "input_audio_buffer.clear", "reason": "input_audio_buffer.clear"} + + +@pytest.mark.asyncio +async def test_native_realtime_protocol_preserves_input_turn_policy_hints(): + ws = TimedWebSocket() + protocol = NativeRealtimeSessionProtocol(ws) # type: ignore[arg-type] + protocol.bind_sender(ws.send_json) + + translated = await protocol._to_duplex_event( + { + "type": "input_audio_buffer.append", + "audio": "AAAA", + "format": "pcm_f32le", + "duration_ms": 240, + "vad": {"is_speech": True, "speech_probability": 0.9}, + "overlap_action": "listen", + } + ) + + assert translated is not None + assert translated["duration_ms"] == 240 + assert translated["vad"] == {"is_speech": True, "speech_probability": 0.9} + assert translated["overlap_action"] == "listen" + + +@pytest.mark.asyncio +async def test_native_realtime_protocol_rejects_unimplemented_server_vad(): + ws = TimedWebSocket() + protocol = NativeRealtimeSessionProtocol(ws) # type: ignore[arg-type] + protocol.bind_sender(ws.send_json) + + translated = await protocol._to_duplex_event( + { + "type": "session.update", + "model": "test-model", + "session_id": "rt-turn-detection", + "turn_detection": { + "type": "server_vad", + "interrupt_response": False, + "silence_duration_ms": 900, + "threshold": 0.4, + }, + } + ) + + assert translated is None + error = ws.sent[-1] + assert error["type"] == "error" + assert error["error"]["code"] == "unsupported_turn_detection" + assert "server_vad" in error["error"]["message"] + + +@pytest.mark.asyncio +async def test_native_realtime_protocol_accepts_disabled_turn_detection(): + ws = TimedWebSocket() + protocol = NativeRealtimeSessionProtocol(ws) # type: ignore[arg-type] + protocol.bind_sender(ws.send_json) + + ws.put( + { + "type": "session.update", + "model": "test-model", + "session_id": "rt-no-turn-detection", + "turn_detection": None, + } + ) + translated = json.loads(await protocol.receive_internal_event_text(ws)) + + assert translated["type"] == "session.create" + assert translated["session"]["extra_body"]["realtime_session_payload"]["turn_detection"] is None + + +@pytest.mark.asyncio +async def test_native_realtime_protocol_rejects_nested_vad_even_when_top_level_is_disabled(): + ws = TimedWebSocket() + protocol = NativeRealtimeSessionProtocol(ws) # type: ignore[arg-type] + protocol.bind_sender(ws.send_json) + + translated = await protocol._to_duplex_event( + { + "type": "session.update", + "model": "test-model", + "turn_detection": None, + "audio": { + "input": { + "turn_detection": { + "type": "server_vad", + } + } + }, + } + ) + + assert translated is None + assert ws.sent[-1]["error"]["code"] == "unsupported_turn_detection" + + +def test_native_duplex_handler_has_no_fixed_session_admission_cap(): + handler = OmniDuplexSessionHandler(chat_service=FakeChatService(FakeEngineClient())) + client_config = DuplexSessionConfig( + extra_body={"max_duplex_sessions": 99, "duplex_max_sessions": 99}, + ) + + assert not hasattr(handler, "_max_native_duplex_sessions") + assert not hasattr(handler, "_max_duplex_sessions") + assert client_config.extra_body["max_duplex_sessions"] == 99 + + +def test_native_response_options_ignore_private_runtime_config(): + session = DuplexSession( + session_id="sid-response-private-config", + config=DuplexSessionConfig(), + capabilities=DuplexCapabilities.minicpmo45_native(), + ) + + handler = OmniDuplexSessionHandler(chat_service=FakeChatService(FakeEngineClient())) + handler._apply_response_create_options( + session, + { + "extra_body": { + "duplex_stage_max_tokens": {"0": 999}, + "client_trace_id": "trace-1", + } + }, + ) + session.begin_response() + + assert "duplex_stage_max_tokens" not in session.config.extra_body + assert "client_trace_id" not in session.config.extra_body + assert session.response_config.extra_body["client_trace_id"] == "trace-1" + + +@pytest.mark.parametrize( + "payload", + [ + {"instructions": "not wired"}, + {"voice": "not-wired"}, + {"temperature": 0.2}, + {"max_output_tokens": 8}, + {"tools": [{"type": "function", "name": "lookup"}]}, + ], +) +def test_native_response_create_rejects_unwired_generation_options(payload): + session = DuplexSession( + session_id="sid-response-unwired-options", + config=DuplexSessionConfig(), + capabilities=DuplexCapabilities.minicpmo45_native(), + ) + + handler = OmniDuplexSessionHandler(chat_service=FakeChatService(FakeEngineClient())) + error = handler._apply_response_create_options(session, payload) + + assert error == "unsupported_native_response_options" + session.begin_response() + assert session.response_config == session.config + + +def test_chat_fallback_excludes_realtime_response_metadata_from_model_request(): + handler = OmniDuplexSessionHandler(chat_service=FakeChatService(FakeEngineClient())) + session = DuplexSession( + session_id="sid-response-protocol-metadata", + config=DuplexSessionConfig(), + ) + session.reserve_response_options( + ResponseCreateOptions( + extra_body={ + "user": "trace-1", + "realtime_response_conversation": "none", + "realtime_response_metadata": {"source": "test"}, + "realtime_response_prompt": {"id": "prompt-1"}, + "realtime_response_tools": [ + { + "type": "function", + "function": {"name": "lookup", "parameters": {}}, + } + ], + "realtime_response_tool_choice": "auto", + } + ) + ) + session.begin_response() + + request = handler._build_chat_request(session, "request-1") + created = handler._response_created_payload(session, session.active_response_id or "response-1", epoch=0) + dumped = request.model_dump() + + assert dumped["user"] == "trace-1" + assert dumped["tools"][0]["function"]["name"] == "lookup" + assert dumped["tool_choice"] == "auto" + assert not any(key.startswith("realtime_response_") for key in dumped) + assert created["conversation"] == "none" + assert created["metadata"] == {"source": "test"} + assert created["prompt"] == {"id": "prompt-1"} + assert session.response_config.extra_body["realtime_response_conversation"] == "none" + + +@pytest.mark.asyncio +async def test_native_response_create_surfaces_unwired_generation_options_error(): + handler = OmniDuplexSessionHandler( + chat_service=FakeChatService(FakeEngineClient()), + config_timeout_s=0.1, + idle_timeout_s=1, + ) + ws = TimedWebSocket() + ws.put(_native_session_create("sid-response-unwired-options-event")) + ws.put( + { + "type": "response.create", + "response": {"instructions": "not wired"}, + } + ) + ws.put({"type": "session.close"}) + + await handler.handle_session(ws) + + error = next(message for message in ws.sent if message.get("type") == "error") + assert error["code"] == "unsupported_native_response_options" + assert "response_create_without_input" not in {message.get("code") for message in ws.sent} + + +@pytest.mark.asyncio +async def test_realtime_session_update_preserves_tools_and_metadata(): + engine = FakeEngineClient() + chat_service = FakeChatService(engine) + handler = OmniDuplexSessionHandler(chat_service=chat_service, config_timeout_s=0.1, idle_timeout_s=1) + ws = TimedWebSocket() + ws.put(_session_create("sid-rt-update-fields")) + ws.put( + { + "type": "turn.signal", + "event": "session.update", + "payload": { + "tools": [{"type": "function", "name": "lookup"}], + "tool_choice": "auto", + "metadata": {"demo": "yes"}, + }, + } + ) + ws.put({"type": "session.close"}) + + await handler.handle_session(ws) + + updated = next(m for m in ws.sent if m.get("type") == "session.updated") + session = updated["session"] + assert session["tools"] == [{"type": "function", "name": "lookup"}] + assert session["tool_choice"] == "auto" + assert session["metadata"] == {"demo": "yes"} + assert "turn_detection" not in session + runtime_config = engine.signal_session_configs[-1] + assert runtime_config is not None + assert runtime_config["extra_body"]["realtime_tools"] == [{"type": "function", "name": "lookup"}] + assert runtime_config["extra_body"]["realtime_metadata"] == {"demo": "yes"} + + +@pytest.mark.asyncio +async def test_native_realtime_protocol_non_speech_append_does_not_emit_speech_started(): + ws = TimedWebSocket() + protocol = NativeRealtimeSessionProtocol(ws) # type: ignore[arg-type] + protocol.bind_sender(ws.send_json) + + translated = await protocol._to_duplex_event( + { + "type": "input_audio_buffer.append", + "audio": "AAAA", + "format": "pcm_f32le", + "vad": {"is_speech": False}, + } + ) + + assert translated is not None + assert translated["vad"] == {"is_speech": False} + assert not any(event.get("type") == "input_audio_buffer.speech_started" for event in ws.sent) + + +@pytest.mark.asyncio +async def test_native_session_update_rebuilds_server_runtime_policy(): + engine = FakeEngineClient() + handler = OmniDuplexSessionHandler( + chat_service=FakeChatService(engine), + config_timeout_s=0.1, + idle_timeout_s=1, + ) + ws = TimedWebSocket() + ws.put(_native_session_create("sid-runtime-update")) + ws.put( + { + "type": "turn.signal", + "event": "session.update", + "payload": {"temperature": 0.2, "max_output_tokens": 7}, + } + ) + ws.put({"type": "session.close"}) + + await handler.handle_session(ws) + + assert "session.updated" in ws.sent_types() + public_config = engine.signal_session_configs[-1] + runtime_config = engine.signal_runtime_configs[-1] + assert public_config is not None + assert runtime_config is not None + assert "duplex_stage_sampling_params" not in public_config["extra_body"] + assert runtime_config["duplex_stage_max_tokens"]["0"] == 7 + assert runtime_config["duplex_stage_sampling_params"]["0"]["temperature"] == 0.2 + assert runtime_config["duplex_stage_sampling_params"]["0"]["top_k"] == 20 + assert runtime_config["duplex_stage_sampling_params"]["0"]["top_p"] == 0.8 + + +@pytest.mark.asyncio +async def test_native_session_update_rejects_client_runtime_config(): + engine = FakeEngineClient() + handler = OmniDuplexSessionHandler( + chat_service=FakeChatService(engine), + config_timeout_s=0.1, + idle_timeout_s=1, + ) + ws = TimedWebSocket() + ws.put(_native_session_create("sid-runtime-update-reject")) + ws.put( + { + "type": "turn.signal", + "event": "session.update", + "payload": { + "extra_body": {"duplex_stage_max_tokens": {"0": 999}}, + }, + } + ) + ws.put({"type": "session.close"}) + + await handler.handle_session(ws) + + error = next(message for message in ws.sent if message.get("type") == "error") + assert error["code"] == "invalid_duplex_runtime_config" + assert ("sid-runtime-update-reject", "session.update") not in engine.signals + + +@pytest.mark.asyncio +async def test_minicpmo_native_session_update_requires_ref_audio_before_enabling_audio(): + engine = FakeEngineClient() + handler = OmniDuplexSessionHandler( + chat_service=FakeChatService(engine), + config_timeout_s=0.1, + idle_timeout_s=1, + ) + ws = TimedWebSocket() + ws.put(_native_session_create("sid-runtime-update-ref-required", modalities=["text"])) + ws.put( + { + "type": "turn.signal", + "event": "session.update", + "payload": {"modalities": ["text", "audio"]}, + } + ) + ws.put({"type": "session.close"}) + + await handler.handle_session(ws) + + error = next(message for message in ws.sent if message.get("type") == "error") + assert error["code"] == "ref_audio_required" + assert "session.updated" not in ws.sent_types() + assert ("sid-runtime-update-ref-required", "session.update") not in engine.signals + + +def test_native_realtime_protocol_audio_delta_preserves_sample_rate_hz(): + ws = TimedWebSocket() + protocol = NativeRealtimeSessionProtocol(ws) # type: ignore[arg-type] + + payloads = protocol._from_duplex_event( + { + "type": "response.output_audio.delta", + "response_id": "resp-a", + "audio": "AAAA", + "format": "pcm", + "sample_rate_hz": 24000, + } + ) + + audio_events = [ + payload for payload in payloads if payload["type"] in {"response.output_audio.delta", "response.audio.delta"} + ] + assert {payload["type"] for payload in audio_events} == {"response.audio.delta"} + assert {payload["format"] for payload in audio_events} == {"pcm16"} + assert {payload["sample_rate_hz"] for payload in audio_events} == {24000} + + +def test_native_realtime_protocol_ignores_removed_legacy_event_switches(): + ws = TimedWebSocket() + ws.query_params = { + "legacy_audio_events": "true", + "vllm_omni_legacy_events": "true", + "output_audio_events": "true", + "vllm_omni_output_audio_events": "true", + } + protocol = NativeRealtimeSessionProtocol(ws) # type: ignore[arg-type] + + payloads = protocol._from_duplex_event( + { + "type": "response.output_audio.delta", + "response_id": "resp-canonical", + "audio": "AAAA", + "text": "hello", + "format": "pcm", + "sample_rate_hz": 24000, + "end_of_turn": True, + } + ) + + event_types = {payload["type"] for payload in payloads} + assert "response.audio.delta" in event_types + assert "response.audio.done" in event_types + assert "response.audio_transcript.delta" in event_types + assert "response.audio_transcript.done" in event_types + assert "response.output_audio.delta" not in event_types + assert "response.output_audio.done" not in event_types + assert "response.output_audio_transcript.delta" not in event_types + assert "response.output_audio_transcript.done" not in event_types + assert "response.output_item.created" not in event_types + assert "response.text.delta" not in event_types + + +def test_native_realtime_protocol_emits_terminal_audio_transcript_events(): + ws = TimedWebSocket() + protocol = NativeRealtimeSessionProtocol(ws) # type: ignore[arg-type] + + payloads = protocol._from_duplex_event( + { + "type": "response.output_audio.delta", + "response_id": "resp-a", + "audio": "AAAA", + "text": "hello", + "format": "pcm", + "sample_rate_hz": 24000, + "end_of_turn": True, + } + ) + + by_type = {payload["type"]: payload for payload in payloads} + assert "response.audio.done" in by_type + assert "response.audio_transcript.delta" in by_type + assert "response.audio_transcript.done" in by_type + assert by_type["response.content_part.done"]["part"]["transcript"] == "hello" + assert by_type["response.output_item.done"]["item"]["object"] == "realtime.item" + assert by_type["response.done"]["response"]["output"][0]["object"] == "realtime.item" + + +def test_native_realtime_protocol_terminal_transcript_equals_joined_deltas(): + protocol = NativeRealtimeSessionProtocol(TimedWebSocket()) # type: ignore[arg-type] + response_id = "resp-transcript-contract" + events: list[dict[str, object]] = [] + + for text, end_of_turn in (("It's Canberra.", False), (" Next question.", True)): + events.extend( + protocol._from_duplex_event( + { + "type": "response.output_audio.delta", + "response_id": response_id, + "audio": "AAAA", + "text": text, + "format": "pcm", + "sample_rate_hz": 24000, + "end_of_turn": end_of_turn, + } + ) + ) + + transcript = "".join(str(event["delta"]) for event in events if event["type"] == "response.audio_transcript.delta") + done = [str(event["transcript"]) for event in events if event["type"] == "response.audio_transcript.done"] + + assert transcript == "It's Canberra. Next question." + assert done == [transcript] + + +def test_native_realtime_protocol_updates_in_progress_item_for_audio_truncate(): + ws = TimedWebSocket() + protocol = NativeRealtimeSessionProtocol(ws) # type: ignore[arg-type] + + protocol._from_duplex_event({"type": "response.created", "response_id": "resp-truncate"}) + protocol._from_duplex_event( + { + "type": "response.output_audio.delta", + "response_id": "resp-truncate", + "audio": "AAAA", + "text": "hello", + "format": "pcm", + "sample_rate_hz": 24000, + "audio_duration_ms": 100, + "audio_text_marks": [{"text_chars": 5, "audio_end_ms": 100}], + } + ) + + item = protocol._conversation_items["item_resp-truncate"] + assert item["content"][0]["type"] == "output_audio" + assert item["content"][0]["transcript"] == "hello" + assert item["content"][0]["audio_duration_ms"] == 100 + + translated = asyncio.run( + protocol._to_duplex_event( + { + "type": "conversation.item.truncate", + "item_id": "item_resp-truncate", + "content_index": 0, + "audio_end_ms": 50, + } + ) + ) + + assert translated is None + assert protocol._conversation_items["item_resp-truncate"]["content"][0]["transcript"] == "he" + + +@pytest.mark.asyncio +async def test_native_realtime_protocol_rejects_truncate_for_user_item(): + ws = TimedWebSocket() + protocol = NativeRealtimeSessionProtocol(ws) # type: ignore[arg-type] + protocol.bind_sender(ws.send_json) + protocol._conversation_items["item-user"] = { + "id": "item-user", + "object": "realtime.item", + "type": "message", + "role": "user", + "status": "completed", + "content": [{"type": "input_audio", "transcript": "hello"}], + } + + translated = await protocol._to_duplex_event( + { + "type": "conversation.item.truncate", + "item_id": "item-user", + "content_index": 0, + "audio_end_ms": 10, + } + ) + + assert translated is None + error = next(event for event in ws.sent if event.get("type") == "error") + assert error["error"]["code"] == "bad_event" + + +def test_native_realtime_protocol_response_done_emits_audio_done_before_terminal_events(): + ws = TimedWebSocket() + protocol = NativeRealtimeSessionProtocol(ws) # type: ignore[arg-type] + protocol._from_duplex_event({"type": "response.created", "response_id": "resp-done"}) + protocol._from_duplex_event( + { + "type": "response.output_audio.delta", + "response_id": "resp-done", + "audio": "AAAA", + "format": "pcm", + "sample_rate_hz": 24000, + } + ) + + payloads = protocol._from_duplex_event({"type": "response.done", "response_id": "resp-done"}) + event_types = [payload["type"] for payload in payloads] + + assert event_types.index("response.audio.done") < event_types.index("response.content_part.done") + assert event_types.index("response.content_part.done") < event_types.index("response.output_item.done") + assert event_types.index("conversation.item.done") < event_types.index("response.done") + assert event_types.index("response.output_item.done") < event_types.index("response.done") + + +def test_native_realtime_protocol_audio_cancelled_uses_active_response_id(): + ws = TimedWebSocket() + protocol = NativeRealtimeSessionProtocol(ws) # type: ignore[arg-type] + protocol._from_duplex_event({"type": "response.created", "response_id": "resp-cancel"}) + + payloads = protocol._from_duplex_event( + { + "type": "audio.cancelled", + "reason": "barge_in", + "committed_ms": 0, + } + ) + + done = [payload for payload in payloads if payload.get("type") == "response.done"] + assert len(done) == 1 + assert done[0]["response_id"] == "resp-cancel" + assert done[0]["response"]["id"] == "resp-cancel" + + +def test_native_realtime_protocol_audio_cancelled_does_not_reopen_completed_response(): + ws = TimedWebSocket() + protocol = NativeRealtimeSessionProtocol(ws) # type: ignore[arg-type] + protocol._from_duplex_event({"type": "response.created", "response_id": "resp-complete"}) + protocol._from_duplex_event( + { + "type": "response.output_audio.delta", + "response_id": "resp-complete", + "audio": "AAAA", + "format": "pcm", + "sample_rate_hz": 24000, + } + ) + protocol._from_duplex_event({"type": "response.done", "response_id": "resp-complete"}) + + payloads = protocol._from_duplex_event( + { + "type": "audio.cancelled", + "reason": "session_close", + "committed_ms": 999999, + } + ) + + assert [payload.get("type") for payload in payloads] == [] + + +def test_native_audio_text_marks_are_normalized_to_session_cumulative_offsets(): + marks = OmniDuplexSessionHandler._normalize_native_audio_text_marks( + [{"text_chars": 3, "audio_end_ms": 200}], + audio_offset_ms=800, + text_offset_chars=5, + ) + + assert marks == [{"text_chars": 8, "audio_end_ms": 1000}] + + +def test_duplex_session_playback_commit_uses_multi_delta_audio_text_marks(): + session = DuplexSession( + session_id="sid-marks", + config=DuplexSessionConfig(playback_commit_policy=DuplexPlaybackCommitPolicy.ACK_ONLY.value), + ) + session.begin_response() + session.append_assistant_text("hello ") + session.mark_audio_sent(1000, text_chars=6) + session.append_assistant_text("world") + session.mark_audio_sent(2000, text_chars=11) + session.acknowledge_playback(played_ms=1500, committed_ms=1500) + + committed = session.end_response(commit_text=True) + + assert committed == {"role": "assistant", "content": "hello wo"} + assert session.history == (committed,) + + +@pytest.mark.asyncio +async def test_duplex_session_actor_preserves_wire_order_before_control(): + ws = TimedWebSocket() + actor = DuplexWebSocketActor(ws) + + await actor.enqueue_event({"type": "input_audio_buffer.append", "audio": "AAAA"}) + await actor.enqueue_event({"type": "response.cancel"}) + + first = await actor.next_event() + second = await actor.next_event() + + assert first["type"] == "input_audio_buffer.append" + assert second["type"] == "response.cancel" + + +def _pcm_f32_b64(samples: int, *, value: float = 0.05) -> str: + return base64.b64encode(struct.pack(f"<{samples}f", *([value] * samples))).decode("ascii") + + +def _pcm16_b64(samples: int = 1, *, value: int = 1000) -> str: + return base64.b64encode(struct.pack(f"<{samples}h", *([value] * samples))).decode("ascii") + + +def _session_create(session_id: str = "duplex-test") -> dict[str, Any]: + return { + "type": "session.create", + "session_id": session_id, + "session": { + "model": "test-model", + "modalities": ["text", "audio"], + "idle_timeout_s": 1, + }, + } + + +def _native_session_create( + session_id: str = "duplex-native", + *, + modalities: list[str] | None = None, +) -> dict[str, Any]: + event = _session_create(session_id) + event["session"]["model"] = "openbmb/MiniCPM-o-4_5" + event["session"]["modalities"] = list(modalities or ["text"]) + event["session"]["instructions"] = "You are a concise assistant." + event["session"]["extra_body"] = {"minicpmo45_native_duplex": True} + return event + + +def _native_realtime_session_update( + session_id: str, + *, + modalities: list[str] | None = None, +) -> dict[str, Any]: + return { + "type": "session.update", + "session": { + "session_id": session_id, + "model": "openbmb/MiniCPM-o-4_5", + "modalities": list(modalities or ["text"]), + "instructions": "You are a concise assistant.", + "idle_timeout_s": 1, + "extra_body": {"minicpmo45_native_duplex": True}, + }, + } + + +def test_native_stage0_request_id_matches_engine_after_same_session_id_reopen(): + handler = OmniDuplexSessionHandler(chat_service=FakeChatService(FakeEngineClient())) + first = handler._registry.create(session_id="sid-reused") + handler._registry.close(first.session_id) + reopened = handler._registry.create(session_id="sid-reused") + expected = duplex_resource_request_id( + DuplexFence( + reopened.session_id, + epoch=reopened.epoch, + incarnation=reopened.incarnation, + ), + "stage0", + ) + + assert reopened.incarnation == 1 + assert handler._native_stage0_request_id(reopened, reopened.epoch) == expected + + +def _native_audio_payload( + *, + samples: int = 16000, + value: float = 0.05, + is_speech: bool | None = True, + force_listen: bool | None = None, +) -> dict[str, Any]: + payload: dict[str, Any] = { + "type": "audio", + "audio": _pcm_f32_b64(samples, value=value), + "format": "pcm_f32le", + "sample_rate_hz": 16000, + } + if is_speech is not None: + payload["is_speech"] = is_speech + if force_listen is not None: + payload["force_listen"] = force_listen + return payload + + +def _auto_response_context( + session_id: str, + *, + playback_active: bool = False, +) -> tuple[OmniDuplexSessionHandler, DuplexSession]: + handler = OmniDuplexSessionHandler( + chat_service=FakeChatService(FakeEngineClient()), + config_timeout_s=0.1, + idle_timeout_s=1, + ) + session = DuplexSession( + session_id=session_id, + config=DuplexSessionConfig(extra_body={"auto_response": True}), + ) + if playback_active: + session.config.playback_commit_policy = DuplexPlaybackCommitPolicy.ACK_ONLY.value + session.begin_response() + session.mark_audio_sent(duration_ms=1000) + return handler, session + + +def _install_direct_silence_scheduler( + handler: OmniDuplexSessionHandler, + session: DuplexSession, +) -> None: + native = handler._minicpmo_session_state(session) + + async def _schedule(payload: object, **kwargs: Any) -> bool: + if handler._native_silence_continuation_is_stale( + session, + request_id=kwargs["request_id"], + response_id=kwargs["response_id"], + response_owned=kwargs["response_owned"], + expected_epoch=kwargs["expected_epoch"], + expected_incarnation=kwargs["expected_incarnation"], + expected_model_turn_id=kwargs["expected_model_turn_id"], + ): + return False + append_ok, _ = await handler._append_runtime_input( + session, + payload, + final=False, + send_json=kwargs["send_json"], + mode="append_audio_chunk", + expected_epoch=kwargs["expected_epoch"], + ) + return append_ok + + native.silence_continuation_scheduler = _schedule + + +def test_minicpmo_pcm_append_buffer_flush_preserves_accumulated_speech_marker(): + buffer = MiniCPMO45PcmAppendBuffer() + payload = _native_audio_payload(samples=8000) + + assert buffer.append(payload, chunk_period_ms=1000) is None + flushed = buffer.flush(chunk_period_ms=1000) + + assert flushed is not None + assert flushed["is_speech"] is True + + +def test_minicpmo_pcm_append_buffer_drops_serving_new_user_turn_marker(): + buffer = MiniCPMO45PcmAppendBuffer() + first = _native_audio_payload(samples=8000) + first.update( + new_user_turn=True, + new_user_turn_prefix_variant="clean_response_done", + force_speak=True, + ) + second = _native_audio_payload(samples=8000) + + assert buffer.append(first, chunk_period_ms=1000) is None + emitted = buffer.append(second, chunk_period_ms=1000) + + assert emitted is not None + assert emitted["is_speech"] is True + assert "new_user_turn" not in emitted + assert "new_user_turn_prefix_variant" not in emitted + assert "force_speak" not in emitted + + +def test_minicpmo_merge_native_audio_payloads_preserves_speech_marker(): + first = _native_audio_payload(samples=8000) + second = _native_audio_payload(samples=8000, value=0.0, is_speech=False) + + merged = OmniDuplexSessionHandler._merge_native_audio_payloads(first, second) + + assert merged["is_speech"] is True + + +def test_minicpmo_merge_drops_serving_new_user_turn_marker(): + first = _native_audio_payload(samples=8000) + first.update( + new_user_turn=True, + new_user_turn_prefix_variant="clean_response_done", + force_speak=True, + ) + second = _native_audio_payload(samples=8000, is_speech=False) + + merged = OmniDuplexSessionHandler._merge_native_audio_payloads(first, second) + + assert merged["is_speech"] is True + assert "new_user_turn" not in merged + assert "new_user_turn_prefix_variant" not in merged + assert "force_speak" not in merged + + +@pytest.mark.asyncio +async def test_minicpmo_clear_continuation_does_not_cancel_pending_silence_task(): + async def _pending() -> bool: + await asyncio.sleep(3600) + return True + + native = MiniCPMO45ServingSessionState() + task = asyncio.create_task(_pending()) + native.continuation_owner_id = "owner" + native.continuation_units = 1 + native.pending_silence_task = task + native.pending_silence_owner_id = "owner" + + native.clear_continuation() + + assert native.continuation_owner_id is None + assert native.continuation_units == 0 + assert native.pending_silence_task is None + assert native.pending_silence_owner_id is None + assert not task.cancelled() + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + + +def test_auto_response_playback_overlap_keeps_model_owned_listen_speak_decision(): + handler, session = _auto_response_context("sid-auto-silent-overlap", playback_active=True) + session.mark_audio_sent(duration_ms=1000) + silent_payload = _native_audio_payload(value=0.0, is_speech=None, force_listen=False) + speech_payload = _native_audio_payload(is_speech=None, force_listen=False) + + assert handler._should_force_listen_for_auto_response_overlap(session, {}, silent_payload) is False + assert handler._should_force_listen_for_auto_response_overlap(session, {}, speech_payload) is False + assert ( + handler._should_force_listen_for_auto_response_overlap( + session, + {"force_listen": True}, + speech_payload, + ) + is True + ) + assert ( + handler._should_force_listen_for_auto_response_overlap( + session, + {"force_barge_in": True}, + speech_payload, + ) + is False + ) + + session.acknowledge_playback(played_ms=1000, committed_ms=1000) + assert handler._should_force_listen_for_auto_response_overlap(session, {}, speech_payload) is False + + +def test_auto_response_overlap_silence_advances_model_unit_and_preserves_realtime_input(): + handler, session = _auto_response_context( + "sid-auto-silent-wire-buffer", + playback_active=True, + ) + decision = handler._overlap_decision( + session, + {"is_speech": False}, + _native_audio_payload(samples=3200, is_speech=False), + ) + + assert decision["action"] == "listen" + assert decision["buffer_audio"] is True + assert decision["defer_runtime_append"] is False + assert decision["force_listen"] is False + assert decision["preserve_realtime_input"] is True + + +def test_auto_response_playback_overlap_admits_model_units_and_tracks_speech(): + handler, session = _auto_response_context("sid-auto-deferred-overlap", playback_active=True) + payload = _native_audio_payload(samples=8640) + + decision = handler._overlap_decision( + session, + {"duration_ms": 540, "is_speech": True}, + payload, + ) + + assert decision["action"] == "listen" + assert decision["defer_runtime_append"] is False + assert decision["force_listen"] is False + assert decision["buffer_audio"] is True + assert session.overlap_speech_ms == 540 + + +@pytest.mark.asyncio +async def test_auto_response_overlap_exact_unit_commit_does_not_block_or_replay_next_unit(): + class ModelUnitEngine(FakeEngineClient): + def __init__(self) -> None: + super().__init__() + self.append_started = asyncio.Event() + self.active_response_ids_at_append: list[str | None] = [] + self.response_done_seen_at_append = False + self.session: DuplexSession | None = None + self.websocket: TimedWebSocket | None = None + + async def append_duplex_input_async(self, session_id: str, **kwargs): + assert self.session is not None + assert self.websocket is not None + self.active_response_ids_at_append.append(self.session.active_response_id) + self.response_done_seen_at_append = "response.done" in self.websocket.sent_types() + kwargs.pop("expected_epoch", None) + result = await super().append_duplex_input_async(session_id, **kwargs) + self.append_started.set() + return result + + engine = ModelUnitEngine() + handler = OmniDuplexSessionHandler( + chat_service=FakeChatService(engine), + config_timeout_s=0.1, + idle_timeout_s=1, + ) + ws = TimedWebSocket() + engine.websocket = ws + create = _native_session_create("sid-auto-overlap-model-unit") + create["session"]["extra_body"]["auto_response"] = True + ws.put(create) + handler_task = asyncio.create_task(handler.handle_session(ws)) + + try: + for _ in range(100): + session = handler._registry.get("sid-auto-overlap-model-unit") + if session is not None and "session.created" in ws.sent_types(): + break + await asyncio.sleep(0.01) + else: + pytest.fail("native duplex session did not open") + + engine.session = session + session.config.playback_commit_policy = DuplexPlaybackCommitPolicy.ACK_ONLY.value + response_id = session.begin_response() + session.mark_audio_sent(duration_ms=2000) + request_id = handler._native_stage0_request_id(session, session.epoch) + session.bind_request(request_id) + expected_fence = DuplexFence( + session.session_id, + epoch=session.epoch, + turn_id=session.turn_id, + incarnation=session.incarnation, + ) + + chunk = { + "type": "input_audio_buffer.append", + "audio": _pcm_f32_b64(3200, value=0.05), + "format": "pcm_f32le", + "sample_rate_hz": 16000, + "duration_ms": 200, + "is_speech": True, + } + for _ in range(4): + ws.put(chunk) + await asyncio.sleep(0.05) + assert engine.appended == [] + + ws.put(chunk) + await asyncio.wait_for(engine.append_started.wait(), timeout=0.5) + + ws.put({"type": "input_audio_buffer.commit", "final": True}) + for _ in range(100): + if "input.committed" in ws.sent_types(): + break + await asyncio.sleep(0.01) + else: + pytest.fail("exact model unit input did not commit") + + engine.append_started.clear() + for _ in range(5): + ws.put(chunk) + await asyncio.wait_for(engine.append_started.wait(), timeout=0.5) + await asyncio.sleep(0.05) + assert "error" not in ws.sent_types() + finally: + ws.put({"type": "session.close"}) + await asyncio.wait_for(handler_task, timeout=2) + + assert len(engine.appended) == 2 + for appended_session_id, mode, payload, final in engine.appended: + assert appended_session_id == session.session_id + assert mode == "append_audio_chunk" + assert final is False + assert isinstance(payload, dict) + assert np.frombuffer(base64.b64decode(payload["audio"]), dtype=" None: + nonlocal overlap_sent + if data.get("type") != "response.audio.delta" or overlap_sent: + return + overlap_sent = True + chunk = { + "type": "input_audio_buffer.append", + "audio": _pcm_f32_b64(3200, value=0.05), + "format": "pcm_f32le", + "sample_rate_hz": 16000, + "duration_ms": 200, + "is_speech": True, + } + for _ in range(4): + ws.put(chunk) + ws.put({"type": "input_audio_buffer.commit", "final": True}) + + ws = TimedWebSocket(on_send=commit_overlap_after_first_response, receive_timeout_s=3) + create = _native_realtime_session_update(session_id) + create["session"]["extra_body"]["auto_response"] = True + ws.put(create) + ws.put( + { + "type": "input_audio_buffer.append", + "audio": _pcm_f32_b64(16000, value=0.05), + "format": "pcm_f32le", + "sample_rate_hz": 16000, + "duration_ms": 1000, + "is_speech": True, + } + ) + ws.put({"type": "input_audio_buffer.commit", "final": True}) + + handler_task = asyncio.create_task(handler.handle_realtime_session(ws)) + try: + for _ in range(300): + if len(engine.appended) >= 2: + break + await asyncio.sleep(0.01) + else: + pytest.fail( + f"committed overlap was not promoted after response.done: " + f"appends={len(engine.appended)}, events={ws.sent_types()}" + ) + + assert overlap_sent is True + assert len(engine.appended) == 2 + assert engine.appended[1][3] is True + assert ws.sent_types().count("response.created") == 1 + finally: + ws.put({"type": "session.close"}) + await asyncio.wait_for(handler_task, timeout=3) + + assert ws.sent_types().count("response.done") >= 1 + + +def test_auto_response_barge_in_does_not_mark_serving_new_user_turn_payload(): + handler, session = _auto_response_context("sid-auto-barge-new-turn", playback_active=True) + payload = _native_audio_payload() + + assert handler._should_force_listen_for_auto_response_overlap(session, {"force_barge_in": True}, payload) is False + assert "new_user_turn" not in payload + assert "new_user_turn_prefix_variant" not in payload + assert session.turn_id == 0 + assert payload.get("force_speak") is not True + + +def test_native_realtime_barge_in_payload_does_not_create_serving_turn_boundary(): + payload: dict[str, object] = { + "type": "audio", + "audio": _pcm_f32_b64(16000, value=0.05), + "format": "pcm_f32le", + "sample_rate_hz": 16000, + "is_speech": True, + } + session = DuplexSession(session_id="sid-realtime-barge", config=DuplexSessionConfig()) + session.turn_id = 2 + + assert "new_user_turn" not in payload + assert "new_user_turn_prefix_variant" not in payload + assert session.turn_id == 2 + + +def test_auto_response_force_barge_in_has_no_waiting_turn_variant(): + handler, session = _auto_response_context("sid-auto-barge-keeps-variant") + payload = _native_audio_payload() + + assert ( + handler._should_force_listen_for_auto_response_overlap( + session, + {"force_barge_in": True}, + payload, + ) + is False + ) + + +def test_duplex_endpoint_requires_explicit_session_mode_duplex(): + assert should_enable_duplex_endpoint(None) is False + assert should_enable_duplex_endpoint([]) is False + assert should_enable_duplex_endpoint([SimpleNamespace(session_mode="turn")]) is False + assert should_enable_duplex_endpoint([{"session_mode": "turn"}]) is False + assert ( + should_enable_duplex_endpoint( + [ + SimpleNamespace(session_mode="turn"), + SimpleNamespace(session_mode="duplex"), + ] + ) + is True + ) + assert should_enable_duplex_endpoint([{"session_mode": "duplex"}]) is True + + +def test_duplex_endpoint_supports_top_level_session_mode(tmp_path): + config_path = tmp_path / "stage_config.yaml" + config_path.write_text( + """ +session_mode: duplex +stage_args: + - stage_id: 0 + engine_args: {} +""", + encoding="utf-8", + ) + + assert should_enable_duplex_endpoint([], config_path=str(config_path)) is True + + +def test_duplex_handler_splits_data_plane_audio_list_into_deltas(): + import torch + + data_plane = _test_data_plane() + output = SimpleNamespace( + finished=True, + outputs=[ + SimpleNamespace( + text="hello", + multimodal_output={ + "audio": [ + torch.zeros(10, dtype=torch.float32), + torch.zeros(20, dtype=torch.float32), + ], + "sr": 24000, + }, + ) + ], + ) + + native_results = list(data_plane.project_output(output)) + + assert [result["audio_data"] for result in native_results] == ["wav-10", "wav-20"] + assert [result["text"] for result in native_results] == ["hello", ""] + assert [result["end_of_turn"] for result in native_results] == [False, True] + + +def test_duplex_listen_latent_does_not_poison_cumulative_audio_offset(): + import torch + + data_plane = _test_data_plane() + request_id = "duplex-duplex-sess-stage0" + + # A model-listen decision wraps the segment with a latent tensor that is + # NOT reply audio; it must not advance the cumulative audio offset. + listen_output = SimpleNamespace( + request_id=request_id, + finished=True, + outputs=[], + multimodal_output={ + "duplex_native_decision": "listen", + "model_listen": True, + "latent": torch.zeros(331776, dtype=torch.float32), + "meta": {"sr": 24000}, + }, + ) + listen_results = list(data_plane.project_output(listen_output)) + assert [result.get("is_listen") for result in listen_results] == [True] + + # The first speak unit carries cumulative stage-1 audio far smaller than + # the listen latent; it must still be delivered from sample 0. + speak_output = SimpleNamespace( + request_id=request_id, + finished=False, + outputs=[ + SimpleNamespace( + text=" It was a very", + multimodal_output={}, + ) + ], + multimodal_output={ + "audio": torch.zeros(32768, dtype=torch.float32), + "sr": 24000, + }, + ) + speak_results = list(data_plane.project_output(speak_output)) + assert [result.get("audio_data") for result in speak_results] == ["wav-32768"] + assert speak_results[0]["text"] == " It was a very" + + +def test_direct_listen_decision_survives_inner_completion_metadata(): + inner_output = SimpleNamespace( + outputs=[ + SimpleNamespace( + multimodal_output={"special_token_ids": {"listen_token_id": 151705}}, + ) + ] + ) + decision = MiniCPMO45DuplexRuntimeExtension().decide_output( + stage_id=0, + final_stage_id=1, + segment_finished=True, + segment_token_ids=(151705,), + segment_output_metadata={"special_token_ids": {"listen_token_id": 151705}}, + output=inner_output, + ) + assert decision is not None + output = attach_duplex_output_decision( + OmniRequestOutput( + request_id="duplex-direct-listen", + finished=True, + stage_id=0, + final_output_type=decision.final_output_type, + request_output=inner_output, + metrics={ + "stage_metrics": { + "0": { + "vllm_ttft_ms": 125.0, + "vllm_tpot_ms": 18.0, + "vllm_itl_ms": 17.5, + "vllm_itls_ms": [17.0, 18.0], + } + } + }, + ), + decision, + ) + + results = list(_test_data_plane().project_output(output)) + + assert len(results) == 1 + assert results[0]["is_listen"] is True + assert results[0]["model_listen"] is True + assert results[0]["listen_source"] == "model_listen" + assert results[0]["stage_metrics"] == output.metrics["stage_metrics"] + + +def test_duplex_segment_text_is_attached_once_across_streaming_batches(): + import torch + + class _ChatService: + model_config = _ModelConfig() + + def create_audio(self, audio_obj): + return SimpleNamespace(audio_data=f"wav-{int(audio_obj.audio_tensor.shape[0])}") + + data_plane = _test_data_plane() + request_id = "duplex-duplex-sess-stage0" + session = DuplexSession( + session_id="sid-auto-respond", + config=DuplexSessionConfig(extra_body={"auto_response": True}), + ) + + def _speak(total_samples: int, text: str, *, finished: bool): + output = SimpleNamespace( + request_id=request_id, + finished=finished, + outputs=[SimpleNamespace(text=text, multimodal_output={})], + multimodal_output={ + "audio": torch.zeros(total_samples, dtype=torch.float32), + "sr": 24000, + }, + ) + return _project_data_plane(data_plane, {"data_plane_outputs": [output]}, session=session) + + def _audio(results): + return [r for r in results if r.get("is_listen") is False] + + def _listen(results): + return [r for r in results if r.get("is_listen") is True] + + # One talker segment streams several cumulative-audio batches, each + # carrying the SAME segment text; only the first may attach it. + assert [r["text"] for r in _audio(_speak(100, " movie called", finished=False))] == [" movie called"] + assert [r["text"] for r in _audio(_speak(220, " movie called", finished=False))] == [""] + finished_results = _speak(300, " movie called", finished=True) + assert [r["text"] for r in _audio(finished_results)] == [""] + assert _listen(finished_results) == [] + + # Continuation units re-run the talker with the SAME handed text past a + # finished boundary (every engine segment ends finished=True); the text + # must stay suppressed or each continuation duplicates the transcript. + assert [r["text"] for r in _audio(_speak(400, " movie called", finished=False))] == [""] + assert [r["text"] for r in _audio(_speak(450, " movie called", finished=True))] == [""] + + # A genuinely new segment text is attached in full. + assert [r["text"] for r in _audio(_speak(500, " Titanic", finished=True))] == [" Titanic"] + + # Text growing within a segment is delivered as suffix deltas. + assert [r["text"] for r in _audio(_speak(600, " by", finished=False))] == [" by"] + assert [r["text"] for r in _audio(_speak(700, " by James", finished=True))] == [" James"] + + # A segment whose finished batch slices to an EMPTY audio delta (all + # samples already delivered) must not block the next segment's text. + assert [r["text"] for r in _audio(_speak(800, " James Cameron. The", finished=False))] == [" James Cameron. The"] + assert _speak(800, " James Cameron. The", finished=True) == [] + assert [r["text"] for r in _audio(_speak(900, " 997.", finished=False))] == [" 997."] + + +def test_duplex_data_plane_output_prefers_audio_segment_text_metadata(): + data_plane = _test_data_plane() + request_id = "duplex-sid-native-segment-text-e0-stage0" + segment_text = "你好,有什么想聊的吗?" + cumulative_text = segment_text * 2 + output = SimpleNamespace( + request_id=request_id, + finished=False, + outputs=[SimpleNamespace(text=cumulative_text, multimodal_output={})], + multimodal_output={ + "audio": np.zeros(24000, dtype=np.float32), + "sr": 24000, + "meta.llm_output_text_utf8": np.frombuffer(segment_text.encode("utf-8"), dtype=np.uint8), + }, + ) + + results = list(data_plane.project_output(output)) + + assert len(results) == 1 + assert results[0]["audio_data"] == "wav-24000" + assert results[0]["text"] == segment_text + + next_output_without_segment_metadata = SimpleNamespace( + request_id=request_id, + finished=False, + outputs=[SimpleNamespace(text=segment_text * 3, multimodal_output={})], + multimodal_output={ + "audio": np.zeros(30720, dtype=np.float32), + "sr": 24000, + }, + ) + next_results = list(data_plane.project_output(next_output_without_segment_metadata)) + + assert len(next_results) == 1 + assert next_results[0]["audio_data"] == "wav-6720" + assert next_results[0]["text"] == "" + + +@pytest.mark.parametrize( + ("request_id", "second_samples", "expected_second_audio"), + [ + ("duplex-sid-model-turn-audio-e0-stage0", 36000, "wav-12000"), + ("duplex-sid-model-turn-audio-restart-e0-stage0", 12000, "wav-12000"), + ], + ids=["cumulative-across-model-turns", "upstream-cumulative-audio-restarts"], +) +def test_duplex_audio_cursor_across_model_turns( + request_id: str, + second_samples: int, + expected_second_audio: str, +): + data_plane = _test_data_plane() + + def output(turn_id: int, samples: int): + return SimpleNamespace( + request_id=request_id, + finished=False, + outputs=[SimpleNamespace(text="", multimodal_output={})], + multimodal_output={ + "audio": np.zeros(samples, dtype=np.float32), + "sr": 24000, + "meta.duplex_turn_id": np.array([turn_id], dtype=np.int32), + }, + ) + + first = list(data_plane.project_output(output(0, 24000))) + second = list(data_plane.project_output(output(1, second_samples))) + + assert first[0]["audio_data"] == "wav-24000" + assert second[0]["audio_data"] == expected_second_audio + + +def test_duplex_data_plane_accepts_model_turn_output_across_client_commits(): + data_plane = _test_data_plane() + session = DuplexSession( + session_id="sid-auto-stale-turn", + config=DuplexSessionConfig(extra_body={"auto_response": True}), + ) + session.commit_native_audio_input() + session.commit_native_audio_input() + session.turn_id = 1 + session.epoch = 0 + request_id = "duplex-sid-auto-stale-turn-stage0" + output = SimpleNamespace( + request_id=request_id, + finished=False, + outputs=[SimpleNamespace(text="old tail", multimodal_output={})], + multimodal_output={ + "audio": np.zeros(24000, dtype=np.float32), + "sr": 24000, + "meta.duplex_turn_id": np.array([1], dtype=np.int32), + "meta.llm_output_text_utf8": np.frombuffer("旧尾巴".encode(), dtype=np.uint8), + }, + ) + + results = list(data_plane.project_output(output, context=_data_plane_context(session))) + + assert len(results) == 1 + assert results[0]["audio_data"] == "wav-24000" + assert results[0]["model_turn_id"] == 1 + + +def test_duplex_data_plane_accepts_active_response_turn_identity(): + data_plane = _test_data_plane() + session = DuplexSession( + session_id="sid-active-response-turn", + config=DuplexSessionConfig(extra_body={"auto_response": True}), + ) + session.turn_id = 1 + session.epoch = 0 + session.begin_response(turn_id=0) + output = SimpleNamespace( + request_id="duplex-sid-active-response-turn-stage0", + finished=True, + outputs=[SimpleNamespace(text="", multimodal_output={})], + multimodal_output={ + "audio": np.zeros(10, dtype=np.float32), + "sr": 24000, + "meta.duplex_turn_id": np.array([0], dtype=np.int32), + "meta.duplex_epoch": np.array([0], dtype=np.int32), + }, + ) + + results = _project_data_plane(data_plane, {"data_plane_outputs": [output]}, session=session) + + assert len(results) == 1 + + +def test_duplex_data_plane_listen_preserves_model_turn_identity(): + data_plane = _test_data_plane() + session = DuplexSession( + session_id="sid-listen-turn", + config=DuplexSessionConfig(extra_body={"auto_response": True}), + ) + output = SimpleNamespace( + request_id="duplex-sid-listen-turn-stage0", + finished=True, + outputs=[SimpleNamespace(text="", token_ids=[], multimodal_output={})], + multimodal_output={ + "duplex_native_decision": "listen", + "meta.duplex_turn_id": np.array([2], dtype=np.int32), + "meta.duplex_epoch": np.array([0], dtype=np.int32), + }, + ) + + results = _project_data_plane(data_plane, {"data_plane_outputs": [output]}, session=session) + + assert len(results) == 1 + assert results[0]["is_listen"] is True + assert results[0]["model_turn_id"] == 2 + + +def test_duplex_data_plane_does_not_drop_future_model_turn_while_response_active(): + data_plane = _test_data_plane() + session = DuplexSession( + session_id="sid-future-response-turn", + config=DuplexSessionConfig(extra_body={"auto_response": True}), + ) + session.turn_id = 0 + session.epoch = 0 + session.begin_response(turn_id=0) + output = SimpleNamespace( + request_id="duplex-sid-future-response-turn-stage0", + finished=False, + outputs=[SimpleNamespace(text="", multimodal_output={})], + multimodal_output={ + "audio": np.zeros(10, dtype=np.float32), + "sr": 24000, + "meta.duplex_turn_id": np.array([1], dtype=np.int32), + "meta.duplex_epoch": np.array([0], dtype=np.int32), + }, + ) + + results = _project_data_plane(data_plane, {"data_plane_outputs": [output]}, session=session) + + assert len(results) == 1 + assert results[0]["model_turn_id"] == 1 + assert results[0]["audio_data"] == "wav-10" + + +def _duplex_tts_output( + *, + request_id: str, + samples: int, + finished: bool, + text: str = "hello", + tts_is_last_chunk: bool = False, + turn_end: bool = False, + turn_id: int = 0, + token_ids: list[int] | None = None, +): + return SimpleNamespace( + request_id=request_id, + finished=finished, + outputs=[SimpleNamespace(text=text, token_ids=list(token_ids or []), multimodal_output={})], + multimodal_output={ + "audio": np.zeros(samples, dtype=np.float32), + "sr": 24000, + "meta.tts_is_last_chunk": np.array([int(tts_is_last_chunk)], dtype=np.int32), + "meta.turn_end": np.array([int(turn_end)], dtype=np.int32), + "meta.duplex_turn_id": np.array([turn_id], dtype=np.int32), + "meta.duplex_epoch": np.array([0], dtype=np.int32), + }, + ) + + +@pytest.mark.parametrize( + ( + "finished", + "tts_is_last_chunk", + "turn_end", + "consume_audio_first", + "expected_end_of_turn", + "expected_audio", + ), + [ + (False, True, False, False, False, "wav-24000"), + (True, True, False, True, False, ""), + (True, True, True, True, True, ""), + ], + ids=[ + "last-audio-batch-does-not-end-response", + "scheduler-eos-only-ends-tts-segment", + "explicit-turn-end-metadata-ends-response", + ], +) +def test_duplex_auto_response_tts_boundary( + finished: bool, + tts_is_last_chunk: bool, + turn_end: bool, + consume_audio_first: bool, + expected_end_of_turn: bool, + expected_audio: str, +): + data_plane = _test_data_plane() + request_id = "duplex-sid-tts-boundary-e0-stage0" + session = DuplexSession( + session_id="sid-tts-boundary", + config=DuplexSessionConfig(extra_body={"auto_response": True}), + ) + session.bind_response_turn(0) + if consume_audio_first: + data_plane.slice_cumulative_audio(request_id, np.zeros(24000, dtype=np.float32)) + output = _duplex_tts_output( + request_id=request_id, + samples=24000, + finished=finished, + tts_is_last_chunk=tts_is_last_chunk, + turn_end=turn_end, + token_ids=[151645] if finished else None, + ) + + results = _project_data_plane(data_plane, {"data_plane_outputs": [output]}, session=session) + + assert len(results) == 1 + boundary = results[0] + assert boundary["stage_role"] == "tts" + assert boundary["end_of_turn"] is expected_end_of_turn + assert boundary["abort_data_plane_request"] is True + assert boundary["audio_data"] == expected_audio + + +def test_duplex_auto_response_tts_scheduler_eos_fallback(): + data_plane = _test_data_plane() + request_id = "duplex-sid-tts-eos-no-profile-e0-stage0" + session = DuplexSession( + session_id="sid-tts-eos-no-profile", + config=DuplexSessionConfig(extra_body={"auto_response": True}), + ) + session.bind_response_turn(0) + data_plane.slice_cumulative_audio(request_id, np.zeros(24000, dtype=np.float32)) + output = _duplex_tts_output( + request_id=request_id, + samples=24000, + finished=True, + tts_is_last_chunk=False, + token_ids=[151645], + ) + + results = _project_data_plane(data_plane, {"data_plane_outputs": [output]}, session=session) + + assert len(results) == 1 + assert results[0]["stage_role"] == "tts" + assert results[0]["abort_data_plane_request"] is True + + +def test_duplex_turn_end_is_not_swallowed_by_finished_segment_fallback(): + data_plane = _test_data_plane() + request_id = "duplex-sid-turn-end-after-segment-e0-stage0" + session = DuplexSession( + session_id="sid-turn-end-after-segment", + config=DuplexSessionConfig(extra_body={"auto_response": True}), + ) + session.begin_response(turn_id=0) + session.bind_request(request_id) + _project_data_plane( + data_plane, + { + "data_plane_outputs": [ + _duplex_tts_output( + request_id=request_id, + samples=24000, + finished=False, + tts_is_last_chunk=True, + ) + ] + }, + session=session, + ) + output = _duplex_tts_output( + request_id=request_id, + samples=24000, + finished=True, + text="", + tts_is_last_chunk=False, + turn_end=True, + token_ids=[151645], + ) + + results = _project_data_plane(data_plane, {"data_plane_outputs": [output]}, session=session) + + assert len(results) == 1 + assert results[0]["stage_role"] == "tts" + assert results[0]["end_of_turn"] is True + + duplicate = _project_data_plane(data_plane, {"data_plane_outputs": [output]}, session=session) + + assert not any(result.get("end_of_turn") is True for result in duplicate) + + +def test_duplex_auto_response_discards_terminal_only_audio_before_response_creation(): + data_plane = _test_data_plane() + request_id = "duplex-sid-terminal-only-audio-e0-stage0" + session = DuplexSession( + session_id="sid-terminal-only-audio", + config=DuplexSessionConfig(extra_body={"auto_response": True}), + ) + session.bind_request(request_id) + first_audio = _duplex_tts_output( + request_id=request_id, + samples=3840, + finished=False, + text="", + ) + + assert _project_data_plane(data_plane, {"data_plane_outputs": [first_audio]}, session=session) == [] + + terminal = _duplex_tts_output( + request_id=request_id, + samples=3840, + finished=True, + text="", + tts_is_last_chunk=True, + turn_end=True, + token_ids=[151645], + ) + results = _project_data_plane(data_plane, {"data_plane_outputs": [terminal]}, session=session) + + assert len(results) == 1 + assert results[0]["audio_data"] == "" + assert results[0]["end_of_turn"] is True + assert session.active_request_id == request_id + assert not data_plane.is_terminal(request_id) + + +def test_duplex_auto_response_releases_buffered_audio_when_transcript_arrives(): + data_plane = _test_data_plane() + request_id = "duplex-sid-delayed-transcript-e0-stage0" + session = DuplexSession( + session_id="sid-delayed-transcript", + config=DuplexSessionConfig(extra_body={"auto_response": True}), + ) + audio_before_text = _duplex_tts_output( + request_id=request_id, + samples=100, + finished=False, + text="", + ) + + assert _project_data_plane(data_plane, {"data_plane_outputs": [audio_before_text]}, session=session) == [] + + audio_with_text = _duplex_tts_output( + request_id=request_id, + samples=200, + finished=False, + text="hello", + ) + results = _project_data_plane(data_plane, {"data_plane_outputs": [audio_with_text]}, session=session) + + assert [result["audio_data"] for result in results] == ["wav-100", "wav-100"] + assert [result["text"] for result in results] == ["", "hello"] + + +def test_duplex_auto_response_releases_buffered_audio_on_text_only_terminal(): + data_plane = _test_data_plane() + request_id = "duplex-sid-delayed-terminal-text-e0-stage0" + session = DuplexSession( + session_id="sid-delayed-terminal-text", + config=DuplexSessionConfig(extra_body={"auto_response": True}), + ) + audio_before_text = _duplex_tts_output( + request_id=request_id, + samples=100, + finished=False, + text="", + turn_id=1, + ) + + assert _project_data_plane(data_plane, {"data_plane_outputs": [audio_before_text]}, session=session) == [] + + textless_turn_terminal = _duplex_tts_output( + request_id=request_id, + samples=100, + finished=True, + text="", + tts_is_last_chunk=True, + turn_end=True, + turn_id=1, + token_ids=[151645], + ) + terminal_results = list( + data_plane.project( + {"data_plane_outputs": [textless_turn_terminal]}, + context=_data_plane_context(session), + ) + ) + + assert len(terminal_results) == 1 + assert terminal_results[0]["audio_data"] == "" + assert terminal_results[0]["end_of_turn"] is True + + text_only_terminal = SimpleNamespace( + request_id=request_id, + finished=True, + outputs=[SimpleNamespace(text="hello", token_ids=[151645], multimodal_output={})], + multimodal_output={ + "sr": 24000, + "meta.tts_is_last_chunk": np.array([1], dtype=np.int32), + "meta.turn_end": np.array([1], dtype=np.int32), + "meta.duplex_turn_id": np.array([2], dtype=np.int32), + "meta.duplex_epoch": np.array([0], dtype=np.int32), + }, + ) + results = _project_data_plane(data_plane, {"data_plane_outputs": [text_only_terminal]}, session=session) + + assert len(results) == 1 + assert results[0]["audio_data"] == "wav-100" + assert results[0]["text"] == "hello" + assert results[0]["end_of_turn"] is True + assert results[0]["abort_data_plane_request"] is True + assert not data_plane.has_pending_audio(request_id) + + +@pytest.mark.asyncio +async def test_native_append_propagates_current_turn_fence_to_engine(): + engine = FakeEngineClient() + handler = OmniDuplexSessionHandler(chat_service=FakeChatService(engine)) + session = DuplexSession( + session_id="sid-fenced-append", + config=DuplexSessionConfig(extra_body={"auto_response": True}), + ) + session.capabilities = DuplexCapabilities.minicpmo45_native() + session.turn_id = 2 + session.begin_response(turn_id=2) + ws = TimedWebSocket() + + await handler._append_runtime_input( + session, + {"duplex_turn_id": 2, "audio": "", "format": "pcm_f32le"}, + final=True, + send_json=ws.send_json, + mode="append_audio_chunk", + expected_epoch=0, + ) + + assert engine.appended_fences == [DuplexFence("sid-fenced-append", epoch=0, turn_id=2)] + + +@pytest.mark.asyncio +async def test_minicpmo_auto_response_tts_segment_boundary_appends_silence_unit(): + request_id = "duplex-sid-segment-boundary-e0-stage0" + engine = FakeEngineClient() + handler = OmniDuplexSessionHandler( + chat_service=FakeChatService(engine), + config_timeout_s=0.1, + idle_timeout_s=1, + ) + session = DuplexSession( + session_id="sid-segment-boundary", + config=DuplexSessionConfig(extra_body={"auto_response": True}), + ) + session.capabilities = DuplexCapabilities.minicpmo45_native() + session.begin_response(turn_id=0) + session.turn_id = 1 + session.bind_request(request_id) + session.mark_audio_sent(100) + _install_direct_silence_scheduler(handler, session) + ws = TimedWebSocket() + + await handler._send_one_native_duplex_event( + ws.send_json, + { + "supported": True, + "stage_role": "tts", + "is_listen": False, + "data_plane_request_id": request_id, + "text": "", + "audio_data": "", + "audio_format": "pcm16", + "end_of_turn": False, + "abort_data_plane_request": True, + "uses_model_runner_scheduler": True, + "runner_kv_backed": True, + "runtime_impl": "scheduler_data_plane", + }, + session=session, + expected_epoch=session.epoch, + ) + await asyncio.sleep(0.01) + + assert len(engine.appended) == 1 + _, mode, payload, final = engine.appended[0] + assert mode == "append_audio_chunk" + assert payload["format"] == "pcm_f32le" + assert payload["duplex_turn_id"] == 0 + assert "force_speak" not in payload + assert final is False + assert session.active_response_id is not None + assert "response.done" not in ws.sent_types() + + +@pytest.mark.asyncio +async def test_minicpmo_auto_response_pre_response_tts_boundary_continues_model_turn(): + request_id = "duplex-sid-pre-response-boundary-e0-stage0" + engine = FakeEngineClient() + handler = OmniDuplexSessionHandler(chat_service=FakeChatService(engine)) + session = DuplexSession( + session_id="sid-pre-response-boundary", + config=DuplexSessionConfig(extra_body={"auto_response": True}), + ) + session.capabilities = DuplexCapabilities.minicpmo45_native() + session.bind_request(request_id) + _install_direct_silence_scheduler(handler, session) + ws = TimedWebSocket() + + close_reason, emitted = await handler._send_one_native_duplex_event( + ws.send_json, + { + "supported": True, + "stage_role": "tts", + "is_listen": False, + "data_plane_request_id": request_id, + "model_turn_id": session.turn_id, + "text": "", + "audio_data": "", + "audio_format": "pcm16", + "end_of_turn": False, + "abort_data_plane_request": True, + "uses_model_runner_scheduler": True, + "runner_kv_backed": True, + "runtime_impl": "scheduler_data_plane", + }, + session=session, + expected_epoch=session.epoch, + ) + await asyncio.sleep(0.01) + + assert close_reason is None + assert emitted is False + assert session.active_response_id is None + assert ws.sent == [] + assert len(engine.appended) == 1 + _, mode, payload, final = engine.appended[0] + assert mode == "append_audio_chunk" + assert payload["duplex_turn_id"] == session.turn_id + assert final is False + + +@pytest.mark.asyncio +async def test_minicpmo_pre_response_continuation_drops_after_model_turn_ends(): + request_id = "duplex-sid-stale-pre-response-boundary-e0-stage0" + engine = FakeEngineClient() + handler = OmniDuplexSessionHandler(chat_service=FakeChatService(engine)) + session = DuplexSession( + session_id="sid-stale-pre-response-boundary", + config=DuplexSessionConfig(extra_body={"auto_response": True}), + ) + session.capabilities = DuplexCapabilities.minicpmo45_native() + session.bind_request(request_id) + model_turn_id = session.turn_id + + native = handler._minicpmo_session_state(session) + + async def _stale_before_append(payload: object, **kwargs: Any) -> bool: + session.complete_model_turn(model_turn_id) + if handler._native_silence_continuation_is_stale( + session, + request_id=kwargs["request_id"], + response_id=kwargs["response_id"], + response_owned=kwargs["response_owned"], + expected_epoch=kwargs["expected_epoch"], + expected_incarnation=kwargs["expected_incarnation"], + expected_model_turn_id=kwargs["expected_model_turn_id"], + ): + return False + append_ok, _ = await handler._append_runtime_input( + session, + payload, + final=False, + send_json=kwargs["send_json"], + mode="append_audio_chunk", + expected_epoch=kwargs["expected_epoch"], + ) + return append_ok + + native.silence_continuation_scheduler = _stale_before_append + + await handler._maybe_continue_native_response( + TimedWebSocket().send_json, + session=session, + expected_epoch=session.epoch, + expected_model_turn_id=model_turn_id, + ) + await asyncio.sleep(0.01) + + assert engine.appended == [] + + +@pytest.mark.asyncio +async def test_minicpmo_auto_response_continuation_has_no_semantic_unit_cap(): + request_id = "duplex-sid-long-response-e0-stage0" + engine = FakeEngineClient() + handler = OmniDuplexSessionHandler(chat_service=FakeChatService(engine)) + session = DuplexSession( + session_id="sid-long-response", + config=DuplexSessionConfig(extra_body={"auto_response": True}), + ) + session.capabilities = DuplexCapabilities.minicpmo45_native() + session.begin_response(turn_id=0) + session.turn_id = 1 + session.bind_request(request_id) + session.mark_audio_sent(100) + _install_direct_silence_scheduler(handler, session) + ws = TimedWebSocket() + + for _ in range(9): + await handler._maybe_continue_native_response( + ws.send_json, + session=session, + expected_epoch=session.epoch, + ) + await asyncio.sleep(0.02) + + assert len(engine.appended) == 9 + assert all(payload["duplex_turn_id"] == 0 for _, _, payload, _ in engine.appended) + assert all("force_speak" not in payload for _, _, payload, _ in engine.appended) + + +@pytest.mark.asyncio +async def test_minicpmo_auto_response_pre_speak_listen_continues_same_response(): + request_id = "duplex-sid-pre-speak-listen-e0-stage0" + engine = FakeEngineClient() + handler = OmniDuplexSessionHandler(chat_service=FakeChatService(engine)) + session = DuplexSession( + session_id="sid-pre-speak-listen", + config=DuplexSessionConfig(extra_body={"auto_response": True}), + ) + session.capabilities = DuplexCapabilities.minicpmo45_native() + response_id = session.begin_response(turn_id=0) + session.turn_id = 1 + session.bind_request(request_id) + _install_direct_silence_scheduler(handler, session) + ws = TimedWebSocket() + + await handler._send_one_native_duplex_event( + ws.send_json, + { + "supported": True, + "stage_role": "llm", + "is_listen": True, + "model_listen": True, + "data_plane_request_id": request_id, + "end_of_turn": False, + "uses_model_runner_scheduler": True, + "runner_kv_backed": True, + "runtime_impl": "scheduler_data_plane", + }, + session=session, + expected_epoch=session.epoch, + ) + await asyncio.sleep(0.01) + + assert len(engine.appended) == 1 + assert session.active_response_id == response_id + assert session.active_request_id == request_id + assert "response.listen" not in ws.sent_types() + assert "response.done" not in ws.sent_types() + + +@pytest.mark.asyncio +async def test_minicpmo_auto_response_listen_before_response_keeps_resumable_request(): + request_id = "duplex-sid-listen-before-response-e0-stage0" + handler = OmniDuplexSessionHandler(chat_service=FakeChatService(FakeEngineClient())) + session = DuplexSession( + session_id="sid-listen-before-response", + config=DuplexSessionConfig(extra_body={"auto_response": True}), + ) + session.capabilities = DuplexCapabilities.minicpmo45_native() + session.bind_request(request_id) + ws = TimedWebSocket() + + close_reason, emitted = await handler._send_one_native_duplex_event( + ws.send_json, + { + "supported": True, + "stage_role": "llm", + "is_listen": True, + "model_listen": True, + "data_plane_request_id": request_id, + "end_of_turn": False, + "uses_model_runner_scheduler": True, + "runner_kv_backed": True, + "runtime_impl": "scheduler_data_plane", + }, + session=session, + expected_epoch=session.epoch, + ) + + assert close_reason is None + assert emitted is True + assert session.active_request_id == request_id + assert not handler._minicpmo_data_plane.is_terminal(request_id) + assert ws.sent_types().count("response.listen") == 1 + + +@pytest.mark.asyncio +async def test_minicpmo_auto_response_listen_without_response_does_not_defer_commit(): + session_id = "sid-listen-before-response-commit" + request_id = duplex_resource_request_id(DuplexFence(session_id), "stage0") + inner_output = SimpleNamespace( + outputs=[ + SimpleNamespace( + multimodal_output={"special_token_ids": {"listen_token_id": 151705}}, + ) + ] + ) + decision = MiniCPMO45DuplexRuntimeExtension().decide_output( + stage_id=0, + final_stage_id=1, + segment_finished=True, + segment_token_ids=(151705,), + segment_output_metadata={"special_token_ids": {"listen_token_id": 151705}}, + output=inner_output, + ) + assert decision is not None + listen_output = attach_duplex_output_decision( + OmniRequestOutput( + request_id=request_id, + finished=True, + stage_id=0, + final_output_type=decision.final_output_type, + request_output=inner_output, + ), + decision, + ) + append_result = { + "operation": "append", + "session_id": session_id, + "ok": True, + "unsupported_count": 0, + "error_count": 0, + "stage_results": [ + { + "stage_id": 0, + "replica_id": 0, + "result": { + "supported": True, + "implementation_level": "model_native_duplex", + "data_plane_append": True, + "request_id": request_id, + "response_stage_id": 1, + }, + } + ], + "data_plane_outputs": [listen_output], + } + engine = FakeEngineClient(append_result=append_result) + handler = OmniDuplexSessionHandler( + chat_service=FakeChatService(engine), + config_timeout_s=0.1, + idle_timeout_s=1, + ) + listen_count = 0 + + def append_residual_after_listen(ws: TimedWebSocket, data: dict[str, Any]) -> None: + nonlocal listen_count + if data.get("type") == "response.listen": + listen_count += 1 + if listen_count == 1: + ws.put( + { + "type": "input_audio_buffer.append", + "audio": _pcm_f32_b64(3_200), + "format": "pcm_f32le", + "sample_rate_hz": 16_000, + "duration_ms": 200, + "is_speech": True, + } + ) + ws.put({"type": "input_audio_buffer.commit", "final": True}) + elif data.get("type") == "input_audio_buffer.committed": + asyncio.get_running_loop().call_later(0.05, ws.put, {"type": "session.close"}) + + ws = TimedWebSocket(on_send=append_residual_after_listen, receive_timeout_s=2) + create = _native_realtime_session_update(session_id) + create["session"]["extra_body"]["auto_response"] = True + ws.put(create) + ws.put( + { + "type": "input_audio_buffer.append", + "audio": _pcm_f32_b64(16_000), + "format": "pcm_f32le", + "sample_rate_hz": 16_000, + "duration_ms": 1_000, + "is_speech": True, + } + ) + + await handler.handle_realtime_session(ws) + + committed = [event for event in ws.sent if event.get("type") == "input_audio_buffer.committed"] + assert len(engine.appended) == 2 + assert engine.appended[1][3] is True + assert len(committed) == 1 + assert committed[0].get("event", {}).get("overlap_deferred") is not True + assert committed[0].get("event", {}).get("response_create_deferred") is not True + assert "response.created" not in ws.sent_types() + + +@pytest.mark.asyncio +async def test_minicpmo_pcm_f32_residual_commit_releases_pending_input_bytes(): + session_id = "sid-pcm-residual-byte-accounting" + control_result = { + "operation": "append", + "session_id": session_id, + "ok": True, + "unsupported_count": 0, + "error_count": 0, + "stage_results": [], + } + engine = FakeEngineClient(control_result=control_result) + handler = OmniDuplexSessionHandler( + chat_service=FakeChatService(engine), + config_timeout_s=0.1, + idle_timeout_s=1, + ) + ws = TimedWebSocket(receive_timeout_s=2) + create = _native_realtime_session_update(session_id) + create["session"]["extra_body"]["auto_response"] = True + ws.put(create) + handler_task = asyncio.create_task(handler.handle_realtime_session(ws)) + + try: + for _ in range(100): + session = handler._registry.get(session_id) + if session is not None and "session.created" in ws.sent_types(): + break + await asyncio.sleep(0.01) + else: + pytest.fail("native duplex session did not open") + + pending_bytes_after_turn: list[int] = [] + for turn_index in range(3): + ws.put( + { + "type": "input_audio_buffer.append", + "audio": _pcm_f32_b64(19_200), + "format": "pcm_f32le", + "sample_rate_hz": 16_000, + "duration_ms": 1_200, + "is_speech": True, + } + ) + ws.put({"type": "input_audio_buffer.commit", "final": True}) + expected_appends = (turn_index + 1) * 2 + for _ in range(100): + if len(engine.appended) >= expected_appends: + await asyncio.sleep(0) + break + await asyncio.sleep(0.01) + else: + pytest.fail(f"turn {turn_index} residual append did not complete") + pending_bytes_after_turn.append(session.pending_input_bytes) + finally: + ws.put({"type": "session.close"}) + await asyncio.wait_for(handler_task, timeout=2) + + assert pending_bytes_after_turn == [0, 0, 0] + assert not any( + event.get("type") == "error" + and isinstance(event.get("error"), dict) + and event["error"].get("code") == "input_backpressure" + for event in ws.sent + ) + + +@pytest.mark.asyncio +async def test_minicpmo_auto_response_post_speak_listen_continues_same_response(): + request_id = "duplex-sid-post-speak-listen-e0-stage0" + engine = FakeEngineClient() + handler = OmniDuplexSessionHandler(chat_service=FakeChatService(engine)) + session = DuplexSession( + session_id="sid-post-speak-listen", + config=DuplexSessionConfig(extra_body={"auto_response": True}), + ) + session.capabilities = DuplexCapabilities.minicpmo45_native() + response_id = session.begin_response(turn_id=0) + session.turn_id = 1 + session.bind_request(request_id) + session.mark_audio_sent(100) + _install_direct_silence_scheduler(handler, session) + ws = TimedWebSocket() + + await handler._send_one_native_duplex_event( + ws.send_json, + { + "supported": True, + "stage_role": "llm", + "is_listen": True, + "model_listen": True, + "data_plane_request_id": request_id, + "end_of_turn": False, + "uses_model_runner_scheduler": True, + "runner_kv_backed": True, + "runtime_impl": "scheduler_data_plane", + }, + session=session, + expected_epoch=session.epoch, + ) + await asyncio.sleep(0.01) + + assert len(engine.appended) == 1 + assert session.active_response_id == response_id + assert session.active_request_id == request_id + assert "response.listen" not in ws.sent_types() + assert "response.done" not in ws.sent_types() + + +@pytest.mark.asyncio +async def test_minicpmo_auto_response_turn_end_preserves_resumable_request(): + request_id = "duplex-sid-turn-end-preserve-request-e0-stage0" + handler = OmniDuplexSessionHandler(chat_service=FakeChatService(FakeEngineClient())) + session = DuplexSession( + session_id="sid-turn-end-preserve-request", + config=DuplexSessionConfig(extra_body={"auto_response": True}), + ) + session.capabilities = DuplexCapabilities.minicpmo45_native() + session.begin_response(turn_id=0) + session.bind_request(request_id) + ws = TimedWebSocket() + + await handler._send_one_native_duplex_event( + ws.send_json, + { + "supported": True, + "stage_role": "tts", + "is_listen": False, + "data_plane_request_id": request_id, + "text": "done", + "audio_data": "audio", + "audio_format": "pcm16", + "audio_duration_ms": 100, + "end_of_turn": True, + "model_turn_id": 0, + "uses_model_runner_scheduler": True, + "runner_kv_backed": True, + "runtime_impl": "scheduler_data_plane", + }, + session=session, + expected_epoch=session.epoch, + ) + + assert session.active_response_id is None + assert session.active_request_id == request_id + assert "response.done" in ws.sent_types() + + +@pytest.mark.asyncio +async def test_minicpmo_auto_response_empty_turn_end_emits_model_listen(): + request_id = "duplex-sid-empty-turn-end-e0-stage0" + handler = OmniDuplexSessionHandler(chat_service=FakeChatService(FakeEngineClient())) + session = DuplexSession( + session_id="sid-empty-turn-end", + config=DuplexSessionConfig(extra_body={"auto_response": True}), + ) + session.capabilities = DuplexCapabilities.minicpmo45_native() + session.bind_request(request_id) + ws = TimedWebSocket() + + close_reason, emitted = await handler._send_one_native_duplex_event( + ws.send_json, + { + "supported": True, + "stage_role": "tts", + "is_listen": False, + "data_plane_request_id": request_id, + "text": "", + "audio_data": "", + "audio_format": "pcm16", + "end_of_turn": True, + "abort_data_plane_request": True, + "model_turn_id": 0, + "uses_model_runner_scheduler": True, + "runner_kv_backed": True, + "runtime_impl": "scheduler_data_plane", + }, + session=session, + expected_epoch=session.epoch, + ) + + assert close_reason is None + assert emitted is True + assert session.turn_id == 1 + assert session.active_request_id == request_id + assert session.active_response_id is None + assert ws.sent_types() == ["response.listen"] + assert ws.sent[0]["model_listen"] is True + assert ws.sent[0]["reason"] == "model_turn_completed_without_output" + + +@pytest.mark.asyncio +async def test_minicpmo_auto_response_drops_late_audio_from_completed_model_turn(): + request_id = "duplex-sid-late-completed-turn-e0-stage0" + handler = OmniDuplexSessionHandler(chat_service=FakeChatService(FakeEngineClient())) + session = DuplexSession( + session_id="sid-late-completed-turn", + config=DuplexSessionConfig(extra_body={"auto_response": True}), + ) + session.capabilities = DuplexCapabilities.minicpmo45_native() + session.bind_request(request_id) + session.complete_model_turn(0) + ws = TimedWebSocket() + + close_reason, emitted = await handler._send_one_native_duplex_event( + ws.send_json, + { + "supported": True, + "stage_role": "tts", + "is_listen": False, + "data_plane_request_id": request_id, + "text": "late duplicate", + "audio_data": "late-audio", + "audio_format": "pcm16", + "audio_duration_ms": 100, + "end_of_turn": True, + "model_turn_id": 0, + "uses_model_runner_scheduler": True, + "runner_kv_backed": True, + "runtime_impl": "scheduler_data_plane", + }, + session=session, + expected_epoch=session.epoch, + ) + + assert close_reason is None + assert emitted is False + assert session.turn_id == 1 + assert session.active_response_id is None + assert "response.created" not in ws.sent_types() + assert "response.done" not in ws.sent_types() + + +def test_duplex_data_plane_drops_stale_epoch_audio_after_barge_in(): + class _ChatService: + model_config = _ModelConfig() + + def create_audio(self, audio_obj): + return SimpleNamespace(audio_data=f"wav-{int(audio_obj.audio_tensor.shape[0])}") + + data_plane = _test_data_plane() + session = DuplexSession( + session_id="sid-auto-stale-epoch", + config=DuplexSessionConfig(extra_body={"auto_response": True}), + ) + session.turn_id = 1 + session.epoch = 2 + output = SimpleNamespace( + request_id="duplex-sid-auto-stale-epoch-stage0", + finished=False, + outputs=[SimpleNamespace(text="old epoch", multimodal_output={})], + multimodal_output={ + "audio": np.zeros(24000, dtype=np.float32), + "sr": 24000, + "meta.duplex_epoch": np.array([1], dtype=np.int32), + "meta.duplex_turn_id": np.array([1], dtype=np.int32), + "meta.llm_output_text_utf8": np.frombuffer("旧轮".encode(), dtype=np.uint8), + }, + ) + + assert list(data_plane.project_output(output, context=_data_plane_context(session))) == [] + + +def test_duplex_data_plane_text_delta_preserves_repeated_suffix_growth(): + data_plane = _test_data_plane() + request_id = "duplex-sid-native-repeated-word-e0-stage0" + + assert data_plane.segment_text_delta(request_id, "好的") == "好的" + assert data_plane.segment_text_delta(request_id, "好的好的") == "好的" + + +def test_duplex_data_plane_text_delta_appends_distinct_non_prefix_segments(): + data_plane = _test_data_plane() + request_id = "duplex-sid-native-rewrite-e0-stage0" + + deltas = [ + data_plane.segment_text_delta(request_id, "It's Canberra."), + data_plane.segment_text_delta(request_id, " Next question."), + ] + + assert deltas == ["It's Canberra.", " Next question."] + assert "".join(deltas) == "It's Canberra. Next question." + + +def test_duplex_data_plane_close_session_removes_encoded_request_state(): + data_plane = _test_data_plane() + target_request_id = duplex_resource_request_id( + DuplexFence("sid-to-close", incarnation=1, epoch=2), + "stage0", + ) + other_request_id = duplex_resource_request_id( + DuplexFence("sid-to-keep", incarnation=1, epoch=2), + "stage0", + ) + data_plane.begin_request(target_request_id) + data_plane.begin_request(other_request_id) + + data_plane.close_session("sid-to-close") + + assert data_plane.has_request(target_request_id) is False + assert data_plane.has_request(other_request_id) is True + + +def test_duplex_auto_response_segment_complete_keeps_data_plane_request_open(): + import numpy as np + + class _ChatService: + model_config = _ModelConfig() + + def create_audio(self, audio_obj): + return SimpleNamespace(audio_data=f"wav-{int(audio_obj.audio_tensor.shape[0])}") + + data_plane = _test_data_plane() + request_id = "duplex-duplex-sess-stage0" + session = DuplexSession( + session_id="sid-auto-respond", + config=DuplexSessionConfig(extra_body={"auto_response": True}), + ) + session.begin_response() + session.mark_audio_sent(1280) + data_plane.slice_cumulative_audio(request_id, np.zeros(30720, dtype=np.float32)) + + output = SimpleNamespace( + request_id=request_id, + finished=True, + outputs=[SimpleNamespace(text="hello", multimodal_output={})], + multimodal_output={ + "audio": np.zeros(30720, dtype=np.float32), + "sr": 24000, + }, + ) + + native_results = _project_data_plane(data_plane, {"data_plane_outputs": [output]}, session=session) + + assert len(native_results) == 1 + assert native_results[0]["is_listen"] is True + assert native_results[0]["reason"] == "auto_response_segment_complete" + assert native_results[0]["data_plane_request_id"] == request_id + assert "abort_data_plane_request" not in native_results[0] + + +def test_duplex_auto_response_prior_playback_does_not_abort_before_current_response(): + data_plane = _test_data_plane() + request_id = "duplex-duplex-sess-stage0" + session = DuplexSession( + session_id="sid-auto-respond-next-turn", + config=DuplexSessionConfig(extra_body={"auto_response": True}), + ) + session.mark_audio_sent(1280) + session.acknowledge_playback(played_ms=1280, committed_ms=1280) + + output = SimpleNamespace( + request_id=request_id, + finished=True, + outputs=[SimpleNamespace(text="", multimodal_output={})], + multimodal_output={}, + ) + + native_results = _project_data_plane(data_plane, {"data_plane_outputs": [output]}, session=session) + + assert native_results == [] + + +def test_duplex_auto_response_text_only_flush_does_not_consume_audio_transcript(): + import numpy as np + + data_plane = _test_data_plane() + request_id = "duplex-duplex-sess-stage0" + session = DuplexSession( + session_id="sid-auto-respond-delayed-audio", + config=DuplexSessionConfig(extra_body={"auto_response": True}), + ) + text = "你好呀,有什莫问的吗?" + + text_only_output = SimpleNamespace( + request_id=request_id, + finished=False, + outputs=[SimpleNamespace(text=text, multimodal_output={})], + multimodal_output={"sr": 24000}, + ) + + assert _project_data_plane(data_plane, {"data_plane_outputs": [text_only_output]}, session=session) == [] + + audio_output = SimpleNamespace( + request_id=request_id, + finished=False, + outputs=[SimpleNamespace(text=text, multimodal_output={})], + multimodal_output={ + "audio": np.zeros(24000, dtype=np.float32), + "sr": 24000, + }, + ) + + native_results = _project_data_plane(data_plane, {"data_plane_outputs": [audio_output]}, session=session) + + assert len(native_results) == 1 + assert native_results[0]["audio_data"].startswith("wav-") + assert native_results[0]["text"] == text + + +@pytest.mark.asyncio +async def test_duplex_auto_response_empty_terminal_does_not_create_empty_response(): + handler = OmniDuplexSessionHandler(chat_service=FakeChatService(FakeEngineClient())) + request_id = "duplex-sid-native-empty-terminal-e0-stage0" + session = DuplexSession( + session_id="sid-native-empty-terminal", + config=DuplexSessionConfig(extra_body={"auto_response": True}), + ) + session.mark_audio_sent(1280) + sent: list[dict[str, Any]] = [] + + async def send_json(payload: dict[str, Any]) -> None: + sent.append(payload) + + output = SimpleNamespace( + request_id=request_id, + finished=False, + outputs=[SimpleNamespace(text="hello", multimodal_output={})], + multimodal_output={ + "sr": 24000, + "meta": {"turn_end": True, "duplex_turn_id": 0}, + }, + ) + + native_results = list( + handler._minicpmo_data_plane.project( + {"data_plane_outputs": [output]}, + context=_data_plane_context(session), + ) + ) + assert len(native_results) == 1 + assert native_results[0]["end_of_turn"] is True + assert native_results[0]["model_turn_id"] == 0 + + for native_result in native_results: + await handler._send_one_native_duplex_event(send_json, native_result, session=session) + + assert "response.created" not in [m.get("type") for m in sent] + assert "response.done" not in [m.get("type") for m in sent] + assert session.turn_id == 1 + assert not handler._minicpmo_data_plane.is_terminal(request_id) + + +@pytest.mark.asyncio +async def test_minicpmo_native_duplex_session_close_cleans_auto_response_data_plane_state(): + session_id = "sid-native-cleanup" + request_id = f"duplex-{session_id}-e0-stage0" + engine = FakeEngineClient() + chat_service = FakeChatService(engine) + handler = OmniDuplexSessionHandler(chat_service=chat_service, config_timeout_s=0.1, idle_timeout_s=1) + handler._minicpmo_data_plane.slice_cumulative_audio(request_id, np.zeros(24000, dtype=np.float32)) + handler._minicpmo_data_plane.segment_text_delta(request_id, "hello") + handler._minicpmo_data_plane.mark_terminal(request_id) + native = MiniCPMO45ServingSessionState() + handler._minicpmo_sessions[session_id] = native + + event = _native_session_create(session_id) + event["session"]["extra_body"]["auto_response"] = True + ws = TimedWebSocket() + ws.put(event) + ws.put({"type": "session.close"}) + + await handler.handle_session(ws) + + assert not handler._minicpmo_data_plane.has_request(request_id) + assert session_id not in handler._minicpmo_sessions + + +@pytest.mark.asyncio +async def test_duplex_chat_audio_stream_uses_output_audio_delta_event(): + engine = FakeEngineClient() + chat_service = FakeChatService(engine) + handler = OmniDuplexSessionHandler(chat_service=chat_service, config_timeout_s=0.1, idle_timeout_s=1) + session = DuplexSession(session_id="sid-chat-audio", config=DuplexSessionConfig()) + response_id = session.begin_response() + sent: list[dict[str, Any]] = [] + + async def send_json(data: dict[str, Any]) -> None: + sent.append(data) + + await handler._emit_chat_payload( + session, + { + "modality": "audio", + "choices": [ + { + "delta": { + "content": "AAAA", + } + } + ], + }, + session.epoch, + response_id, + send_json, + ) + + assert sent == [ + { + "type": "response.output_audio.delta", + "session_id": "sid-chat-audio", + "response_id": response_id, + "epoch": 0, + "audio": "AAAA", + "format": "wav", + } + ] + + +@pytest.mark.asyncio +async def test_duplex_chat_rejects_unknown_output_modality(): + handler = OmniDuplexSessionHandler( + chat_service=FakeChatService(FakeEngineClient()), + config_timeout_s=0.1, + idle_timeout_s=1, + ) + session = DuplexSession(session_id="sid-chat-unknown", config=DuplexSessionConfig()) + response_id = session.begin_response() + sent: list[dict[str, Any]] = [] + + async def send_json(data: dict[str, Any]) -> None: + sent.append(data) + + await handler._emit_chat_payload( + session, + { + "modality": "video", + "choices": [{"delta": {"content": "not-text"}}], + }, + session.epoch, + response_id, + send_json, + ) + + assert sent == [ + { + "type": "error", + "session_id": session.session_id, + "response_id": response_id, + "epoch": session.epoch, + "code": "unsupported_response_modality", + "error": "Unsupported chat response modality: video", + } + ] + assert session.assistant_text_buffer == () + + +@pytest.mark.asyncio +async def test_minicpmo_model_name_does_not_auto_enable_experimental_native_duplex(): + event = _session_create("sid-minicpmo-default-protocol") + event["session"]["model"] = "openbmb/MiniCPM-o-4_5" + engine = FakeEngineClient() + chat_service = FakeChatService(engine) + handler = OmniDuplexSessionHandler(chat_service=chat_service, config_timeout_s=0.1, idle_timeout_s=1) + ws = TimedWebSocket() + ws.put(event) + ws.put({"type": "session.close"}) + + await handler.handle_session(ws) + + created = next(m for m in ws.sent if m.get("type") == "session.created") + capabilities = created["session"]["capabilities"] + assert capabilities["implementation_level"] == "serving_session_adapter" + assert capabilities["supports_input_append"] is False + + +@pytest.mark.asyncio +async def test_duplex_handler_aborts_current_chat_request_id_on_barge_in(): + engine = FakeEngineClient() + chat_service = FakeChatService(engine) + handler = OmniDuplexSessionHandler(chat_service=chat_service, config_timeout_s=0.1, idle_timeout_s=1) + + def on_send(ws: TimedWebSocket, data: dict[str, Any]) -> None: + if data.get("type") == "response.created": + ws.put({"type": "input.cancel", "reason": "test_barge_in"}) + + ws = TimedWebSocket(on_send=on_send) + ws.put(_session_create("sid-a")) + ws.put({"type": "input.text.append", "text": "hello"}) + ws.put({"type": "input.commit"}) + + await handler.handle_session(ws) + + assert ws.accepted + assert "response.created" in ws.sent_types() + assert "audio.cancelled" in ws.sent_types() + assert chat_service.seen_request_ids == ["duplex-sid-a-0-1"] + assert engine.aborted == ["chatcmpl-duplex-sid-a-0-1"] + assert engine.internal_abort_batches == [["chatcmpl-duplex-sid-a-0-1"]] + + +@pytest.mark.asyncio +async def test_duplex_handler_runtime_open_failure_is_reported_to_client(): + engine = FakeEngineClient(fail_open=True) + chat_service = FakeChatService(engine) + handler = OmniDuplexSessionHandler(chat_service=chat_service, config_timeout_s=0.1, idle_timeout_s=1) + ws = TimedWebSocket() + ws.put(_session_create("sid-open-fail")) + ws.put({"type": "session.close"}) + + await handler.handle_session(ws) + + assert ws.sent_types() == ["error"] + assert ws.sent[0]["code"] == "runtime_open_failed" + assert engine.opened == [] + + +@pytest.mark.asyncio +async def test_duplex_handler_preserves_typed_admission_error(): + class AdmissionRejectingEngine(FakeEngineClient): + async def open_duplex_session_async(self, session_id: str, **kwargs): + del kwargs + raise DuplexControlRequestError( + { + "operation": "open", + "session_id": session_id, + "error": { + "code": "resource_exhausted", + "message": "duplex session capacity exhausted", + "retryable": True, + }, + "accepted_fence": None, + "lease_generation": None, + } + ) + + handler = OmniDuplexSessionHandler( + chat_service=FakeChatService(AdmissionRejectingEngine()), + config_timeout_s=0.1, + idle_timeout_s=1, + ) + ws = TimedWebSocket() + ws.put(_session_create("sid-admission-rejected")) + + await handler.handle_session(ws) + + assert ws.sent_types() == ["error"] + assert ws.sent[0]["code"] == "resource_exhausted" + assert ws.sent[0]["retryable"] is True + + +@pytest.mark.asyncio +async def test_duplex_cancel_reports_playback_committed_cursor(): + engine = FakeEngineClient() + chat_service = FakeChatService(engine) + handler = OmniDuplexSessionHandler(chat_service=chat_service, config_timeout_s=0.1, idle_timeout_s=1) + + def on_send(ws: TimedWebSocket, data: dict[str, Any]) -> None: + if data.get("type") == "response.created": + ws.put({"type": "playback.ack", "played_ms": 1200, "committed_ms": 1000}) + ws.put({"type": "input.cancel", "reason": "test_barge_in"}) + + ws = TimedWebSocket(on_send=on_send) + ws.put(_session_create("sid-playback")) + ws.put({"type": "input.text.append", "text": "hello"}) + ws.put({"type": "input.commit"}) + + await handler.handle_session(ws) + + ack = next(m for m in ws.sent if m.get("type") == "playback.acknowledged") + cancelled = next(m for m in ws.sent if m.get("type") == "audio.cancelled") + assert ack["playback"]["played_ms"] == 1200 + assert ack["playback"]["committed_ms"] == 1000 + assert cancelled["committed_ms"] == 1000 + assert cancelled["epoch"] == 1 + + +@pytest.mark.asyncio +async def test_playback_ack_response_id_selects_matching_pending_history_item(): + handler = OmniDuplexSessionHandler( + chat_service=FakeChatService(FakeEngineClient()), + config_timeout_s=0.1, + idle_timeout_s=1, + ) + session = DuplexSession( + session_id="sid-playback-response-id", + config=DuplexSessionConfig(), + ) + first_item_id = "item_resp-first" + second_item_id = "item_resp-second" + session.stage_pending_history_item( + first_item_id, + {"role": "assistant", "content": "first response"}, + ) + session.stage_pending_history_item( + second_item_id, + {"role": "assistant", "content": "second response"}, + ) + ws = TimedWebSocket() + + await handler._handle_playback_ack( + session, + { + "type": "playback.ack", + "response_id": "resp-second", + "played_ms": 1000, + "committed_ms": 1000, + }, + ws.send_json, + ) + + ack = next(message for message in ws.sent if message.get("type") == "playback.acknowledged") + assert ack["item_id"] == second_item_id + assert ack["history_committed"] is True + assert first_item_id in session.pending_history_item_ids + assert second_item_id not in session.pending_history_item_ids + + +@pytest.mark.asyncio +async def test_late_playback_ack_does_not_advance_new_response_cursor(): + handler = OmniDuplexSessionHandler( + chat_service=FakeChatService(FakeEngineClient()), + config_timeout_s=0.1, + idle_timeout_s=1, + ) + session = DuplexSession( + session_id="sid-late-playback-ack", + config=DuplexSessionConfig(), + ) + first_response_id = session.begin_response() + session.append_assistant_text("first response") + session.mark_audio_sent(11480, text_chars=len("first response")) + session.end_response(commit_text=False, preserve_request=True) + session.register_history_item(f"item_{first_response_id}", None) + + second_response_id = session.begin_response() + session.mark_audio_sent(2200, text_chars=len("second response")) + ws = TimedWebSocket() + + await handler._handle_playback_ack( + session, + { + "type": "playback.ack", + "response_id": first_response_id, + "item_id": f"item_{first_response_id}", + "played_ms": 11480, + "committed_ms": 11480, + }, + ws.send_json, + ) + + assert session.active_response_id == second_response_id + assert session.playback.as_dict() == { + "generated_ms": 2200, + "sent_ms": 2200, + "played_ms": 0, + "committed_ms": 0, + } + ack = next(message for message in ws.sent if message.get("type") == "playback.acknowledged") + assert ack["playback"] == { + "generated_ms": 11480, + "sent_ms": 11480, + "played_ms": 11480, + "committed_ms": 11480, + } + assert first_response_id not in session.response_playbacks + assert session.response_playbacks[second_response_id] == session.playback + + +@pytest.mark.asyncio +async def test_late_playback_ack_truncates_history_with_matching_response_cursor(): + handler = OmniDuplexSessionHandler( + chat_service=FakeChatService(FakeEngineClient()), + config_timeout_s=0.1, + idle_timeout_s=1, + ) + session = DuplexSession( + session_id="sid-late-playback-history", + config=DuplexSessionConfig(), + ) + first_response_id = session.begin_response() + session.append_assistant_text("abcdefghij") + session.mark_audio_sent(1000) + session.end_response(commit_text=False, preserve_request=True) + session.register_history_item(f"item_{first_response_id}", None) + + session.begin_response() + session.mark_audio_sent(10000) + ws = TimedWebSocket() + + await handler._handle_playback_ack( + session, + { + "type": "playback.ack", + "response_id": first_response_id, + "item_id": f"item_{first_response_id}", + "played_ms": 500, + "committed_ms": 500, + }, + ws.send_json, + ) + + assert session.history[-1] == {"role": "assistant", "content": "abcde"} + assert first_response_id in session.response_playbacks + + +@pytest.mark.asyncio +async def test_cancel_active_response_commits_only_played_assistant_history(): + engine = FakeEngineClient() + chat_service = FakeChatService(engine) + handler = OmniDuplexSessionHandler(chat_service=chat_service, config_timeout_s=0.1, idle_timeout_s=1) + ws = TimedWebSocket() + session = DuplexSession( + session_id="sid-playback-partial", + config=DuplexSessionConfig(playback_commit_policy=DuplexPlaybackCommitPolicy.COMMIT_ALL_ON_DONE.value), + ) + response_id = session.begin_response() + session.append_assistant_text("hello world") + session.mark_audio_sent(1000, text_chars=len("hello world")) + session.acknowledge_playback(played_ms=500, committed_ms=500) + + cancelled = await handler._cancel_active_response(session, None, ws.send_json, reason="barge_in") + + assert cancelled is True + assert session.epoch == 1 + assert session.history == ({"role": "assistant", "content": "hello"},) + assert session.history_item_ids[f"item_{response_id}"] == session.history[0] + event = next(m for m in ws.sent if m.get("type") == "audio.cancelled") + assert event["committed_ms"] == 500 + + +@pytest.mark.asyncio +async def test_cancel_active_response_keeps_truncated_history_in_serving(): + engine = FakeEngineClient() + chat_service = FakeChatService(engine) + handler = OmniDuplexSessionHandler(chat_service=chat_service, config_timeout_s=0.1, idle_timeout_s=1) + ws = TimedWebSocket() + session = DuplexSession( + session_id="sid-playback-runtime-rebuild", + config=DuplexSessionConfig(playback_commit_policy=DuplexPlaybackCommitPolicy.COMMIT_ALL_ON_DONE.value), + ) + response_id = session.begin_response() + session.append_assistant_text("hello world") + session.mark_audio_sent(1000, text_chars=len("hello world")) + session.acknowledge_playback(played_ms=500, committed_ms=500) + + cancelled = await handler._cancel_active_response( + session, + None, + ws.send_json, + reason="barge_in", + ) + + assert cancelled is True + assert session.history == ({"role": "assistant", "content": "hello"},) + assert session.history_item_ids[f"item_{response_id}"] == session.history[0] + assert engine.signals == [] + assert "error" not in ws.sent_types() + + +@pytest.mark.asyncio +async def test_cancel_active_native_data_plane_request_aborts_stage_request_id(): + engine = FakeEngineClient() + chat_service = FakeChatService(engine) + handler = OmniDuplexSessionHandler(chat_service=chat_service, config_timeout_s=0.1, idle_timeout_s=1) + ws = TimedWebSocket() + session = DuplexSession(session_id="sid-native-abort", config=DuplexSessionConfig()) + session.bind_request("duplex-sid-native-abort-e0-stage0-s1") + + cancelled = await handler._cancel_active_response(session, None, ws.send_json, reason="barge_in") + + assert cancelled is True + assert engine.aborted == ["duplex-sid-native-abort-e0-stage0-s1"] + assert engine.internal_abort_batches == [["duplex-sid-native-abort-e0-stage0-s1"]] + assert "audio.cancelled" in ws.sent_types() + + +@pytest.mark.asyncio +async def test_duplex_handler_explicit_close_closes_runtime_once_with_client_reason(): + engine = FakeEngineClient() + chat_service = FakeChatService(engine) + handler = OmniDuplexSessionHandler(chat_service=chat_service, config_timeout_s=0.1, idle_timeout_s=1) + ws = TimedWebSocket() + ws.put(_session_create("sid-close")) + ws.put({"type": "session.close"}) + + await handler.handle_session(ws) + + assert ws.sent_types() == ["session.created", "session.closed"] + assert engine.closed == [("sid-close", "session_close")] + + +@pytest.mark.asyncio +async def test_realtime_disconnect_detaches_resumable_session_without_runtime_close(): + engine = FakeEngineClient() + handler = OmniDuplexSessionHandler( + chat_service=FakeChatService(engine), + config_timeout_s=0.1, + idle_timeout_s=1, + ) + ws = TimedWebSocket(receive_timeout_s=0.01) + ws.put(_native_realtime_session_update("sid-resumable-disconnect")) + + await handler.handle_realtime_session(ws) + + created = next(message for message in ws.sent if message.get("type") == "session.created") + assert created["session"]["id"] == "sid-resumable-disconnect" + assert isinstance(created["resume_token"], str) + assert created["resume_token"] + assert created["attachment_generation"] == 1 + assert engine.closed == [] + assert engine.touched == [ + ("sid-resumable-disconnect", DuplexLeaseActivity.DETACH), + ] + assert handler._registry.get("sid-resumable-disconnect") is not None + assert "sid-resumable-disconnect" in handler._minicpmo_sessions + + +@pytest.mark.asyncio +async def test_realtime_initial_resume_token_delivery_failure_closes_unrecoverable_session(): + class FailingCreatedWebSocket(TimedWebSocket): + async def send_json(self, data: dict[str, Any]): + if data.get("type") == "session.created": + raise WebSocketDisconnect(code=1006) + await super().send_json(data) + + engine = FakeEngineClient() + handler = OmniDuplexSessionHandler( + chat_service=FakeChatService(engine), + config_timeout_s=0.1, + idle_timeout_s=1, + ) + ws = FailingCreatedWebSocket(receive_timeout_s=0.01) + ws.put(_native_realtime_session_update("sid-created-delivery-failure")) + + await handler.handle_realtime_session(ws) + + assert engine.closed == [("sid-created-delivery-failure", "disconnect")] + assert handler._registry.get("sid-created-delivery-failure") is None + assert "sid-created-delivery-failure" not in handler._minicpmo_sessions + + +@pytest.mark.asyncio +async def test_realtime_send_after_created_disconnect_preserves_resumable_session(): + class ClosedAfterCreatedWebSocket(TimedWebSocket): + def __init__(self) -> None: + super().__init__(receive_timeout_s=0.01) + self.transport_closed = False + + async def send_json(self, data: dict[str, Any]): + if self.transport_closed: + raise RuntimeError( + "Unexpected ASGI message 'websocket.send', " + "after sending 'websocket.close' or response already completed." + ) + await super().send_json(data) + if data.get("type") == "session.created": + self.transport_closed = True + + engine = FakeEngineClient() + handler = OmniDuplexSessionHandler( + chat_service=FakeChatService(engine), + config_timeout_s=0.1, + idle_timeout_s=1, + ) + ws = ClosedAfterCreatedWebSocket() + ws.put(_native_realtime_session_update("sid-created-then-disconnected")) + + await handler.handle_realtime_session(ws) + + assert engine.closed == [] + assert engine.touched == [ + ("sid-created-then-disconnected", DuplexLeaseActivity.DETACH), + ] + assert handler._registry.get("sid-created-then-disconnected") is not None + assert "sid-created-then-disconnected" in handler._minicpmo_sessions + + +@pytest.mark.asyncio +async def test_realtime_resume_rotates_token_replays_and_preserves_runtime_identity(): + engine = FakeEngineClient() + handler = OmniDuplexSessionHandler( + chat_service=FakeChatService(engine), + config_timeout_s=0.1, + idle_timeout_s=1, + ) + first = TimedWebSocket(receive_timeout_s=0.01) + first.put(_native_realtime_session_update("sid-realtime-resume")) + first.put({"type": "session.heartbeat"}) + + await handler.handle_realtime_session(first) + + created = next(message for message in first.sent if message.get("type") == "session.created") + heartbeat = next(message for message in first.sent if message.get("type") == "session.heartbeat_ack") + token = created["resume_token"] + incarnation = created["incarnation"] + second = TimedWebSocket(receive_timeout_s=0.1) + second.query_params = { + "model": "openbmb/MiniCPM-o-4_5", + "minicpmo45_native_duplex": "1", + "resume": "1", + } + second.put( + { + "type": "session.resume", + "session_id": "sid-realtime-resume", + "incarnation": incarnation, + "resume_token": token, + "last_received_server_event_seq": heartbeat["server_event_seq"] - 1, + } + ) + second.put({"type": "session.close"}) + + await handler.handle_realtime_session(second) + + resumed = next(message for message in second.sent if message.get("type") == "session.resumed") + replayed = [ + message + for message in second.sent + if message.get("type") == "session.heartbeat_ack" + and message.get("server_event_seq") == heartbeat["server_event_seq"] + ] + assert resumed["session_id"] == "sid-realtime-resume" + assert resumed["incarnation"] == incarnation + assert resumed["attachment_generation"] == 2 + assert resumed["resume_token"] != token + assert replayed == [heartbeat] + assert engine.opened == ["sid-realtime-resume"] + assert engine.resumed == [("sid-realtime-resume", 0)] + assert engine.resume_fences[0] == engine.opened_fences[0] + assert engine.closed == [("sid-realtime-resume", "session_close")] + + +@pytest.mark.asyncio +async def test_realtime_resume_preserves_append_tail_order_across_connections(): + class SlowFirstAppendEngine(FakeEngineClient): + def __init__(self) -> None: + super().__init__() + self.first_append_started = asyncio.Event() + self.second_append_started = asyncio.Event() + self.release_first_append = asyncio.Event() + self.append_start_order: list[int] = [] + + async def append_duplex_input_async(self, session_id: str, **kwargs): + call_index = len(self.append_start_order) + 1 + self.append_start_order.append(call_index) + if call_index == 1: + self.first_append_started.set() + await self.release_first_append.wait() + else: + self.second_append_started.set() + kwargs.pop("expected_epoch", None) + return await super().append_duplex_input_async(session_id, **kwargs) + + engine = SlowFirstAppendEngine() + handler = OmniDuplexSessionHandler( + chat_service=FakeChatService(engine), + config_timeout_s=0.1, + idle_timeout_s=1, + ) + first = TimedWebSocket(receive_timeout_s=1.0) + session_update = _native_realtime_session_update("sid-resume-append-order") + session_update["session"]["extra_body"]["auto_response"] = True + first.put(session_update) + first_task = asyncio.create_task(handler.handle_realtime_session(first)) + for _ in range(100): + created = next((message for message in first.sent if message.get("type") == "session.created"), None) + if created is not None: + break + await asyncio.sleep(0.01) + else: + pytest.fail("resumable session did not open") + session = handler._registry.get("sid-resume-append-order") + assert session is not None + native_state = handler._minicpmo_session_state(session) + first_scheduler = native_state.silence_continuation_scheduler + assert first_scheduler is not None + first.put( + { + "type": "input_audio_buffer.append", + "audio": _pcm16_b64(16000), + "sample_rate_hz": 16000, + "is_speech": True, + } + ) + await asyncio.wait_for(engine.first_append_started.wait(), timeout=1) + await first.close(code=1006) + await asyncio.wait_for(first_task, timeout=1) + + second = TimedWebSocket(receive_timeout_s=0.5) + second.query_params = { + "model": "openbmb/MiniCPM-o-4_5", + "minicpmo45_native_duplex": "1", + "resume": "1", + } + second.put( + { + "type": "session.resume", + "session_id": "sid-resume-append-order", + "incarnation": created["incarnation"], + "resume_token": created["resume_token"], + "last_received_server_event_seq": created.get("server_event_seq", 0), + } + ) + second_task = asyncio.create_task(handler.handle_realtime_session(second)) + for _ in range(100): + if "session.resumed" in second.sent_types(): + break + await asyncio.sleep(0.01) + else: + pytest.fail("session did not resume") + assert native_state.silence_continuation_scheduler is not first_scheduler + second.put( + { + "type": "input_audio_buffer.append", + "audio": _pcm16_b64(16000), + "sample_rate_hz": 16000, + "is_speech": True, + } + ) + try: + await asyncio.sleep(0.05) + assert not engine.second_append_started.is_set() + finally: + engine.release_first_append.set() + + await asyncio.wait_for(engine.second_append_started.wait(), timeout=1) + second.put({"type": "session.close"}) + await asyncio.wait_for(second_task, timeout=2) + assert engine.append_start_order == [1, 2] + + +@pytest.mark.asyncio +async def test_realtime_takeover_ignores_late_detach_from_replaced_attachment(): + engine = FakeEngineClient() + handler = OmniDuplexSessionHandler( + chat_service=FakeChatService(engine), + config_timeout_s=0.1, + idle_timeout_s=1, + ) + first = TimedWebSocket(receive_timeout_s=1.0) + first.put(_native_realtime_session_update("sid-takeover")) + first_task = asyncio.create_task(handler.handle_realtime_session(first)) + for _ in range(100): + created = next((message for message in first.sent if message.get("type") == "session.created"), None) + if created is not None: + break + await asyncio.sleep(0.001) + else: + raise AssertionError("first attachment did not create the session") + + second = TimedWebSocket(receive_timeout_s=0.03) + second.put( + { + "type": "session.resume", + "session_id": "sid-takeover", + "incarnation": created["incarnation"], + "resume_token": created["resume_token"], + "last_received_server_event_seq": 0, + } + ) + await handler.handle_realtime_session(second) + await first_task + + detach_touches = [item for item in engine.touched if item[1] is DuplexLeaseActivity.DETACH] + assert detach_touches == [("sid-takeover", DuplexLeaseActivity.DETACH)] + assert "session.replaced" in first.sent_types() + assert engine.resumed == [("sid-takeover", 0)] + assert engine.closed == [] + + +@pytest.mark.asyncio +async def test_realtime_takeover_does_not_clear_new_attachment_pending_turns(): + class DelayedReplacementCloseWebSocket(TimedWebSocket): + def __init__(self) -> None: + super().__init__(receive_timeout_s=1.0) + self.replacement_close_started = asyncio.Event() + self.release_replacement_close = asyncio.Event() + + async def close(self, code: int = 1000, reason: str | None = None) -> None: + if not self.replacement_close_started.is_set(): + self.replacement_close_started.set() + await self.release_replacement_close.wait() + await super().close(code=code, reason=reason) + + engine = FakeEngineClient() + handler = OmniDuplexSessionHandler( + chat_service=FakeChatService(engine), + config_timeout_s=0.1, + idle_timeout_s=1, + ) + first = DelayedReplacementCloseWebSocket() + first.put(_native_realtime_session_update("sid-takeover-pending-turn")) + first_task = asyncio.create_task(handler.handle_realtime_session(first)) + for _ in range(100): + created = next((message for message in first.sent if message.get("type") == "session.created"), None) + if created is not None: + break + await asyncio.sleep(0.001) + else: + pytest.fail("first attachment did not create the session") + + second = TimedWebSocket(receive_timeout_s=1.0) + second.put( + { + "type": "session.resume", + "session_id": "sid-takeover-pending-turn", + "incarnation": created["incarnation"], + "resume_token": created["resume_token"], + "last_received_server_event_seq": 0, + } + ) + second_task = asyncio.create_task(handler.handle_realtime_session(second)) + await asyncio.wait_for(first.replacement_close_started.wait(), timeout=1) + + session = handler._registry.get("sid-takeover-pending-turn") + assert session is not None + assert session.reserve_pending_turn(limit=2) + first.release_replacement_close.set() + await asyncio.wait_for(first_task, timeout=1) + + assert session.pending_input_turns == 1 + session.release_pending_turn() + second.put({"type": "session.close"}) + await asyncio.wait_for(second_task, timeout=2) + + +@pytest.mark.asyncio +async def test_realtime_takeover_drops_input_queued_by_replaced_attachment(): + class DelayedReplacementCloseWebSocket(TimedWebSocket): + def __init__(self) -> None: + super().__init__(receive_timeout_s=1.0) + self.replacement_close_started = asyncio.Event() + self.release_replacement_close = asyncio.Event() + self.stale_heartbeat_read = asyncio.Event() + + async def receive_text(self) -> str: + payload = await super().receive_text() + if json.loads(payload).get("type") == "session.heartbeat": + self.stale_heartbeat_read.set() + return payload + + async def close(self, code: int = 1000, reason: str | None = None) -> None: + if not self.replacement_close_started.is_set(): + self.replacement_close_started.set() + await self.release_replacement_close.wait() + await super().close(code=code, reason=reason) + + engine = FakeEngineClient() + handler = OmniDuplexSessionHandler( + chat_service=FakeChatService(engine), + config_timeout_s=0.1, + idle_timeout_s=1, + ) + first = DelayedReplacementCloseWebSocket() + first.put(_native_realtime_session_update("sid-takeover-stale-input")) + first_task = asyncio.create_task(handler.handle_realtime_session(first)) + for _ in range(100): + created = next((message for message in first.sent if message.get("type") == "session.created"), None) + if created is not None: + break + await asyncio.sleep(0.001) + else: + pytest.fail("first attachment did not create the session") + + second = TimedWebSocket(receive_timeout_s=1.0) + second.put( + { + "type": "session.resume", + "session_id": "sid-takeover-stale-input", + "incarnation": created["incarnation"], + "resume_token": created["resume_token"], + "last_received_server_event_seq": 0, + } + ) + second_task = asyncio.create_task(handler.handle_realtime_session(second)) + await asyncio.wait_for(first.replacement_close_started.wait(), timeout=1) + + first.put({"type": "session.heartbeat"}) + await asyncio.wait_for(first.stale_heartbeat_read.wait(), timeout=1) + first.release_replacement_close.set() + await asyncio.wait_for(first_task, timeout=1) + + second.put({"type": "session.close"}) + await asyncio.wait_for(second_task, timeout=2) + assert all(activity is not DuplexLeaseActivity.HEARTBEAT for _, activity in engine.touched) + assert "session.heartbeat_ack" not in second.sent_types() + + +@pytest.mark.asyncio +async def test_realtime_resume_rejects_invalid_token_before_engine_control(): + engine = FakeEngineClient() + handler = OmniDuplexSessionHandler( + chat_service=FakeChatService(engine), + config_timeout_s=0.1, + idle_timeout_s=1, + ) + first = TimedWebSocket(receive_timeout_s=0.01) + first.put(_native_realtime_session_update("sid-invalid-resume")) + await handler.handle_realtime_session(first) + + created = next(message for message in first.sent if message.get("type") == "session.created") + second = TimedWebSocket(receive_timeout_s=0.01) + second.put( + { + "type": "session.resume", + "session_id": "sid-invalid-resume", + "incarnation": created["incarnation"], + "resume_token": "invalid-token", + "last_received_server_event_seq": 0, + } + ) + + await handler.handle_realtime_session(second) + + error = next(message for message in second.sent if message.get("type") == "error") + assert error["error"]["code"] == "invalid_resume_token" + assert engine.resumed == [] + assert engine.closed == [] + + +@pytest.mark.asyncio +async def test_realtime_resume_recovers_when_rotated_token_delivery_is_lost(): + class FailingResumeWebSocket(TimedWebSocket): + async def send_json(self, data: dict[str, Any]): + if data.get("type") == "session.resumed": + raise RuntimeError("resume transport disappeared") + await super().send_json(data) + + engine = FakeEngineClient() + handler = OmniDuplexSessionHandler( + chat_service=FakeChatService(engine), + config_timeout_s=0.1, + idle_timeout_s=1, + ) + first = TimedWebSocket(receive_timeout_s=0.01) + first.put(_native_realtime_session_update("sid-token-delivery")) + await handler.handle_realtime_session(first) + created = next(message for message in first.sent if message.get("type") == "session.created") + resume_event = { + "type": "session.resume", + "session_id": "sid-token-delivery", + "incarnation": created["incarnation"], + "resume_token": created["resume_token"], + "last_received_server_event_seq": 0, + } + + failed_transport = FailingResumeWebSocket(receive_timeout_s=0.01) + failed_transport.put(resume_event) + await handler.handle_realtime_session(failed_transport) + + recovered_transport = TimedWebSocket(receive_timeout_s=0.1) + recovered_transport.put(resume_event) + recovered_transport.put({"type": "session.close"}) + await handler.handle_realtime_session(recovered_transport) + + assert "session.resumed" in recovered_transport.sent_types() + assert engine.resumed == [ + ("sid-token-delivery", 0), + ("sid-token-delivery", 1), + ] + assert engine.closed == [("sid-token-delivery", "session_close")] + + +@pytest.mark.asyncio +async def test_realtime_disconnect_grace_cancels_only_orphan_response(): + engine = FakeEngineClient() + handler = OmniDuplexSessionHandler( + chat_service=FakeChatService(engine), + config_timeout_s=0.1, + idle_timeout_s=1, + duplex_session_config=DuplexSessionRuntimeConfig(disconnect_grace_s=0.02), + ) + ws = TimedWebSocket(receive_timeout_s=0.01) + ws.put(_native_realtime_session_update("sid-grace-response")) + + await handler.handle_realtime_session(ws) + session = handler._registry.get("sid-grace-response") + assert session is not None + session.begin_response() + session.bind_request("orphan-request") + orphan_task = asyncio.create_task(asyncio.sleep(10)) + handler._session_tasks[session.session_id].active_response_task = orphan_task + assert engine.closed == [] + + await asyncio.sleep(0.05) + + assert engine.closed == [] + assert engine.aborted == ["orphan-request"] + assert orphan_task.cancelled() + assert session.active_response_id is None + assert "sid-grace-response" in handler._minicpmo_sessions + + +@pytest.mark.asyncio +async def test_realtime_heartbeat_control_failure_is_non_terminal(): + engine = FakeEngineClient(fail_touch=True) + handler = OmniDuplexSessionHandler( + chat_service=FakeChatService(engine), + config_timeout_s=0.1, + idle_timeout_s=1, + ) + ws = TimedWebSocket(receive_timeout_s=0.1) + ws.put(_native_realtime_session_update("sid-heartbeat-failure")) + ws.put({"type": "session.heartbeat"}) + ws.put({"type": "session.close"}) + + await handler.handle_realtime_session(ws) + + error = next(message for message in ws.sent if message.get("type") == "error") + assert error["error"]["code"] == "runtime_touch_failed" + assert engine.closed == [("sid-heartbeat-failure", "session_close")] + + +@pytest.mark.asyncio +async def test_realtime_idle_ttl_lifecycle_removes_detached_serving_projection(): + engine = FakeEngineClient() + handler = OmniDuplexSessionHandler( + chat_service=FakeChatService(engine), + config_timeout_s=0.1, + idle_timeout_s=1, + ) + ws = TimedWebSocket(receive_timeout_s=0.01) + ws.put(_native_realtime_session_update("sid-idle-expired")) + + await handler.handle_realtime_session(ws) + fence = engine.opened_fences[0] + assert fence is not None + await engine.duplex_lifecycle_events.put( + DuplexSessionLifecycleMessage( + fence=fence, + session_id="sid-idle-expired", + event="expired", + reason="idle_ttl_expired", + lease_generation=1, + submitted_request_ids=[], + reserved_request_ids=[], + ) + ) + await asyncio.sleep(0.05) + + assert handler._registry.get("sid-idle-expired") is None + assert "sid-idle-expired" not in handler._minicpmo_sessions + assert "sid-idle-expired" not in handler._session_tasks + assert engine.closed == [] + + +@pytest.mark.asyncio +async def test_realtime_resume_reports_resync_required_after_journal_ttl_gap(): + engine = FakeEngineClient() + handler = OmniDuplexSessionHandler( + chat_service=FakeChatService(engine), + config_timeout_s=0.1, + idle_timeout_s=1, + duplex_session_config=DuplexSessionRuntimeConfig(resume_replay_ttl_s=0.01), + ) + first = TimedWebSocket(receive_timeout_s=0.01) + first.put(_native_realtime_session_update("sid-journal-gap")) + first.put({"type": "session.heartbeat"}) + await handler.handle_realtime_session(first) + created = next(message for message in first.sent if message.get("type") == "session.created") + await asyncio.sleep(0.02) + + second = TimedWebSocket(receive_timeout_s=0.01) + second.put( + { + "type": "session.resume", + "session_id": "sid-journal-gap", + "incarnation": created["incarnation"], + "resume_token": created["resume_token"], + "last_received_server_event_seq": 0, + } + ) + await handler.handle_realtime_session(second) + + assert "session.resync_required" in second.sent_types() + assert engine.resumed == [] + assert engine.closed == [] + + +@pytest.mark.asyncio +async def test_realtime_journal_overflow_degrades_to_live_resync_without_closing_runtime(): + engine = FakeEngineClient() + handler = OmniDuplexSessionHandler( + chat_service=FakeChatService(engine), + config_timeout_s=0.1, + idle_timeout_s=1, + duplex_session_config=DuplexSessionRuntimeConfig(resume_replay_max_bytes_per_session=128), + ) + ws = TimedWebSocket(receive_timeout_s=0.01) + ws.put(_native_realtime_session_update("sid-journal-overflow")) + + await handler.handle_realtime_session(ws) + + assert "session.created" in ws.sent_types() + assert "session.resync_required" in ws.sent_types() + assert engine.closed == [] + assert handler._registry.get("sid-journal-overflow") is not None + + +@pytest.mark.asyncio +async def test_realtime_input_backpressure_is_per_session_and_rejects_before_mutation(): + engine = FakeEngineClient() + handler = OmniDuplexSessionHandler( + chat_service=FakeChatService(engine), + config_timeout_s=0.1, + idle_timeout_s=1, + duplex_session_config=DuplexSessionRuntimeConfig( + max_pending_input_bytes_per_session=100, + ), + ) + first = TimedWebSocket(receive_timeout_s=0.1) + first.put(_native_realtime_session_update("sid-backpressure-a")) + for _ in range(2): + first.put( + { + "type": "input_audio_buffer.append", + "audio": _pcm16_b64(20), + "format": "pcm16", + "sample_rate_hz": 16000, + } + ) + first.put({"type": "session.close"}) + + await handler.handle_realtime_session(first) + + backpressure = [ + event + for event in first.sent + if event.get("type") == "error" + and isinstance(event.get("error"), dict) + and event["error"].get("code") == "input_backpressure" + ] + assert len(backpressure) == 1 + + second = TimedWebSocket(receive_timeout_s=0.1) + second.put(_native_realtime_session_update("sid-backpressure-b")) + second.put( + { + "type": "input_audio_buffer.append", + "audio": _pcm16_b64(20), + "format": "pcm16", + "sample_rate_hz": 16000, + } + ) + second.put({"type": "session.close"}) + + await handler.handle_realtime_session(second) + + assert not any( + event.get("type") == "error" + and isinstance(event.get("error"), dict) + and event["error"].get("code") == "input_backpressure" + for event in second.sent + ) + + +@pytest.mark.asyncio +async def test_duplex_handler_idle_timeout_close_does_not_emit_runtime_control(): + control_result = { + "operation": "close", + "session_id": "sid-disconnect", + "ok": True, + "unsupported_count": 1, + "error_count": 0, + "stage_results": [{"stage_id": 0, "replica_id": 0, "result": {"supported": False}}], + } + engine = FakeEngineClient(append_result=control_result) + chat_service = FakeChatService(engine) + handler = OmniDuplexSessionHandler(chat_service=chat_service, config_timeout_s=0.1, idle_timeout_s=0.1) + ws = TimedWebSocket() + ws.put(_session_create("sid-disconnect")) + + await handler.handle_session(ws) + + assert engine.closed == [("sid-disconnect", "timeout")] + + +@pytest.mark.asyncio +async def test_idle_output_audio_clear_does_not_advance_runtime_fence(): + engine = FakeEngineClient() + handler = OmniDuplexSessionHandler( + chat_service=FakeChatService(engine), + config_timeout_s=0.1, + idle_timeout_s=1, + ) + ws = TimedWebSocket() + ws.put(_session_create("sid-idle-output-clear")) + ws.put({"type": "output_audio_buffer.clear"}) + ws.put({"type": "session.close"}) + + await handler.handle_session(ws) + + cancelled = next(event for event in ws.sent if event.get("type") == "audio.cancelled") + assert cancelled["cancelled_epoch"] == cancelled["epoch"] == 0 + assert engine.signals == [] + + +@pytest.mark.asyncio +async def test_duplex_handler_runtime_close_failure_is_reported_without_closed_ack(): + engine = FakeEngineClient(fail_close=True) + chat_service = FakeChatService(engine) + handler = OmniDuplexSessionHandler(chat_service=chat_service, config_timeout_s=0.1, idle_timeout_s=1) + ws = TimedWebSocket() + ws.put(_session_create("sid-close-fail")) + ws.put({"type": "session.close"}) + + await handler.handle_session(ws) + + assert ws.sent_types() == ["session.created", "error"] + assert ws.sent[-1]["code"] == "runtime_close_failed" + + +@pytest.mark.asyncio +async def test_duplex_handler_control_close_failure_is_reported_without_closed_ack(): + control_result = { + "operation": "close", + "session_id": "sid-control-close-fail", + "ok": False, + "unsupported_count": 0, + "error_count": 1, + "stage_results": [ + { + "stage_id": 0, + "replica_id": 0, + "result": {"supported": False, "error": "stage close failed"}, + } + ], + } + engine = FakeEngineClient(close_result=control_result) + chat_service = FakeChatService(engine) + handler = OmniDuplexSessionHandler(chat_service=chat_service, config_timeout_s=0.1, idle_timeout_s=1) + ws = TimedWebSocket() + ws.put(_session_create("sid-control-close-fail")) + ws.put({"type": "session.close"}) + + await handler.handle_session(ws) + + assert ws.sent_types() == ["session.created", "error"] + assert ws.sent[-1]["code"] == "runtime_close_failed" + assert ws.sent[-1]["runtime_control"]["error_count"] == 1 + + +@pytest.mark.asyncio +async def test_duplex_cancel_without_active_response_clears_pending_input_and_acks(): + engine = FakeEngineClient() + chat_service = FakeChatService(engine) + handler = OmniDuplexSessionHandler(chat_service=chat_service, config_timeout_s=0.1, idle_timeout_s=1) + ws = TimedWebSocket() + ws.put(_session_create("sid-pending-cancel")) + ws.put({"type": "input.text.append", "text": "hello"}) + ws.put({"type": "input.cancel", "reason": "user_cancel"}) + ws.put({"type": "input.commit"}) + + await handler.handle_session(ws) + + cancelled = next(m for m in ws.sent if m.get("type") == "input.cancelled") + assert cancelled["cancelled"] == {"text_chunks": 1, "audio_chunks": 0} + assert cancelled["epoch"] == 1 + assert chat_service.seen_request_ids == [] + assert "response.created" not in ws.sent_types() + + +@pytest.mark.asyncio +async def test_turn_signal_input_cancel_uses_epoch_fence_transition(): + engine = FakeEngineClient() + handler = OmniDuplexSessionHandler( + chat_service=FakeChatService(engine), + config_timeout_s=0.1, + idle_timeout_s=1, + ) + ws = TimedWebSocket() + ws.put(_session_create("sid-turn-signal-cancel")) + ws.put({"type": "input.text.append", "text": "discard me"}) + ws.put({"type": "turn.signal", "event": "input.cancel"}) + ws.put({"type": "session.close"}) + + await handler.handle_session(ws) + + assert engine.signal_fences[-1] == DuplexFence("sid-turn-signal-cancel") + assert engine.signal_next_fences[-1] == DuplexFence("sid-turn-signal-cancel", epoch=1) + assert any(message.get("type") == "input.cancelled" for message in ws.sent) + + +@pytest.mark.asyncio +async def test_duplex_handler_local_turn_signal_does_not_round_trip_runtime(): + engine = FakeEngineClient(fail_signal=True) + chat_service = FakeChatService(engine) + handler = OmniDuplexSessionHandler(chat_service=chat_service, config_timeout_s=0.1, idle_timeout_s=1) + ws = TimedWebSocket() + ws.put(_session_create("sid-signal-fail")) + ws.put({"type": "turn.signal", "event": "user_started"}) + ws.put({"type": "session.close"}) + + await handler.handle_session(ws) + + turn_event = next(m for m in ws.sent if m.get("type") == "turn.event") + assert turn_event["event"] == "user_started" + assert not engine.signals + assert "runtime_signal_failed" not in {m.get("code") for m in ws.sent} + + +@pytest.mark.asyncio +async def test_duplex_barge_in_aborts_active_response_when_runtime_signal_fails(): + engine = FakeEngineClient(fail_signal_events={"barge_in"}) + chat_service = FakeChatService(engine) + handler = OmniDuplexSessionHandler(chat_service=chat_service, config_timeout_s=0.1, idle_timeout_s=1) + + def on_send(ws: TimedWebSocket, data: dict[str, Any]) -> None: + if data.get("type") == "response.created": + ws.put({"type": "input.cancel", "reason": "test_barge_in"}) + + ws = TimedWebSocket(on_send=on_send) + ws.put(_session_create("sid-barge-signal-fail")) + ws.put({"type": "input.text.append", "text": "hello"}) + ws.put({"type": "input.commit"}) + + await handler.handle_session(ws) + + assert "audio.cancelled" in ws.sent_types() + assert engine.aborted == ["chatcmpl-duplex-sid-barge-signal-fail-0-1"] + assert engine.internal_abort_batches == [["chatcmpl-duplex-sid-barge-signal-fail-0-1"]] + error = next(m for m in ws.sent if m.get("type") == "error") + assert error["code"] == "runtime_signal_failed" + + +@pytest.mark.asyncio +async def test_duplex_handler_surfaces_stage_unsupported_result_to_client(): + control_result = { + "operation": "open", + "session_id": "sid-unsupported", + "ok": True, + "unsupported_count": 1, + "error_count": 0, + "stage_results": [ + { + "stage_id": 0, + "replica_id": 0, + "result": {"supported": False, "reason": "not implemented"}, + } + ], + } + engine = FakeEngineClient(open_result=control_result) + chat_service = FakeChatService(engine) + handler = OmniDuplexSessionHandler(chat_service=chat_service, config_timeout_s=0.1, idle_timeout_s=1) + ws = TimedWebSocket() + ws.put(_session_create("sid-unsupported")) + ws.put({"type": "session.close"}) + + await handler.handle_session(ws) + + created = next(m for m in ws.sent if m.get("type") == "session.created") + assert created["runtime_control"]["unsupported_count"] == 1 + + +@pytest.mark.asyncio +async def test_minicpmo_native_duplex_append_without_format_defaults_to_pcm16(): + engine = FakeEngineClient() + chat_service = FakeChatService(engine) + handler = OmniDuplexSessionHandler(chat_service=chat_service, config_timeout_s=0.1, idle_timeout_s=1) + ws = TimedWebSocket() + ws.put(_native_session_create("sid-native-default-pcm16")) + ws.put( + { + "type": "input_audio_buffer.append", + "audio": _pcm16_b64(16000), + "sample_rate_hz": 16000, + "is_speech": True, + } + ) + ws.put({"type": "session.close"}) + + await handler.handle_session(ws) + + assert len(engine.appended) == 1 + _, mode, payload, final = engine.appended[0] + assert mode == "append_audio_chunk" + assert final is False + assert isinstance(payload, dict) + assert payload["format"] == "pcm_f32le" + assert payload["sample_rate_hz"] == 16000 + samples = np.frombuffer(base64.b64decode(payload["audio"]), dtype=" None: + super().__init__() + self.append_started = asyncio.Event() + self.append_cancelled = asyncio.Event() + self.release_append = asyncio.Event() + + async def append_duplex_input_async( + self, + session_id: str, + *, + mode: str, + payload: object, + final: bool = False, + timeout: float | None = None, + collect_outputs: bool = True, + fence: DuplexFence | None = None, + ): + self.append_started.set() + try: + await self.release_append.wait() + except asyncio.CancelledError: + self.append_cancelled.set() + raise + return await super().append_duplex_input_async( + session_id, + mode=mode, + payload=payload, + final=final, + timeout=timeout, + collect_outputs=collect_outputs, + fence=fence, + ) + + engine = SlowAppendEngine() + handler = OmniDuplexSessionHandler( + chat_service=FakeChatService(engine), + config_timeout_s=0.1, + idle_timeout_s=1, + ) + ws = TimedWebSocket() + ws.put(_native_session_create("sid-cancel-slow-append")) + ws.put( + { + "type": "input_audio_buffer.append", + "audio": _pcm16_b64(16000), + "sample_rate_hz": 16000, + "is_speech": True, + } + ) + ws.put({"type": "input.cancel"}) + ws.put({"type": "session.close"}) + + handler_task = asyncio.create_task(handler.handle_session(ws)) + await asyncio.wait_for(engine.append_started.wait(), timeout=1) + try: + await asyncio.wait_for(engine.append_cancelled.wait(), timeout=0.2) + except TimeoutError: + pass + finally: + engine.release_append.set() + await asyncio.wait_for(handler_task, timeout=1) + + assert engine.append_cancelled.is_set() + assert engine.signal_fences[-1] == DuplexFence("sid-cancel-slow-append") + assert engine.signal_next_fences[-1] == DuplexFence("sid-cancel-slow-append", epoch=1) + + +@pytest.mark.asyncio +async def test_minicpmo_native_duplex_append_control_error_does_not_emit_model_delta(): + control_result = { + "operation": "append", + "session_id": "sid-native-append-error", + "ok": False, + "unsupported_count": 0, + "error_count": 1, + "stage_results": [ + { + "stage_id": -1, + "replica_id": -1, + "result": { + "supported": False, + "error": "duplex_stage_handoff_target_missing:tts", + }, + }, + ], + } + engine = FakeEngineClient(append_result=control_result) + chat_service = FakeChatService(engine) + handler = OmniDuplexSessionHandler(chat_service=chat_service, config_timeout_s=0.1, idle_timeout_s=1) + ws = TimedWebSocket() + ws.put(_native_session_create("sid-native-append-error")) + ws.put({"type": "input_audio_buffer.append", "audio": "AAAA", "format": "pcm_f32le"}) + ws.put({"type": "session.close"}) + + await handler.handle_session(ws) + + assert "response.output_audio.delta" not in ws.sent_types() + error = next(m for m in ws.sent if m.get("type") == "error") + assert error["code"] == "runtime_append_failed" + assert error["runtime_control"]["error_count"] == 1 + + +@pytest.mark.asyncio +async def test_duplex_handler_session_update_control_failure_rolls_back_config(): + control_result = { + "operation": "signal", + "session_id": "sid-control-signal-fail", + "ok": False, + "unsupported_count": 0, + "error_count": 1, + "stage_results": [ + { + "stage_id": 0, + "replica_id": 0, + "result": {"supported": False, "error": "stage signal failed"}, + } + ], + } + engine = FakeEngineClient(signal_result=control_result) + chat_service = FakeChatService(engine) + handler = OmniDuplexSessionHandler(chat_service=chat_service, config_timeout_s=0.1, idle_timeout_s=1) + ws = TimedWebSocket() + ws.put(_session_create("sid-control-signal-fail")) + ws.put( + { + "type": "turn.signal", + "event": "session.update", + "payload": {"instructions": "must roll back"}, + } + ) + ws.put({"type": "session.close"}) + + await handler.handle_session(ws) + + assert "session.updated" not in ws.sent_types() + error = next(m for m in ws.sent if m.get("type") == "error") + assert error["code"] == "runtime_signal_failed" + assert error["runtime_control"]["error_count"] == 1 + assert engine.signal_session_configs[-1]["instructions"] == "must roll back" + + +@pytest.mark.asyncio +async def test_native_session_update_commits_config_only_after_runtime_ack(): + class BlockingUpdateEngine(FakeEngineClient): + def __init__(self): + super().__init__() + self.update_started = asyncio.Event() + self.release_update = asyncio.Event() + + async def signal_duplex_turn_async(self, session_id: str, **kwargs): + if kwargs.get("event") == "session.update": + self.update_started.set() + await self.release_update.wait() + return await super().signal_duplex_turn_async(session_id, **kwargs) + + engine = BlockingUpdateEngine() + handler = OmniDuplexSessionHandler( + chat_service=FakeChatService(engine), + config_timeout_s=0.1, + idle_timeout_s=1, + ) + ws = TimedWebSocket() + ws.put(_session_create("sid-update-two-phase")) + ws.put( + { + "type": "turn.signal", + "event": "session.update", + "payload": {"instructions": "candidate instructions"}, + } + ) + + handler_task = asyncio.create_task(handler.handle_session(ws)) + await asyncio.wait_for(engine.update_started.wait(), timeout=1) + live_session = handler._registry.get("sid-update-two-phase") + assert live_session is not None + assert live_session.config.instructions != "candidate instructions" + + engine.release_update.set() + ws.put({"type": "session.close"}) + await asyncio.wait_for(handler_task, timeout=2) + + updated = next(message for message in ws.sent if message.get("type") == "session.updated") + assert updated["session"]["instructions"] == "candidate instructions" + + +@pytest.mark.asyncio +async def test_native_session_update_waits_for_prior_append_effect(): + class OrderedUpdateEngine(FakeEngineClient): + def __init__(self): + super().__init__() + self.append_started = asyncio.Event() + self.release_append = asyncio.Event() + self.update_started = asyncio.Event() + self.effects: list[str] = [] + + async def append_duplex_input_async(self, session_id: str, **kwargs): + self.effects.append("append.started") + self.append_started.set() + await self.release_append.wait() + kwargs.pop("expected_epoch", None) + result = await super().append_duplex_input_async(session_id, **kwargs) + self.effects.append("append.done") + return result + + async def signal_duplex_turn_async(self, session_id: str, **kwargs): + if kwargs.get("event") == "session.update": + self.effects.append("session.update") + self.update_started.set() + return await super().signal_duplex_turn_async(session_id, **kwargs) + + engine = OrderedUpdateEngine() + handler = OmniDuplexSessionHandler( + chat_service=FakeChatService(engine), + config_timeout_s=0.1, + idle_timeout_s=1, + ) + ws = TimedWebSocket() + ws.put(_native_session_create("sid-ordered-update")) + ws.put( + { + "type": "input_audio_buffer.append", + "audio": _pcm_f32_b64(16000, value=0.05), + "format": "pcm_f32le", + } + ) + ws.put( + { + "type": "turn.signal", + "event": "session.update", + "payload": {"temperature": 0.2}, + } + ) + + handler_task = asyncio.create_task(handler.handle_session(ws)) + await asyncio.wait_for(engine.append_started.wait(), timeout=1) + try: + await asyncio.sleep(0.05) + assert not engine.update_started.is_set() + finally: + engine.release_append.set() + ws.put({"type": "session.close"}) + await asyncio.wait_for(handler_task, timeout=2) + + assert engine.effects.index("append.done") < engine.effects.index("session.update") + + +@pytest.mark.asyncio +async def test_failed_native_append_prevents_queued_append_from_running(): + failed_result = { + "operation": "append", + "session_id": "sid-append-chain-failure", + "ok": False, + "unsupported_count": 0, + "error_count": 1, + "stage_results": [ + { + "stage_id": 0, + "replica_id": 0, + "result": {"supported": False, "error": "append failed"}, + } + ], + } + + class BlockingFailureEngine(FakeEngineClient): + def __init__(self): + super().__init__(append_result=failed_result) + self.first_started = asyncio.Event() + self.release_first = asyncio.Event() + + async def append_duplex_input_async(self, session_id: str, **kwargs): + if not self.appended: + self.first_started.set() + await self.release_first.wait() + kwargs.pop("expected_epoch", None) + return await super().append_duplex_input_async(session_id, **kwargs) + + engine = BlockingFailureEngine() + handler = OmniDuplexSessionHandler( + chat_service=FakeChatService(engine), + config_timeout_s=0.1, + idle_timeout_s=1, + ) + ws = TimedWebSocket() + create = _native_session_create("sid-append-chain-failure") + create["session"]["extra_body"]["auto_response"] = True + ws.put(create) + for _ in range(2): + ws.put( + { + "type": "input_audio_buffer.append", + "audio": _pcm_f32_b64(16_000, value=0.05), + "format": "pcm_f32le", + } + ) + + handler_task = asyncio.create_task(handler.handle_session(ws)) + await asyncio.wait_for(engine.first_started.wait(), timeout=1) + engine.release_first.set() + await asyncio.sleep(0.1) + ws.put({"type": "session.close"}) + await asyncio.wait_for(handler_task, timeout=2) + + assert len(engine.appended) == 1 + + +@pytest.mark.asyncio +async def test_failed_final_append_keeps_committed_audio_for_response_retry(): + failed_result = { + "operation": "append", + "session_id": "sid-final-append-retry", + "ok": False, + "unsupported_count": 0, + "error_count": 1, + "stage_results": [], + } + successful_result = { + "operation": "append", + "session_id": "sid-final-append-retry", + "ok": True, + "unsupported_count": 0, + "error_count": 0, + "stage_results": [], + } + engine = FakeEngineClient(append_results=[failed_result, successful_result]) + handler = OmniDuplexSessionHandler( + chat_service=FakeChatService(engine), + config_timeout_s=0.1, + idle_timeout_s=1, + ) + + def retry_after_runtime_error(ws: TimedWebSocket, data: dict[str, Any]) -> None: + if data.get("type") == "error" and data.get("code") == "runtime_append_failed": + ws.put({"type": "response.create"}) + + ws = TimedWebSocket(on_send=retry_after_runtime_error) + ws.put(_native_session_create("sid-final-append-retry")) + ws.put( + { + "type": "input_audio_buffer.append", + "audio": _pcm_f32_b64(8_000, value=0.05), + "format": "pcm_f32le", + } + ) + ws.put({"type": "input_audio_buffer.commit", "final": True}) + ws.put({"type": "response.create"}) + + handler_task = asyncio.create_task(handler.handle_session(ws)) + for _ in range(100): + if len(engine.appended) >= 2: + break + await asyncio.sleep(0.01) + ws.put({"type": "session.close"}) + await asyncio.wait_for(handler_task, timeout=2) + + assert len(engine.appended) == 2 + assert engine.appended[0][2]["audio"] == engine.appended[1][2]["audio"] + assert engine.append_operation_ids[0] == engine.append_operation_ids[1] + + +@pytest.mark.asyncio +async def test_duplex_handler_signal_unsupported_workers_with_data_plane_ack_is_not_fatal(): + control_result = { + "operation": "signal", + "session_id": "sid-control-signal-data-plane", + "ok": False, + "unsupported_count": 2, + "error_count": 0, + "stage_results": [ + { + "stage_id": 0, + "replica_id": 0, + "result": {"supported": False, "reason": "worker_duplex_signal_not_implemented"}, + }, + { + "stage_id": 1, + "replica_id": 0, + "result": {"supported": False, "reason": "worker_duplex_signal_not_implemented"}, + }, + { + "stage_id": -1, + "replica_id": -1, + "result": { + "supported": True, + "data_plane_signal": True, + "event": "barge_in", + "epoch": 1, + }, + }, + ], + } + engine = FakeEngineClient(signal_result=control_result) + chat_service = FakeChatService(engine) + handler = OmniDuplexSessionHandler(chat_service=chat_service, config_timeout_s=0.1, idle_timeout_s=1) + ws = TimedWebSocket() + ws.put(_session_create("sid-control-signal-data-plane")) + ws.put({"type": "turn.signal", "event": "barge_in"}) + ws.put({"type": "session.close"}) + + await handler.handle_session(ws) + + assert "runtime_signal_failed" not in {m.get("code") for m in ws.sent} + assert "input.cancelled" in ws.sent_types() + runtime_control = next(m for m in ws.sent if m.get("type") == "runtime.control") + assert runtime_control["result"]["unsupported_count"] == 2 + + +@pytest.mark.asyncio +async def test_minicpmo_native_duplex_open_unsupported_fails_session_create(): + control_result = { + "operation": "open", + "session_id": "sid-native-busy", + "ok": True, + "unsupported_count": 1, + "error_count": 0, + "stage_results": [ + { + "stage_id": 0, + "replica_id": 0, + "result": {"supported": False, "reason": "native_duplex_session_busy"}, + } + ], + } + engine = FakeEngineClient(control_result=control_result) + chat_service = FakeChatService(engine) + handler = OmniDuplexSessionHandler(chat_service=chat_service, config_timeout_s=0.1, idle_timeout_s=1) + ws = TimedWebSocket() + ws.put(_native_session_create("sid-native-busy")) + ws.put({"type": "session.close"}) + + await handler.handle_session(ws) + + assert ws.sent_types() == ["error"] + assert ws.sent[0]["code"] == "runtime_open_unsupported" + assert ws.sent[0]["runtime_control"]["unsupported_count"] == 1 + + +@pytest.mark.asyncio +async def test_minicpmo_native_duplex_rejects_engine_without_fence_contract(): + class LegacyEngineClient(FakeEngineClient): + async def open_duplex_session_async( + self, + session_id: str, + *, + session_mode: str = "duplex", + capabilities: dict[str, object] | None = None, + session_config: dict[str, object] | None = None, + timeout: float | None = None, + ) -> None: + return await super().open_duplex_session_async( + session_id, + session_mode=session_mode, + capabilities=capabilities, + session_config=session_config, + timeout=timeout, + ) + + engine = LegacyEngineClient() + handler = OmniDuplexSessionHandler( + chat_service=FakeChatService(engine), + config_timeout_s=0.1, + idle_timeout_s=1, + ) + ws = TimedWebSocket() + ws.put(_native_session_create("sid-native-no-fence-contract")) + ws.put({"type": "session.close"}) + + await handler.handle_session(ws) + + assert ws.sent_types() == ["error"] + assert ws.sent[0]["code"] == "runtime_contract_invalid" + assert "open_duplex_session_async" in ws.sent[0]["error"] + assert "fence" in ws.sent[0]["error"] + assert engine.opened == [] + + +@pytest.mark.asyncio +async def test_minicpmo_native_duplex_audio_append_does_not_retain_pending_audio(): + engine = FakeEngineClient() + chat_service = FakeChatService(engine) + handler = OmniDuplexSessionHandler(chat_service=chat_service, config_timeout_s=0.1, idle_timeout_s=1) + ws = TimedWebSocket() + ws.put(_native_session_create("sid-native-no-pending")) + short_chunk = base64.b64encode(b"\x00" * (800 * 4)).decode("ascii") + ws.put({"type": "input_audio_buffer.append", "audio": short_chunk, "format": "pcm_f32le"}) + ws.put({"type": "input.cancel", "reason": "barge_in"}) + ws.put({"type": "session.close"}) + + await handler.handle_session(ws) + + cancelled = next(m for m in ws.sent if m.get("type") == "input.cancelled") + assert cancelled["cancelled"] == {"text_chunks": 0, "audio_chunks": 0} + assert cancelled["epoch"] == 1 + + +@pytest.mark.asyncio +async def test_minicpmo_native_duplex_rejects_text_append_before_runtime_call(): + engine = FakeEngineClient() + chat_service = FakeChatService(engine) + handler = OmniDuplexSessionHandler(chat_service=chat_service, config_timeout_s=0.1, idle_timeout_s=1) + ws = TimedWebSocket() + ws.put(_native_session_create("sid-native-text")) + ws.put({"type": "input.text.append", "text": "hello"}) + ws.put({"type": "session.close"}) + + await handler.handle_session(ws) + + error = next(m for m in ws.sent if m.get("type") == "error") + assert error["code"] == "native_text_append_unsupported" + assert engine.appended == [] + + +@pytest.mark.asyncio +async def test_minicpmo_native_duplex_separates_public_and_runtime_config(monkeypatch): + engine = FakeEngineClient() + chat_service = FakeChatService(engine) + handler = OmniDuplexSessionHandler(chat_service=chat_service, config_timeout_s=0.1, idle_timeout_s=1) + + async def fake_resolve_ref_audio(_, *, model_config): + del model_config + return [0.25] * 1600, 16000 + + monkeypatch.setattr( + "vllm_omni.experimental.fullduplex.minicpmo45.MiniCPMO45NativeDuplexServingAdapter.resolve_ref_audio", + fake_resolve_ref_audio, + ) + event = _native_session_create("sid-native-ref-audio", modalities=["text", "audio"]) + event["session"]["ref_audio"] = "data:audio/wav;base64,AAAA" + ws = TimedWebSocket() + ws.put(event) + ws.put({"type": "session.close"}) + + await handler.handle_session(ws) + + assert ws.sent_types()[0] == "session.created" + opened_config = engine.opened_configs[0] + opened_runtime_config = engine.opened_runtime_configs[0] + extra_body = opened_config["extra_body"] + assert "ref_audio_data" not in extra_body + assert "duplex_stage_max_tokens" not in extra_body + assert opened_runtime_config["ref_audio_format"] == "pcm_f32le" + assert opened_runtime_config["ref_audio_sample_rate_hz"] == 16000 + assert opened_runtime_config["duplex_stage_max_tokens"] == {"0": 20, "1": 8192} + assert opened_runtime_config["duplex_stage_sampling_params"]["0"]["top_k"] == 20 + assert opened_runtime_config["duplex_stage_sampling_params"]["0"]["top_p"] == 0.8 + assert base64.b64decode(opened_runtime_config["ref_audio_data"]) == struct.pack("<1600f", *([0.25] * 1600)) + + +@pytest.mark.asyncio +async def test_minicpmo_native_duplex_rejects_client_runtime_config(): + engine = FakeEngineClient() + handler = OmniDuplexSessionHandler( + chat_service=FakeChatService(engine), + config_timeout_s=0.1, + idle_timeout_s=1, + ) + event = _native_session_create("sid-native-private-config") + event["session"]["extra_body"]["duplex_stage_sampling_params"] = {"0": {"temperature": 0.0}} + ws = TimedWebSocket() + ws.put(event) + + await handler.handle_session(ws) + + assert ws.sent_types() == ["error"] + assert ws.sent[0]["code"] == "invalid_duplex_runtime_config" + assert engine.opened == [] + + +@pytest.mark.asyncio +async def test_minicpmo_native_duplex_audio_output_requires_ref_audio(monkeypatch): + monkeypatch.setattr( + MiniCPMO45NativeDuplexServingAdapter, + "_load_native_tokenizer", + staticmethod(lambda model_config: None), + ) + + event = _native_session_create( + "sid-native-ref-required", + modalities=["text", "audio"], + ) + config = DuplexSessionConfig.from_event(event) + + with pytest.raises(ServingRuntimeConfigError) as exc_info: + await MiniCPMO45NativeDuplexServingAdapter.prepare_runtime_config( + config, + model_config=SimpleNamespace(model="openbmb/MiniCPM-o-4_5"), + ) + + assert exc_info.value.code == "ref_audio_required" + + +@pytest.mark.asyncio +async def test_minicpmo_native_duplex_open_session_maps_missing_ref_audio_to_typed_error(): + engine = FakeEngineClient() + handler = OmniDuplexSessionHandler( + chat_service=FakeChatService(engine), + config_timeout_s=0.1, + idle_timeout_s=1, + ) + ws = TimedWebSocket() + ws.put( + _native_session_create( + "sid-native-ref-required-open", + modalities=["text", "audio"], + ) + ) + + await handler.handle_session(ws) + + assert ws.sent_types() == ["error"] + assert ws.sent[0]["code"] == "ref_audio_required" + assert engine.opened == [] + + +@pytest.mark.asyncio +async def test_minicpmo_native_duplex_text_only_omits_ref_audio_when_client_does_not_provide_it(monkeypatch): + monkeypatch.setattr( + MiniCPMO45NativeDuplexServingAdapter, + "_load_native_tokenizer", + staticmethod(lambda model_config: None), + ) + + event = _native_session_create("sid-native-default-ref", modalities=["text"]) + config = DuplexSessionConfig.from_event(event) + + runtime_config = await MiniCPMO45NativeDuplexServingAdapter.prepare_runtime_config( + config, + model_config=SimpleNamespace(model="openbmb/MiniCPM-o-4_5"), + ) + + assert "ref_audio_data" not in runtime_config + assert "ref_audio_format" not in runtime_config + assert "ref_audio_sample_rate_hz" not in runtime_config + + +@pytest.mark.asyncio +async def test_minicpmo_native_duplex_preserves_ref_audio_channels_until_normalize(monkeypatch): + class FakeMediaConnector: + def __init__(self, **kwargs): + self.kwargs = kwargs + + async def fetch_audio_async(self, ref_audio): + assert ref_audio == "data:audio/wav;base64,AAAA" + return np.tile(np.array([[0.25, -0.25], [0.5, -0.5]], dtype=np.float32), (800, 1)), 16000 + + monkeypatch.setattr( + "vllm_omni.experimental.fullduplex.minicpmo45.adapter.MediaConnector", + FakeMediaConnector, + ) + + engine = FakeEngineClient() + chat_service = FakeChatService(engine) + handler = OmniDuplexSessionHandler(chat_service=chat_service, config_timeout_s=0.1, idle_timeout_s=1) + event = _native_session_create("sid-native-ref-audio-stereo", modalities=["text", "audio"]) + event["session"]["ref_audio"] = "data:audio/wav;base64,AAAA" + ws = TimedWebSocket() + ws.put(event) + ws.put({"type": "session.close"}) + + await handler.handle_session(ws) + + runtime_config = engine.opened_runtime_configs[0] + assert base64.b64decode(runtime_config["ref_audio_data"]) == struct.pack("<1600f", *([0.0] * 1600)) + + +@pytest.mark.asyncio +async def test_minicpmo_native_duplex_rejects_ref_audio_path(): + engine = FakeEngineClient() + chat_service = FakeChatService(engine) + handler = OmniDuplexSessionHandler(chat_service=chat_service, config_timeout_s=0.1, idle_timeout_s=1) + event = _native_session_create("sid-native-ref-path") + event["session"]["extra_body"] = { + "minicpmo45_native_duplex": True, + "ref_audio_path": "/tmp/ref.wav", + } + ws = TimedWebSocket() + ws.put(event) + + await handler.handle_session(ws) + + assert ws.sent_types() == ["error"] + assert ws.sent[0]["code"] == "unsupported_ref_audio_path" + assert engine.opened == [] + + +def test_minicpmo_native_duplex_explicit_barge_in_request_is_deferred_as_listen(): + engine = FakeEngineClient() + handler = OmniDuplexSessionHandler( + chat_service=FakeChatService(engine), + config_timeout_s=0.1, + idle_timeout_s=1, + ) + session = DuplexSession( + session_id="sid-native-explicit-barge-disabled", + config=DuplexSessionConfig(overlap_policy=DuplexOverlapPolicy.BARGE_IN_ON_SPEECH.value), + capabilities=DuplexCapabilities.minicpmo45_native(), + ) + payload = { + "type": "audio", + "audio": _pcm_f32_b64(16000, value=0.05), + "format": "pcm_f32le", + "sample_rate_hz": 16000, + "is_speech": True, + } + + decision = handler._overlap_decision( + session, + {"force_barge_in": True}, + payload, + ) + + assert decision == { + "action": "listen", + "reason": "barge_in_unsupported", + "duration_ms": 1000, + "overlap_speech_ms": 1000, + "buffer_audio": True, + "defer_runtime_append": True, + } + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "barge_in_event", + [ + {"type": "barge_in"}, + {"type": "turn.signal", "event": "barge_in"}, + ], +) +async def test_minicpmo_native_duplex_rejects_unadvertised_barge_in_control( + barge_in_event: dict[str, object], +): + engine = FakeEngineClient() + handler = OmniDuplexSessionHandler( + chat_service=FakeChatService(engine), + config_timeout_s=0.1, + idle_timeout_s=1, + ) + ws = TimedWebSocket() + ws.put(_native_session_create("sid-native-barge-control-disabled")) + ws.put(barge_in_event) + ws.put({"type": "session.close"}) + + await handler.handle_session(ws) + + error = next(message for message in ws.sent if message.get("code") == "barge_in_unsupported") + assert error["session_id"] == "sid-native-barge-control-disabled" + assert ("sid-native-barge-control-disabled", "barge_in") not in engine.signals + + +@pytest.mark.asyncio +async def test_minicpmo_native_duplex_explicit_non_speech_stays_listening_without_append(): + engine = FakeEngineClient() + chat_service = FakeChatService(engine) + handler = OmniDuplexSessionHandler(chat_service=chat_service, config_timeout_s=0.1, idle_timeout_s=1) + ws = TimedWebSocket() + ws.put(_native_session_create("sid-native-silence")) + ws.put( + { + "type": "input_audio_buffer.append", + "audio": "AAAA", + "format": "pcm_f32le", + "is_speech": False, + } + ) + ws.put({"type": "session.close"}) + + await handler.handle_session(ws) + + assert engine.appended == [] + assert "response.listen" in ws.sent_types() + assert "response.created" not in ws.sent_types() + assert "response.output_audio.delta" not in ws.sent_types() + + +@pytest.mark.asyncio +async def test_minicpmo_native_duplex_drains_data_plane_stream_until_done(): + def _stage_output(samples: int, *, finished: bool): + return SimpleNamespace( + request_id="duplex-sid-native-stream-e0-stage0-s1", + finished=finished, + outputs=[ + SimpleNamespace( + text="", + multimodal_output={ + "audio": np.zeros(samples, dtype=np.float32), + "sr": 24000, + }, + ) + ], + ) + + control_result = { + "operation": "append", + "session_id": "sid-native-stream", + "ok": True, + "unsupported_count": 0, + "error_count": 0, + "stage_results": [ + { + "stage_id": 0, + "replica_id": 0, + "result": { + "supported": True, + "implementation_level": "model_native_duplex", + "data_plane_append": True, + "request_id": "duplex-sid-native-stream-e0-stage0-s1", + "response_stage_id": 1, + }, + } + ], + "data_plane_outputs": [_stage_output(10, finished=False)], + } + engine = FakeEngineClient( + append_result=control_result, + collect_outputs=[ + [_stage_output(20, finished=False)], + [_stage_output(30, finished=True)], + ], + ) + chat_service = FakeChatService(engine) + handler = OmniDuplexSessionHandler(chat_service=chat_service, config_timeout_s=0.1, idle_timeout_s=1) + + def close_on_second_created(ws: TimedWebSocket, data: dict[str, Any]) -> None: + if ws.sent_types().count("response.created") >= 2: + ws.put({"type": "session.close"}) + + ws = TimedWebSocket(on_send=close_on_second_created) + ws.put(_native_session_create("sid-native-stream")) + ws.put({"type": "input_audio_buffer.append", "audio": "AAAA", "format": "pcm_f32le"}) + + await handler.handle_session(ws) + + deltas = [m for m in ws.sent if m.get("type") == "response.output_audio.delta"] + assert [m["audio"] for m in deltas] == ["wav-10", "wav-10", "wav-10"] + assert [m["end_of_turn"] for m in deltas] == [False, False, True] + assert len([m for m in ws.sent if m.get("type") == "response.done"]) == 1 + assert engine.collected == [ + ("duplex-sid-native-stream-e0-stage0-s1", 1), + ("duplex-sid-native-stream-e0-stage0-s1", 1), + ] + + +@pytest.mark.asyncio +async def test_minicpmo_auto_response_restarts_drain_when_append_races_idle_exit( + monkeypatch: pytest.MonkeyPatch, +): + request_id = "duplex-sid-native-late-output-e0-stage0" + late_output = object() + engine = FakeEngineClient( + collect_outputs=[[], [], [], [late_output]], + collect_delay_s=0.05, + ) + handler = OmniDuplexSessionHandler( + chat_service=FakeChatService(engine), + config_timeout_s=0.1, + idle_timeout_s=1, + ) + session = DuplexSession( + session_id="sid-native-late-output", + config=DuplexSessionConfig(extra_body={"auto_response": True}), + ) + session.capabilities = DuplexCapabilities.minicpmo45_native() + session.bind_request(request_id) + native = handler._minicpmo_session_state(session) + projected_batches: list[object] = [] + projected = asyncio.Event() + + async def project_late_output(_send_json, result, **_kwargs): + projected_batches.append(result) + projected.set() + session.close() + return None, True + + monkeypatch.setattr(handler, "_send_native_duplex_events", project_late_output) + result = { + "stage_results": [ + { + "result": { + "data_plane_append": True, + "request_id": request_id, + "response_stage_id": 1, + } + } + ] + } + + assert ( + await handler._start_native_data_plane_stream_task( + None, + result, + session=session, + expected_epoch=session.epoch, + ) + is True + ) + first_drain = native.data_plane_task + assert first_drain is not None + await asyncio.sleep(0.01) + + assert ( + await handler._start_native_data_plane_stream_task( + None, + result, + session=session, + expected_epoch=session.epoch, + ) + is False + ) + await asyncio.wait_for(projected.wait(), timeout=1) + await asyncio.wait_for(first_drain, timeout=1) + + assert len(engine.collected) == 4 + assert projected_batches == [{"data_plane_outputs": [late_output]}] + + +@pytest.mark.asyncio +async def test_minicpmo_native_duplex_ignores_outputs_after_data_plane_turn_done(): + request_id = "duplex-sid-native-post-done-e0-stage0-s1" + + def _stage_output(samples: int, *, turn_end: bool = False): + return SimpleNamespace( + request_id=request_id, + finished=False, + outputs=[ + SimpleNamespace( + text="hello", + multimodal_output={ + "audio": np.zeros(samples, dtype=np.float32), + "sr": 24000, + "meta": { + "turn_end": turn_end, + "duplex_turn_id": 0, + "duplex_epoch": 0, + }, + }, + ) + ], + ) + + control_result = { + "operation": "append", + "session_id": "sid-native-post-done", + "ok": True, + "unsupported_count": 0, + "error_count": 0, + "stage_results": [ + { + "stage_id": 0, + "replica_id": 0, + "result": { + "supported": True, + "implementation_level": "model_native_duplex", + "data_plane_append": True, + "request_id": request_id, + "response_stage_id": 1, + }, + } + ], + "data_plane_outputs": [_stage_output(10)], + } + engine = FakeEngineClient( + append_result=control_result, + collect_outputs=[ + [_stage_output(20, turn_end=True)], + [_stage_output(30)], + ], + ) + chat_service = FakeChatService(engine) + handler = OmniDuplexSessionHandler(chat_service=chat_service, config_timeout_s=0.1, idle_timeout_s=1) + + def close_on_done(ws: TimedWebSocket, data: dict[str, Any]) -> None: + if data.get("type") == "response.done": + ws.put({"type": "session.close"}) + + event = _native_session_create("sid-native-post-done") + event["session"]["extra_body"]["auto_response"] = True + ws = TimedWebSocket(on_send=close_on_done) + ws.put(event) + ws.put({"type": "input_audio_buffer.append", "audio": "AAAA", "format": "pcm_f32le"}) + + await handler.handle_session(ws) + + assert ws.sent_types().count("response.created") == 1 + assert ws.sent_types().count("response.done") == 1 + deltas = [m for m in ws.sent if m.get("type") == "response.output_audio.delta"] + assert [m["audio"] for m in deltas] == ["wav-10", "wav-10"] + assert engine.collected + assert all(collected == (request_id, 1) for collected in engine.collected) + + +@pytest.mark.asyncio +async def test_minicpmo_native_duplex_accepts_next_append_after_response_done(): + def _stage_output(*, finished: bool): + return SimpleNamespace( + request_id="duplex-sid-native-next-turn-e0-stage0-s1", + finished=finished, + outputs=[ + SimpleNamespace( + text="", + multimodal_output={ + "audio": np.zeros(10, dtype=np.float32), + "sr": 24000, + }, + ) + ], + ) + + control_result = { + "operation": "append", + "session_id": "sid-native-next-turn", + "ok": True, + "unsupported_count": 0, + "error_count": 0, + "stage_results": [ + { + "stage_id": 0, + "replica_id": 0, + "result": { + "supported": True, + "implementation_level": "model_native_duplex", + "data_plane_append": True, + "request_id": "duplex-sid-native-next-turn-e0-stage0-s1", + "response_stage_id": 1, + }, + } + ], + "data_plane_outputs": [_stage_output(finished=True)], + } + engine = FakeEngineClient(append_result=control_result) + chat_service = FakeChatService(engine) + handler = OmniDuplexSessionHandler(chat_service=chat_service, config_timeout_s=0.1, idle_timeout_s=1) + done_count = 0 + + def send_next_turn_or_close(ws: TimedWebSocket, data: dict[str, Any]) -> None: + nonlocal done_count + if data.get("type") != "response.done": + return + done_count += 1 + if done_count == 1: + ws.put({"type": "input_audio_buffer.append", "audio": "BBBB", "format": "pcm_f32le"}) + ws.put({"type": "input_audio_buffer.commit", "final": True}) + ws.put({"type": "response.create"}) + else: + ws.put({"type": "session.close"}) + + ws = TimedWebSocket(on_send=send_next_turn_or_close, receive_timeout_s=5) + create = _native_session_create("sid-native-next-turn") + create["session"]["idle_timeout_s"] = 5 + ws.put(create) + ws.put({"type": "input_audio_buffer.append", "audio": "AAAA", "format": "pcm_f32le"}) + + await handler.handle_session(ws) + + assert len(engine.appended) >= 2, [ + (message.get("type"), message.get("code"), message.get("reason")) for message in ws.sent + ] + assert len([m for m in ws.sent if m.get("type") == "response.done"]) == 2 + + +@pytest.mark.asyncio +async def test_minicpmo_native_auto_response_commit_finalizes_after_streamed_full_chunk(): + control_result = { + "operation": "append", + "session_id": "sid-native-auto-finalize", + "ok": True, + "unsupported_count": 0, + "error_count": 0, + "stage_results": [], + } + engine = FakeEngineClient(control_result=control_result) + handler = OmniDuplexSessionHandler( + chat_service=FakeChatService(engine), + config_timeout_s=0.1, + idle_timeout_s=1, + ) + ws = TimedWebSocket() + event = _native_session_create("sid-native-auto-finalize") + event["session"]["extra_body"]["auto_response"] = True + ws.put(event) + ws.put( + { + "type": "input_audio_buffer.append", + "audio": _pcm_f32_b64(16000), + "format": "pcm_f32le", + "sample_rate_hz": 16000, + } + ) + ws.put({"type": "input_audio_buffer.commit", "final": True}) + ws.put({"type": "session.close"}) + + await handler.handle_session(ws) + + assert len(engine.appended) == 1 + assert engine.appended[0][3] is False + assert "response.created" not in ws.sent_types() + + +@pytest.mark.asyncio +async def test_minicpmo_native_auto_response_commit_finalizes_with_active_data_plane_stream(): + request_id = "duplex-sid-native-auto-active-stream-stage0" + control_result = { + "operation": "append", + "session_id": "sid-native-auto-active-stream", + "ok": True, + "unsupported_count": 0, + "error_count": 0, + "stage_results": [ + { + "result": { + "data_plane_append": True, + "request_id": request_id, + "response_stage_id": 1, + } + } + ], + } + engine = FakeEngineClient( + control_result=control_result, + collect_outputs=[[], [], []], + collect_delay_s=0.05, + ) + handler = OmniDuplexSessionHandler( + chat_service=FakeChatService(engine), + config_timeout_s=0.1, + idle_timeout_s=1, + ) + ws = TimedWebSocket() + event = _native_session_create("sid-native-auto-active-stream") + event["session"]["extra_body"]["auto_response"] = True + ws.put(event) + ws.put( + { + "type": "input_audio_buffer.append", + "audio": _pcm_f32_b64(16000), + "format": "pcm_f32le", + "sample_rate_hz": 16000, + } + ) + ws.put({"type": "input_audio_buffer.commit", "final": True}) + ws.put({"type": "session.close"}) + + await handler.handle_session(ws) + + assert len(engine.appended) == 1 + assert engine.appended[0][3] is False + assert "response.created" not in ws.sent_types() + + +@pytest.mark.asyncio +async def test_minicpmo_native_auto_response_accepts_realtime_commit_after_stream_consumed(): + engine = FakeEngineClient() + handler = OmniDuplexSessionHandler( + chat_service=FakeChatService(engine), + config_timeout_s=0.1, + idle_timeout_s=1, + ) + ws = TimedWebSocket() + event = _native_session_create("sid-native-auto-consumed-stream") + event["session"]["extra_body"]["auto_response"] = True + ws.put(event) + ws.put( + { + "type": "input_audio_buffer.append", + "audio": _pcm_f32_b64(16000), + "format": "pcm_f32le", + "sample_rate_hz": 16000, + } + ) + ws.put( + { + "type": "input_audio_buffer.commit", + "final": True, + "realtime_item_id": "item-consumed-stream", + } + ) + ws.put({"type": "session.close"}) + + await handler.handle_session(ws) + + assert not any(message.get("code") == "input_audio_buffer_empty" for message in ws.sent) + committed = next(message for message in ws.sent if message.get("type") == "input.committed") + assert committed["realtime_item_id"] == "item-consumed-stream" + assert committed["native_audio"] is True + + +@pytest.mark.asyncio +async def test_minicpmo_native_auto_response_post_response_commit_ignores_stale_progress_state(): + session_id = "sid-native-auto-post-response" + engine = FakeEngineClient() + handler = OmniDuplexSessionHandler( + chat_service=FakeChatService(engine), + config_timeout_s=0.1, + idle_timeout_s=1, + ) + + def on_send(ws: TimedWebSocket, data: dict[str, Any]) -> None: + if data.get("type") != "session.created": + return + session = handler._registry.get(session_id) + assert session is not None + ws.put( + { + "type": "input_audio_buffer.append", + "audio": _pcm_f32_b64(8000), + "format": "pcm_f32le", + "sample_rate_hz": 16000, + } + ) + ws.put({"type": "input_audio_buffer.commit", "final": True}) + ws.put({"type": "session.close"}) + + ws = TimedWebSocket(on_send=on_send) + create = _native_session_create(session_id) + create["session"]["extra_body"]["auto_response"] = True + ws.put(create) + + await handler.handle_session(ws) + + assert len(engine.appended) == 1 + assert engine.appended[0][3] is True + assert "input.committed" in ws.sent_types() + assert not any(message.get("code") == "response_already_active" for message in ws.sent) + + +def test_realtime_overlap_commit_uses_current_generation_identity(): + handler, session = _auto_response_context("sid-realtime-reserved-input-turn") + session.turn_id = 1 + session.bind_response_turn(1) + session.turn_id = 3 + committed = handler._commit_native_audio_input(session) + + assert committed.turn_id == 3 + assert session.turn_id == 3 + + +@pytest.mark.asyncio +async def test_minicpmo_native_auto_response_keeps_request_bound_for_segment_continuation(): + request_id = "duplex-sid-native-auto-continuation-e0-stage0" + control_result = { + "operation": "append", + "session_id": "sid-native-auto-continuation", + "ok": True, + "unsupported_count": 0, + "error_count": 0, + "stage_results": [ + { + "result": { + "data_plane_append": True, + "request_id": request_id, + "response_stage_id": 1, + } + } + ], + } + terminal_segment = _duplex_tts_output( + request_id=request_id, + samples=0, + finished=True, + tts_is_last_chunk=True, + token_ids=[151645], + ) + engine = FakeEngineClient( + control_result=control_result, + collect_outputs=[[terminal_segment], [], []], + collect_delay_s=0.05, + ) + handler = OmniDuplexSessionHandler( + chat_service=FakeChatService(engine), + config_timeout_s=0.1, + idle_timeout_s=1, + ) + ws = TimedWebSocket() + event = _native_session_create("sid-native-auto-continuation") + event["session"]["extra_body"]["auto_response"] = True + ws.put(event) + ws.put( + { + "type": "input_audio_buffer.append", + "audio": _pcm_f32_b64(16000), + "format": "pcm_f32le", + "sample_rate_hz": 16000, + } + ) + ws.put({"type": "input_audio_buffer.commit", "final": True}) + + await handler.handle_session(ws) + + assert len(engine.appended) >= 3 + _, mode, payload, final = engine.appended[2] + assert mode == "append_audio_chunk" + assert payload["duplex_turn_id"] == 0 + assert final is False + + +@pytest.mark.asyncio +async def test_minicpmo_native_auto_response_real_input_waits_for_submitted_silence_tail(): + request_id = "duplex-sid-native-silence-tail-e0-stage0" + + class SubmittedSilenceEngine(FakeEngineClient): + def __init__(self) -> None: + control_result = { + "operation": "append", + "session_id": "sid-native-silence-tail", + "ok": True, + "unsupported_count": 0, + "error_count": 0, + "stage_results": [ + { + "result": { + "data_plane_append": True, + "request_id": request_id, + "response_stage_id": 1, + } + } + ], + } + terminal_segment = _duplex_tts_output( + request_id=request_id, + samples=0, + finished=True, + tts_is_last_chunk=True, + token_ids=[151645], + ) + super().__init__( + control_result=control_result, + collect_outputs=[[terminal_segment], [], []], + collect_delay_s=0.01, + ) + self.silence_append_started = asyncio.Event() + self.release_silence_append = asyncio.Event() + self.append_sequence: list[str] = [] + self.real_append_count = 0 + + async def append_duplex_input_async(self, session_id: str, **kwargs): + kwargs.pop("expected_epoch", None) + payload = kwargs.get("payload") + is_silence_continuation = ( + isinstance(payload, dict) + and payload.get("format") == "pcm_f32le" + and payload.get("audio") == OmniDuplexSessionHandler._NATIVE_SILENCE_UNIT_PAYLOAD_AUDIO + ) + if is_silence_continuation: + result = await super().append_duplex_input_async(session_id, **kwargs) + self.append_sequence.append("silence-started") + self.silence_append_started.set() + await self.release_silence_append.wait() + self.append_sequence.append("silence-returned") + return result + result = await super().append_duplex_input_async(session_id, **kwargs) + if ( + isinstance(payload, dict) + and payload.get("audio") != OmniDuplexSessionHandler._NATIVE_SILENCE_UNIT_PAYLOAD_AUDIO + ): + self.real_append_count += 1 + self.append_sequence.append(f"real-{self.real_append_count}") + return result + + engine = SubmittedSilenceEngine() + handler = OmniDuplexSessionHandler( + chat_service=FakeChatService(engine), + config_timeout_s=0.1, + idle_timeout_s=1, + ) + ws = TimedWebSocket(receive_timeout_s=2.0) + event = _native_session_create("sid-native-silence-tail") + event["session"]["extra_body"]["auto_response"] = True + ws.put(event) + ws.put( + { + "type": "input_audio_buffer.append", + "audio": _pcm_f32_b64(16000, value=0.05), + "format": "pcm_f32le", + "sample_rate_hz": 16000, + } + ) + handler_task = asyncio.create_task(handler.handle_session(ws)) + + await asyncio.wait_for(engine.silence_append_started.wait(), timeout=1) + ws.put( + { + "type": "input_audio_buffer.append", + "audio": _pcm_f32_b64(16000, value=0.07), + "format": "pcm_f32le", + "sample_rate_hz": 16000, + } + ) + await asyncio.sleep(0.05) + + assert engine.real_append_count == 1 + assert engine.append_sequence == ["real-1", "silence-started"] + + engine.release_silence_append.set() + ws.put({"type": "session.close"}) + await asyncio.wait_for(handler_task, timeout=2) + + assert engine.real_append_count == 2 + assert engine.append_sequence == ["real-1", "silence-started", "silence-returned", "real-2"] + + +@pytest.mark.asyncio +async def test_minicpmo_native_duplex_uses_segment_text_metadata_for_transcript_cursor(): + request_id = "duplex-sid-native-repeat-text-e0-stage0" + + def _stage_output( + text: str, + samples: int, + *, + segment_text: str, + turn_id: int = 0, + turn_end: bool = False, + token_ids: list[int] | None = None, + ): + return SimpleNamespace( + request_id=request_id, + finished=False, + outputs=[ + SimpleNamespace( + text=text, + token_ids=list(token_ids or []), + multimodal_output={ + "audio": np.zeros(samples, dtype=np.float32), + "sr": 24000, + "meta.llm_output_text_utf8": np.frombuffer(segment_text.encode("utf-8"), dtype=np.uint8), + "meta": {"duplex_turn_id": turn_id, "turn_end": turn_end}, + }, + ) + ], + ) + + handler = OmniDuplexSessionHandler(chat_service=FakeChatService(FakeEngineClient())) + session = DuplexSession( + session_id="sid-native-repeat-text", + config=DuplexSessionConfig(extra_body={"auto_response": True}), + ) + sent: list[dict[str, Any]] = [] + + async def send_json(payload: dict[str, Any]) -> None: + sent.append(payload) + + async def emit(output: object) -> None: + result = {"data_plane_outputs": [output]} + for native_result in handler._minicpmo_data_plane.project( + result, + context=_data_plane_context(session), + ): + await handler._send_one_native_duplex_event(send_json, native_result, session=session) + + await emit(_stage_output("same reply", 24000, segment_text="same reply")) + await emit( + _stage_output("same replysame reply", 24000, segment_text="same reply", turn_end=True, token_ids=[151645]) + ) + + await emit(_stage_output("same replysame replysame reply", 48000, segment_text="same reply", turn_id=1)) + await emit( + _stage_output( + "same replysame replysame replysame reply", + 48000, + segment_text="same reply", + turn_id=1, + turn_end=True, + token_ids=[151645], + ) + ) + + created = [m for m in sent if m.get("type") == "response.created"] + done = [m for m in sent if m.get("type") == "response.done"] + deltas = [m for m in sent if m.get("type") == "response.output_audio.delta" and m.get("audio")] + protocol = NativeRealtimeSessionProtocol(TimedWebSocket()) # type: ignore[arg-type] + realtime_events = [event for payload in sent for event in protocol._from_duplex_event(payload)] + transcript_deltas = [ + m for m in realtime_events if m.get("type") == "response.audio_transcript.delta" and m.get("delta") + ] + assert len(created) == 2 + assert len(done) == 2 + assert len(deltas) == 2 + assert [m["text"] for m in deltas] == ["same reply", "same reply"] + assert [m["audio"] for m in deltas] == ["wav-24000", "wav-24000"] + assert [m["delta"] for m in transcript_deltas] == ["same reply", "same reply"] + assert [m["response_id"] for m in deltas] == [m["response_id"] for m in created] + assert [m["response_id"] for m in done] == [m["response_id"] for m in created] + + +@pytest.mark.asyncio +async def test_minicpmo_native_duplex_continuous_speak_reuses_active_response_until_turn_end(): + request_id = "duplex-sid-native-continuous-speak-e0-stage0" + + def _stage_output( + text: str, + samples: int, + *, + turn_end: bool = False, + token_ids: list[int] | None = None, + ): + return SimpleNamespace( + request_id=request_id, + finished=turn_end, + outputs=[ + SimpleNamespace( + text=text, + token_ids=list(token_ids or []), + multimodal_output={}, + ) + ], + multimodal_output={ + "audio": np.zeros(samples, dtype=np.float32), + "sr": 24000, + "meta.duplex_turn_id": np.array([0], dtype=np.int32), + "meta.duplex_epoch": np.array([0], dtype=np.int32), + "meta.turn_end": np.array([int(turn_end)], dtype=np.int32), + "meta.tts_is_last_chunk": np.array([int(turn_end)], dtype=np.int32), + }, + ) + + handler = OmniDuplexSessionHandler(chat_service=FakeChatService(FakeEngineClient())) + session = DuplexSession( + session_id="sid-native-continuous-speak", + config=DuplexSessionConfig(extra_body={"auto_response": True}), + ) + sent: list[dict[str, Any]] = [] + + async def send_json(payload: dict[str, Any]) -> None: + sent.append(payload) + + async def emit(output: object) -> None: + result = {"data_plane_outputs": [output]} + for native_result in handler._minicpmo_data_plane.project( + result, + context=_data_plane_context(session), + ): + await handler._send_one_native_duplex_event(send_json, native_result, session=session) + + await emit(_stage_output("hello", 24000)) + await emit(_stage_output("hello again", 48000)) + + created = [m for m in sent if m.get("type") == "response.created"] + deltas = [m for m in sent if m.get("type") == "response.output_audio.delta" and m.get("audio")] + assert len(created) == 1 + assert len(deltas) == 2 + assert {m["response_id"] for m in deltas} == {created[0]["response_id"]} + assert "response.done" not in [m.get("type") for m in sent] + + await emit(_stage_output("hello again", 48000, turn_end=True, token_ids=[151645])) + + done = [m for m in sent if m.get("type") == "response.done"] + assert len(done) == 1 + assert done[0]["response_id"] == created[0]["response_id"] + assert session.active_response_id is None + assert session.turn_id == 1 + + +@pytest.mark.asyncio +async def test_continuous_response_metrics_accumulate_only_owned_model_units(): + handler = OmniDuplexSessionHandler(chat_service=FakeChatService(FakeEngineClient())) + session = DuplexSession( + session_id="sid-response-metrics", + config=DuplexSessionConfig(extra_body={"auto_response": True}), + ) + sent: list[dict[str, Any]] = [] + + async def send_json(payload: dict[str, Any]) -> None: + sent.append(payload) + + async def emit(*, text: str, audio: str, tokens: int, ttft_ms: float, itls: list[float]) -> None: + await handler._send_one_native_duplex_event( + send_json, + { + "supported": True, + "stage_role": "tts", + "is_listen": False, + "data_plane_request_id": "duplex-sid-response-metrics-e0-stage0", + "text": text, + "audio_data": audio, + "audio_format": "pcm16", + "audio_duration_ms": 100, + "end_of_turn": False, + "model_turn_id": 0, + "stage_metrics": { + "0": { + "num_tokens_out": tokens, + "stage_gen_time_ms": 80.0, + "vllm_ttft_ms": ttft_ms, + "vllm_tpot_ms": 9.0, + "vllm_itls_ms": itls, + } + }, + }, + session=session, + ) + + await emit(text="first", audio="audio-a", tokens=4, ttft_ms=50.0, itls=[10.0, 11.0]) + await emit(text="second", audio="audio-b", tokens=6, ttft_ms=70.0, itls=[12.0, 13.0, 14.0]) + + deltas = [payload for payload in sent if payload.get("type") == "response.output_audio.delta"] + assert len(deltas) == 2 + first_metrics = deltas[0]["vllm_omni"]["stage_metrics"]["0"] + second_metrics = deltas[1]["vllm_omni"]["stage_metrics"]["0"] + assert first_metrics["num_tokens_out"] == 4 + assert second_metrics["num_tokens_out"] == 10 + assert second_metrics["vllm_ttft_ms"] == 50.0 + assert second_metrics["vllm_itls_ms"] == [10.0, 11.0, 12.0, 13.0, 14.0] + + first_response_id = deltas[0]["response_id"] + session.end_response() + await emit(text="third", audio="audio-c", tokens=3, ttft_ms=90.0, itls=[15.0, 16.0]) + + third_delta = [payload for payload in sent if payload.get("type") == "response.output_audio.delta"][-1] + third_metrics = third_delta["vllm_omni"]["stage_metrics"]["0"] + assert third_delta["response_id"] != first_response_id + assert third_metrics["num_tokens_out"] == 3 + assert third_metrics["vllm_ttft_ms"] == 90.0 + assert third_metrics["vllm_itls_ms"] == [15.0, 16.0] + + +@pytest.mark.asyncio +async def test_minicpmo_native_duplex_drops_old_turn_audio_while_new_response_is_active(): + request_id = "duplex-sid-native-old-turn-audio-e0-stage0" + handler = OmniDuplexSessionHandler(chat_service=FakeChatService(FakeEngineClient())) + session = DuplexSession( + session_id="sid-native-old-turn-audio", + config=DuplexSessionConfig(extra_body={"auto_response": True}), + ) + response_id = session.begin_response(turn_id=1) + session.turn_id = 2 + session.bind_request(request_id) + ws = TimedWebSocket() + + close_reason, emitted = await handler._send_one_native_duplex_event( + ws.send_json, + { + "supported": True, + "stage_role": "tts", + "is_listen": False, + "data_plane_request_id": request_id, + "text": "stale", + "audio_data": "old-audio", + "audio_format": "pcm16", + "audio_duration_ms": 100, + "end_of_turn": False, + "model_turn_id": 0, + "uses_model_runner_scheduler": True, + "runner_kv_backed": True, + "runtime_impl": "scheduler_data_plane", + }, + session=session, + expected_epoch=session.epoch, + ) + + assert close_reason is None + assert emitted is False + assert session.active_response_id == response_id + assert ws.sent == [] + + +@pytest.mark.asyncio +async def test_minicpmo_native_duplex_future_turn_audio_starts_next_response(): + request_id = "duplex-sid-native-future-turn-audio-e0-stage0" + handler = OmniDuplexSessionHandler(chat_service=FakeChatService(FakeEngineClient())) + session = DuplexSession( + session_id="sid-native-future-turn-audio", + config=DuplexSessionConfig(extra_body={"auto_response": True}), + ) + first_response_id = session.begin_response(turn_id=0) + session.bind_request(request_id) + ws = TimedWebSocket() + + close_reason, emitted = await handler._send_one_native_duplex_event( + ws.send_json, + { + "supported": True, + "stage_role": "tts", + "is_listen": False, + "data_plane_request_id": request_id, + "text": "next", + "audio_data": "next-audio", + "audio_format": "pcm16", + "audio_duration_ms": 100, + "end_of_turn": False, + "model_turn_id": 1, + "uses_model_runner_scheduler": True, + "runner_kv_backed": True, + "runtime_impl": "scheduler_data_plane", + }, + session=session, + expected_epoch=session.epoch, + ) + + assert close_reason is None + assert emitted is True + sent_types = [event["type"] for event in ws.sent] + assert sent_types == [ + "response.done", + "response.created", + "response.speak", + "response.output_audio.delta", + ] + assert ws.sent[0]["response_id"] == first_response_id + assert ws.sent[1]["response_id"] != first_response_id + assert ws.sent[3]["response_id"] == ws.sent[1]["response_id"] + assert ws.sent[3]["vllm_omni"]["model_turn_id"] == 1 + assert session.active_response_turn_id == 1 + assert session.turn_id == 1 + + +@pytest.mark.asyncio +async def test_minicpmo_native_duplex_drops_old_turn_listen_while_new_response_is_active(): + request_id = "duplex-sid-native-old-turn-listen-e0-stage0" + handler = OmniDuplexSessionHandler(chat_service=FakeChatService(FakeEngineClient())) + session = DuplexSession( + session_id="sid-native-old-turn-listen", + config=DuplexSessionConfig(extra_body={"auto_response": True}), + ) + response_id = session.begin_response(turn_id=1) + session.turn_id = 2 + session.bind_request(request_id) + ws = TimedWebSocket() + + close_reason, emitted = await handler._send_one_native_duplex_event( + ws.send_json, + { + "supported": True, + "stage_role": "llm", + "is_listen": True, + "model_listen": True, + "data_plane_request_id": request_id, + "end_of_turn": False, + "model_turn_id": 0, + "uses_model_runner_scheduler": True, + "runner_kv_backed": True, + "runtime_impl": "scheduler_data_plane", + }, + session=session, + expected_epoch=session.epoch, + ) + + assert close_reason is None + assert emitted is False + assert session.active_response_id == response_id + assert ws.sent == [] + + +@pytest.mark.asyncio +async def test_minicpmo_native_duplex_silence_continuation_requires_active_request(): + engine = FakeEngineClient() + chat_service = FakeChatService(engine) + handler = OmniDuplexSessionHandler(chat_service=chat_service, config_timeout_s=0.1, idle_timeout_s=1) + session = DuplexSession( + session_id="sid-native-stale-continuation", + config=DuplexSessionConfig(extra_body={"auto_response": True}), + ) + session.begin_response() + session.bind_request("duplex-sid-native-stale-continuation-stage0") + session.mark_audio_sent(100) + sent: list[dict[str, Any]] = [] + + async def send_json(data: dict[str, Any]) -> None: + sent.append(data) + + _install_direct_silence_scheduler(handler, session) + + await handler._maybe_continue_native_response(send_json, session=session, expected_epoch=session.epoch) + session.clear_request() + await asyncio.sleep(0.01) + + assert engine.appended == [] + + +@pytest.mark.asyncio +async def test_minicpmo_auto_response_segment_complete_continues_until_model_listen(): + engine = FakeEngineClient() + chat_service = FakeChatService(engine) + handler = OmniDuplexSessionHandler(chat_service=chat_service, config_timeout_s=0.1, idle_timeout_s=1) + session = DuplexSession( + session_id="sid-native-segment-complete", + config=DuplexSessionConfig(extra_body={"auto_response": True}), + ) + session.capabilities = DuplexCapabilities.minicpmo45_native() + response_id = session.begin_response() + request_id = "duplex-sid-native-segment-complete-stage0" + session.bind_request(request_id) + session.mark_audio_sent(1000) + _install_direct_silence_scheduler(handler, session) + sent: list[dict[str, Any]] = [] + + async def send_json(data: dict[str, Any]) -> None: + sent.append(data) + + close_reason, emitted = await handler._send_one_native_duplex_event( + send_json, + { + "supported": True, + "stage_role": "llm", + "is_listen": True, + "model_listen": False, + "listen_source": "auto_response_segment_complete", + "reason": "auto_response_segment_complete", + "data_plane_request_id": request_id, + "end_of_turn": False, + "uses_model_runner_scheduler": True, + "runner_kv_backed": True, + "runtime_impl": "scheduler_data_plane", + "owned_runtime": False, + }, + session=session, + expected_epoch=session.epoch, + ) + await asyncio.sleep(0.01) + + assert close_reason is None + assert emitted is False + assert session.active_response_id == response_id + assert session.active_request_id == request_id + assert not handler._minicpmo_data_plane.is_terminal(request_id) + assert sent == [] + assert len(engine.appended) == 1 + _, mode, payload, final = engine.appended[0] + assert mode == "append_audio_chunk" + assert payload["type"] == "audio" + assert payload.get("force_listen") is not True + assert final is False + + +@pytest.mark.parametrize( + ("steps", "expected_deltas"), + [ + ( + [("你好呀", 1), ("你好呀", 1), ("你好呀", 2)], + ["你好呀", "", "你好呀"], + ), + ( + [("你好呀", 1), ("你好呀", 2)], + ["你好呀", "你好呀"], + ), + ], + ids=["turn-scoped", "exact-repeated-turn-text"], +) +def test_minicpmo_data_plane_text_cursor( + steps: list[tuple[str, int]], + expected_deltas: list[str], +): + data_plane = _test_data_plane() + request_id = "duplex-sid-native-e0-stage0" + + deltas = [data_plane.segment_text_delta(request_id, text, turn_id=turn_id) for text, turn_id in steps] + + assert deltas == expected_deltas + + +def test_minicpmo_data_plane_text_cursor_does_not_clip_prior_turn_overlap(): + data_plane = _test_data_plane() + request_id = "duplex-sid-native-e0-stage0" + + assert data_plane.segment_text_delta(request_id, "你好呀。你有什么想聊的吗?", turn_id=1) + assert ( + data_plane.segment_text_delta(request_id, "你有什么想聊的吗?你好呀。", turn_id=2) + == "你有什么想聊的吗?你好呀。" + ) + + +@pytest.mark.asyncio +async def test_minicpmo_native_duplex_cancel_interrupts_background_data_plane_stream(): + control_result = { + "operation": "append", + "session_id": "sid-native-cancel-stream", + "ok": True, + "unsupported_count": 0, + "error_count": 0, + "stage_results": [ + { + "stage_id": 0, + "replica_id": 0, + "result": { + "supported": True, + "implementation_level": "model_native_duplex", + "data_plane_append": True, + "request_id": "duplex-sid-native-cancel-stream-e0-stage0-s1", + "response_stage_id": 1, + }, + } + ], + } + engine = FakeEngineClient( + append_result=control_result, + collect_delay_s=0.2, + collect_outputs=[ + [ + SimpleNamespace( + request_id="duplex-sid-native-cancel-stream-e0-stage0-s1", + finished=True, + outputs=[ + SimpleNamespace( + text="", + multimodal_output={"audio": np.zeros(10, dtype=np.float32), "sr": 24000}, + ) + ], + ) + ] + ], + ) + chat_service = FakeChatService(engine) + handler = OmniDuplexSessionHandler(chat_service=chat_service, config_timeout_s=0.1, idle_timeout_s=1) + ws = TimedWebSocket() + ws.put(_native_session_create("sid-native-cancel-stream")) + ws.put({"type": "input_audio_buffer.append", "audio": "AAAA", "format": "pcm_f32le"}) + ws.put({"type": "input.cancel"}) + ws.put({"type": "session.close"}) + + await handler.handle_session(ws) + + assert "audio.cancelled" in ws.sent_types() + assert "response.output_audio.delta" not in ws.sent_types() + assert engine.signals == [("sid-native-cancel-stream", "barge_in")] + assert engine.signal_fences == [DuplexFence("sid-native-cancel-stream")] + + +@pytest.mark.asyncio +async def test_minicpmo_native_duplex_session_close_discards_partial_pcm_tail(): + engine = FakeEngineClient() + chat_service = FakeChatService(engine) + handler = OmniDuplexSessionHandler(chat_service=chat_service, config_timeout_s=0.1, idle_timeout_s=1) + ws = TimedWebSocket() + ws.put(_native_session_create("sid-native-short-chunks")) + chunk = _pcm_f32_b64(3200) + for _ in range(6): + ws.put( + { + "type": "input_audio_buffer.append", + "audio": chunk, + "format": "pcm_f32le", + "sample_rate_hz": 16000, + } + ) + ws.put({"type": "session.close"}) + + await handler.handle_session(ws) + + assert len(engine.appended) == 1 + _, mode, payload, final = engine.appended[0] + assert mode == "append_audio_chunk" + assert final is False + assert isinstance(payload, dict) + assert len(base64.b64decode(payload["audio"])) == 16000 * 4 + assert "duplex_num_input_tokens" not in payload + + +@pytest.mark.asyncio +async def test_minicpmo_native_duplex_idle_timeout_closes_runtime_with_timeout_reason(): + engine = FakeEngineClient() + chat_service = FakeChatService(engine) + handler = OmniDuplexSessionHandler(chat_service=chat_service, config_timeout_s=0.1, idle_timeout_s=0.1) + ws = TimedWebSocket() + ws.put(_native_session_create("sid-native-timeout")) + + await handler.handle_session(ws) + + assert ws.sent_types() == ["session.created", "session.closed"] + assert ws.sent[-1]["reason"] == "timeout" + assert engine.closed == [("sid-native-timeout", "timeout")] diff --git a/tests/entrypoints/openai_api/test_serving_video_stream.py b/tests/entrypoints/openai_api/test_serving_video_stream.py index de2fcc24d..fc8d83da6 100644 --- a/tests/entrypoints/openai_api/test_serving_video_stream.py +++ b/tests/entrypoints/openai_api/test_serving_video_stream.py @@ -138,6 +138,112 @@ async def test_receive_config_accepts_client_legacy_aliases(): assert config.frame_filter_threshold == 0.87 +@pytest.mark.asyncio +async def test_video_frame_ack_reports_receiver_buffer_state(): + ws = MockWebSocket( + [ + json.dumps( + { + "type": "session.config", + "model": "test", + "enable_frame_filter": False, + } + ), + json.dumps( + { + "type": "video.frame", + "data": _b64(_make_jpeg()), + "frame_id": "frame-7", + "pts_ms": 700, + "capture_ts_ms": 1234.5, + } + ), + json.dumps({"type": "video.done"}), + ] + ) + handler = QwenOmniStreamingVideoHandler(chat_service=object()) + + await handler.handle_session(ws) + + ack = next(message for message in ws.sent if message.get("type") == "video.frame.ack") + assert ack["frame_id"] == "frame-7" + assert ack["pts_ms"] == 700 + assert ack["capture_ts_ms"] == 1234.5 + assert ack["accepted"] is True + assert ack["buffered_frames"] == 1 + assert ack["server_receive_ts_ms"] > 0 + + +@pytest.mark.asyncio +async def test_video_frames_consumed_is_emitted_after_engine_uses_frame_prompt(): + class OneOutputEngine: + def generate(self, **_kwargs): + async def _gen(): + await asyncio.sleep(0.05) + yield _text_result("visible") + + return _gen() + + class CapturingHandler(QwenOmniStreamingVideoHandler): + async def _preprocess_to_engine_prompt(self, request): + return {"prompt": "with-video"} + + ws = MockWebSocket( + [ + json.dumps( + { + "type": "session.config", + "model": "test", + "modalities": ["text"], + "num_frames": 1, + "enable_frame_filter": False, + } + ), + json.dumps( + { + "type": "video.frame", + "data": _b64(_make_jpeg()), + "frame_id": "frame-9", + "pts_ms": 900, + "source_pts_ms": 880, + "quality_profile": "balanced", + } + ), + json.dumps({"type": "video.query", "text": "describe"}), + json.dumps({"type": "video.done"}), + ] + ) + handler = CapturingHandler( + chat_service=object(), + engine_client=OneOutputEngine(), + idle_timeout=2.0, + ) + + await handler.handle_session(ws) + + consumed = next(message for message in ws.sent if message.get("type") == "video.frames.consumed") + assert consumed["frame_ids"] == ["frame-9"] + assert consumed["latest_pts_ms"] == 900 + assert consumed["request_id"].startswith("video-") + assert consumed["model_selected_ts_ms"] > 0 + assert consumed["frames"] == [ + { + "frame_id": "frame-9", + "pts_ms": 900, + "source_pts_ms": 880, + "quality_profile": "balanced", + "receiver_received_ts_ms": consumed["frames"][0]["receiver_received_ts_ms"], + "decoded_ready_ts_ms": consumed["frames"][0]["decoded_ready_ts_ms"], + } + ] + assert consumed["frames"][0]["receiver_received_ts_ms"] > 0 + assert consumed["frames"][0]["decoded_ready_ts_ms"] >= consumed["frames"][0]["receiver_received_ts_ms"] + assert consumed["model_selected_ts_ms"] >= consumed["frames"][0]["decoded_ready_ts_ms"] + assert ws.sent.index(consumed) < next( + index for index, message in enumerate(ws.sent) if message.get("type") == "response.text.delta" + ) + + @pytest.mark.asyncio async def test_audio_in_video_sets_mm_processor_kwargs(): captured_requests = [] diff --git a/tests/entrypoints/test_async_omni_duplex.py b/tests/entrypoints/test_async_omni_duplex.py new file mode 100644 index 000000000..62f3f49ad --- /dev/null +++ b/tests/entrypoints/test_async_omni_duplex.py @@ -0,0 +1,600 @@ +from types import SimpleNamespace + +import pytest + +from vllm_omni.engine.messages import OutputMessage +from vllm_omni.entrypoints.async_omni import AsyncOmni +from vllm_omni.entrypoints.client_request_state import ClientRequestState +from vllm_omni.experimental.fullduplex.engine.duplex_runtime import ( + DuplexOutputAction, + DuplexOutputDecision, + duplex_resource_request_id, +) +from vllm_omni.experimental.fullduplex.engine.lease import DuplexLeaseActivity +from vllm_omni.experimental.fullduplex.engine.messages import DuplexFence +from vllm_omni.experimental.fullduplex.output import attach_duplex_output_decision +from vllm_omni.experimental.fullduplex.request_client import DuplexRequestClient +from vllm_omni.metrics.stats import OrchestratorAggregator, StageRequestStats, StageStats +from vllm_omni.outputs import OmniRequestOutput + + +def test_async_omni_uses_extracted_duplex_request_client(): + app = object.__new__(AsyncOmni) + app.engine = SimpleNamespace(num_stages=2) + app.request_states = {} + app._duplex_request_client = None + + client = app._get_duplex_request_client() + + assert isinstance(client, DuplexRequestClient) + assert not hasattr(client, "_async_omni") + + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +@pytest.mark.asyncio +async def test_async_omni_open_duplex_forwards_session_config_and_timeout(): + calls = [] + output_handler_starts = 0 + + async def open_duplex_session_async(session_id, **kwargs): + calls.append((session_id, kwargs)) + return {"ok": True} + + app = object.__new__(AsyncOmni) + app.engine = SimpleNamespace(open_duplex_session_async=open_duplex_session_async) + + def start_output_handler(): + nonlocal output_handler_starts + output_handler_starts += 1 + + app._final_output_handler = start_output_handler + + result = await app.open_duplex_session_async( + "sid", + fence=DuplexFence("sid"), + capabilities={"implementation_level": "model_native_duplex"}, + session_config={"instructions": "Be brief.", "idle_timeout_s": 30}, + timeout=7.5, + ) + + assert result == {"ok": True} + assert output_handler_starts == 1 + assert calls == [ + ( + "sid", + { + "session_mode": "duplex", + "capabilities": {"implementation_level": "model_native_duplex"}, + "session_config": {"instructions": "Be brief.", "idle_timeout_s": 30}, + "fence": DuplexFence("sid"), + "timeout": 7.5, + }, + ) + ] + + +@pytest.mark.asyncio +async def test_async_omni_duplex_runtime_controls_forward_timeout(): + calls = [] + cancelled_fence = DuplexFence("sid", epoch=2, turn_id=3) + next_fence = DuplexFence("sid", epoch=3, turn_id=3) + + async def append_duplex_input_async(session_id, **kwargs): + calls.append(("append", session_id, kwargs)) + return {"ok": True} + + async def signal_duplex_turn_async(session_id, **kwargs): + calls.append(("signal", session_id, kwargs)) + return {"ok": True} + + async def close_duplex_session_async(session_id, **kwargs): + calls.append(("close", session_id, kwargs)) + return {"ok": True} + + async def touch_duplex_session_async(session_id, **kwargs): + calls.append(("touch", session_id, kwargs)) + return {"ok": True} + + async def resume_duplex_session_async(session_id, **kwargs): + calls.append(("resume", session_id, kwargs)) + return {"ok": True} + + app = object.__new__(AsyncOmni) + app.request_states = {} + app._final_output_handler = lambda: None + app.engine = SimpleNamespace( + append_duplex_input_async=append_duplex_input_async, + signal_duplex_turn_async=signal_duplex_turn_async, + close_duplex_session_async=close_duplex_session_async, + touch_duplex_session_async=touch_duplex_session_async, + resume_duplex_session_async=resume_duplex_session_async, + ) + + await app.append_duplex_input_async( + "sid", + mode="append_audio_chunk", + payload={}, + fence=cancelled_fence, + timeout=12.5, + ) + await app.signal_duplex_turn_async( + "sid", + event="barge_in", + fence=cancelled_fence, + next_fence=next_fence, + timeout=13.5, + ) + await app.close_duplex_session_async("sid", reason="done", fence=next_fence, timeout=14.5) + await app.touch_duplex_session_async( + "sid", + fence=next_fence, + activity=DuplexLeaseActivity.HEARTBEAT, + timeout=15.5, + ) + await app.resume_duplex_session_async( + "sid", + fence=next_fence, + expected_lease_generation=3, + timeout=16.5, + ) + + assert calls == [ + ( + "append", + "sid", + { + "mode": "append_audio_chunk", + "payload": {}, + "final": False, + "expected_epoch": None, + "fence": cancelled_fence, + "timeout": 12.5, + }, + ), + ( + "signal", + "sid", + { + "event": "barge_in", + "fence": cancelled_fence, + "next_fence": next_fence, + "timeout": 13.5, + }, + ), + ( + "close", + "sid", + { + "reason": "done", + "fence": next_fence, + "timeout": 14.5, + }, + ), + ( + "touch", + "sid", + { + "fence": next_fence, + "activity": DuplexLeaseActivity.HEARTBEAT, + "timeout": 15.5, + }, + ), + ( + "resume", + "sid", + { + "fence": next_fence, + "expected_lease_generation": 3, + "timeout": 16.5, + }, + ), + ] + + +@pytest.mark.asyncio +async def test_async_omni_duplex_append_can_defer_data_plane_output_collection(): + app = object.__new__(AsyncOmni) + request_id = duplex_resource_request_id(DuplexFence("sid"), "stage0") + + async def append_duplex_input_async(session_id, **kwargs): + del session_id, kwargs + request_state = app.request_states.get(request_id) + assert request_state is not None + assert request_state.metrics is not None + return { + "stage_results": [ + { + "result": { + "data_plane_append": True, + "request_id": request_id, + "response_stage_id": 1, + } + } + ] + } + + app.engine = SimpleNamespace(append_duplex_input_async=append_duplex_input_async) + app.request_states = {} + app._final_output_handler = lambda: None + + result = await app.append_duplex_input_async( + "sid", + mode="append_audio_chunk", + payload={}, + fence=DuplexFence("sid"), + collect_outputs=False, + ) + + assert "data_plane_outputs" not in result + + +@pytest.mark.asyncio +async def test_async_omni_duplex_append_forwards_fence(): + calls = [] + + async def append_duplex_input_async(session_id, **kwargs): + calls.append((session_id, kwargs)) + return {"ok": True} + + app = object.__new__(AsyncOmni) + app.request_states = {} + app._final_output_handler = lambda: None + app.engine = SimpleNamespace(append_duplex_input_async=append_duplex_input_async) + fence = DuplexFence("sid", epoch=1, turn_id=2) + + await app.append_duplex_input_async( + "sid", + mode="append_audio_chunk", + payload={}, + fence=fence, + collect_outputs=False, + ) + + assert calls[0][1]["fence"] is fence + + +@pytest.mark.asyncio +async def test_async_omni_duplex_collect_waits_for_response_stage(): + app = object.__new__(AsyncOmni) + req_state = ClientRequestState("duplex-sid-e0-stage0") + stage0_output = OmniRequestOutput( + request_id="duplex-sid-e0-stage0", + stage_id=0, + finished=False, + ) + stage1_output = OmniRequestOutput( + request_id="duplex-sid-e0-stage0", + stage_id=1, + final_output_type="audio", + finished=False, + ) + await req_state.queue.put( + OutputMessage( + request_id="duplex-sid-e0-stage0", + stage_id=0, + engine_outputs=stage0_output, + finished=False, + ) + ) + await req_state.queue.put( + OutputMessage( + request_id="duplex-sid-e0-stage0", + stage_id=1, + engine_outputs=stage1_output, + finished=False, + ) + ) + + outputs = await app._collect_duplex_data_plane_outputs( + "duplex-sid-e0-stage0", + req_state, + response_stage_id=1, + timeout=1.0, + ) + + assert outputs == [stage1_output] + + +@pytest.mark.asyncio +async def test_duplex_request_client_retains_output_route_at_segment_terminal(): + request_id = "duplex-sid-e0-stage0" + request_state = ClientRequestState(request_id) + request_states = {request_id: request_state} + output = OmniRequestOutput( + request_id=request_id, + stage_id=1, + final_output_type="audio", + finished=True, + ) + await request_state.queue.put( + OutputMessage( + request_id=request_id, + stage_id=1, + engine_outputs=output, + finished=True, + ) + ) + client = DuplexRequestClient( + SimpleNamespace(), + SimpleNamespace( + request_states=request_states, + num_stages=2, + log_stats=False, + ), + ) + + outputs = await client.collect_outputs( + request_id, + request_state, + response_stage_id=1, + timeout=1.0, + ) + + assert outputs == [output] + assert request_states[request_id] is request_state + + +@pytest.mark.asyncio +async def test_async_omni_duplex_direct_output_keeps_stage_metrics(): + app = object.__new__(AsyncOmni) + app._duplex_request_client = None + app._final_output_handler = lambda: None + request_id = "duplex-sid-e0-stage0" + req_state = ClientRequestState(request_id) + decision = DuplexOutputDecision( + action=DuplexOutputAction.DIRECT_RESPONSE, + metadata={ + "duplex_direct_response": True, + "duplex_native_decision": "speak", + }, + ) + direct_output = attach_duplex_output_decision( + OmniRequestOutput( + request_id=request_id, + stage_id=0, + final_output_type="text", + finished=True, + ), + decision, + ) + await req_state.queue.put( + OutputMessage( + request_id=request_id, + stage_id=0, + engine_outputs=direct_output, + metrics=StageRequestStats( + batch_id=1, + batch_size=1, + num_tokens_in=160, + num_tokens_out=8, + stage_gen_time_ms=80.0, + rx_transfer_bytes=0, + rx_decode_time_ms=0.0, + rx_in_flight_time_ms=0.0, + stage_stats=StageStats(total_token=8, total_gen_time_ms=80.0), + vllm_ttft_ms=12.5, + vllm_tpot_ms=9.0, + vllm_itl_ms=8.75, + vllm_itls_ms=[8.0, 9.5], + ), + finished=True, + ) + ) + + outputs = await app._collect_duplex_data_plane_outputs( + request_id, + req_state, + response_stage_id=1, + timeout=1.0, + ) + + assert outputs == [direct_output] + assert direct_output.metrics["stage_metrics"]["0"]["vllm_ttft_ms"] == 12.5 + assert direct_output.metrics["stage_metrics"]["0"]["vllm_tpot_ms"] == 9.0 + assert direct_output.metrics["stage_metrics"]["0"]["vllm_itls_ms"] == [8.0, 9.5] + + +@pytest.mark.asyncio +async def test_duplex_metrics_cursor_isolates_resumable_collect_windows(): + app = object.__new__(AsyncOmni) + app._duplex_request_client = None + app._final_output_handler = lambda: None + request_id = "duplex-sid-e0-stage0" + req_state = ClientRequestState(request_id) + req_state.metrics = OrchestratorAggregator( + num_stages=2, + log_stats=False, + wall_start_ts=0.0, + final_stage_id_for_e2e=1, + ) + + def stage_message(*, batch_id: int, tokens: int, ttft_ms: float, itls: list[float]) -> OutputMessage: + decision = DuplexOutputDecision( + action=DuplexOutputAction.DIRECT_RESPONSE, + metadata={"duplex_direct_response": True, "duplex_native_decision": "speak"}, + ) + output = attach_duplex_output_decision( + OmniRequestOutput( + request_id=request_id, + stage_id=0, + final_output_type="text", + finished=True, + ), + decision, + ) + return OutputMessage( + request_id=request_id, + stage_id=0, + engine_outputs=output, + metrics=StageRequestStats( + batch_id=batch_id, + batch_size=1, + num_tokens_in=160, + num_tokens_out=tokens, + stage_gen_time_ms=80.0, + rx_transfer_bytes=0, + rx_decode_time_ms=0.0, + rx_in_flight_time_ms=0.0, + stage_stats=StageStats(total_token=tokens, total_gen_time_ms=80.0), + vllm_ttft_ms=ttft_ms, + vllm_tpot_ms=9.0, + vllm_itl_ms=sum(itls) / len(itls), + vllm_itls_ms=itls, + ), + finished=True, + ) + + await req_state.queue.put(stage_message(batch_id=1, tokens=71, ttft_ms=62.5, itls=[10.0, 11.0])) + first = await app._collect_duplex_data_plane_outputs( + request_id, + req_state, + response_stage_id=1, + timeout=1.0, + ) + await req_state.queue.put(stage_message(batch_id=2, tokens=155, ttft_ms=81.0, itls=[12.0, 13.0, 14.0])) + second = await app._collect_duplex_data_plane_outputs( + request_id, + req_state, + response_stage_id=1, + timeout=1.0, + ) + + assert first[0].metrics["stage_metrics"]["0"]["num_tokens_out"] == 71 + second_stage0 = second[0].metrics["stage_metrics"]["0"] + assert second_stage0["num_tokens_out"] == 155 + assert second_stage0["vllm_ttft_ms"] == 81.0 + assert second_stage0["vllm_itls_ms"] == [12.0, 13.0, 14.0] + + +def test_async_omni_duplex_direct_output_prefers_outer_control_metadata(): + decision = DuplexOutputDecision( + action=DuplexOutputAction.DIRECT_RESPONSE, + metadata={ + "duplex_direct_response": True, + "duplex_native_decision": "listen", + }, + ) + output = attach_duplex_output_decision( + OmniRequestOutput( + request_id="duplex-sid-e0-stage0", + stage_id=0, + request_output=SimpleNamespace( + outputs=[], + multimodal_output=SimpleNamespace(kind="processed-payload"), + _custom_output={"inner": "completion"}, + ), + ), + decision, + ) + + assert AsyncOmni._is_direct_duplex_data_plane_response(output) + + +def test_async_omni_drops_unregistered_duplex_prefixed_output(): + app = object.__new__(AsyncOmni) + app.request_states = {} + output = OmniRequestOutput(request_id="duplex-unregistered-e0-stage0", stage_id=0) + + keep_processing, request_id, stage_id, request_state = app._handle_output_message( + OutputMessage( + request_id=output.request_id, + stage_id=0, + engine_outputs=output, + finished=False, + ) + ) + + assert keep_processing is True + assert request_id is stage_id is request_state is None + assert app.request_states == {} + + +@pytest.mark.asyncio +async def test_async_omni_duplex_collect_wraps_raw_response_stage_output(): + app = object.__new__(AsyncOmni) + app.log_stats = False + app._enable_ar_profiler = False + app.request_states = {} + app.prom_metrics = SimpleNamespace( + set_running=lambda value: None, + set_waiting=lambda value: None, + ) + app.engine = SimpleNamespace( + num_stages=2, + get_stage_metadata=lambda stage_id: SimpleNamespace( + final_output=stage_id == 1, + final_output_type="audio", + ), + ) + req_state = ClientRequestState("duplex-sid-e0-stage0") + raw_stage1_output = SimpleNamespace( + finished=False, + outputs=[], + stage_durations={}, + peak_memory_mb=0.0, + final_output_type="audio", + ) + await req_state.queue.put( + OutputMessage( + request_id="duplex-sid-e0-stage0", + stage_id=1, + engine_outputs=raw_stage1_output, + finished=False, + ) + ) + + outputs = await app._collect_duplex_data_plane_outputs( + "duplex-sid-e0-stage0", + req_state, + response_stage_id=1, + timeout=1.0, + ) + + assert len(outputs) == 1 + assert outputs[0].request_id == "duplex-sid-e0-stage0" + assert outputs[0].finished is False + assert outputs[0].stage_id == 1 + assert outputs[0].final_output_type == "audio" + assert outputs[0].request_output is raw_stage1_output + + +def test_async_omni_duplex_request_info_includes_response_stage(): + request_id, response_stage_id = AsyncOmni._duplex_data_plane_request_info( + { + "stage_results": [ + { + "result": { + "data_plane_append": True, + "request_id": "duplex-sid-e0-stage0", + "response_stage_id": 1, + } + } + ] + } + ) + + assert request_id == "duplex-sid-e0-stage0" + assert response_stage_id == 1 + + +@pytest.mark.asyncio +async def test_duplex_request_client_retains_output_route_when_close_fails(): + fence = DuplexFence("sid-close-route") + request_id = duplex_resource_request_id(fence, "stage0") + request_states = {request_id: ClientRequestState(request_id)} + + async def close_duplex_session_async(session_id, **kwargs): + del session_id, kwargs + raise RuntimeError("close failed") + + client = DuplexRequestClient( + SimpleNamespace(close_duplex_session_async=close_duplex_session_async), + SimpleNamespace(request_states=request_states), + ) + + with pytest.raises(RuntimeError, match="close failed"): + await client.close(fence.session_id, reason="test", fence=fence, timeout=1.0) + + assert request_id in request_states diff --git a/tests/entrypoints/test_duplex_fence_propagation.py b/tests/entrypoints/test_duplex_fence_propagation.py new file mode 100644 index 000000000..f7b8cdcfc --- /dev/null +++ b/tests/entrypoints/test_duplex_fence_propagation.py @@ -0,0 +1,123 @@ +import pytest + +from tests.entrypoints.openai_api.test_duplex_handler import FakeChatService, TimedWebSocket +from vllm_omni.entrypoints.async_omni import AsyncOmni +from vllm_omni.experimental.fullduplex.engine.messages import DuplexFence +from vllm_omni.experimental.fullduplex.openai.protocol import DuplexSession, DuplexSessionConfig +from vllm_omni.experimental.fullduplex.openai.serving import OmniDuplexSessionHandler + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +class FenceRecordingEngine: + output_modalities = ["text", "audio"] + + def __init__(self) -> None: + self.calls: list[tuple[str, DuplexFence | None]] = [] + self.signal_next_fences: list[DuplexFence | None] = [] + + async def open_duplex_session_async(self, session_id, *, fence=None, **kwargs): + del session_id, kwargs + self.calls.append(("open", fence)) + return {"ok": True} + + async def append_duplex_input_async(self, session_id, *, fence=None, **kwargs): + del session_id, kwargs + self.calls.append(("append", fence)) + return {"ok": True} + + async def signal_duplex_turn_async( + self, + session_id, + *, + event, + fence=None, + next_fence=None, + timeout=None, + ): + del session_id, event, timeout + self.calls.append(("signal", fence)) + self.signal_next_fences.append(next_fence) + return {"ok": True} + + async def close_duplex_session_async(self, session_id, *, fence=None, **kwargs): + del session_id, kwargs + self.calls.append(("close", fence)) + return {"ok": True} + + +@pytest.mark.asyncio +async def test_async_omni_forwards_fence_for_all_runtime_operations(): + engine = FenceRecordingEngine() + app = object.__new__(AsyncOmni) + app.engine = engine + app.request_states = {} + app._final_output_handler = lambda: None + fence = DuplexFence("sid", epoch=1, turn_id=2) + + await app.open_duplex_session_async("sid", fence=fence) + await app.append_duplex_input_async( + "sid", mode="append_audio_chunk", payload={}, fence=fence, collect_outputs=False + ) + await app.signal_duplex_turn_async("sid", event="turn.end", fence=fence) + await app.close_duplex_session_async("sid", reason="done", fence=fence) + + assert engine.calls == [(operation, fence) for operation in ("open", "append", "signal", "close")] + + +@pytest.mark.asyncio +async def test_openai_handler_passes_current_session_fence_for_runtime_controls(): + engine = FenceRecordingEngine() + handler = OmniDuplexSessionHandler(chat_service=FakeChatService(engine)) + session = DuplexSession( + session_id="sid", + config=DuplexSessionConfig(), + epoch=1, + turn_id=2, + ) + ws = TimedWebSocket() + fence = DuplexFence("sid", epoch=1, turn_id=2) + + await handler._open_runtime_session(session, ws.send_json) + await handler._signal_runtime_session(session, "turn.end", send_json=ws.send_json) + await handler._close_runtime_session(session, reason="done", send_json=ws.send_json) + + assert engine.calls == [(operation, fence) for operation in ("open", "signal", "close")] + + +@pytest.mark.asyncio +async def test_runtime_signal_uses_fence_captured_before_session_turn_advances(): + engine = FenceRecordingEngine() + handler = OmniDuplexSessionHandler(chat_service=FakeChatService(engine)) + session = DuplexSession(session_id="sid", config=DuplexSessionConfig()) + captured_fence = DuplexFence("sid", epoch=0, turn_id=0) + session.turn_id = 1 + + await handler._signal_runtime_session( + session, + "input.commit", + fence=captured_fence, + ) + + assert engine.calls == [("signal", captured_fence)] + + +@pytest.mark.asyncio +async def test_openai_handler_forwards_cancel_fence_through_async_omni_facade(): + engine = FenceRecordingEngine() + app = object.__new__(AsyncOmni) + app.engine = engine + handler = OmniDuplexSessionHandler(chat_service=FakeChatService(app)) + session = DuplexSession(session_id="sid-cancel", config=DuplexSessionConfig()) + cancelled_fence = DuplexFence("sid-cancel", epoch=2, turn_id=3) + next_fence = DuplexFence("sid-cancel", epoch=3, turn_id=3) + + assert await handler._signal_runtime_session( + session, + "input.cancel", + fence=cancelled_fence, + next_fence=next_fence, + ) + + assert engine.calls == [("signal", cancelled_fence)] + assert engine.signal_next_fences == [next_fence] diff --git a/tests/entrypoints/test_omni_entrypoints.py b/tests/entrypoints/test_omni_entrypoints.py index 4da82390b..5d9b85938 100644 --- a/tests/entrypoints/test_omni_entrypoints.py +++ b/tests/entrypoints/test_omni_entrypoints.py @@ -108,6 +108,8 @@ class FakeAsyncOmniEngine: self.output_processors = [SimpleNamespace(tokenizer=None) for _ in range(self.num_stages)] self.input_processor = None self.endpoint_restrictions = () + self.duplex_session_config = None + self.duplex_serving_adapter_path = None self.output_q: queue.Queue[Any] = queue.Queue() self.submitted: list[dict[str, Any]] = [] diff --git a/tests/examples/test_minicpmo_realtime_duplex_simple_demo.py b/tests/examples/test_minicpmo_realtime_duplex_simple_demo.py new file mode 100644 index 000000000..93fc4db38 --- /dev/null +++ b/tests/examples/test_minicpmo_realtime_duplex_simple_demo.py @@ -0,0 +1,211 @@ +import base64 +import importlib.util +import sys +import wave +from pathlib import Path + +import pytest + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + +DEMO_PATH = Path(__file__).resolve().parents[2] / "examples/online_serving/minicpmo/realtime_duplex_demo.py" + + +def _load_demo_module(): + spec = importlib.util.spec_from_file_location( + "minicpmo_realtime_duplex_simple_demo_test", + DEMO_PATH, + ) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def test_post_commit_decision_ignores_streaming_listens_before_commit_ack(): + demo = _load_demo_module() + events = [ + {"type": "session.created"}, + {"type": "response.listen"}, + {"type": "response.listen"}, + {"type": "input_audio_buffer.committed"}, + ] + + commit_index = demo._input_committed_index(events, after_index=0) + + assert commit_index == 3 + assert demo._post_commit_model_decision(events, commit_index) is None + + +def test_post_commit_decision_accepts_listen_after_commit_ack(): + demo = _load_demo_module() + events = [ + {"type": "response.listen"}, + {"type": "input_audio_buffer.committed"}, + {"type": "response.listen"}, + ] + + commit_index = demo._input_committed_index(events, after_index=0) + + assert demo._post_commit_model_decision(events, commit_index) == "listen" + + +def test_post_commit_decision_accepts_drained_speak_after_commit_ack(): + demo = _load_demo_module() + events = [ + {"type": "input_audio_buffer.committed"}, + {"type": "response.created", "response": {"id": "resp-a"}}, + {"type": "response.audio.delta", "response_id": "resp-a", "delta": "AAAA"}, + {"type": "response.done", "response_id": "resp-a"}, + ] + + commit_index = demo._input_committed_index(events, after_index=0) + + assert demo._post_commit_model_decision(events, commit_index) == "speak" + + +def test_post_commit_decision_ignores_cancelled_response(): + demo = _load_demo_module() + events = [ + {"type": "input_audio_buffer.committed"}, + {"type": "response.done", "response": {"status": "cancelled"}}, + ] + + commit_index = demo._input_committed_index(events, after_index=0) + + assert demo._post_commit_model_decision(events, commit_index) is None + + +def test_commit_ack_search_starts_after_stream_cursor(): + demo = _load_demo_module() + events = [ + {"type": "input_audio_buffer.committed"}, + {"type": "response.listen"}, + {"type": "input_audio_buffer.committed"}, + ] + + assert demo._input_committed_index(events, after_index=1) == 2 + + +def test_exact_model_unit_boundary_does_not_require_an_extra_decision(): + demo = _load_demo_module() + unit_bytes = demo.PCM16_SAMPLE_RATE * demo.PCM16_BYTES_PER_SAMPLE + + assert not demo._has_residual_model_unit(b"\0" * (unit_bytes * 6), chunk_period_ms=1000) + + +def test_partial_model_unit_requires_a_post_commit_decision(): + demo = _load_demo_module() + unit_bytes = demo.PCM16_SAMPLE_RATE * demo.PCM16_BYTES_PER_SAMPLE + + assert demo._has_residual_model_unit(b"\0" * (unit_bytes * 6 + 512), chunk_period_ms=1000) + + +def test_latest_streaming_decision_is_used_at_exact_boundary(): + demo = _load_demo_module() + events = [ + {"type": "response.listen"}, + {"type": "response.listen"}, + {"type": "input_audio_buffer.committed"}, + ] + + assert demo._latest_model_decision(events, after_index=0) == "listen" + + +def test_ref_audio_data_url_encodes_explicit_wav(tmp_path): + demo = _load_demo_module() + ref = tmp_path / "ref.wav" + ref.write_bytes(b"RIFFfake") + + assert demo._ref_audio_data_url(str(ref)) == "data:audio/wav;base64,UklGRmZha2U=" + assert demo._ref_audio_data_url(None) is None + + +def test_realtime_duplex_demo_requires_ref_audio(monkeypatch): + demo = _load_demo_module() + monkeypatch.setattr( + demo.sys, + "argv", + [ + "realtime_duplex_demo.py", + "--input-wav", + "input.wav", + ], + ) + + with pytest.raises(SystemExit): + demo.parse_args() + + +def test_realtime_duplex_demo_accepts_explicit_ref_audio(monkeypatch): + demo = _load_demo_module() + monkeypatch.setattr( + demo.sys, + "argv", + [ + "realtime_duplex_demo.py", + "--input-wav", + "input.wav", + "--ref-audio", + "ref.wav", + ], + ) + + args = demo.parse_args() + + assert args.ref_audio == "ref.wav" + + +def test_open_streaming_response_requires_post_commit_drain(): + demo = _load_demo_module() + + assert demo._response_in_progress([{"type": "response.created"}]) + assert not demo._response_in_progress( + [ + {"type": "response.created"}, + {"type": "response.done"}, + ] + ) + + +def test_streaming_output_writer_persists_audio_deltas_as_they_arrive(tmp_path, capsys): + demo = _load_demo_module() + writer = demo._StreamingOutputWriter(tmp_path) + first_pcm = b"\x01\x00\x02\x00" + second_pcm = b"\x03\x00" + + writer.handle( + { + "type": "response.audio.delta", + "response_id": "resp-a", + "delta": base64.b64encode(first_pcm).decode("ascii"), + "sample_rate_hz": 24_000, + } + ) + writer.handle( + { + "type": "response.audio_transcript.delta", + "response_id": "resp-a", + "delta": "你好", + } + ) + writer.handle( + { + "type": "response.audio.delta", + "response_id": "resp-a", + "delta": base64.b64encode(second_pcm).decode("ascii"), + "sample_rate_hz": 24_000, + } + ) + + assert (tmp_path / "output.pcm").read_bytes() == first_pcm + second_pcm + chunk_paths = sorted((tmp_path / "audio_chunks").glob("*.wav")) + assert [path.name for path in chunk_paths] == ["chunk_0001.wav", "chunk_0002.wav"] + with wave.open(str(chunk_paths[0]), "rb") as chunk_wav: + assert chunk_wav.getframerate() == 24_000 + assert chunk_wav.readframes(chunk_wav.getnframes()) == first_pcm + assert writer.audio_chunk_paths == chunk_paths + stderr = capsys.readouterr().err + assert "audio chunk 1" in stderr + assert "你好" in stderr diff --git a/tests/examples/test_minicpmo_realtime_web.py b/tests/examples/test_minicpmo_realtime_web.py new file mode 100644 index 000000000..03fbc6d6d --- /dev/null +++ b/tests/examples/test_minicpmo_realtime_web.py @@ -0,0 +1,1663 @@ +import asyncio +import base64 +import importlib.util +import subprocess +import sys +import wave +from pathlib import Path +from types import SimpleNamespace +from urllib.parse import parse_qs, urlsplit + +import pytest + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + +DEMO_PATH = Path(__file__).resolve().parents[1] / "e2e/online_serving/minicpmo_realtime_duplex_scenarios.py" +MULTI_DEMO_PATH = ( + Path(__file__).resolve().parents[1] / "e2e/online_serving/run_minicpmo_realtime_duplex_multi_session.py" +) +PAIR_DEMO_PATH = Path(__file__).resolve().parents[1] / "e2e/online_serving/run_minicpmo_realtime_duplex_demo_pair.py" +SOFT_INTERRUPT_DEMO_PATH = ( + Path(__file__).resolve().parents[1] / "e2e/online_serving/run_minicpmo_realtime_duplex_soft_interrupt.py" +) + + +def _load_demo_module(): + spec = importlib.util.spec_from_file_location("minicpmo_realtime_duplex_demo_test", DEMO_PATH) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def _load_multi_demo_module(): + spec = importlib.util.spec_from_file_location("minicpmo_realtime_duplex_multi_session_test", MULTI_DEMO_PATH) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def _load_pair_demo_module(): + spec = importlib.util.spec_from_file_location("minicpmo_realtime_duplex_demo_pair_test", PAIR_DEMO_PATH) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def _load_soft_interrupt_demo_module(): + spec = importlib.util.spec_from_file_location( + "minicpmo_realtime_duplex_soft_interrupt_test", + SOFT_INTERRUPT_DEMO_PATH, + ) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def test_realtime_duplex_multi_session_script_is_directly_executable(): + result = subprocess.run( + [sys.executable, str(MULTI_DEMO_PATH), "--help"], + cwd=MULTI_DEMO_PATH.parents[3], + capture_output=True, + text=True, + timeout=20, + check=False, + ) + + assert result.returncode == 0, result.stderr + + +def test_realtime_duplex_demo_pair_script_is_directly_executable(): + result = subprocess.run( + [sys.executable, str(PAIR_DEMO_PATH), "--help"], + cwd=PAIR_DEMO_PATH.parents[3], + capture_output=True, + text=True, + timeout=20, + check=False, + ) + + assert result.returncode == 0, result.stderr + + +def test_realtime_duplex_soft_interrupt_script_is_directly_executable(): + result = subprocess.run( + [sys.executable, str(SOFT_INTERRUPT_DEMO_PATH), "--help"], + cwd=SOFT_INTERRUPT_DEMO_PATH.parents[3], + capture_output=True, + text=True, + timeout=20, + check=False, + ) + + assert result.returncode == 0, result.stderr + + +def test_realtime_duplex_demo_pair_requires_distinct_inputs_and_outputs(tmp_path): + demo = _load_pair_demo_module() + wav_a = tmp_path / "request_a.wav" + wav_b = tmp_path / "request_b.wav" + wav_a.write_bytes(b"same") + wav_b.write_bytes(b"same") + + args = SimpleNamespace( + input_wav_a=str(wav_a), + input_wav_b=str(wav_b), + output_dir_a=str(tmp_path / "session_a"), + output_dir_b=str(tmp_path / "session_a"), + ) + + with pytest.raises(ValueError, match="output directories must be different"): + demo._validate_pair_args(args) + + args.output_dir_b = str(tmp_path / "session_b") + with pytest.raises(ValueError, match="input WAV files must have different content"): + demo._validate_pair_args(args) + + +def test_realtime_duplex_demo_pair_launches_demo_processes_concurrently(tmp_path, monkeypatch): + demo = _load_pair_demo_module() + fake_demo = tmp_path / "fake_realtime_duplex_demo.py" + start_log = tmp_path / "starts.log" + fake_demo.write_text( + "\n".join( + [ + "import argparse, json, os, time", + "from pathlib import Path", + "parser = argparse.ArgumentParser()", + "parser.add_argument('--url')", + "parser.add_argument('--model')", + "parser.add_argument('--input-wav')", + "parser.add_argument('--output-dir')", + "parser.add_argument('--chunk-ms')", + "parser.add_argument('--timeout-s')", + "parser.add_argument('--session-id')", + "parser.add_argument('--ref-audio')", + "parser.add_argument('--require-audio', action='store_true')", + "parser.add_argument('--no-realtime-pacing', action='store_true')", + "args = parser.parse_args()", + "if args.ref_audio != os.environ.get('EXPECTED_REF_AUDIO'):", + " raise SystemExit(f'unexpected ref audio: {args.ref_audio}')", + "output = Path(args.output_dir)", + "output.mkdir(parents=True, exist_ok=True)", + "response_id = 'resp-' + output.name", + "with open(os.environ['PAIR_START_LOG'], 'a', encoding='utf-8') as f:", + " f.write(f'{output.name} {time.time()}\\n')", + "time.sleep(0.25)", + "events = [", + " {'type': 'session.created'},", + " {'type': 'response.created', 'response': {'id': response_id}},", + " {'type': 'response.audio.delta', 'response_id': response_id, " + "'delta': 'AAAA', 'sample_rate_hz': 24000, '_client_received_at_s': 10.1},", + " {'type': 'response.audio.delta', 'response_id': response_id, " + "'delta': 'AAAA', 'sample_rate_hz': 24000, '_client_received_at_s': 10.6},", + " {'type': 'response.done', 'response_id': response_id},", + " {'type': 'session.closed'},", + "]", + "Path(output / 'events.jsonl').write_text(", + " ''.join(json.dumps(event) + '\\n' for event in events),", + " encoding='utf-8',", + ")", + "result = {'ok': True, 'response_ids': [response_id], 'audio_bytes': 1, 'output_dir': str(output)}", + "Path(output / 'result.json').write_text(json.dumps(result), encoding='utf-8')", + "print(json.dumps(result))", + ] + ), + encoding="utf-8", + ) + monkeypatch.setattr(demo, "DEMO_PATH", fake_demo) + monkeypatch.setenv("PAIR_START_LOG", str(start_log)) + monkeypatch.setenv("EXPECTED_REF_AUDIO", str(tmp_path / "ref.wav")) + wav_a = tmp_path / "request_a.wav" + wav_b = tmp_path / "request_b.wav" + wav_a.write_bytes(b"request a") + wav_b.write_bytes(b"request b") + + result = asyncio.run( + demo.run_pair( + SimpleNamespace( + url="ws://localhost:28889/v1/realtime?duplex=1", + model="/data/why/MiniCPM-o-4_5", + input_wav_a=str(wav_a), + input_wav_b=str(wav_b), + output_dir_a=str(tmp_path / "duplex_session_a"), + output_dir_b=str(tmp_path / "duplex_session_b"), + ref_audio=str(tmp_path / "ref.wav"), + summary_output=str(tmp_path / "summary.json"), + chunk_ms=200, + timeout_s=5.0, + no_realtime_pacing=False, + require_audio=True, + min_audio_deltas_per_session=2, + ) + ) + ) + + starts = [float(line.split()[1]) for line in start_log.read_text(encoding="utf-8").splitlines()] + assert len(starts) == 2 + assert max(starts) - min(starts) < 0.2 + assert result["ok"] is True + assert result["input_wavs_distinct"] is True + assert result["output_dirs_distinct"] is True + assert result["identity_isolation_ok"] is True + assert result["response_audio_contract_ok"] is True + assert {session["audio_delta_count"] for session in result["sessions"]} == {2} + for session in result["sessions"]: + assert session["all_audio_deltas_same_response"] is True + assert session["response_id_nonempty"] is True + assert session["first_audio_delta_before_done"] is True + assert len(session["audio_delta_timings"]) == 2 + + +def test_realtime_duplex_demo_pair_rejects_false_green_audio_contract(tmp_path): + demo = _load_pair_demo_module() + output = tmp_path / "session" + output.mkdir() + response_id = "resp-a" + events = [ + {"type": "response.created", "response": {"id": response_id}}, + {"type": "response.audio.delta", "response_id": response_id, "delta": "AAAA", "sample_rate_hz": 24000}, + {"type": "response.audio.delta", "response_id": "resp-other", "delta": "AAAA", "sample_rate_hz": 24000}, + {"type": "response.done", "response_id": response_id}, + ] + (output / "events.jsonl").write_text( + "".join(demo.json.dumps(event) + "\n" for event in events), + encoding="utf-8", + ) + (output / "result.json").write_text( + demo.json.dumps({"ok": True, "response_ids": [response_id], "audio_bytes": 4}), + encoding="utf-8", + ) + + session = demo._summarize_session( + label="a", + input_wav="request.wav", + output_dir=str(output), + returncode=0, + stdout="", + stderr="", + ) + + assert session["ok"] is False + assert session["all_audio_deltas_same_response"] is False + assert session["response_audio_contract_ok"] is False + + +def test_realtime_duplex_demo_pair_default_requires_multiple_audio_deltas(monkeypatch): + demo = _load_pair_demo_module() + monkeypatch.setattr( + demo.sys, + "argv", + [ + "pair.py", + "--input-wav-a", + "a.wav", + "--input-wav-b", + "b.wav", + "--output-dir-a", + "a", + "--output-dir-b", + "b", + "--ref-audio", + "ref.wav", + ], + ) + + args = demo.parse_args() + + assert args.ref_audio == "ref.wav" + assert args.min_audio_deltas_per_session == 2 + + +def test_realtime_duplex_demo_pair_requires_ref_audio(monkeypatch): + demo = _load_pair_demo_module() + monkeypatch.setattr( + demo.sys, + "argv", + [ + "pair.py", + "--input-wav-a", + "a.wav", + "--input-wav-b", + "b.wav", + "--output-dir-a", + "a", + "--output-dir-b", + "b", + ], + ) + + with pytest.raises(SystemExit): + demo.parse_args() + + +def test_realtime_duplex_soft_interrupt_requires_ref_audio(monkeypatch): + demo = _load_soft_interrupt_demo_module() + monkeypatch.setattr( + demo.sys, + "argv", + [ + "soft.py", + "--input-wav", + "input.wav", + "--output-dir", + "out", + ], + ) + + with pytest.raises(SystemExit): + demo.parse_args() + + +def test_realtime_duplex_soft_interrupt_accepts_explicit_ref_audio(monkeypatch): + demo = _load_soft_interrupt_demo_module() + monkeypatch.setattr( + demo.sys, + "argv", + [ + "soft.py", + "--input-wav", + "input.wav", + "--output-dir", + "out", + "--ref-audio", + "ref.wav", + ], + ) + + args = demo.parse_args() + + assert args.ref_audio == "ref.wav" + assert args.validation_mode == "model-policy" + + +@pytest.mark.parametrize( + "extra_args", + [ + ["--expect-second-response-substring", "一加一等于二"], + ["--input-sha256", "a" * 64], + ], +) +def test_realtime_duplex_soft_interrupt_response_required_needs_bound_fixture(monkeypatch, extra_args): + demo = _load_soft_interrupt_demo_module() + monkeypatch.setattr( + demo.sys, + "argv", + [ + "soft.py", + "--input-wav", + "input.wav", + "--output-dir", + "out", + "--ref-audio", + "ref.wav", + "--validation-mode", + "response-required", + *extra_args, + ], + ) + + with pytest.raises(SystemExit): + demo.parse_args() + + +def test_realtime_duplex_soft_interrupt_rejects_input_checksum_mismatch(tmp_path): + demo = _load_soft_interrupt_demo_module() + input_wav = tmp_path / "input.wav" + input_wav.write_bytes(b"two-turn fixture") + + with pytest.raises(ValueError, match="input WAV SHA256 mismatch"): + demo._validate_input_sha256(input_wav, "0" * 64) + + +def test_realtime_duplex_soft_interrupt_accepts_multi_delta_handoff_sequence(tmp_path): + demo = _load_soft_interrupt_demo_module() + output = tmp_path / "soft_interrupt" + output.mkdir() + first_response_id = "resp-first" + second_response_id = "resp-second" + events = [ + {"type": "response.listen", "_client_received_at_s": 1.0}, + {"type": "response.created", "response": {"id": first_response_id}, "_client_received_at_s": 2.0}, + {"type": "response.speak", "response_id": first_response_id, "_client_received_at_s": 2.0}, + { + "type": "response.audio.delta", + "response_id": first_response_id, + "delta": "AAAA", + "_client_received_at_s": 2.1, + }, + { + "type": "response.audio_transcript.delta", + "response_id": first_response_id, + "delta": "中国古代四大发明", + "_client_received_at_s": 2.1, + }, + { + "type": "response.audio.delta", + "response_id": first_response_id, + "delta": "AAAA", + "_client_received_at_s": 2.6, + }, + { + "type": "response.audio_transcript.delta", + "response_id": first_response_id, + "delta": "是造纸术。", + "_client_received_at_s": 2.6, + }, + {"type": "response.done", "response_id": first_response_id, "_client_received_at_s": 3.0}, + {"type": "response.listen", "_client_received_at_s": 3.2}, + {"type": "response.created", "response": {"id": second_response_id}, "_client_received_at_s": 4.0}, + {"type": "response.speak", "response_id": second_response_id, "_client_received_at_s": 4.0}, + { + "type": "response.audio.delta", + "response_id": second_response_id, + "delta": "AAAA", + "_client_received_at_s": 4.1, + }, + { + "type": "response.audio_transcript.delta", + "response_id": second_response_id, + "delta": "一加一等于", + "_client_received_at_s": 4.1, + }, + { + "type": "response.audio.delta", + "response_id": second_response_id, + "delta": "AAAA", + "_client_received_at_s": 4.6, + }, + { + "type": "response.audio_transcript.delta", + "response_id": second_response_id, + "delta": "二。", + "_client_received_at_s": 4.6, + }, + {"type": "response.done", "response_id": second_response_id, "_client_received_at_s": 5.0}, + {"type": "response.listen", "_client_received_at_s": 5.2}, + {"type": "input_audio_buffer.committed", "_client_received_at_s": 6.0}, + {"type": "response.listen", "_client_received_at_s": 6.1}, + ] + (output / "events.jsonl").write_text( + "".join(demo.json.dumps(event) + "\n" for event in events), + encoding="utf-8", + ) + (output / "result.json").write_text( + demo.json.dumps({"ok": True, "response_ids": [first_response_id, second_response_id]}), + encoding="utf-8", + ) + + summary = demo.summarize_artifacts( + output_dir=output, + validation_mode="response-required", + min_responses=2, + min_audio_deltas_per_response=2, + expect_second_response_substring="一加一等于二", + ) + + assert summary["ok"] is True + assert summary["second_response_before_final_commit"] is True + assert summary["final_listen_after_commit"] is True + assert summary["response_audio_contract_ok"] is True + assert summary["response_summaries"][0]["audio_delta_count"] == 2 + assert summary["response_summaries"][1]["transcript"] == "一加一等于二。" + + +def test_realtime_duplex_soft_interrupt_model_policy_accepts_single_response(tmp_path): + demo = _load_soft_interrupt_demo_module() + output = tmp_path / "model_policy" + output.mkdir() + response_id = "resp-only" + events = [ + {"type": "response.listen", "_client_received_at_s": 1.0}, + {"type": "response.created", "response": {"id": response_id}, "_client_received_at_s": 2.0}, + {"type": "response.audio.delta", "response_id": response_id, "delta": "AAAA", "_client_received_at_s": 2.1}, + {"type": "response.audio.delta", "response_id": response_id, "delta": "AAAA", "_client_received_at_s": 2.15}, + {"type": "response.done", "response_id": response_id, "_client_received_at_s": 2.2}, + {"type": "response.listen", "_client_received_at_s": 2.3}, + {"type": "input_audio_buffer.committed", "_client_received_at_s": 3.0}, + {"type": "response.listen", "_client_received_at_s": 3.1}, + ] + (output / "events.jsonl").write_text( + "".join(demo.json.dumps(event) + "\n" for event in events), + encoding="utf-8", + ) + (output / "result.json").write_text( + demo.json.dumps({"ok": True, "response_ids": [response_id]}), + encoding="utf-8", + ) + + summary = demo.summarize_artifacts( + output_dir=output, + validation_mode="model-policy", + min_responses=2, + min_audio_deltas_per_response=2, + expect_second_response_substring=None, + ) + + assert summary["ok"] is True + assert summary["validation_mode"] == "model-policy" + assert summary["enough_responses"] is True + assert summary["response_before_final_commit"] is True + assert summary["listen_after_response_before_commit"] is True + + +def test_realtime_duplex_soft_interrupt_response_required_rejects_single_response(tmp_path): + demo = _load_soft_interrupt_demo_module() + output = tmp_path / "response_required" + output.mkdir() + response_id = "resp-only" + events = [ + {"type": "response.listen", "_client_received_at_s": 1.0}, + {"type": "response.created", "response": {"id": response_id}, "_client_received_at_s": 2.0}, + {"type": "response.audio.delta", "response_id": response_id, "delta": "AAAA", "_client_received_at_s": 2.1}, + {"type": "response.audio.delta", "response_id": response_id, "delta": "AAAA", "_client_received_at_s": 2.15}, + {"type": "response.done", "response_id": response_id, "_client_received_at_s": 2.2}, + {"type": "response.listen", "_client_received_at_s": 2.3}, + {"type": "input_audio_buffer.committed", "_client_received_at_s": 3.0}, + {"type": "response.listen", "_client_received_at_s": 3.1}, + ] + (output / "events.jsonl").write_text( + "".join(demo.json.dumps(event) + "\n" for event in events), + encoding="utf-8", + ) + (output / "result.json").write_text( + demo.json.dumps({"ok": True, "response_ids": [response_id]}), + encoding="utf-8", + ) + + summary = demo.summarize_artifacts( + output_dir=output, + validation_mode="response-required", + min_responses=2, + min_audio_deltas_per_response=2, + expect_second_response_substring="一加一等于二", + ) + + assert summary["ok"] is False + assert summary["enough_responses"] is False + + +def test_realtime_duplex_soft_interrupt_checks_expected_text_in_second_response(tmp_path): + demo = _load_soft_interrupt_demo_module() + output = tmp_path / "second_response_text" + output.mkdir() + first_response_id = "resp-first" + second_response_id = "resp-second" + events = [ + {"type": "response.listen", "_client_received_at_s": 1.0}, + {"type": "response.created", "response": {"id": first_response_id}, "_client_received_at_s": 2.0}, + { + "type": "response.audio.delta", + "response_id": first_response_id, + "delta": "AAAA", + "_client_received_at_s": 2.1, + }, + { + "type": "response.audio.delta", + "response_id": first_response_id, + "delta": "AAAA", + "_client_received_at_s": 2.2, + }, + { + "type": "response.audio_transcript.delta", + "response_id": first_response_id, + "delta": "一加一等于二", + "_client_received_at_s": 2.2, + }, + {"type": "response.done", "response_id": first_response_id, "_client_received_at_s": 2.3}, + {"type": "response.listen", "_client_received_at_s": 2.4}, + {"type": "response.created", "response": {"id": second_response_id}, "_client_received_at_s": 3.0}, + { + "type": "response.audio.delta", + "response_id": second_response_id, + "delta": "AAAA", + "_client_received_at_s": 3.1, + }, + { + "type": "response.audio.delta", + "response_id": second_response_id, + "delta": "AAAA", + "_client_received_at_s": 3.2, + }, + { + "type": "response.audio_transcript.delta", + "response_id": second_response_id, + "delta": "不知道", + "_client_received_at_s": 3.2, + }, + {"type": "response.done", "response_id": second_response_id, "_client_received_at_s": 3.3}, + {"type": "response.listen", "_client_received_at_s": 3.4}, + {"type": "input_audio_buffer.committed", "_client_received_at_s": 4.0}, + {"type": "response.listen", "_client_received_at_s": 4.1}, + ] + (output / "events.jsonl").write_text( + "".join(demo.json.dumps(event) + "\n" for event in events), + encoding="utf-8", + ) + (output / "result.json").write_text( + demo.json.dumps({"ok": True, "response_ids": [first_response_id, second_response_id]}), + encoding="utf-8", + ) + + summary = demo.summarize_artifacts( + output_dir=output, + validation_mode="response-required", + min_responses=2, + min_audio_deltas_per_response=2, + expect_second_response_substring="一加一等于二", + ) + + assert summary["ok"] is False + assert summary["second_response_transcript_expectation_ok"] is False + + +def test_realtime_duplex_multi_session_resume_url_disables_autostart(): + demo = _load_multi_demo_module() + + url = demo._with_resume_mode("ws://localhost:8113/v1/realtime?duplex=1&model=openbmb%2FMiniCPM-o-4_5") + + query = parse_qs(urlsplit(url).query) + assert query["model"] == ["openbmb/MiniCPM-o-4_5"] + assert query["resume"] == ["1"] + + +def test_realtime_duplex_multi_session_rejects_cross_session_response_identity(): + demo = _load_multi_demo_module() + + assert demo._validate_identity_isolation( + [ + {"completed_response_ids": ["resp-a"]}, + {"completed_response_ids": ["resp-b"]}, + ] + ) + assert not demo._validate_identity_isolation( + [ + {"completed_response_ids": ["resp-shared"]}, + {"completed_response_ids": ["resp-shared"]}, + ] + ) + + +def test_realtime_duplex_multi_session_reads_nested_terminal_identity(): + demo = _load_multi_demo_module() + + assert demo._event_session_id({"type": "session.heartbeat_ack", "session_id": "sid"}) == "sid" + assert ( + demo._event_session_id({"type": "session.closed", "event": {"type": "session.closed", "session_id": "sid"}}) + == "sid" + ) + + +def test_realtime_duplex_demo_resolves_distinct_turn_inputs(): + demo = _load_demo_module() + primary = Path("first.wav") + + assert demo._turn_input_paths(primary, [], turns=3) == [primary, primary, primary] + assert demo._turn_input_paths(primary, ["second.wav", "third.wav"], turns=3) == [ + primary, + Path("second.wav"), + Path("third.wav"), + ] + with pytest.raises(ValueError, match="one --turn-input-wav"): + demo._turn_input_paths(primary, ["second.wav"], turns=3) + + +def test_realtime_duplex_demo_explicitly_enables_native_runtime_before_connect(): + demo = _load_demo_module() + + url = demo._url_with_model( + "ws://localhost:8099/v1/realtime?duplex=1", + "openbmb/MiniCPM-o-4_5", + ) + + query = parse_qs(urlsplit(url).query) + assert query["minicpmo45_native_duplex"] == ["1"] + + +def test_realtime_duplex_demo_explicit_session_id_reaches_autostart_query(): + demo = _load_demo_module() + + url = demo._url_with_model( + "ws://localhost:8099/v1/realtime?duplex=1", + "openbmb/MiniCPM-o-4_5", + session_id="reopen-e2e", + ) + + query = parse_qs(urlsplit(url).query) + assert query["session_id"] == ["reopen-e2e"] + + +def test_realtime_duplex_demo_session_update_uses_explicit_session_id(): + demo = _load_demo_module() + + event = demo._session_update_event( + SimpleNamespace( + model="openbmb/MiniCPM-o-4_5", + output_audio_format="pcm16", + short_ack_ms=1200, + session_id="reopen-e2e", + ) + ) + + assert event["type"] == "session.update" + assert "session_id" not in event + assert event["session"]["session_id"] == "reopen-e2e" + assert event["session"]["extra_body"]["minicpmo45_native_duplex"] is True + + +def test_realtime_duplex_demo_response_required_uses_deterministic_sampling(): + demo = _load_demo_module() + + event = demo._session_update_event( + SimpleNamespace( + model="openbmb/MiniCPM-o-4_5", + output_audio_format="pcm16", + short_ack_ms=1200, + session_id=None, + validation_mode="response-required", + temperature=None, + ) + ) + + assert event["session"]["temperature"] == 0.0 + + +def test_realtime_duplex_demo_model_policy_preserves_default_sampling(): + demo = _load_demo_module() + + event = demo._session_update_event( + SimpleNamespace( + model="openbmb/MiniCPM-o-4_5", + output_audio_format="pcm16", + short_ack_ms=1200, + session_id=None, + validation_mode="model-policy", + temperature=None, + ) + ) + + assert "temperature" not in event["session"] + + +def test_realtime_duplex_demo_resolves_explicit_turn_durations(): + demo = _load_demo_module() + + assert demo._turn_durations([], turns=3, first_turn_ms=3000) == [3000, 1200, 1200] + assert demo._turn_durations([0, 900, 1500], turns=3, first_turn_ms=3000) == [None, 900, 1500] + with pytest.raises(ValueError, match="one --turn-duration-ms"): + demo._turn_durations([0, 900], turns=3, first_turn_ms=3000) + with pytest.raises(ValueError, match="non-negative"): + demo._turn_durations([0, -1, 900], turns=3, first_turn_ms=3000) + + +def test_realtime_duplex_demo_resolves_transcript_labels_for_requested_turns(): + demo = _load_demo_module() + + assert demo._turn_transcripts("first", turns=1) == ["first"] + assert demo._turn_transcripts("first", turns=4) == ["first", "继续", "再说一次", "turn-4"] + + +def test_realtime_duplex_demo_reads_response_playback_cursor(): + demo = _load_demo_module() + state = demo.DemoState() + state.add( + { + "type": "response.done", + "response_id": "resp-1", + "response": { + "id": "resp-1", + "metadata": { + "playback": { + "sent_ms": 27920, + "played_ms": 0, + } + }, + }, + } + ) + + assert state.response_playback_sent_ms("resp-1") == 27920 + + +def test_realtime_duplex_demo_partitions_timing_by_response_identity(): + demo = _load_demo_module() + state = demo.DemoState() + for response_id, created_at_s, audio_at_s, token_count in ( + ("resp-1", 10.0, 10.1, 3), + ("resp-2", 20.0, 20.2, 5), + ): + state.add( + {"type": "response.created", "response": {"id": response_id}}, + received_at_s=created_at_s, + ) + state.add( + { + "type": "response.audio.delta", + "response_id": response_id, + "delta": base64.b64encode(b"audio").decode("ascii"), + "metadata": { + "audio_duration_ms": 80, + "vllm_omni": { + "stage_metrics": { + "0": { + "num_tokens_out": token_count, + "vllm_itls_ms": [10.0], + } + } + }, + }, + }, + received_at_s=audio_at_s, + ) + + timings = state.response_timing_summaries() + + assert timings["resp-1"]["stage0_tokens"]["output_token_count"] == 3 + assert timings["resp-1"]["audio_output"]["response_created_to_first_audio_ms"] == 100.0 + assert timings["resp-1"]["audio_output"]["commit_to_first_audio_ms"] is None + assert timings["resp-2"]["stage0_tokens"]["output_token_count"] == 5 + assert timings["resp-2"]["audio_output"]["response_created_to_first_audio_ms"] == 200.0 + assert timings["resp-2"]["audio_output"]["commit_to_first_audio_ms"] is None + + +def test_realtime_duplex_demo_model_policy_accepts_one_listen_per_streamed_turn(): + demo = _load_demo_module() + state = demo.DemoState() + state.add({"type": "session.created"}) + for turn in range(3): + state.add({"type": "input_audio_buffer.speech_started", "turn": turn}) + state.add( + { + "type": "response.listen", + "response": { + "status": "listening", + "metadata": {"model_listen": True}, + }, + } + ) + state.add({"type": "input_audio_buffer.committed", "turn": turn}) + + assert state.model_policy_event_order_ok(expected_turns=3) + + +def test_realtime_duplex_demo_model_policy_accepts_continuous_input_without_commits(): + demo = _load_demo_module() + state = demo.DemoState() + state.add({"type": "session.created"}) + state.add({"type": "input_audio_buffer.speech_started"}) + for _ in range(3): + state.add( + { + "type": "response.listen", + "response": {"metadata": {"model_listen": True}}, + } + ) + + assert not state.model_policy_event_order_ok(expected_turns=3) + assert state.model_policy_event_order_ok(expected_turns=3, require_input_commit=False) + + +def test_realtime_duplex_demo_accepts_overlap_unit_terminating_continuous_response(): + demo = _load_demo_module() + state = demo.DemoState() + state.add({"type": "response.created", "response_id": "resp-1"}) + state.add({"type": "response.done", "response_id": "resp-1"}) + + assert demo._continuous_overlap_terminal_is_outcome( + state, + validation_mode="model-policy", + model_unit_ready_while_active=True, + before_created=1, + before_model_listen=0, + ) + assert not demo._continuous_overlap_terminal_is_outcome( + state, + validation_mode="response-required", + model_unit_ready_while_active=True, + before_created=1, + before_model_listen=0, + ) + + +def test_realtime_duplex_demo_full_turn_duration_does_not_slice_audio(): + demo = _load_demo_module() + pcm16 = b"\x01\x00" * (demo.PCM16_SAMPLE_RATE * 2) + + assert demo._select_turn_audio(pcm16, None) == pcm16 + assert len(demo._select_turn_audio(pcm16, 1000)) == demo.PCM16_SAMPLE_RATE * demo.PCM16_BYTES_PER_SAMPLE + + +def test_realtime_duplex_demo_distinct_inputs_compare_audio_content(): + demo = _load_demo_module() + paths = [Path("first.wav"), Path("second.wav"), Path("third.wav")] + + assert demo._turn_inputs_are_distinct(paths, [b"first", b"second", b"third"]) is True + assert demo._turn_inputs_are_distinct(paths, [b"same", b"same", b"third"]) is False + assert demo._turn_inputs_are_distinct([paths[0], paths[0]], [b"first", b"second"]) is False + + +def _add_response_transcript(state, response_id, *, transcript, audio=True): + state.add( + { + "type": "response.audio.delta", + "response_id": response_id, + "delta": "YQ==" if audio else "", + } + ) + if transcript: + state.add( + { + "type": "response.audio_transcript.delta", + "response_id": response_id, + "delta": transcript, + } + ) + state.add( + { + "type": "response.audio_transcript.done", + "response_id": response_id, + "transcript": transcript, + } + ) + state.add( + { + "type": "response.done", + "response_id": response_id, + "response": {"id": response_id, "status": "completed"}, + } + ) + + +def test_realtime_duplex_demo_gate_checks_delta_done_and_turn_independence(): + demo = _load_demo_module() + state = demo.DemoState() + _add_response_transcript( + state, + "resp-1", + transcript="第一轮回答", + ) + _add_response_transcript( + state, + "resp-2", + transcript="第二轮回答", + ) + + result = demo._evaluate_transcript_integrity( + state, + ["resp-1", "resp-2"], + expected_empty_response_ids=set(), + require_cross_turn_independence=True, + ) + + assert result["transcript_delta_done_ok"] is True + assert result["cross_turn_independent_ok"] is True + + +def test_realtime_duplex_demo_speak_gate_accepts_one_decision_event_per_response(): + demo = _load_demo_module() + state = demo.DemoState() + state.add({"type": "response.created", "response": {"id": "resp-1"}}) + state.add( + { + "type": "response.speak", + "response_id": "resp-1", + "metadata": {"model_speak": True}, + } + ) + + assert demo._evaluate_response_speak_contract(state)["response_speak_contract_ok"] is True + + +def test_realtime_duplex_demo_speak_gate_rejects_duplicate_event(): + demo = _load_demo_module() + state = demo.DemoState() + state.add({"type": "response.created", "response": {"id": "resp-1"}}) + state.add({"type": "response.speak", "response_id": "resp-1"}) + state.add({"type": "response.speak", "response_id": "resp-1"}) + + result = demo._evaluate_response_speak_contract(state) + + assert result["response_speak_contract_ok"] is False + assert result["duplicate_response_speak_ids"] == ["resp-1"] + + +def test_realtime_duplex_demo_speak_gate_rejects_text_channel(): + demo = _load_demo_module() + state = demo.DemoState() + state.add({"type": "response.created", "response": {"id": "resp-1"}}) + state.add( + { + "type": "response.speak", + "response_id": "resp-1", + "text": "must be emitted through response.audio_transcript.delta", + } + ) + + result = demo._evaluate_response_speak_contract(state) + + assert result["response_speak_contract_ok"] is False + assert result["text_bearing_response_speak_count"] == 1 + + +def test_realtime_duplex_demo_gate_rejects_cross_turn_tail_reuse(): + demo = _load_demo_module() + state = demo.DemoState() + _add_response_transcript(state, "resp-1", transcript="第一轮回答") + _add_response_transcript(state, "resp-2", transcript="第二轮仍然带着第一轮回答") + + result = demo._evaluate_transcript_integrity( + state, + ["resp-1", "resp-2"], + expected_empty_response_ids=set(), + require_cross_turn_independence=True, + ) + + assert result["cross_turn_independent_ok"] is False + + +def test_realtime_duplex_demo_gate_rejects_terminal_only_previous_tail(): + demo = _load_demo_module() + state = demo.DemoState() + _add_response_transcript(state, "resp-1", transcript="上一轮回答结尾是的吗?") + _add_response_transcript(state, "resp-2", transcript="的吗?") + + result = demo._evaluate_transcript_integrity( + state, + ["resp-1", "resp-2"], + expected_empty_response_ids=set(), + require_cross_turn_independence=True, + ) + + assert result["cross_turn_independent_ok"] is False + + +def test_realtime_duplex_demo_gate_rejects_delta_done_mismatch(): + demo = _load_demo_module() + state = demo.DemoState() + _add_response_transcript(state, "resp-1", transcript="delta文本") + done = next(event for event in state.events if event.get("type") == "response.audio_transcript.done") + done["transcript"] = "另一个done文本" + + result = demo._evaluate_transcript_integrity( + state, + ["resp-1"], + expected_empty_response_ids=set(), + require_cross_turn_independence=False, + ) + + assert result["transcript_delta_done_ok"] is False + + +def test_realtime_duplex_demo_gate_rejects_terminal_only_stale_tail(): + demo = _load_demo_module() + state = demo.DemoState() + _add_response_transcript( + state, + "resp-empty", + transcript="的吗?", + audio=False, + ) + + result = demo._evaluate_transcript_integrity( + state, + ["resp-empty"], + expected_empty_response_ids={"resp-empty"}, + require_cross_turn_independence=False, + ) + + assert result["empty_turns_ok"] is False + + +def test_realtime_duplex_demo_gate_rejects_audio_without_transcript(): + demo = _load_demo_module() + state = demo.DemoState() + _add_response_transcript( + state, + "resp-audio-no-text", + transcript="", + audio=True, + ) + + result = demo._evaluate_transcript_integrity( + state, + ["resp-audio-no-text"], + expected_empty_response_ids=set(), + require_cross_turn_independence=False, + ) + + assert result["nonempty_audio_has_transcript_ok"] is False + + +def test_realtime_duplex_demo_response_required_rejects_unselected_audio_only_response(): + demo = _load_demo_module() + state = demo.DemoState() + _add_response_transcript(state, "resp-audio-only", transcript="", audio=True) + _add_response_transcript(state, "resp-with-text", transcript="你好", audio=True) + + assert ( + demo._all_audio_responses_have_transcript( + state, + ["resp-audio-only", "resp-with-text"], + ) + is False + ) + + +def test_realtime_duplex_demo_gate_rejects_incomplete_model_turn_sentence(): + demo = _load_demo_module() + state = demo.DemoState() + _add_response_transcript(state, "resp-1", transcript="哎,不是说好不") + + result = demo._evaluate_transcript_integrity( + state, + ["resp-1"], + expected_empty_response_ids=set(), + require_cross_turn_independence=False, + require_terminal_punctuation=True, + ) + + assert result["terminal_punctuation_ok"] is False + + +def test_realtime_duplex_demo_gate_accepts_short_interjection_without_punctuation(): + demo = _load_demo_module() + state = demo.DemoState() + _add_response_transcript(state, "resp-1", transcript="哈哈") + + result = demo._evaluate_transcript_integrity( + state, + ["resp-1"], + expected_empty_response_ids=set(), + require_cross_turn_independence=False, + require_terminal_punctuation=True, + ) + + assert result["terminal_punctuation_ok"] is True + + +def test_realtime_duplex_demo_response_required_skips_audio_only_model_turn(monkeypatch): + demo = _load_demo_module() + state = demo.DemoState() + + async def fake_send_pcm16(*args, **kwargs): + state.add({"type": "response.created", "response": {"id": "resp-audio-only"}}) + _add_response_transcript(state, "resp-audio-only", transcript="", audio=True) + state.add({"type": "response.created", "response": {"id": "resp-with-text"}}) + _add_response_transcript(state, "resp-with-text", transcript="你好", audio=True) + + class FakeWebSocket: + async def send(self, payload): + return None + + monkeypatch.setattr(demo, "_send_pcm16", fake_send_pcm16) + + response_id, outcome = asyncio.run( + demo._send_clean_turn( + FakeWebSocket(), + state, + b"\x00\x00", + transcript="fixture", + duration_ms=None, + chunk_ms=200, + timeout_s=0.1, + require_audio=True, + validation_mode="response-required", + ) + ) + + assert response_id == "resp-with-text" + assert outcome == "speak" + + +def test_realtime_duplex_demo_no_hint_realtime_turn_omits_transcript_hint(monkeypatch): + demo = _load_demo_module() + state = demo.DemoState() + captured = {} + + async def fake_send_pcm16(*args, **kwargs): + del args + captured.update(kwargs) + state.add( + { + "type": "response.listen", + "response": {"metadata": {"model_listen": True}}, + } + ) + + class FakeWebSocket: + async def send(self, payload): + del payload + + monkeypatch.setattr(demo, "_send_pcm16", fake_send_pcm16) + + response_id, outcome = asyncio.run( + demo._send_clean_turn( + FakeWebSocket(), + state, + b"\x00\x00", + transcript="turn label only", + duration_ms=None, + chunk_ms=200, + timeout_s=0.1, + require_audio=False, + validation_mode="model-policy", + send_transcript_hint=False, + realtime_input=True, + model_policy_settle_s=0.01, + ) + ) + + assert response_id is None + assert outcome == "listen" + assert captured["hints"] == {} + assert captured["realtime_delay"] is True + + +def test_realtime_duplex_demo_continuous_input_omits_browser_commit(monkeypatch): + demo = _load_demo_module() + state = demo.DemoState() + + async def fake_send_pcm16(*args, **kwargs): + del args, kwargs + state.add({"type": "response.created", "response": {"id": "resp-continuous"}}) + _add_response_transcript(state, "resp-continuous", transcript="连续输入", audio=True) + + class FakeWebSocket: + def __init__(self): + self.messages = [] + + async def send(self, payload): + self.messages.append(demo.json.loads(payload)) + + monkeypatch.setattr(demo, "_send_pcm16", fake_send_pcm16) + ws = FakeWebSocket() + + response_id, outcome = asyncio.run( + demo._send_clean_turn( + ws, + state, + b"\x00\x00", + transcript="fixture", + duration_ms=None, + chunk_ms=200, + timeout_s=0.1, + require_audio=True, + validation_mode="response-required", + commit_input=False, + ) + ) + + assert response_id == "resp-continuous" + assert outcome == "speak" + assert not any(message.get("type") == "input_audio_buffer.commit" for message in ws.messages) + + +def test_realtime_duplex_demo_no_hint_gate_does_not_require_transcription_event(): + demo = _load_demo_module() + + assert demo._input_transcription_ok(0, transcript_hints_enabled=False) + assert not demo._input_transcription_ok(0, transcript_hints_enabled=True) + assert demo._input_transcription_ok(3, transcript_hints_enabled=True) + + +def test_realtime_duplex_demo_gate_rejects_server_errors(): + demo = _load_demo_module() + state = demo.DemoState() + state.add({"type": "error", "code": "runtime_signal_failed", "error": "signal failed"}) + + errors = demo._unexpected_error_events(state) + + assert errors == [{"type": "error", "code": "runtime_signal_failed", "error": "signal failed"}] + + +def test_realtime_duplex_demo_model_policy_waits_for_speak_after_intermediate_listen(monkeypatch): + demo = _load_demo_module() + state = demo.DemoState() + + async def fake_send_pcm16(*args, **kwargs): + del args, kwargs + state.add( + { + "type": "response.listen", + "response": {"metadata": {"model_listen": True}}, + } + ) + + async def delayed_speak(): + await asyncio.sleep(0.01) + state.add({"type": "response.created", "response": {"id": "resp-delayed"}}) + _add_response_transcript(state, "resp-delayed", transcript="延迟回答", audio=False) + + asyncio.create_task(delayed_speak()) + + class FakeWebSocket: + async def send(self, payload): + del payload + + monkeypatch.setattr(demo, "_send_pcm16", fake_send_pcm16) + + response_id, outcome = asyncio.run( + demo._send_clean_turn( + FakeWebSocket(), + state, + b"\x00\x00", + transcript="fixture", + duration_ms=None, + chunk_ms=200, + timeout_s=0.2, + require_audio=False, + validation_mode="model-policy", + model_policy_settle_s=0.05, + ) + ) + + assert response_id == "resp-delayed" + assert outcome == "speak" + + +def test_realtime_duplex_demo_playback_gate_covers_unassigned_completed_response(): + demo = _load_demo_module() + state = demo.DemoState() + for response_id in ("resp-assigned", "resp-unassigned"): + state.add({"type": "response.created", "response": {"id": response_id}}) + state.add( + { + "type": "response.done", + "response": { + "id": response_id, + "metadata": {"playback": {"sent_ms": 1200}}, + }, + } + ) + state.add( + { + "type": "playback.acknowledged", + "event": { + "item_id": "item_resp-assigned", + "history_committed": True, + }, + } + ) + + assert not demo._all_response_playback_history_committed( + state, + state.completed_response_ids(), + ) + + +def test_realtime_duplex_demo_acks_unassigned_completed_response(): + demo = _load_demo_module() + state = demo.DemoState() + for response_id in ("resp-assigned", "resp-unassigned"): + state.add({"type": "response.created", "response": {"id": response_id}}) + state.add( + { + "type": "response.done", + "response": { + "id": response_id, + "metadata": {"playback": {"sent_ms": 1200}}, + }, + } + ) + state.add( + { + "type": "playback.acknowledged", + "event": { + "item_id": "item_resp-assigned", + "history_committed": True, + }, + } + ) + + class FakeWebSocket: + def __init__(self): + self.messages = [] + + async def send(self, payload): + message = demo.json.loads(payload) + self.messages.append(message) + state.add( + { + "type": "playback.acknowledged", + "event": { + "item_id": message["item_id"], + "history_committed": True, + }, + } + ) + + ws = FakeWebSocket() + asyncio.run(demo._ack_all_completed_response_playback(ws, state, timeout_s=0.1)) + + assert [message["response_id"] for message in ws.messages] == ["resp-unassigned"] + assert demo._all_response_playback_history_committed(state, state.completed_response_ids()) + + +def test_realtime_duplex_demo_model_policy_does_not_assume_one_response_per_physical_input(): + demo = _load_demo_module() + + assert demo._response_cardinality_ok( + ["resp-1", "resp-2", "resp-3", "resp-4"], + expected_turns=3, + validation_mode="model-policy", + ) + assert not demo._response_cardinality_ok( + ["resp-1", "resp-2", "resp-3", "resp-4"], + expected_turns=3, + validation_mode="response-required", + ) + + +def test_realtime_duplex_demo_only_requires_cross_turn_independence_for_response_required(): + demo = _load_demo_module() + + assert not demo._requires_cross_turn_independence( + validation_mode="model-policy", + distinct_inputs_required=True, + ) + assert demo._requires_cross_turn_independence( + validation_mode="response-required", + distinct_inputs_required=True, + ) + assert not demo._requires_cross_turn_independence( + validation_mode="response-required", + distinct_inputs_required=False, + ) + + +def test_realtime_duplex_demo_drains_late_model_response_before_close(): + demo = _load_demo_module() + state = demo.DemoState() + state.add({"type": "response.created", "response": {"id": "resp-first"}}) + state.add({"type": "response.done", "response": {"id": "resp-first"}}) + + class FakeWebSocket: + def __init__(self): + self.messages = [] + + async def send(self, payload): + self.messages.append(demo.json.loads(payload)) + + async def run_fixture(): + async def finish_late_response(): + await asyncio.sleep(0.01) + state.add({"type": "response.created", "response": {"id": "resp-late"}}) + await asyncio.sleep(0.01) + state.add({"type": "response.done", "response": {"id": "resp-late"}}) + + late_response = asyncio.create_task(finish_late_response()) + ws = FakeWebSocket() + await demo._drain_model_policy_responses( + ws, + state, + timeout_s=0.2, + settle_s=0.03, + ) + await late_response + return ws + + ws = asyncio.run(run_fixture()) + + assert state.completed_response_ids() == ["resp-first", "resp-late"] + assert not [message for message in ws.messages if message.get("type") == "session.close"] + + +def test_realtime_duplex_demo_model_response_drain_times_out_with_active_ids(): + demo = _load_demo_module() + state = demo.DemoState() + state.add({"type": "response.created", "response": {"id": "resp-active"}}) + + class FakeWebSocket: + async def send(self, payload): + del payload + + with pytest.raises(TimeoutError, match="resp-active"): + asyncio.run( + demo._drain_model_policy_responses( + FakeWebSocket(), + state, + timeout_s=0.03, + settle_s=0.01, + ) + ) + + +def test_realtime_duplex_demo_waits_at_each_model_unit_and_stops_after_speak(): + demo = _load_demo_module() + + class FakeWebSocket: + def __init__(self): + self.messages = [] + + async def send(self, payload): + self.messages.append(demo.json.loads(payload)) + + async def run_fixture(): + ws = FakeWebSocket() + model_unit_message_counts = [] + + async def on_model_unit_ready(): + model_unit_message_counts.append(len(ws.messages)) + await asyncio.sleep(0) + return len(model_unit_message_counts) < 2 + + await demo._send_pcm16( + ws, + b"\x01\x00" * (demo.PCM16_SAMPLE_RATE * 3), + chunk_ms=200, + realtime_delay=False, + on_model_unit_ready=on_model_unit_ready, + ) + return ws, model_unit_message_counts + + ws, model_unit_message_counts = asyncio.run(run_fixture()) + + assert model_unit_message_counts == [5, 10] + assert len(ws.messages) == 10 + + +def test_realtime_duplex_demo_listen_only_overlap_sends_next_turn_before_first_done(monkeypatch): + demo = _load_demo_module() + state = demo.DemoState() + send_calls = 0 + + async def fake_send_pcm16(*args, **kwargs): + nonlocal send_calls + send_calls += 1 + if send_calls == 1: + assert len(args[1]) == 2 + demo.PCM16_SAMPLE_RATE * demo.PCM16_BYTES_PER_SAMPLE + state.add({"type": "input_audio_buffer.speech_started", "turn": 1}) + state.add({"type": "response.created", "response": {"id": "resp-first"}}) + state.add( + { + "type": "response.audio.delta", + "response_id": "resp-first", + "delta": "YQ==", + } + ) + assert await kwargs["on_model_unit_ready"]() is False + return + + del args + assert not state.response_done("resp-first") + kwargs["on_model_unit_ready"]() + state.add({"type": "input_audio_buffer.speech_started", "turn": 2}) + _add_response_transcript(state, "resp-first", transcript="第一轮完成", audio=False) + state.add({"type": "response.created", "response": {"id": "resp-second"}}) + _add_response_transcript(state, "resp-second", transcript="第二轮完成", audio=True) + + class FakeWebSocket: + def __init__(self): + self.messages = [] + + async def send(self, payload): + self.messages.append(payload) + + monkeypatch.setattr(demo, "_send_pcm16", fake_send_pcm16) + ws = FakeWebSocket() + + response_ids, outcomes, overlap_ok = asyncio.run( + demo._send_listen_only_overlap_pair( + ws, + state, + b"\x00\x00", + b"\x01\x00", + transcripts=("first", "second"), + durations_ms=(None, None), + chunk_ms=200, + timeout_s=0.2, + send_transcript_hint=False, + realtime_input=True, + model_policy_settle_s=0.01, + validation_mode="response-required", + silence_ms=1000, + ) + ) + + assert response_ids == ["resp-first", "resp-second"] + assert outcomes == ["speak", "speak"] + assert overlap_ok is True + assert sum(demo.json.loads(message)["type"] == "input_audio_buffer.commit" for message in ws.messages) == 2 + + +def test_realtime_duplex_demo_playback_ack_identifies_and_commits_response(): + demo = _load_demo_module() + state = demo.DemoState() + response_id = "resp-overlap-second" + state.add( + { + "type": "response.done", + "response": { + "id": response_id, + "metadata": {"playback": {"sent_ms": 1200}}, + }, + } + ) + + class FakeWebSocket: + def __init__(self): + self.messages = [] + + async def send(self, payload): + message = demo.json.loads(payload) + self.messages.append(message) + state.add( + { + "type": "playback.acknowledged", + "event": { + "item_id": message.get("item_id"), + "history_committed": message.get("item_id") == f"item_{response_id}", + }, + } + ) + + ws = FakeWebSocket() + asyncio.run( + demo._ack_response_playback( + ws, + state, + response_id, + timeout_s=0.1, + label="overlap second", + ) + ) + + assert ws.messages == [ + { + "type": "playback.ack", + "response_id": response_id, + "item_id": f"item_{response_id}", + "played_ms": 1200, + "committed_ms": 1200, + } + ] + assert state.playback_history_committed_count == 1 + + +def test_realtime_duplex_demo_writes_audio_per_response(tmp_path): + demo = _load_demo_module() + state = demo.DemoState() + for response_id, payload in (("resp-1", b"\x01\x00"), ("resp-2", b"\x02\x00")): + state.add({"type": "response.created", "response": {"id": response_id}}) + state.add( + { + "type": "response.audio.delta", + "response_id": response_id, + "delta": demo.base64.b64encode(payload).decode(), + "sample_rate_hz": 24000, + } + ) + + demo._write_demo_artifacts(state, tmp_path, output_audio_format="pcm16") + + for index, expected in enumerate((b"\x01\x00", b"\x02\x00"), start=1): + with wave.open(str(tmp_path / f"response_{index:02d}.wav"), "rb") as wf: + assert wf.getframerate() == 24000 + assert wf.readframes(wf.getnframes()) == expected diff --git a/tests/examples/test_minicpmo_realtime_web_server.py b/tests/examples/test_minicpmo_realtime_web_server.py new file mode 100644 index 000000000..6c35760a7 --- /dev/null +++ b/tests/examples/test_minicpmo_realtime_web_server.py @@ -0,0 +1,99 @@ +import hashlib +import importlib.util +import sys +from pathlib import Path + +import pytest +from fastapi.testclient import TestClient + +SERVER_PATH = Path(__file__).resolve().parents[2] / "examples/online_serving/minicpmo/realtime_web/server.py" +spec = importlib.util.spec_from_file_location( + "minicpmo_realtime_web_server_test", + SERVER_PATH, +) +assert spec is not None and spec.loader is not None +server = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = server +spec.loader.exec_module(server) + +APP_DIR = server.APP_DIR +STATIC_DIR = server.STATIC_DIR +_join_ws_url = server._join_ws_url +build_app = server.build_app + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +def test_join_ws_url_preserves_realtime_query(): + assert ( + _join_ws_url( + "ws://127.0.0.1:8099/", + "/v1/realtime", + "duplex=1&model=openbmb%2FMiniCPM-o-4_5", + ) + == "ws://127.0.0.1:8099/v1/realtime?duplex=1&model=openbmb%2FMiniCPM-o-4_5" + ) + + +def test_build_app_serves_health_and_injected_client_config(): + app = build_app( + ws_backend="ws://127.0.0.1:9001", + model="local/MiniCPM-o-4_5", + ) + client = TestClient(app) + + health = client.get("/healthz") + assert health.status_code == 200 + assert health.text == "ok" + + index = client.get("/") + assert index.status_code == 200 + assert '"model": "local/MiniCPM-o-4_5"' in index.text + assert '"realtimePath": "v1/realtime"' in index.text + assert "__FULL_DUPLEX_CONFIG__" not in index.text + + +def test_build_app_injects_public_realtime_url(): + public_realtime_url = "wss://proxy.example.test/backend/v1/realtime" + app = build_app( + ws_backend="ws://127.0.0.1:9001", + public_realtime_url=public_realtime_url, + ) + client = TestClient(app) + + index = client.get("/") + + assert index.status_code == 200 + assert f'"realtimePath": "{public_realtime_url}"' in index.text + + +def test_build_app_versions_client_bundle_and_disables_index_cache(): + client = TestClient(build_app()) + expected_version_hash = hashlib.sha256() + for asset_name in ("app.js", "pcm_worklet.js", "playback_worklet.js"): + expected_version_hash.update((STATIC_DIR / asset_name).read_bytes()) + expected_version = expected_version_hash.hexdigest()[:12] + + assert (APP_DIR / "index.html").is_file() + assert (STATIC_DIR / "app.js").is_file() + assert not (APP_DIR.parent / "static").exists() + assert not (STATIC_DIR / "index.html").exists() + + index = client.get("/") + + assert index.status_code == 200 + assert f'src="static/app.js?v={expected_version}"' in index.text + assert index.headers["cache-control"] == "no-store" + + +def test_build_app_exposes_realtime_websocket_and_static_assets(): + app = build_app() + paths = {route.path for route in app.routes} + + assert "/v1/realtime" in paths + assert "/static" in paths + + client = TestClient(app) + assert client.get("/static/app.js").status_code == 200 + assert client.get("/static/pcm_worklet.js").status_code == 200 + assert client.get("/static/playback_worklet.js").status_code == 200 diff --git a/tests/examples/test_minicpmo_realtime_web_static.py b/tests/examples/test_minicpmo_realtime_web_static.py new file mode 100644 index 000000000..aa9f60fe2 --- /dev/null +++ b/tests/examples/test_minicpmo_realtime_web_static.py @@ -0,0 +1,209 @@ +import re +import shutil +import subprocess +import textwrap +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[2] / "examples" / "online_serving" / "minicpmo" / "realtime_web" +APP_ROOT = ROOT / "app" +STATIC_ROOT = APP_ROOT / "static" + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +def test_page_exposes_focused_call_conversation_and_log_surfaces(): + html = (APP_ROOT / "index.html").read_text(encoding="utf-8") + + assert 'id="callButton"' in html + assert 'id="muteButton"' in html + assert 'id="connectionState"' in html + assert 'id="modelState"' in html + assert 'id="conversation"' in html + assert 'id="eventLog"' in html + assert ".*?)\n\s*\)", + source, + re.DOTALL, + ) + assert ref_audio_arg is not None + assert "required=True" in ref_audio_arg.group("body") + assert "Optional reference voice" not in ref_audio_arg.group("body") + + +def test_client_sends_ref_audio_in_realtime_session_contract(): + source = (STATIC_ROOT / "app.js").read_text(encoding="utf-8") + + assert "if (config.refAudio) session.ref_audio = config.refAudio;" in source + assert "extraBody.ref_audio" not in source + + +def test_client_has_transactional_cleanup_and_visible_event_logging(): + source = (STATIC_ROOT / "app.js").read_text(encoding="utf-8") + + assert "async function stopSession" in source + assert "type: 'session.close'" in source + assert "waitForSessionClosed" in source + assert "SESSION_CLOSE_TIMEOUT_MS" in source + assert "case 'session.closed':" in source + assert "track.stop()" in source + assert "clearInterval(sendTimer)" in source + assert "appendEventLog(event)" in source + assert "stopSession({ terminal: false })" in source + + +def test_client_keeps_microphone_upload_active_during_assistant_playback(): + source = (STATIC_ROOT / "app.js").read_text(encoding="utf-8") + upload_gate = re.search(r"function microphoneUploadEnabled\(\) \{(?P.*?)\n \}", source, re.DOTALL) + begin_assistant = re.search(r"function beginAssistant\(responseId\) \{(?P.*?)\n \}", source, re.DOTALL) + + assert upload_gate is not None + assert "return running && !muted;" in upload_gate.group("body") + assert "assistantActive" not in upload_gate.group("body") + assert begin_assistant is not None + assert "pendingCapture = []" not in begin_assistant.group("body") + + +def test_audio_worklets_define_capture_and_playback_processors(): + capture = (STATIC_ROOT / "pcm_worklet.js").read_text(encoding="utf-8") + playback = (STATIC_ROOT / "playback_worklet.js").read_text(encoding="utf-8") + + assert "registerProcessor('fullduplex-pcm-capture'" in capture + assert "Int16Array" in capture + assert "registerProcessor('fullduplex-pcm-playback'" in playback + assert "playback-drained" in playback + assert "clear" in playback + + +def test_audio_worklet_urls_use_the_static_asset_version(): + server = (ROOT / "server.py").read_text(encoding="utf-8") + app = (STATIC_ROOT / "app.js").read_text(encoding="utf-8") + + assert '"appVersion": app_version' in server + assert 'STATIC_DIR / "playback_worklet.js"' in server + assert 'STATIC_DIR / "pcm_worklet.js"' in server + assert "staticAssetUrl('static/playback_worklet.js')" in app + assert "staticAssetUrl('static/pcm_worklet.js')" in app + + +def test_playback_worklet_buffers_first_400ms_and_reports_underruns(): + app = (STATIC_ROOT / "app.js").read_text(encoding="utf-8") + playback = (STATIC_ROOT / "playback_worklet.js").read_text(encoding="utf-8") + + assert "INITIAL_PLAYBACK_BUFFER_MS = 400" in app + assert "initialBufferMs" in app + assert "responseId" in app + assert "playback-underrun" in app + assert "underrunMs" in app + assert "initialBufferFrames" in playback + assert "playback-underrun" in playback + assert "underrunFrames" in playback + assert "underrunMs" in playback + + +def test_playback_worklet_waits_before_playing_and_rebuffers_after_underrun(): + node = shutil.which("node") + if node is None: + pytest.skip("node is required for the AudioWorklet regression test") + + script = textwrap.dedent( + """ + const fs = require('fs'); + const vm = require('vm'); + + global.sampleRate = 1000; + global.AudioWorkletProcessor = class { + constructor() { + this.port = { onmessage: null, postMessage: () => {} }; + } + }; + let Processor = null; + global.registerProcessor = (_name, processor) => { Processor = processor; }; + vm.runInThisContext(fs.readFileSync(process.argv[1], 'utf8')); + + const processor = new Processor(); + const render = () => { + const output = new Float32Array(100); + processor.process([], [[output]]); + return output; + }; + const assert = (condition, message) => { + if (!condition) throw new Error(message); + }; + + const first = new Int16Array(150); + first.fill(16384); + processor.handleMessage({ + type: 'audio', + pcm: first, + responseId: 'response-1', + initialBufferMs: 400, + }); + assert(!processor.started, 'large first delta must not bypass wall-clock prebuffer'); + assert(render().every((sample) => sample === 0), 'first prebuffer render must be silent'); + assert(render().every((sample) => sample === 0), 'second prebuffer render must be silent'); + assert(render().every((sample) => sample === 0), 'third prebuffer render must be silent'); + assert(render().every((sample) => sample === 0), 'fourth prebuffer render must be silent'); + const firstPlayback = render(); + assert(firstPlayback.some((sample) => sample !== 0), 'playback must start after prebuffer'); + + const underrun = render(); + assert(!processor.started, 'an empty queue must return to buffering'); + assert(underrun[underrun.length - 1] === 0, 'underrun boundary must fade to zero'); + + const resumed = new Int16Array(300); + resumed.fill(8192); + processor.handleMessage({ + type: 'audio', + pcm: resumed, + responseId: 'response-1', + initialBufferMs: 400, + }); + assert(render().every((sample) => sample === 0), 'resume must rebuild jitter buffer'); + assert(render().every((sample) => sample === 0), 'resume must keep rebuilding jitter buffer'); + assert(render().every((sample) => sample === 0), 'resume must keep waiting'); + assert(render().every((sample) => sample === 0), 'resume must wait the full buffer interval'); + const resumedPlayback = render(); + assert(resumedPlayback.some((sample) => sample !== 0), 'playback must resume after rebuffer'); + assert( + Math.abs(resumedPlayback[0]) < Math.abs(resumedPlayback[50]), + 'resumed playback must fade in instead of jumping from silence', + ); + """ + ) + subprocess.run( + [node, "-e", script, str(STATIC_ROOT / "playback_worklet.js")], + check=True, + capture_output=True, + text=True, + ) diff --git a/tests/fullduplex/engine/test_duplex_control_client.py b/tests/fullduplex/engine/test_duplex_control_client.py new file mode 100644 index 000000000..c1fa9a592 --- /dev/null +++ b/tests/fullduplex/engine/test_duplex_control_client.py @@ -0,0 +1,110 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from __future__ import annotations + +import pytest + +from vllm_omni.experimental.fullduplex.engine.duplex_control_client import ( + DuplexControlClient, + DuplexControlRequestError, +) +from vllm_omni.experimental.fullduplex.engine.lease import DuplexLeaseActivity +from vllm_omni.experimental.fullduplex.engine.messages import ( + DuplexControlError, + DuplexControlResultMessage, + DuplexFence, + ResumeDuplexSessionMessage, + TouchDuplexSessionMessage, +) + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +class _Transport: + def __init__(self) -> None: + self.calls = [] + + def execute(self, key, message, **kwargs): + self.calls.append((key, message, kwargs)) + return DuplexControlResultMessage( + control_id=message.control_id, + fence=message.fence, + operation="touch" if isinstance(message, TouchDuplexSessionMessage) else "resume", + session_id=message.session_id, + ok=True, + stage_results=[], + ) + + +class _UnsupportedTransport: + def execute(self, key, message, **kwargs): + del key, kwargs + return DuplexControlResultMessage( + control_id=message.control_id, + fence=message.fence, + operation="touch", + session_id=message.session_id, + ok=False, + stage_results=[{"result": {"supported": False}}], + unsupported_count=1, + error=DuplexControlError(code="invalid_capability", message="unsupported", retryable=False), + accepted_fence=DuplexFence(message.session_id, epoch=2), + lease_generation=7, + ) + + +def test_control_client_routes_touch_and_resume_by_control_id() -> None: + transport = _Transport() + control_ids = iter(("touch-id", "resume-id")) + client = DuplexControlClient(transport, control_id_factory=lambda: next(control_ids)) + fence = DuplexFence("sid-client") + + assert ( + client.touch( + fence.session_id, + fence=fence, + activity=DuplexLeaseActivity.PLAYBACK_ACK, + timeout=2.0, + )["ok"] + is True + ) + assert ( + client.resume( + fence.session_id, + fence=fence, + expected_lease_generation=7, + timeout=3.0, + )["ok"] + is True + ) + + touch_key, touch_message, touch_kwargs = transport.calls[0] + assert touch_key == ("duplex", "touch-id") + assert isinstance(touch_message, TouchDuplexSessionMessage) + assert touch_message.activity == DuplexLeaseActivity.PLAYBACK_ACK.value + assert touch_kwargs["timeout"] == 2.0 + + resume_key, resume_message, resume_kwargs = transport.calls[1] + assert resume_key == ("duplex", "resume-id") + assert isinstance(resume_message, ResumeDuplexSessionMessage) + assert resume_message.expected_lease_generation == 7 + assert resume_kwargs["timeout"] == 3.0 + + +def test_control_client_treats_ok_false_as_authoritative_failure() -> None: + client = DuplexControlClient(_UnsupportedTransport(), control_id_factory=lambda: "unsupported") + fence = DuplexFence("sid-unsupported") + + with pytest.raises(DuplexControlRequestError, match="duplex touch failed") as exc_info: + client.touch( + fence.session_id, + fence=fence, + activity=DuplexLeaseActivity.HEARTBEAT, + timeout=1.0, + ) + + assert exc_info.value.code == "invalid_capability" + assert exc_info.value.retryable is False + assert exc_info.value.accepted_fence == DuplexFence(fence.session_id, epoch=2) + assert exc_info.value.lease_generation == 7 diff --git a/tests/fullduplex/engine/test_duplex_control_plane.py b/tests/fullduplex/engine/test_duplex_control_plane.py new file mode 100644 index 000000000..46df33ffa --- /dev/null +++ b/tests/fullduplex/engine/test_duplex_control_plane.py @@ -0,0 +1,1270 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from __future__ import annotations + +import asyncio +import logging +from dataclasses import FrozenInstanceError + +import msgspec +import pytest + +from vllm_omni.experimental.fullduplex.engine.duplex_control_plane import ( + DuplexControlPlane, + DuplexOutputContext, + DuplexRequestIdentity, + DuplexStageRequestContext, + DuplexStageSubmission, + DuplexStageSubmissionResult, +) +from vllm_omni.experimental.fullduplex.engine.duplex_runtime import ( + DuplexAppendPlan, + DuplexInputMode, + DuplexRuntimeCapabilities, +) +from vllm_omni.experimental.fullduplex.engine.lease import DuplexLeaseActivity, DuplexLeaseConfig +from vllm_omni.experimental.fullduplex.engine.messages import ( + AppendDuplexInputMessage, + CloseDuplexSessionMessage, + DuplexFence, + DuplexSessionLifecycleMessage, + OpenDuplexSessionMessage, + ResumeDuplexSessionMessage, + SignalDuplexTurnMessage, + TouchDuplexSessionMessage, +) + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +class _Extension: + def configure_sampling_params(self, *, runtime_config, defaults): + del runtime_config + return tuple(f"configured-{stage_id}" for stage_id, _ in enumerate(defaults)) + + def plan_append( + self, + *, + request_id, + fence, + session_config, + runtime_config, + seq, + turn_seq, + mode, + payload, + final, + sampling_params, + ): + del request_id, fence, session_config, runtime_config, seq, turn_seq, mode, payload, final + assert sampling_params == "configured-0" + return DuplexAppendPlan(prompt={"prompt_token_ids": [1, 2, 3]}) + + def decide_output(self, **kwargs): + del kwargs + return None + + +class _TypedStagePort: + stage_count = 2 + + def __init__(self) -> None: + self.ensure_calls: list[DuplexStageRequestContext] = [] + self.submit_calls: list[DuplexStageSubmission] = [] + self.cleanup_calls: list[tuple[list[str], bool]] = [] + + def sampling_defaults(self) -> tuple[object, ...]: + return ("default-0", "default-1") + + def ensure_request(self, context: DuplexStageRequestContext) -> None: + self.ensure_calls.append(context) + + async def submit(self, submission: DuplexStageSubmission) -> DuplexStageSubmissionResult: + self.submit_calls.append(submission) + return DuplexStageSubmissionResult( + request_id=submission.context.request_id, + stage_id=submission.context.stage_id, + replica_id=3, + ) + + async def cleanup(self, request_ids: list[str], *, abort: bool = False) -> None: + self.cleanup_calls.append((request_ids, abort)) + + +class _Clock: + def __init__(self) -> None: + self.value = 0.0 + + def __call__(self) -> float: + return self.value + + def advance(self, seconds: float) -> None: + self.value += seconds + + +@pytest.mark.asyncio +async def test_control_plane_uses_frozen_typed_stage_context_without_request_state() -> None: + stage_port = _TypedStagePort() + result_sink: asyncio.Queue = asyncio.Queue() + plane = DuplexControlPlane( + extension=_Extension(), + stage_port=stage_port, + result_sink=result_sink, + ) + fence = DuplexFence("typed-port") + + await plane.handle( + OpenDuplexSessionMessage( + control_id="open", + fence=fence, + session_id=fence.session_id, + capabilities={ + "input_modes": [DuplexInputMode.APPEND_AUDIO_CHUNK.value], + }, + session_config={"voice": "test"}, + runtime_config={"runtime": "test"}, + ) + ) + assert (await result_sink.get()).ok is True + + context = stage_port.ensure_calls[-1] + assert context.session_id == fence.session_id + assert context.fence == fence + assert context.stage_id == 0 + assert context.final_stage_id == 1 + assert context.sampling_params == ("configured-0", "configured-1") + assert context.session_config == {"voice": "test"} + assert context.runtime_config == {"runtime": "test"} + with pytest.raises(FrozenInstanceError): + context.stage_id = 1 # type: ignore[misc] + + await plane.handle( + AppendDuplexInputMessage( + control_id="append", + fence=fence, + session_id=fence.session_id, + mode=DuplexInputMode.APPEND_AUDIO_CHUNK.value, + payload={"audio": b"pcm"}, + final=True, + ) + ) + append_result = await result_sink.get() + assert append_result.ok is True + assert append_result.stage_results[0]["replica_id"] == 3 + assert append_result.stage_results[0]["result"]["response_stage_id"] == 1 + + submission = stage_port.submit_calls[-1] + assert submission.context == stage_port.ensure_calls[-1] + assert submission.prompt == {"prompt_token_ids": [1, 2, 3]} + assert submission.already_submitted is False + assert not hasattr(submission, "request_state") + + +def test_control_plane_accepts_only_typed_duplex_messages() -> None: + plane = DuplexControlPlane( + extension=None, + stage_port=_TypedStagePort(), + result_sink=asyncio.Queue(), + ) + + assert plane.accepts( + OpenDuplexSessionMessage( + control_id="open", + fence=DuplexFence("typed-message"), + session_id="typed-message", + capabilities=DuplexRuntimeCapabilities().__dict__, + ) + ) + assert plane.accepts(type("Lookalike", (), {"type": "open_duplex_session"})()) is False + + +@pytest.mark.parametrize( + "message", + [ + TouchDuplexSessionMessage( + control_id="touch-1", + fence=DuplexFence("sid-message"), + session_id="sid-message", + activity=DuplexLeaseActivity.HEARTBEAT.value, + ), + ResumeDuplexSessionMessage( + control_id="resume-1", + fence=DuplexFence("sid-message"), + session_id="sid-message", + expected_lease_generation=3, + ), + DuplexSessionLifecycleMessage( + fence=DuplexFence("sid-message"), + session_id="sid-message", + event="expired", + reason="idle_ttl_expired", + lease_generation=4, + submitted_request_ids=["req-submitted"], + reserved_request_ids=["req-reserved"], + ), + ], +) +def test_duplex_lease_messages_round_trip(message) -> None: + encoded = msgspec.json.encode(message) + decoded = msgspec.json.decode(encoded, type=type(message)) + + assert decoded == message + + +@pytest.mark.asyncio +async def test_control_plane_touches_append_signal_and_explicit_activity() -> None: + clock = _Clock() + stage_port = _TypedStagePort() + result_sink: asyncio.Queue = asyncio.Queue() + lifecycle_sink: asyncio.Queue = asyncio.Queue() + plane = DuplexControlPlane( + extension=_Extension(), + stage_port=stage_port, + result_sink=result_sink, + lifecycle_sink=lifecycle_sink, + lease_config=DuplexLeaseConfig(idle_ttl_s=30.0, disconnect_grace_s=5.0), + clock=clock, + ) + fence = DuplexFence("sid-touch") + await plane.handle( + OpenDuplexSessionMessage( + control_id="open", + fence=fence, + session_id=fence.session_id, + capabilities={"input_modes": [DuplexInputMode.APPEND_AUDIO_CHUNK.value]}, + ) + ) + await result_sink.get() + session = plane.sessions.require(fence.session_id) + + clock.advance(1.0) + await plane.handle( + AppendDuplexInputMessage( + control_id="append", + fence=fence, + session_id=fence.session_id, + mode=DuplexInputMode.APPEND_AUDIO_CHUNK.value, + payload={"audio": b"pcm"}, + ) + ) + assert (await result_sink.get()).ok is True + assert session.lease.last_activity == 1.0 + + clock.advance(1.0) + await plane.handle( + SignalDuplexTurnMessage( + control_id="signal", + fence=fence, + session_id=fence.session_id, + event="session.update", + ) + ) + assert (await result_sink.get()).ok is True + assert session.lease.last_activity == 2.0 + + submit_count = len(stage_port.submit_calls) + clock.advance(1.0) + await plane.handle( + TouchDuplexSessionMessage( + control_id="heartbeat", + fence=fence, + session_id=fence.session_id, + activity=DuplexLeaseActivity.HEARTBEAT.value, + ) + ) + touch_result = await result_sink.get() + assert touch_result.ok is True + assert session.lease.last_activity == 3.0 + assert len(stage_port.submit_calls) == submit_count + assert lifecycle_sink.empty() + + +@pytest.mark.asyncio +async def test_control_plane_resume_requires_expected_lease_generation() -> None: + clock = _Clock() + result_sink: asyncio.Queue = asyncio.Queue() + plane = DuplexControlPlane( + extension=None, + stage_port=_TypedStagePort(), + result_sink=result_sink, + lifecycle_sink=asyncio.Queue(), + lease_config=DuplexLeaseConfig(), + clock=clock, + ) + fence = DuplexFence("sid-resume-control") + await plane.handle( + OpenDuplexSessionMessage( + control_id="open", + fence=fence, + session_id=fence.session_id, + capabilities={}, + ) + ) + await result_sink.get() + await plane.handle( + TouchDuplexSessionMessage( + control_id="detach", + fence=fence, + session_id=fence.session_id, + activity=DuplexLeaseActivity.DETACH.value, + ) + ) + await result_sink.get() + + await plane.handle( + ResumeDuplexSessionMessage( + control_id="resume", + fence=fence, + session_id=fence.session_id, + expected_lease_generation=0, + ) + ) + result = await result_sink.get() + + assert result.ok is True + assert result.stage_results[0]["result"]["lease_generation"] == 1 + + await plane.handle( + ResumeDuplexSessionMessage( + control_id="stale-resume", + fence=fence, + session_id=fence.session_id, + expected_lease_generation=0, + ) + ) + stale_result = await result_sink.get() + assert stale_result.ok is False + assert stale_result.accepted_fence == fence + assert stale_result.lease_generation == 1 + + +@pytest.mark.asyncio +async def test_control_plane_reaps_one_session_through_cleanup_and_lifecycle_sink() -> None: + clock = _Clock() + stage_port = _TypedStagePort() + result_sink: asyncio.Queue = asyncio.Queue() + lifecycle_sink: asyncio.Queue = asyncio.Queue() + plane = DuplexControlPlane( + extension=None, + stage_port=stage_port, + result_sink=result_sink, + lifecycle_sink=lifecycle_sink, + lease_config=DuplexLeaseConfig(idle_ttl_s=2.0, disconnect_grace_s=1.0), + clock=clock, + ) + for session_id in ("sid-a", "sid-b"): + fence = DuplexFence(session_id) + await plane.handle( + OpenDuplexSessionMessage( + control_id=f"open-{session_id}", + fence=fence, + session_id=session_id, + capabilities={}, + ) + ) + await result_sink.get() + clock.advance(1.0) + plane.sessions.require("sid-b").touch(DuplexFence("sid-b"), DuplexLeaseActivity.HEARTBEAT) + clock.advance(1.1) + + expired_count = await plane.reap_expired() + + assert expired_count == 1 + assert plane.sessions.get("sid-a") is None + assert plane.sessions.get("sid-b") is not None + assert stage_port.cleanup_calls == [] + lifecycle = await lifecycle_sink.get() + assert isinstance(lifecycle, DuplexSessionLifecycleMessage) + assert lifecycle.session_id == "sid-a" + assert lifecycle.reason == "idle_ttl_expired" + assert result_sink.empty() + + +@pytest.mark.asyncio +async def test_control_plane_retries_expired_cleanup_before_publishing_lifecycle() -> None: + class _FailOnceStagePort(_TypedStagePort): + def __init__(self) -> None: + super().__init__() + self.failures_remaining = 1 + + async def cleanup(self, request_ids: list[str], *, abort: bool = False) -> None: + if self.failures_remaining: + self.failures_remaining -= 1 + raise RuntimeError("transient cleanup failure") + await super().cleanup(request_ids, abort=abort) + + clock = _Clock() + stage_port = _FailOnceStagePort() + result_sink: asyncio.Queue = asyncio.Queue() + lifecycle_sink: asyncio.Queue = asyncio.Queue() + plane = DuplexControlPlane( + extension=None, + stage_port=stage_port, + result_sink=result_sink, + lifecycle_sink=lifecycle_sink, + lease_config=DuplexLeaseConfig(idle_ttl_s=1.0, disconnect_grace_s=1.0), + clock=clock, + ) + fence = DuplexFence("sid-retry-expiry") + await plane.handle( + OpenDuplexSessionMessage( + control_id="open-retry", + fence=fence, + session_id=fence.session_id, + capabilities={}, + ) + ) + await result_sink.get() + request_id = plane.stage_request_id(fence, stage_id=0) + plane.sessions.require(fence.session_id).reserve_stage_request(0, request_id, fence=fence) + clock.advance(2.0) + + assert await plane.reap_expired() == 0 + assert lifecycle_sink.empty() + + assert await plane.reap_expired() == 1 + assert (await lifecycle_sink.get()).session_id == fence.session_id + + +@pytest.mark.asyncio +async def test_expired_session_holds_admission_slot_while_cleanup_is_blocked() -> None: + class _BlockedCleanupStagePort(_TypedStagePort): + def __init__(self) -> None: + super().__init__() + self.cleanup_started = asyncio.Event() + self.release_cleanup = asyncio.Event() + + async def cleanup(self, request_ids: list[str], *, abort: bool = False) -> None: + self.cleanup_started.set() + await self.release_cleanup.wait() + await super().cleanup(request_ids, abort=abort) + + clock = _Clock() + stage_port = _BlockedCleanupStagePort() + result_sink: asyncio.Queue = asyncio.Queue() + plane = DuplexControlPlane( + extension=None, + stage_port=stage_port, + result_sink=result_sink, + lease_config=DuplexLeaseConfig(idle_ttl_s=1.0, disconnect_grace_s=1.0), + clock=clock, + max_sessions=1, + ) + old_fence = DuplexFence("sid-expired-blocked") + await plane.handle( + OpenDuplexSessionMessage( + control_id="open-expired-blocked", + fence=old_fence, + session_id=old_fence.session_id, + capabilities={}, + ) + ) + assert (await result_sink.get()).ok is True + old_session = plane.sessions.require(old_fence.session_id) + old_session.bind_stage_request(0, "req-expired-blocked", fence=old_fence) + clock.advance(2.0) + + reap_task = asyncio.create_task(plane.reap_expired()) + await asyncio.wait_for(stage_port.cleanup_started.wait(), timeout=1.0) + replacement_fence = DuplexFence("sid-replacement-blocked") + await plane.handle( + OpenDuplexSessionMessage( + control_id="open-replacement-blocked", + fence=replacement_fence, + session_id=replacement_fence.session_id, + capabilities={}, + ) + ) + + blocked_result = await result_sink.get() + assert blocked_result.ok is False + assert blocked_result.error is not None + assert blocked_result.error.code == "resource_exhausted" + assert plane.sessions.get(old_fence.session_id) is old_session + + stage_port.release_cleanup.set() + assert await reap_task == 1 + assert plane.sessions.get(old_fence.session_id) is None + + await plane.handle( + OpenDuplexSessionMessage( + control_id="open-replacement-after-cleanup", + fence=replacement_fence, + session_id=replacement_fence.session_id, + capabilities={}, + ) + ) + assert (await result_sink.get()).ok is True + + +@pytest.mark.asyncio +async def test_failed_expiry_cleanup_retains_admission_slot_for_retry() -> None: + class _FailingCleanupStagePort(_TypedStagePort): + async def cleanup(self, request_ids: list[str], *, abort: bool = False) -> None: + del request_ids, abort + raise RuntimeError("cleanup failed") + + clock = _Clock() + result_sink: asyncio.Queue = asyncio.Queue() + plane = DuplexControlPlane( + extension=None, + stage_port=_FailingCleanupStagePort(), + result_sink=result_sink, + lease_config=DuplexLeaseConfig(idle_ttl_s=1.0, disconnect_grace_s=1.0), + clock=clock, + max_sessions=1, + ) + old_fence = DuplexFence("sid-expired-failed") + await plane.handle( + OpenDuplexSessionMessage( + control_id="open-expired-failed", + fence=old_fence, + session_id=old_fence.session_id, + capabilities={}, + ) + ) + assert (await result_sink.get()).ok is True + old_session = plane.sessions.require(old_fence.session_id) + old_session.bind_stage_request(0, "req-expired-failed", fence=old_fence) + clock.advance(2.0) + + assert await plane.reap_expired() == 0 + replacement_fence = DuplexFence("sid-replacement-failed") + await plane.handle( + OpenDuplexSessionMessage( + control_id="open-replacement-failed", + fence=replacement_fence, + session_id=replacement_fence.session_id, + capabilities={}, + ) + ) + + blocked_result = await result_sink.get() + assert blocked_result.ok is False + assert blocked_result.error is not None + assert blocked_result.error.code == "resource_exhausted" + assert plane.sessions.get(old_fence.session_id) is old_session + + +@pytest.mark.asyncio +async def test_capacity_rejection_is_an_info_event(caplog: pytest.LogCaptureFixture) -> None: + result_sink: asyncio.Queue = asyncio.Queue() + plane = DuplexControlPlane( + extension=None, + stage_port=_TypedStagePort(), + result_sink=result_sink, + max_sessions=1, + ) + first_fence = DuplexFence("sid-capacity-first") + await plane.handle( + OpenDuplexSessionMessage( + control_id="open-capacity-first", + fence=first_fence, + session_id=first_fence.session_id, + capabilities={}, + ) + ) + assert (await result_sink.get()).ok is True + caplog.clear() + + rejected_fence = DuplexFence("sid-capacity-rejected") + await plane.handle( + OpenDuplexSessionMessage( + control_id="open-capacity-rejected", + fence=rejected_fence, + session_id=rejected_fence.session_id, + capabilities={}, + ) + ) + + result = await result_sink.get() + assert result.error.code == "resource_exhausted" + assert not [record for record in caplog.records if record.levelno >= logging.ERROR] + + +@pytest.mark.asyncio +async def test_failed_request_cleanup_retries_before_releasing_admission() -> None: + class _FailOnceCleanupStagePort(_TypedStagePort): + def __init__(self) -> None: + super().__init__() + self.attempts = 0 + + async def cleanup(self, request_ids: list[str], *, abort: bool = False) -> None: + self.cleanup_calls.append((list(request_ids), abort)) + self.attempts += 1 + if self.attempts == 1: + raise RuntimeError("transient request cleanup failure") + + stage_port = _FailOnceCleanupStagePort() + result_sink: asyncio.Queue = asyncio.Queue() + plane = DuplexControlPlane( + extension=None, + stage_port=stage_port, + result_sink=result_sink, + max_sessions=1, + ) + old_fence = DuplexFence("sid-request-cleanup") + await plane.handle( + OpenDuplexSessionMessage( + control_id="open-request-cleanup", + fence=old_fence, + session_id=old_fence.session_id, + capabilities={}, + ) + ) + assert (await result_sink.get()).ok is True + old_session = plane.sessions.require(old_fence.session_id) + old_session.bind_stage_request(0, "req-request-cleanup", fence=old_fence) + + closed = plane.close_sessions_for_request_ids( + ["req-request-cleanup"], + abort=True, + ) + assert closed == {old_fence.session_id: ["req-request-cleanup"]} + assert plane.sessions.get(old_fence.session_id) is old_session + with pytest.raises(RuntimeError, match="transient request cleanup failure"): + await stage_port.cleanup(closed[old_fence.session_id], abort=True) + + replacement_fence = DuplexFence("sid-request-cleanup-replacement") + await plane.handle( + OpenDuplexSessionMessage( + control_id="open-request-cleanup-replacement-blocked", + fence=replacement_fence, + session_id=replacement_fence.session_id, + capabilities={}, + ) + ) + assert (await result_sink.get()).error.code == "resource_exhausted" + + assert await plane.reap_expired() == 1 + assert plane.sessions.get(old_fence.session_id) is None + assert stage_port.cleanup_calls == [ + (["req-request-cleanup"], True), + (["req-request-cleanup"], True), + ] + + await plane.handle( + OpenDuplexSessionMessage( + control_id="open-request-cleanup-replacement", + fence=replacement_fence, + session_id=replacement_fence.session_id, + capabilities={}, + ) + ) + assert (await result_sink.get()).ok is True + + +@pytest.mark.asyncio +async def test_stale_request_cleanup_does_not_finalize_reopened_incarnation() -> None: + stage_port = _TypedStagePort() + result_sink: asyncio.Queue = asyncio.Queue() + plane = DuplexControlPlane( + extension=None, + stage_port=stage_port, + result_sink=result_sink, + max_sessions=1, + ) + old_fence = DuplexFence("sid-request-cleanup-reopen") + await plane.handle( + OpenDuplexSessionMessage( + control_id="open-request-cleanup-old", + fence=old_fence, + session_id=old_fence.session_id, + capabilities={}, + ) + ) + assert (await result_sink.get()).ok is True + old_session = plane.sessions.require(old_fence.session_id) + old_session.bind_stage_request(0, "req-request-cleanup-old", fence=old_fence) + closed = plane.close_sessions_for_request_ids( + ["req-request-cleanup-old"], + abort=True, + cleanup_in_progress=True, + ) + assert closed == {old_fence.session_id: ["req-request-cleanup-old"]} + plane.defer_request_cleanups([old_fence.session_id]) + + await plane.handle( + CloseDuplexSessionMessage( + control_id="close-request-cleanup-old", + fence=old_fence, + session_id=old_fence.session_id, + reason="client_close", + ) + ) + assert (await result_sink.get()).ok is True + assert plane.sessions.get(old_fence.session_id) is None + + new_fence = DuplexFence(old_fence.session_id, incarnation=1) + await plane.handle( + OpenDuplexSessionMessage( + control_id="open-request-cleanup-new", + fence=new_fence, + session_id=new_fence.session_id, + capabilities={}, + ) + ) + assert (await result_sink.get()).ok is True + new_session = plane.sessions.require(new_fence.session_id) + assert new_session.resume(new_fence, expected_lease_generation=0) == 1 + + assert await plane.reap_expired() == 1 + assert plane.sessions.get(new_fence.session_id) is new_session + + +@pytest.mark.asyncio +async def test_failed_open_cleanup_is_retried_before_releasing_admission() -> None: + class _FailOpenAndFirstCleanupStagePort(_TypedStagePort): + def __init__(self) -> None: + super().__init__() + self.ensure_attempts = 0 + self.cleanup_attempts = 0 + + def ensure_request(self, context: DuplexStageRequestContext) -> None: + self.ensure_attempts += 1 + if self.ensure_attempts == 1: + raise RuntimeError("initial stage setup failed") + super().ensure_request(context) + + async def cleanup(self, request_ids: list[str], *, abort: bool = False) -> None: + self.cleanup_calls.append((list(request_ids), abort)) + self.cleanup_attempts += 1 + if self.cleanup_attempts == 1: + raise RuntimeError("transient open rollback failure") + + stage_port = _FailOpenAndFirstCleanupStagePort() + result_sink: asyncio.Queue = asyncio.Queue() + plane = DuplexControlPlane( + extension=_Extension(), + stage_port=stage_port, + result_sink=result_sink, + max_sessions=1, + ) + failed_fence = DuplexFence("sid-open-rollback") + + await plane.handle( + OpenDuplexSessionMessage( + control_id="open-fails-during-stage-setup", + fence=failed_fence, + session_id=failed_fence.session_id, + capabilities={"input_modes": ["append_audio_chunk"]}, + ) + ) + + failed_result = await result_sink.get() + assert failed_result.ok is False + assert plane.sessions.get(failed_fence.session_id) is not None + + replacement_fence = DuplexFence("sid-open-rollback-replacement") + await plane.handle( + OpenDuplexSessionMessage( + control_id="open-replacement-before-rollback", + fence=replacement_fence, + session_id=replacement_fence.session_id, + capabilities={"input_modes": ["append_audio_chunk"]}, + ) + ) + assert (await result_sink.get()).error.code == "resource_exhausted" + + assert await plane.reap_expired() == 0 + assert plane.sessions.get(failed_fence.session_id) is None + + await plane.handle( + OpenDuplexSessionMessage( + control_id="open-replacement-after-rollback", + fence=replacement_fence, + session_id=replacement_fence.session_id, + capabilities={"input_modes": ["append_audio_chunk"]}, + ) + ) + assert (await result_sink.get()).ok is True + + +@pytest.mark.asyncio +async def test_expired_cleanup_failure_does_not_block_other_sessions() -> None: + class _OneStuckExpiryStagePort(_TypedStagePort): + async def cleanup(self, request_ids: list[str], *, abort: bool = False) -> None: + if request_ids == [stuck_request_id]: + raise RuntimeError("stuck expiry cleanup") + await super().cleanup(request_ids, abort=abort) + + clock = _Clock() + stage_port = _OneStuckExpiryStagePort() + result_sink: asyncio.Queue = asyncio.Queue() + lifecycle_sink: asyncio.Queue = asyncio.Queue() + plane = DuplexControlPlane( + extension=None, + stage_port=stage_port, + result_sink=result_sink, + lifecycle_sink=lifecycle_sink, + lease_config=DuplexLeaseConfig(idle_ttl_s=1.0, disconnect_grace_s=1.0), + clock=clock, + ) + stuck_fence = DuplexFence("sid-expiry-stuck") + ready_fence = DuplexFence("sid-expiry-ready") + stuck_request_id = plane.stage_request_id(stuck_fence, stage_id=0) + ready_request_id = plane.stage_request_id(ready_fence, stage_id=0) + for fence in (stuck_fence, ready_fence): + await plane.handle( + OpenDuplexSessionMessage( + control_id=f"open-{fence.session_id}", + fence=fence, + session_id=fence.session_id, + capabilities={}, + ) + ) + assert (await result_sink.get()).ok is True + plane.sessions.require(fence.session_id).reserve_stage_request( + 0, + plane.stage_request_id(fence, stage_id=0), + fence=fence, + ) + clock.advance(2.0) + + assert await plane.reap_expired() == 1 + + assert (await lifecycle_sink.get()).session_id == ready_fence.session_id + assert stage_port.cleanup_calls == [([ready_request_id], False)] + + +@pytest.mark.asyncio +async def test_cancel_retains_request_resources_until_abort_succeeds() -> None: + class _FailOnceStagePort(_TypedStagePort): + def __init__(self) -> None: + super().__init__() + self.failures_remaining = 1 + + async def cleanup(self, request_ids: list[str], *, abort: bool = False) -> None: + if self.failures_remaining: + self.failures_remaining -= 1 + raise RuntimeError("transient abort failure") + await super().cleanup(request_ids, abort=abort) + + stage_port = _FailOnceStagePort() + result_sink: asyncio.Queue = asyncio.Queue() + plane = DuplexControlPlane(extension=None, stage_port=stage_port, result_sink=result_sink) + cancelled_fence = DuplexFence("sid-cancel-retry") + next_fence = DuplexFence("sid-cancel-retry", epoch=1) + session = plane.sessions.open_session(cancelled_fence) + session.bind_stage_request(0, "req-cancel-retry", fence=cancelled_fence) + + def signal(control_id: str) -> SignalDuplexTurnMessage: + return SignalDuplexTurnMessage( + control_id=control_id, + fence=cancelled_fence, + next_fence=next_fence, + session_id=cancelled_fence.session_id, + event="input.cancel", + ) + + await plane.handle(signal("cancel-first")) + assert (await result_sink.get()).ok is False + assert session.fence == next_fence + assert session.resource_request_ids(cancelled_fence) == ["req-cancel-retry"] + + await plane.handle(signal("cancel-retry")) + assert (await result_sink.get()).ok is True + assert session.resource_request_ids(cancelled_fence) == [] + assert stage_port.cleanup_calls == [(["req-cancel-retry"], True)] + + +@pytest.mark.asyncio +async def test_close_retains_session_resources_until_cleanup_succeeds() -> None: + class _FailOnceStagePort(_TypedStagePort): + def __init__(self) -> None: + super().__init__() + self.failures_remaining = 1 + + async def cleanup(self, request_ids: list[str], *, abort: bool = False) -> None: + if self.failures_remaining: + self.failures_remaining -= 1 + raise RuntimeError("transient close cleanup failure") + await super().cleanup(request_ids, abort=abort) + + stage_port = _FailOnceStagePort() + result_sink: asyncio.Queue = asyncio.Queue() + plane = DuplexControlPlane(extension=None, stage_port=stage_port, result_sink=result_sink) + fence = DuplexFence("sid-close-retry") + session = plane.sessions.open_session(fence) + session.bind_stage_request(0, "req-close-retry", fence=fence) + + def close(control_id: str) -> CloseDuplexSessionMessage: + return CloseDuplexSessionMessage( + control_id=control_id, + fence=fence, + session_id=fence.session_id, + reason="client_close", + ) + + await plane.handle(close("close-first")) + assert (await result_sink.get()).ok is False + assert plane.sessions.get(fence.session_id) is session + assert session.resource_request_ids() == ["req-close-retry"] + + await plane.handle(close("close-retry")) + assert (await result_sink.get()).ok is True + assert plane.sessions.get(fence.session_id) is None + assert stage_port.cleanup_calls == [(["req-close-retry"], True)] + + +@pytest.mark.asyncio +async def test_control_cleanup_failure_does_not_block_other_sessions() -> None: + class _IndependentFailureStagePort(_TypedStagePort): + def __init__(self) -> None: + super().__init__() + self.attempts: dict[str, int] = {} + + async def cleanup(self, request_ids: list[str], *, abort: bool = False) -> None: + request_id = request_ids[0] + self.attempts[request_id] = self.attempts.get(request_id, 0) + 1 + if request_id == "req-stuck" or self.attempts[request_id] == 1: + raise RuntimeError(f"cleanup failed for {request_id}") + await super().cleanup(request_ids, abort=abort) + + stage_port = _IndependentFailureStagePort() + result_sink: asyncio.Queue = asyncio.Queue() + plane = DuplexControlPlane(extension=None, stage_port=stage_port, result_sink=result_sink) + + for session_id, request_id in (("sid-stuck", "req-stuck"), ("sid-ready", "req-ready")): + fence = DuplexFence(session_id) + session = plane.sessions.open_session(fence) + session.bind_stage_request(0, request_id, fence=fence) + await plane.handle( + CloseDuplexSessionMessage( + control_id=f"close-{session_id}", + fence=fence, + session_id=session_id, + reason="client_close", + ) + ) + assert (await result_sink.get()).ok is False + + await plane.reap_expired() + + assert plane.sessions.get("sid-stuck") is not None + assert plane.sessions.get("sid-ready") is None + assert stage_port.cleanup_calls == [(["req-ready"], True)] + + +@pytest.mark.asyncio +async def test_control_cleanup_is_single_flight_with_concurrent_reaper() -> None: + class _SlowCleanupStagePort(_TypedStagePort): + def __init__(self) -> None: + super().__init__() + self.cleanup_started = asyncio.Event() + self.release_cleanup = asyncio.Event() + self.cleanup_attempts = 0 + + async def cleanup(self, request_ids: list[str], *, abort: bool = False) -> None: + self.cleanup_attempts += 1 + self.cleanup_started.set() + await self.release_cleanup.wait() + await super().cleanup(request_ids, abort=abort) + + stage_port = _SlowCleanupStagePort() + result_sink: asyncio.Queue = asyncio.Queue() + plane = DuplexControlPlane(extension=None, stage_port=stage_port, result_sink=result_sink) + fence = DuplexFence("sid-concurrent-cleanup") + session = plane.sessions.open_session(fence) + session.bind_stage_request(0, "req-concurrent-cleanup", fence=fence) + + close_task = asyncio.create_task( + plane.handle( + CloseDuplexSessionMessage( + control_id="close-concurrent-cleanup", + fence=fence, + session_id=fence.session_id, + reason="client_close", + ) + ) + ) + await asyncio.wait_for(stage_port.cleanup_started.wait(), timeout=1) + + reaper_task = asyncio.create_task(plane.reap_expired()) + await asyncio.sleep(0) + await asyncio.sleep(0) + assert stage_port.cleanup_attempts == 1 + + stage_port.release_cleanup.set() + await asyncio.gather(close_task, reaper_task) + + assert (await result_sink.get()).ok is True + assert plane.sessions.get(fence.session_id) is None + assert stage_port.cleanup_calls == [(["req-concurrent-cleanup"], True)] + + +@pytest.mark.asyncio +async def test_unknown_input_mode_rejects_open_without_registering_session() -> None: + result_sink: asyncio.Queue = asyncio.Queue() + plane = DuplexControlPlane(extension=None, stage_port=_TypedStagePort(), result_sink=result_sink) + fence = DuplexFence("sid-unknown-mode") + + await plane.handle( + OpenDuplexSessionMessage( + control_id="open-unknown-mode", + fence=fence, + session_id=fence.session_id, + capabilities={"input_modes": ["future_mode"]}, + ) + ) + + result = await result_sink.get() + assert result.ok is False + assert result.error is not None + assert result.error.code == "invalid_capability" + assert plane.sessions.get(fence.session_id) is None + + +@pytest.mark.asyncio +async def test_extension_free_turn_commit_does_not_allocate_or_submit_stage_request() -> None: + stage_port = _TypedStagePort() + result_sink: asyncio.Queue = asyncio.Queue() + plane = DuplexControlPlane(extension=None, stage_port=stage_port, result_sink=result_sink) + fence = DuplexFence("sid-turn-commit") + + await plane.handle( + OpenDuplexSessionMessage( + control_id="open-turn-commit", + fence=fence, + session_id=fence.session_id, + capabilities={"input_modes": [DuplexInputMode.TURN_COMMIT_ONLY.value]}, + ) + ) + assert (await result_sink.get()).ok is True + + await plane.handle( + AppendDuplexInputMessage( + control_id="append-turn-commit", + fence=fence, + session_id=fence.session_id, + mode=DuplexInputMode.TURN_COMMIT_ONLY.value, + payload=None, + final=True, + ) + ) + result = await result_sink.get() + + assert result.ok is True + assert result.stage_results[0]["result"]["data_plane_append"] is False + assert stage_port.ensure_calls == [] + assert stage_port.submit_calls == [] + + +@pytest.mark.asyncio +async def test_session_update_replaces_both_configs_with_one_generation() -> None: + result_sink: asyncio.Queue = asyncio.Queue() + plane = DuplexControlPlane(extension=_Extension(), stage_port=_TypedStagePort(), result_sink=result_sink) + fence = DuplexFence("sid-atomic-config") + session = plane.sessions.open_session(fence) + + await plane.handle( + SignalDuplexTurnMessage( + control_id="update-config", + fence=fence, + session_id=fence.session_id, + event="session.update", + session_config={"voice": "new"}, + runtime_config={"temperature": 0.5}, + ) + ) + + assert (await result_sink.get()).ok is True + assert session.config_generation == 1 + assert session.session_config == {"voice": "new"} + assert session.runtime_config == {"temperature": 0.5} + + +@pytest.mark.asyncio +async def test_stage_submission_is_compensated_when_local_commit_fails() -> None: + stage_port = _TypedStagePort() + result_sink: asyncio.Queue = asyncio.Queue() + plane = DuplexControlPlane(extension=_Extension(), stage_port=stage_port, result_sink=result_sink) + fence = DuplexFence("sid-submit-compensation") + session = plane.sessions.open_session( + fence, + capabilities=DuplexRuntimeCapabilities(input_modes={DuplexInputMode.APPEND_AUDIO_CHUNK}), + ) + request_id = plane.stage_request_id(fence, stage_id=0) + + def fail_commit(_reservation): + raise RuntimeError("local commit failed") + + session.commit_append = fail_commit # type: ignore[method-assign] + await plane.handle( + AppendDuplexInputMessage( + control_id="append-compensation", + fence=fence, + session_id=fence.session_id, + mode=DuplexInputMode.APPEND_AUDIO_CHUNK.value, + payload={"audio": b"pcm"}, + ) + ) + + assert (await result_sink.get()).ok is False + assert stage_port.cleanup_calls == [([request_id], True)] + + +@pytest.mark.asyncio +async def test_failed_submission_compensation_blocks_append_until_reaper_cleans_it() -> None: + class _FailingCleanupStagePort(_TypedStagePort): + def __init__(self) -> None: + super().__init__() + self.cleanup_failures = 2 + + async def cleanup(self, request_ids: list[str], *, abort: bool = False) -> None: + await super().cleanup(request_ids, abort=abort) + if self.cleanup_failures > 0: + self.cleanup_failures -= 1 + raise RuntimeError("stage abort failed") + + stage_port = _FailingCleanupStagePort() + result_sink: asyncio.Queue = asyncio.Queue() + plane = DuplexControlPlane(extension=_Extension(), stage_port=stage_port, result_sink=result_sink) + fence = DuplexFence("sid-durable-submit-compensation") + session = plane.sessions.open_session( + fence, + capabilities=DuplexRuntimeCapabilities(input_modes={DuplexInputMode.APPEND_AUDIO_CHUNK}), + ) + original_commit = session.commit_append + + def fail_commit(_reservation): + raise RuntimeError("local commit failed") + + session.commit_append = fail_commit # type: ignore[method-assign] + first = AppendDuplexInputMessage( + control_id="append-compensation-1", + fence=fence, + session_id=fence.session_id, + mode=DuplexInputMode.APPEND_AUDIO_CHUNK.value, + payload={"audio": b"pcm"}, + ) + await plane.handle(first) + assert (await result_sink.get()).ok is False + assert plane.pending_submission_cleanup_count == 1 + assert len(stage_port.submit_calls) == 1 + + await plane.handle( + AppendDuplexInputMessage( + control_id="append-compensation-2", + fence=fence, + session_id=fence.session_id, + mode=DuplexInputMode.APPEND_AUDIO_CHUNK.value, + payload={"audio": b"pcm"}, + ) + ) + assert (await result_sink.get()).ok is False + assert plane.pending_submission_cleanup_count == 1 + assert len(stage_port.submit_calls) == 1 + + session.commit_append = original_commit # type: ignore[method-assign] + stage_port.cleanup_failures = 0 + await plane.reap_expired() + + assert plane.pending_submission_cleanup_count == 0 + assert session.resource_request_ids() == [] + + +def test_stale_output_is_rejected_before_extension_decision() -> None: + class _CountingExtension(_Extension): + def __init__(self) -> None: + self.decision_calls = 0 + + def decide_output(self, **kwargs): + del kwargs + self.decision_calls += 1 + return None + + extension = _CountingExtension() + plane = DuplexControlPlane(extension=extension, stage_port=_TypedStagePort(), result_sink=asyncio.Queue()) + current = DuplexFence("sid-stale-output", epoch=1) + plane.sessions.open_session(current) + context = DuplexOutputContext( + identity=DuplexRequestIdentity( + session_id=current.session_id, + fence=DuplexFence(current.session_id, epoch=0), + ), + final_stage_id=1, + segment_finished=True, + ) + + assert plane.decide_output(0, object(), context) is None + assert extension.decision_calls == 0 + + +@pytest.mark.asyncio +async def test_control_dispatch_is_ordered_per_session_without_blocking_other_sessions() -> None: + class _BlockingStagePort(_TypedStagePort): + def __init__(self) -> None: + super().__init__() + self.started = asyncio.Event() + self.release = asyncio.Event() + + async def submit(self, submission: DuplexStageSubmission) -> DuplexStageSubmissionResult: + if submission.context.session_id == "sid-blocked": + self.started.set() + await self.release.wait() + return await super().submit(submission) + + stage_port = _BlockingStagePort() + result_sink: asyncio.Queue = asyncio.Queue() + plane = DuplexControlPlane(extension=_Extension(), stage_port=stage_port, result_sink=result_sink) + blocked = DuplexFence("sid-blocked") + independent = DuplexFence("sid-independent") + capabilities = DuplexRuntimeCapabilities(input_modes={DuplexInputMode.APPEND_AUDIO_CHUNK}) + plane.sessions.open_session(blocked, capabilities=capabilities) + plane.sessions.open_session(independent, capabilities=capabilities) + + plane.dispatch( + AppendDuplexInputMessage( + control_id="blocked-append", + fence=blocked, + session_id=blocked.session_id, + mode=DuplexInputMode.APPEND_AUDIO_CHUNK.value, + payload={"audio": b"pcm"}, + operation_id="blocked-operation", + ) + ) + await asyncio.wait_for(stage_port.started.wait(), timeout=1) + plane.dispatch( + AppendDuplexInputMessage( + control_id="blocked-append-retry", + fence=blocked, + session_id=blocked.session_id, + mode=DuplexInputMode.APPEND_AUDIO_CHUNK.value, + payload={"audio": b"pcm"}, + operation_id="blocked-operation", + ) + ) + plane.dispatch( + TouchDuplexSessionMessage( + control_id="blocked-touch", + fence=blocked, + session_id=blocked.session_id, + activity=DuplexLeaseActivity.HEARTBEAT.value, + ) + ) + plane.dispatch( + TouchDuplexSessionMessage( + control_id="independent-touch", + fence=independent, + session_id=independent.session_id, + activity=DuplexLeaseActivity.HEARTBEAT.value, + ) + ) + + independent_result = await asyncio.wait_for(result_sink.get(), timeout=1) + assert independent_result.control_id == "independent-touch" + + stage_port.release.set() + await plane.drain() + assert [result_sink.get_nowait().control_id for _ in range(3)] == [ + "blocked-append", + "blocked-append-retry", + "blocked-touch", + ] + assert [ + submission.context.session_id + for submission in stage_port.submit_calls + if submission.context.session_id == blocked.session_id + ] == [blocked.session_id] diff --git a/tests/fullduplex/engine/test_duplex_deploy_config.py b/tests/fullduplex/engine/test_duplex_deploy_config.py new file mode 100644 index 000000000..abb387573 --- /dev/null +++ b/tests/fullduplex/engine/test_duplex_deploy_config.py @@ -0,0 +1,68 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Focused tests for typed deployment-only runtime configuration.""" + +from __future__ import annotations + +from dataclasses import FrozenInstanceError + +import pytest + +from vllm_omni.config.stage_config import ( + DuplexSessionRuntimeConfig, + load_deploy_config, +) + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +def test_duplex_session_runtime_defaults_are_typed_and_immutable(tmp_path) -> None: + deploy_path = tmp_path / "duplex.yaml" + deploy_path.write_text("duplex_session: {}\nstages: []\n", encoding="utf-8") + + deploy = load_deploy_config(deploy_path) + + assert isinstance(deploy.duplex_session, DuplexSessionRuntimeConfig) + assert deploy.duplex_session.idle_ttl_s == 300.0 + assert deploy.duplex_session.disconnect_grace_s == 30.0 + assert deploy.duplex_session.reaper_interval_s == 5.0 + assert deploy.duplex_session.resume_replay_ttl_s == 60.0 + assert deploy.duplex_session.resume_replay_max_bytes_per_session == 8 * 1024 * 1024 + assert deploy.duplex_session.max_pending_input_bytes_per_session == 16 * 1024 * 1024 + assert deploy.duplex_session.max_pending_turns_per_session == 4 + assert deploy.duplex_session.max_sessions == 1 + assert deploy.duplex_session.completed_append_cache_size == 256 + with pytest.raises(FrozenInstanceError): + deploy.duplex_session.idle_ttl_s = 1.0 # type: ignore[misc] + + +def test_duplex_session_runtime_accepts_disabled_idle_expiry(tmp_path) -> None: + deploy_path = tmp_path / "duplex.yaml" + deploy_path.write_text("duplex_session:\n idle_ttl_s: null\nstages: []\n", encoding="utf-8") + + deploy = load_deploy_config(deploy_path) + + assert deploy.duplex_session.idle_ttl_s is None + + +@pytest.mark.parametrize( + ("name", "value"), + [ + ("idle_ttl_s", 0), + ("idle_ttl_s", -1), + ("disconnect_grace_s", 0), + ("reaper_interval_s", -1), + ("resume_replay_ttl_s", 0), + ("resume_replay_max_bytes_per_session", -1), + ("max_pending_input_bytes_per_session", 0), + ("max_pending_turns_per_session", -1), + ("max_sessions", 0), + ("completed_append_cache_size", 0), + ], +) +def test_duplex_session_runtime_rejects_non_positive_values(tmp_path, name: str, value: int) -> None: + deploy_path = tmp_path / "duplex.yaml" + deploy_path.write_text(f"duplex_session:\n {name}: {value}\nstages: []\n", encoding="utf-8") + + with pytest.raises(ValueError, match=rf"duplex_session\.{name} must be positive"): + load_deploy_config(deploy_path) diff --git a/tests/fullduplex/engine/test_duplex_intermediate.py b/tests/fullduplex/engine/test_duplex_intermediate.py new file mode 100644 index 000000000..37602e62e --- /dev/null +++ b/tests/fullduplex/engine/test_duplex_intermediate.py @@ -0,0 +1,14 @@ +import pytest + +from vllm_omni.experimental.fullduplex.engine.intermediate import get_stream_request_key + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +def test_get_stream_request_key_requires_stable_identifier(): + with pytest.raises(ValueError, match="stable request id"): + get_stream_request_key({"ids": {"tts": [1, 2, 3]}}) + + +def test_get_stream_request_key_accepts_global_request_id(): + assert get_stream_request_key({"global_request_id": ["duplex-sid-stage0"]}) == "duplex-sid-stage0" diff --git a/tests/fullduplex/engine/test_duplex_lease.py b/tests/fullduplex/engine/test_duplex_lease.py new file mode 100644 index 000000000..80e7a73eb --- /dev/null +++ b/tests/fullduplex/engine/test_duplex_lease.py @@ -0,0 +1,224 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from __future__ import annotations + +from dataclasses import FrozenInstanceError + +import pytest + +from vllm_omni.experimental.fullduplex.engine.duplex_runtime import DuplexInputMode +from vllm_omni.experimental.fullduplex.engine.duplex_session import DuplexSessionRuntimeManager +from vllm_omni.experimental.fullduplex.engine.lease import ( + DuplexLeaseActivity, + DuplexLeaseConfig, +) +from vllm_omni.experimental.fullduplex.engine.messages import DuplexFence + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +class FakeMonotonicClock: + def __init__(self, value: float) -> None: + self.value = value + + def __call__(self) -> float: + return self.value + + def advance(self, seconds: float) -> None: + self.value += seconds + + +def _lease_config(*, idle_ttl_s: float | None = 300.0) -> DuplexLeaseConfig: + return DuplexLeaseConfig( + idle_ttl_s=idle_ttl_s, + disconnect_grace_s=30.0, + ) + + +def test_open_touch_detach_and_idle_expiry_use_monotonic_time() -> None: + clock = FakeMonotonicClock(100.0) + manager = DuplexSessionRuntimeManager(clock=clock) + fence = DuplexFence("sid-expiry") + session = manager.open_session(fence, lease_config=_lease_config()) + + assert session.lease.last_activity == 100.0 + clock.advance(10.0) + session.touch(fence, DuplexLeaseActivity.HEARTBEAT) + session.detach(fence) + clock.advance(29.0) + assert session.lease.disconnect_grace_expired(clock()) is False + assert manager.collect_expired() == [] + + clock.advance(272.0) + expired = manager.collect_expired() + + assert [item.session_id for item in expired] == ["sid-expiry"] + assert expired[0].reason == "disconnect_grace_expired" + assert manager.get("sid-expiry") is session + manager.finalize_close_session(session) + assert manager.get("sid-expiry") is None + + +def test_detach_grace_and_resume_advance_token_independent_lease_generation() -> None: + clock = FakeMonotonicClock(10.0) + manager = DuplexSessionRuntimeManager(clock=clock) + fence = DuplexFence("sid-resume") + session = manager.open_session(fence, lease_config=_lease_config()) + + session.detach(fence) + clock.advance(29.0) + assert session.lease.disconnect_grace_expired(clock()) is False + clock.advance(2.0) + assert session.lease.disconnect_grace_expired(clock()) is True + + generation = session.resume(fence, expected_lease_generation=0) + + assert generation == 1 + assert session.lease.detached_at is None + assert session.lease.disconnect_grace_expired(clock()) is False + with pytest.raises(ValueError, match="lease generation mismatch"): + session.resume(fence, expected_lease_generation=0) + + +def test_active_operation_prevents_mid_transaction_expiry() -> None: + clock = FakeMonotonicClock(0.0) + manager = DuplexSessionRuntimeManager(clock=clock) + fence = DuplexFence("sid-operation") + session = manager.open_session(fence, lease_config=_lease_config(idle_ttl_s=5.0)) + + session.begin_operation(fence, "append-1") + clock.advance(10.0) + assert manager.collect_expired() == [] + + session.end_operation(fence, "append-1") + assert manager.collect_expired() == [] + clock.advance(6.0) + assert [item.session_id for item in manager.collect_expired()] == ["sid-operation"] + + +def test_sessions_have_independent_activity_deadlines_and_resources() -> None: + clock = FakeMonotonicClock(0.0) + manager = DuplexSessionRuntimeManager(clock=clock) + fence_a = DuplexFence("sid-a") + fence_b = DuplexFence("sid-b") + session_a = manager.open_session(fence_a, lease_config=_lease_config(idle_ttl_s=10.0)) + session_b = manager.open_session(fence_b, lease_config=_lease_config(idle_ttl_s=10.0)) + session_a.reserve_stage_request(0, "req-a-reserved", fence=fence_a) + session_a.bind_stage_request(1, "req-a-submitted", fence=fence_a) + session_b.bind_stage_request(0, "req-b", fence=fence_b) + + clock.advance(6.0) + session_b.touch(fence_b, DuplexLeaseActivity.MODEL_OUTPUT) + clock.advance(5.0) + expired = manager.collect_expired() + + assert len(expired) == 1 + assert expired[0].session_id == "sid-a" + assert expired[0].reserved_request_ids == ("req-a-reserved",) + assert expired[0].submitted_request_ids == ("req-a-submitted",) + assert manager.get("sid-b") is session_b + assert session_b.resource_request_ids() == ["req-b"] + + +def test_close_and_reaper_select_exactly_one_terminal_transition() -> None: + clock = FakeMonotonicClock(0.0) + manager = DuplexSessionRuntimeManager(clock=clock) + fence = DuplexFence("sid-race") + manager.open_session(fence, lease_config=_lease_config(idle_ttl_s=1.0)) + clock.advance(2.0) + + assert len(manager.collect_expired()) == 1 + assert manager.close_session(fence, reason="explicit_close") is None + assert manager.collect_expired() == [] + + fence_2 = DuplexFence("sid-race-2") + manager.open_session(fence_2, lease_config=_lease_config(idle_ttl_s=1.0)) + assert manager.close_session(fence_2, reason="explicit_close") is not None + clock.advance(2.0) + assert manager.collect_expired() == [] + + +def test_stale_fence_cannot_touch_detach_or_resume_lease() -> None: + clock = FakeMonotonicClock(0.0) + manager = DuplexSessionRuntimeManager(clock=clock) + current = DuplexFence("sid-fence", epoch=1) + stale = DuplexFence("sid-fence", epoch=0) + session = manager.open_session(current, lease_config=_lease_config()) + + with pytest.raises(RuntimeError, match="fence mismatch"): + session.touch(stale, DuplexLeaseActivity.APPEND) + with pytest.raises(RuntimeError, match="fence mismatch"): + session.detach(stale) + with pytest.raises(RuntimeError, match="fence mismatch"): + session.resume(stale, expected_lease_generation=0) + + +def test_disabled_idle_expiry_never_collects_session() -> None: + clock = FakeMonotonicClock(0.0) + manager = DuplexSessionRuntimeManager(clock=clock) + manager.open_session(DuplexFence("sid-no-expiry"), lease_config=_lease_config(idle_ttl_s=None)) + + clock.advance(1_000_000.0) + + assert manager.collect_expired() == [] + + +def test_detached_session_expires_at_disconnect_grace_when_idle_ttl_is_disabled() -> None: + clock = FakeMonotonicClock(0.0) + manager = DuplexSessionRuntimeManager(clock=clock) + fence = DuplexFence("sid-disconnect-grace") + session = manager.open_session(fence, lease_config=_lease_config(idle_ttl_s=None)) + + session.detach(fence) + clock.advance(29.0) + assert manager.collect_expired() == [] + + clock.advance(1.0) + expired = manager.collect_expired() + + assert [item.session_id for item in expired] == [fence.session_id] + assert expired[0].reason == "disconnect_grace_expired" + + +def test_runtime_manager_enforces_server_owned_session_admission() -> None: + manager = DuplexSessionRuntimeManager(max_sessions=2) + manager.open_session(DuplexFence("sid-admission-a")) + manager.open_session(DuplexFence("sid-admission-b")) + + with pytest.raises(RuntimeError, match="duplex_session_capacity_exhausted"): + manager.open_session(DuplexFence("sid-admission-c")) + + +def test_completed_append_cache_is_bounded() -> None: + manager = DuplexSessionRuntimeManager(completed_append_limit=2) + fence = DuplexFence("sid-completed-cache") + session = manager.open_session(fence) + + for index in range(3): + session.record_completed_append( + f"operation-{index}", + fence=fence, + mode=DuplexInputMode.TURN_COMMIT_ONLY, + final=True, + stage_results=[{"index": index}], + ) + + assert list(session.completed_appends) == ["operation-1", "operation-2"] + + session.accept_fence(DuplexFence(fence.session_id, epoch=1)) + assert session.completed_appends == {} + + +def test_lease_config_and_expiry_record_are_immutable() -> None: + config = _lease_config() + with pytest.raises(FrozenInstanceError): + config.idle_ttl_s = 1.0 # type: ignore[misc] + + clock = FakeMonotonicClock(0.0) + manager = DuplexSessionRuntimeManager(clock=clock) + manager.open_session(DuplexFence("sid-record"), lease_config=_lease_config(idle_ttl_s=1.0)) + clock.advance(2.0) + record = manager.collect_expired()[0] + with pytest.raises(FrozenInstanceError): + record.reason = "changed" # type: ignore[misc] diff --git a/tests/fullduplex/engine/test_duplex_runtime.py b/tests/fullduplex/engine/test_duplex_runtime.py new file mode 100644 index 000000000..7cce93161 --- /dev/null +++ b/tests/fullduplex/engine/test_duplex_runtime.py @@ -0,0 +1,480 @@ +from dataclasses import FrozenInstanceError +from types import SimpleNamespace + +import pytest +from vllm.sampling_params import SamplingParams + +from vllm_omni.experimental.fullduplex.engine import duplex_runtime +from vllm_omni.experimental.fullduplex.engine.contracts import ( + duplex_data_plane_request_info, + duplex_resource_request_belongs_to_session, + duplex_resource_request_id, +) +from vllm_omni.experimental.fullduplex.engine.duplex_runtime import ( + DuplexInputMode, + DuplexOutputAction, + DuplexOutputDecision, + DuplexRuntimeCapabilities, + DuplexSessionRuntimeManager, +) +from vllm_omni.experimental.fullduplex.engine.messages import DuplexFence +from vllm_omni.experimental.fullduplex.minicpmo45.runtime import ( + MiniCPMO45DuplexRuntimeExtension, + build_duplex_data_plane_prompt, + duplex_scheduler_token_budget, +) + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +def test_duplex_fence_is_immutable(): + fence = DuplexFence("session") + + with pytest.raises(FrozenInstanceError): + fence.epoch = 1 # type: ignore[misc] + + assert not hasattr(fence, "__dict__") + + +def test_duplex_runtime_tracks_stage_bindings_and_barge_in_epoch(): + manager = DuplexSessionRuntimeManager() + session = manager.open_session( + DuplexFence("sid-1"), + capabilities=DuplexRuntimeCapabilities( + input_modes={DuplexInputMode.APPEND_TOKENS}, + ), + ) + session.bind_stage_request(stage_id=0, request_id="req-stage0", fence=session.fence) + session.bind_stage_request(stage_id=1, request_id="req-stage1", fence=session.fence) + + update = session.append_input(mode=DuplexInputMode.APPEND_TOKENS, fence=session.fence) + next_fence = DuplexFence("sid-1", epoch=1) + stale_request_ids = session.release_fence(session.fence) + session.accept_fence(next_fence) + + assert update.seq == 1 + assert session.fence == next_fence + assert stale_request_ids == ["req-stage0", "req-stage1"] + assert session.stage_bindings == {} + + +def test_duplex_runtime_tracks_same_request_id_for_each_pipeline_stage(): + manager = DuplexSessionRuntimeManager() + fence = DuplexFence("sid-shared-pipeline-request") + session = manager.open_session(fence) + + session.reserve_stage_request(0, "req-shared", fence=fence) + session.bind_stage_request(0, "req-shared", fence=fence) + session.bind_stage_request(1, "req-shared", fence=fence) + + assert session.stage_bindings[0].request_id == "req-shared" + assert session.stage_bindings[1].request_id == "req-shared" + assert session.resource_request_ids() == ["req-shared"] + assert session.input_seq == 0 + + +def test_duplex_runtime_extension_validation_rejects_missing_methods(): + class IncompleteExtension: + def configure_sampling_params(self, *, runtime_config, defaults): + del runtime_config + return defaults + + with pytest.raises(TypeError, match="plan_append"): + duplex_runtime.validate_duplex_runtime_extension(IncompleteExtension()) + + +def test_duplex_runtime_extension_validation_rejects_stage_count_mismatch(): + class WrongStageCountExtension(MiniCPMO45DuplexRuntimeExtension): + def configure_sampling_params(self, *, runtime_config, defaults): + del runtime_config + return defaults[:1] + + with pytest.raises(ValueError, match="one sampling parameter per stage"): + duplex_runtime.validate_duplex_runtime_extension( + WrongStageCountExtension(), + sampling_defaults=(SamplingParams(), SamplingParams()), + ) + + +def test_duplex_runtime_extension_validation_rejects_sampling_type_mismatch(): + class WrongSamplingTypeExtension(MiniCPMO45DuplexRuntimeExtension): + def configure_sampling_params(self, *, runtime_config, defaults): + del runtime_config + return tuple(object() for _ in defaults) + + with pytest.raises(TypeError, match="sampling parameter type"): + duplex_runtime.validate_duplex_runtime_extension( + WrongSamplingTypeExtension(), + sampling_defaults=(SamplingParams(), SamplingParams()), + ) + + +def test_duplex_output_decision_metadata_is_immutable(): + decision = DuplexOutputDecision( + action=DuplexOutputAction.DIRECT_RESPONSE, + metadata={"model_listen": True}, + ) + + with pytest.raises(TypeError): + decision.metadata["model_listen"] = False + + +def _decide_minicpmo_output( + output: object, + *, + segment_token_ids: tuple[int, ...] = (), + segment_output_metadata: dict | None = None, +): + return MiniCPMO45DuplexRuntimeExtension().decide_output( + stage_id=0, + final_stage_id=1, + segment_finished=True, + segment_token_ids=segment_token_ids, + segment_output_metadata=segment_output_metadata or {}, + output=output, + ) + + +def test_minicpmo_extension_owns_stage_sampling_overrides(): + defaults = ( + SamplingParams(max_tokens=4), + SamplingParams(max_tokens=8), + ) + + configured = MiniCPMO45DuplexRuntimeExtension().configure_sampling_params( + runtime_config={ + "duplex_stage_max_tokens": {"0": 20}, + "duplex_stage_sampling_params": {"1": {"stop_token_ids": [151645]}}, + }, + defaults=defaults, + ) + + assert configured[0].max_tokens == 20 + assert configured[1].stop_token_ids == [151645] + assert defaults[0].max_tokens == 4 + assert 151645 not in (defaults[1].stop_token_ids or []) + + +def test_minicpmo_output_decision_uses_raw_streaming_token_snapshot(): + decision = _decide_minicpmo_output( + SimpleNamespace(outputs=[SimpleNamespace()]), + segment_token_ids=(151705,), + segment_output_metadata={"special_token_ids": {"listen_token_id": 151705}}, + ) + + assert decision is not None + assert decision.action is DuplexOutputAction.DIRECT_RESPONSE + assert decision.metadata["duplex_native_decision"] == "listen" + assert decision.metadata["model_listen"] is True + + +@pytest.mark.parametrize("attr", ["token_ids", "cumulative_token_ids"]) +def test_minicpmo_output_decision_ignores_output_level_token_history(attr): + output = SimpleNamespace( + multimodal_output={"special_token_ids": {"listen_token_id": 151705}}, + outputs=[SimpleNamespace()], + **{attr: [42, 151705]}, + ) + + assert _decide_minicpmo_output(output) is None + + +@pytest.mark.parametrize("attr", ["token_ids", "cumulative_token_ids"]) +def test_minicpmo_output_decision_uses_completion_token_ids(attr): + output = SimpleNamespace( + multimodal_output={"special_token_ids": {"listen_token_id": 151705}}, + outputs=[SimpleNamespace(**{attr: [42, 151705]})], + ) + + assert _decide_minicpmo_output(output) is not None + + +def test_minicpmo_output_decision_uses_completion_stop_reason(): + output = SimpleNamespace( + multimodal_output={"special_token_ids": {"listen_token_id": 151705}}, + outputs=[SimpleNamespace(stop_reason=151705)], + ) + + assert _decide_minicpmo_output(output) is not None + + +def test_duplex_runtime_cancel_fence_rejects_late_append_and_accepts_next_epoch(): + manager = DuplexSessionRuntimeManager() + cancelled_fence = DuplexFence("sid-cancel-race") + next_fence = DuplexFence("sid-cancel-race", epoch=1) + session = manager.open_session( + cancelled_fence, + capabilities=DuplexRuntimeCapabilities( + input_modes={DuplexInputMode.APPEND_AUDIO_CHUNK}, + ), + ) + session.bind_stage_request( + stage_id=0, + request_id="req-cancelled", + fence=cancelled_fence, + ) + + stale_request_ids = session.cancel_fence(cancelled_fence, next_fence) + + assert stale_request_ids == ["req-cancelled"] + assert session.fence == next_fence + assert session.stage_bindings == {} + with pytest.raises(RuntimeError, match="fence mismatch"): + session.append_input( + mode=DuplexInputMode.APPEND_AUDIO_CHUNK, + fence=cancelled_fence, + ) + update = session.append_input( + mode=DuplexInputMode.APPEND_AUDIO_CHUNK, + fence=next_fence, + ) + assert update.seq == 1 + + +def test_duplex_runtime_stale_close_preserves_live_session_and_bindings(): + manager = DuplexSessionRuntimeManager() + current_fence = DuplexFence("sid-stale-close", epoch=1) + session = manager.open_session( + current_fence, + capabilities=DuplexRuntimeCapabilities( + input_modes={DuplexInputMode.APPEND_AUDIO_CHUNK}, + ), + ) + session.bind_stage_request(0, "req-live", fence=current_fence) + + with pytest.raises(RuntimeError, match="fence mismatch"): + manager.close_session(DuplexFence("sid-stale-close")) + + assert manager.get("sid-stale-close") is session + assert session.stage_request_ids() == ["req-live"] + + +def test_duplex_runtime_reopen_rejects_late_append_from_old_incarnation(): + manager = DuplexSessionRuntimeManager() + old_fence = DuplexFence("sid-reopen", incarnation=0) + old_session = manager.open_session( + old_fence, + capabilities=DuplexRuntimeCapabilities( + input_modes={DuplexInputMode.APPEND_AUDIO_CHUNK}, + ), + ) + manager.close_session(old_fence) + + new_fence = DuplexFence("sid-reopen", incarnation=1) + new_session = manager.open_session( + new_fence, + capabilities=old_session.capabilities, + ) + + with pytest.raises(RuntimeError, match="fence mismatch"): + new_session.append_input( + mode=DuplexInputMode.APPEND_AUDIO_CHUNK, + fence=old_fence, + ) + assert ( + new_session.append_input( + mode=DuplexInputMode.APPEND_AUDIO_CHUNK, + fence=new_fence, + ).seq + == 1 + ) + + +def test_duplex_prompt_expands_incarnation_metadata(): + fence = DuplexFence("sid-incarnation", incarnation=3) + + prompt = build_duplex_data_plane_prompt( + request_id=duplex_resource_request_id(fence, "stage0"), + fence=fence, + session_config={}, + runtime_config={}, + seq=1, + turn_seq=1, + mode=DuplexInputMode.APPEND_AUDIO_CHUNK, + payload={"is_speech": True}, + final=False, + ) + + assert prompt["model_intermediate_buffer"]["duplex"]["incarnation"] == 3 + + +def test_duplex_runtime_tracks_turn_local_append_sequence(): + manager = DuplexSessionRuntimeManager() + session = manager.open_session( + DuplexFence("sid-turn-seq"), + capabilities=DuplexRuntimeCapabilities( + input_modes={DuplexInputMode.APPEND_AUDIO_CHUNK}, + ), + ) + + first = session.append_input(mode=DuplexInputMode.APPEND_AUDIO_CHUNK, fence=session.fence) + second = session.append_input(mode=DuplexInputMode.APPEND_AUDIO_CHUNK, fence=session.fence) + next_turn = DuplexFence("sid-turn-seq", turn_id=1, response_seq=1) + third = session.append_input( + mode=DuplexInputMode.APPEND_AUDIO_CHUNK, + fence=next_turn, + ) + fourth = session.append_input( + mode=DuplexInputMode.APPEND_AUDIO_CHUNK, + fence=next_turn, + ) + + assert [first.seq, second.seq, third.seq, fourth.seq] == [1, 2, 3, 4] + assert [first.turn_seq, second.turn_seq, third.turn_seq, fourth.turn_seq] == [1, 2, 1, 2] + assert [first.turn_id, second.turn_id, third.turn_id, fourth.turn_id] == [0, 0, 1, 1] + + +def test_duplex_runtime_rejects_unsupported_append_mode(): + manager = DuplexSessionRuntimeManager() + session = manager.open_session( + DuplexFence("sid-2"), + capabilities=DuplexRuntimeCapabilities(input_modes={DuplexInputMode.TURN_COMMIT_ONLY}), + ) + + with pytest.raises(ValueError, match="not supported"): + session.append_input(mode=DuplexInputMode.APPEND_TOKENS, fence=session.fence) + + +def test_duplex_data_plane_request_info_extracts_structured_stage_result(): + request_id, response_stage_id = duplex_data_plane_request_info( + { + "stage_results": [ + {"result": {"supported": True}}, + { + "result": { + "data_plane_append": True, + "request_id": "duplex-sid-e0-stage0-s1", + "response_stage_id": 1, + } + }, + ] + } + ) + + assert request_id == "duplex-sid-e0-stage0-s1" + assert response_stage_id == 1 + + +def test_duplex_data_plane_request_info_rejects_missing_request_id(): + assert duplex_data_plane_request_info( + { + "stage_results": [ + { + "result": { + "data_plane_append": True, + "request_id": "", + "response_stage_id": 1, + } + } + ] + } + ) == (None, None) + + +def test_duplex_scheduler_token_budget_estimates_pcm_slots(): + assert ( + duplex_scheduler_token_budget( + { + "audio": "AAAAAA==", + "format": "pcm_f32le", + "sample_rate_hz": 16000, + } + ) + == 16 + ) + + +def test_duplex_scheduler_token_budget_ignores_client_budget_fields(): + assert ( + duplex_scheduler_token_budget( + { + "audio": "AAAAAA==", + "format": "pcm_f32le", + "duplex_num_input_tokens": 999, + "num_input_tokens": 999, + } + ) + == 16 + ) + + +def test_resource_state_rejects_fence_regression_and_requires_explicit_fence(): + current = DuplexFence("sid", epoch=2, turn_id=3, response_seq=4) + manager = DuplexSessionRuntimeManager() + session = manager.open_session( + current, + capabilities=DuplexRuntimeCapabilities(input_modes={DuplexInputMode.APPEND_AUDIO_CHUNK}), + ) + + for stale in ( + DuplexFence("sid", epoch=1, turn_id=99, response_seq=99), + DuplexFence("sid", epoch=2, turn_id=2, response_seq=4), + DuplexFence("sid", epoch=2, turn_id=3, response_seq=3), + ): + with pytest.raises(RuntimeError, match="fence mismatch"): + session.accept_fence(stale) + assert session.fence == current + + with pytest.raises(TypeError, match="fence"): + session.bind_stage_request(0, "request") + with pytest.raises(TypeError, match="fence"): + session.append_input(mode=DuplexInputMode.APPEND_AUDIO_CHUNK) + with pytest.raises(TypeError, match="DuplexFence"): + manager.open_session("legacy-session") + with pytest.raises(TypeError, match="DuplexFence"): + manager.close_session("legacy-session") + + +def test_resource_request_id_is_derived_from_fence_and_role(): + fence = DuplexFence("sid-with-dashes", epoch=7, turn_id=11, response_seq=13) + + assert duplex_resource_request_id(fence, "stage0") == "duplex-s.c2lkLXdpdGgtZGFzaGVz.i.0.e.7.r.stage0" + assert duplex_resource_request_id(fence, "stage1") == "duplex-s.c2lkLXdpdGgtZGFzaGVz.i.0.e.7.r.stage1" + + +def test_resource_request_id_codec_separates_session_id_from_incarnation(): + embedded_incarnation = duplex_resource_request_id( + DuplexFence("foo-i1", incarnation=0), + "stage0", + ) + actual_incarnation = duplex_resource_request_id( + DuplexFence("foo", incarnation=1), + "stage0", + ) + + assert embedded_incarnation != actual_incarnation + + +def test_resource_request_id_session_membership_uses_encoded_identity(): + request_id = duplex_resource_request_id( + DuplexFence("sid-with-dashes", incarnation=2, epoch=7), + "stage0", + ) + + assert duplex_resource_request_belongs_to_session(request_id, "sid-with-dashes") is True + assert duplex_resource_request_belongs_to_session(request_id, "sid") is False + assert duplex_resource_request_belongs_to_session("duplex-s.invalid.i.x.e.7.r.stage0", "sid") is False + + +def test_placeholder_budget_is_planned_inside_omni_engine_boundary(): + fence = DuplexFence("sid", turn_id=1, response_seq=1) + prompt = build_duplex_data_plane_prompt( + request_id=duplex_resource_request_id(fence, "stage0"), + fence=fence, + session_config={}, + runtime_config={}, + seq=2, + turn_seq=1, + mode=DuplexInputMode.APPEND_AUDIO_CHUNK, + payload={ + "audio": "AAAAAA==", + "format": "pcm_f32le", + "duplex_num_input_tokens": 999, + "num_input_tokens": 999, + }, + final=False, + ) + + assert len(prompt["prompt_token_ids"]) == 16 + assert prompt["model_intermediate_buffer"]["duplex"]["fence"] == fence + assert prompt["model_intermediate_buffer"]["duplex"]["scheduler_token_budget"] == 16 diff --git a/tests/fullduplex/minicpmo45/__init__.py b/tests/fullduplex/minicpmo45/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/fullduplex/minicpmo45/test_commit_policy.py b/tests/fullduplex/minicpmo45/test_commit_policy.py new file mode 100644 index 000000000..5ee66b41a --- /dev/null +++ b/tests/fullduplex/minicpmo45/test_commit_policy.py @@ -0,0 +1,99 @@ +from __future__ import annotations + +import pytest + +from vllm_omni.experimental.fullduplex.openai.commit_policy import ( + CommitAction, + CommitSnapshot, + decide_commit_action, +) + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +@pytest.mark.parametrize( + ("case", "snapshot", "expected"), + [ + ( + "first idle commit", + CommitSnapshot( + auto_responds=True, + speech_since_commit=True, + active_response_id=None, + overlap_speech_ms=0, + native_response_in_progress=False, + playback_active=False, + ), + CommitAction.START_AUTO_RESPONSE, + ), + ( + "native auto-response remains model-owned during playback", + CommitSnapshot( + auto_responds=True, + speech_since_commit=True, + active_response_id=None, + overlap_speech_ms=800, + native_response_in_progress=True, + playback_active=True, + ), + CommitAction.START_AUTO_RESPONSE, + ), + ( + "native auto-response commit without speech", + CommitSnapshot( + auto_responds=True, + speech_since_commit=False, + active_response_id=None, + overlap_speech_ms=0, + native_response_in_progress=True, + playback_active=False, + ), + CommitAction.COMMIT_ONLY, + ), + ( + "non-auto response overlap", + CommitSnapshot( + auto_responds=False, + speech_since_commit=True, + active_response_id=None, + overlap_speech_ms=800, + native_response_in_progress=True, + playback_active=False, + ), + CommitAction.DEFER_ACTIVE_RESPONSE, + ), + ( + "non-auto playback overlap", + CommitSnapshot( + auto_responds=False, + speech_since_commit=True, + active_response_id=None, + overlap_speech_ms=800, + native_response_in_progress=False, + playback_active=True, + ), + CommitAction.DEFER_ACTIVE_RESPONSE, + ), + ( + "non-auto commit without overlap", + CommitSnapshot( + auto_responds=False, + speech_since_commit=True, + active_response_id=None, + overlap_speech_ms=0, + native_response_in_progress=False, + playback_active=False, + ), + CommitAction.COMMIT_ONLY, + ), + ], + ids=lambda value: value if isinstance(value, str) else None, +) +def test_decide_commit_action_covers_realtime_lifecycle_sequences( + case: str, + snapshot: CommitSnapshot, + expected: CommitAction, +) -> None: + del case + + assert decide_commit_action(snapshot) is expected diff --git a/tests/fullduplex/minicpmo45/test_input.py b/tests/fullduplex/minicpmo45/test_input.py new file mode 100644 index 000000000..035ab47ef --- /dev/null +++ b/tests/fullduplex/minicpmo45/test_input.py @@ -0,0 +1,131 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import base64 + +import numpy as np +import pytest + +from vllm_omni.experimental.fullduplex.minicpmo45.input import MiniCPMO45PcmAppendBuffer + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +def pcm_payload(samples: int, *, speech: bool = True) -> dict[str, object]: + audio = np.ones(samples, dtype=np.float32).tobytes() + return { + "type": "audio", + "audio": base64.b64encode(audio).decode("ascii"), + "format": "pcm_f32le", + "sample_rate_hz": 16_000, + "is_speech": speech, + } + + +def test_commit_does_not_add_silence_after_incremental_audio_was_drained(): + buffer = MiniCPMO45PcmAppendBuffer() + + emitted = buffer.append(pcm_payload(16_000), chunk_period_ms=1_000) + committed = buffer.commit(chunk_period_ms=1_000) + + assert emitted is not None + assert not buffer.has_pending() + assert committed is None + + +def test_append_emits_one_model_unit_when_multiple_units_are_buffered(): + buffer = MiniCPMO45PcmAppendBuffer() + + emitted = buffer.append(pcm_payload(32_000), chunk_period_ms=1_000) + + assert emitted is not None + assert len(base64.b64decode(emitted["audio"])) == 16_000 * 4 + assert buffer.pending_byte_count == 16_000 * 4 + + +def test_commit_without_speech_does_not_synthesize_terminal_audio(): + buffer = MiniCPMO45PcmAppendBuffer() + buffer.append(pcm_payload(8_000, speech=False), chunk_period_ms=1_000) + + committed = buffer.commit(chunk_period_ms=1_000) + + assert committed is None + + +def test_commit_resets_cumulative_turn_accounting(): + buffer = MiniCPMO45PcmAppendBuffer() + buffer.append(pcm_payload(16_000), chunk_period_ms=1_000) + buffer.commit(chunk_period_ms=1_000) + + empty = buffer.commit(chunk_period_ms=1_000) + + assert empty is None + + +def test_pcm_append_reservation_rollback_restores_emitted_audio(): + buffer = MiniCPMO45PcmAppendBuffer() + original = pcm_payload(16_000) + + reservation = buffer.prepare_append( + original, + operation_id="append-1", + chunk_period_ms=1_000, + ) + + assert reservation is not None + assert not buffer.has_pending() + reservation.rollback() + assert buffer.has_pending() + + retried = buffer.flush(chunk_period_ms=1_000) + assert retried is not None + assert base64.b64decode(retried["audio"]) == base64.b64decode(original["audio"]) + + +def test_pcm_commit_keeps_prior_append_reservation_active(): + buffer = MiniCPMO45PcmAppendBuffer() + append_reservation = buffer.prepare_append( + pcm_payload(16_000), + operation_id="append-before-commit", + chunk_period_ms=1_000, + ) + + commit_reservation = buffer.prepare_commit( + operation_id="commit-after-append", + chunk_period_ms=1_000, + ) + + assert append_reservation is not None + assert append_reservation.active + assert commit_reservation.payload is None + append_reservation.commit() + commit_reservation.commit() + + +def test_pcm_commit_reservation_rollback_restores_residual_audio(): + buffer = MiniCPMO45PcmAppendBuffer() + original = pcm_payload(8_000) + assert ( + buffer.prepare_append( + original, + operation_id="buffer-half-chunk", + chunk_period_ms=1_000, + ) + is None + ) + + reservation = buffer.prepare_commit( + operation_id="final-half-chunk", + chunk_period_ms=1_000, + ) + + assert reservation.payload is not None + assert reservation.payload["final"] is True + reservation.rollback() + + retried = buffer.prepare_commit( + operation_id="retry-final-half-chunk", + chunk_period_ms=1_000, + ) + assert retried.payload is not None + assert base64.b64decode(retried.payload["audio"]) == (base64.b64decode(original["audio"]) + b"\x00" * (8_000 * 4)) diff --git a/tests/fullduplex/openai/test_duplex_audio.py b/tests/fullduplex/openai/test_duplex_audio.py new file mode 100644 index 000000000..535856976 --- /dev/null +++ b/tests/fullduplex/openai/test_duplex_audio.py @@ -0,0 +1,54 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import base64 + +import numpy as np +import pytest + +from vllm_omni.experimental.fullduplex.openai.audio import convert_input_audio_with_rate + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +def test_pcm16_input_conversion_is_pure_and_reports_target_rate(): + pcm16 = np.array([-32768, 0, 32767], dtype=" None: + repo_root = Path(__file__).resolve().parents[3] + script = f""" +import sys + +import {module_name} + +loaded = sorted( + name + for name in sys.modules + if name == "vllm_omni.experimental.fullduplex.minicpmo45" + or name.startswith("vllm_omni.experimental.fullduplex.minicpmo45.") +) +if loaded: + raise SystemExit("generic OpenAI runtime loaded MiniCPM-o: " + ", ".join(loaded)) +""" + + result = subprocess.run( + [sys.executable, "-c", script], + cwd=repo_root, + capture_output=True, + text=True, + timeout=60, + check=False, + ) + + assert result.returncode == 0, result.stderr + + +def test_generic_handler_requires_explicit_serving_runtime_adapter() -> None: + repo_root = Path(__file__).resolve().parents[3] + script = """ +from types import SimpleNamespace + +from vllm_omni.experimental.fullduplex.openai.serving import OmniDuplexSessionHandler + +chat_service = SimpleNamespace(engine_client=SimpleNamespace()) +try: + OmniDuplexSessionHandler(chat_service=chat_service) +except ValueError as exc: + if "serving runtime adapter" not in str(exc).lower(): + raise +else: + raise SystemExit("generic handler silently selected a model serving adapter") +""" + + result = subprocess.run( + [sys.executable, "-c", script], + cwd=repo_root, + capture_output=True, + text=True, + timeout=60, + check=False, + ) + + assert result.returncode == 0, result.stderr + + +def _valid_runtime_adapter() -> SimpleNamespace: + data_plane = SimpleNamespace( + begin_request=lambda request_id: None, + is_terminal=lambda request_id: False, + mark_terminal=lambda request_id: None, + close_stream=lambda request_id: None, + close_session=lambda session_id, **kwargs: None, + project=lambda result, **kwargs: (), + ) + return SimpleNamespace( + adapter_id="test-adapter", + session_states={}, + data_plane=data_plane, + clean_response_done_prefix="clean", + interrupted_tts_prefix="interrupted", + private_runtime_config_keys=frozenset({"private_key"}), + create_session_state=lambda: object(), + session_state=lambda session_id: object(), + remove_session_state=lambda session_id: None, + is_enabled=lambda config: True, + capabilities=lambda **kwargs: object(), + validate_client_extra_body=lambda extra_body: None, + prepare_runtime_config=lambda config, **kwargs: {}, + runtime_config_for_update=lambda config, current: {}, + data_plane_context=lambda **kwargs: object(), + ) + + +@pytest.mark.parametrize( + ("attribute", "invalid_value"), + [ + ("session_states", None), + ("clean_response_done_prefix", None), + ("interrupted_tts_prefix", None), + ("private_runtime_config_keys", {"private_key"}), + ], +) +def test_runtime_adapter_validator_rejects_invalid_protocol_attributes( + attribute: str, + invalid_value: object, +) -> None: + adapter = _valid_runtime_adapter() + setattr(adapter, attribute, invalid_value) + + with pytest.raises(TypeError, match=attribute): + validate_serving_runtime_adapter(adapter) + + +def test_runtime_adapter_validator_rejects_incomplete_data_plane() -> None: + adapter = _valid_runtime_adapter() + adapter.data_plane.close_session = None + + with pytest.raises(TypeError, match="data_plane.*close_session"): + validate_serving_runtime_adapter(adapter) diff --git a/tests/fullduplex/openai/test_websocket_actor.py b/tests/fullduplex/openai/test_websocket_actor.py new file mode 100644 index 000000000..37924af52 --- /dev/null +++ b/tests/fullduplex/openai/test_websocket_actor.py @@ -0,0 +1,145 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import asyncio + +import pytest + +from vllm_omni.experimental.fullduplex.openai.websocket import ( + DuplexWebSocketActor, + normalize_duplex_input_event, +) + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +class FakeWebSocket: + def __init__(self) -> None: + self.sent: list[dict[str, object]] = [] + + async def send_json(self, payload: dict[str, object]) -> None: + self.sent.append(dict(payload)) + + +@pytest.mark.parametrize( + ("event", "expected"), + [ + ({"type": "signal_turn", "event": "barge_in"}, {"type": "turn.signal", "event": "barge_in"}), + ({"type": "close_session"}, {"type": "session.close"}), + ({"type": "audio.playback_ack", "played_ms": 1}, {"type": "playback.ack", "played_ms": 1}), + ({"type": "input_text.append", "text": "a"}, {"type": "input.text.append", "text": "a"}), + ({"type": "push_text", "text": "b"}, {"type": "input.text.append", "text": "b"}), + ( + {"type": "input.audio.append", "audio": "wav"}, + {"type": "input_audio_buffer.append", "audio": "wav", "format": "wav"}, + ), + ( + {"type": "push_chunk", "audio": "wav", "format": "pcm_f32le"}, + {"type": "input_audio_buffer.append", "audio": "wav", "format": "pcm_f32le"}, + ), + ], +) +def test_input_aliases_normalize_once_at_mailbox_boundary(event, expected): + assert normalize_duplex_input_event(event) == expected + + +def test_actor_uses_one_fifo_mailbox_for_inbound_events(): + actor = DuplexWebSocketActor(FakeWebSocket()) + + assert isinstance(actor.mailbox, asyncio.Queue) + assert not isinstance(actor.mailbox, asyncio.PriorityQueue) + assert not hasattr(actor, "control_queue") + assert not hasattr(actor, "input_queue") + assert not hasattr(actor, "event_queue") + assert not hasattr(actor, "lifecycle_state") + assert not hasattr(actor, "last_response_id") + assert not hasattr(actor, "overlap_speech_ms") + assert not hasattr(actor, "transition") + assert not hasattr(actor, "output_generation_in_flight") + assert not hasattr(actor, "session") + assert not hasattr(actor, "runtime_opened") + assert not hasattr(actor, "runtime_closed") + assert not hasattr(actor, "drain_input_queue") + assert not hasattr(actor, "control_events_seen") + assert not hasattr(actor, "input_events_seen") + assert not hasattr(actor, "cancel_count") + + +@pytest.mark.asyncio +async def test_control_event_does_not_overtake_earlier_audio_input(): + actor = DuplexWebSocketActor(FakeWebSocket()) + await actor.enqueue_event({"type": "input_audio_buffer.append", "audio": "pcm"}) + await actor.enqueue_event({"type": "response.cancel"}) + + assert await actor.next_event() == {"type": "input_audio_buffer.append", "audio": "pcm"} + assert await actor.next_event() == {"type": "response.cancel"} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("event_type", ["input.cancel", "session.close", "close_session"]) +async def test_terminal_control_preserves_wire_order(event_type: str): + actor = DuplexWebSocketActor(FakeWebSocket()) + await actor.enqueue_event({"type": "input_audio_buffer.append", "audio": "pcm"}) + await actor.enqueue_event({"type": event_type}) + + assert await actor.next_event() == {"type": "input_audio_buffer.append", "audio": "pcm"} + assert await actor.next_event() == {"type": event_type} + + +@pytest.mark.asyncio +async def test_mailbox_delivers_every_enqueued_event_exactly_once(): + actor = DuplexWebSocketActor(FakeWebSocket()) + events = [ + {"type": "input_audio_buffer.append", "audio": str(index)} + if index % 2 + else {"type": "response.created", "response_id": f"resp-{index}"} + for index in range(20) + ] + + await asyncio.gather(*(actor.enqueue_event(dict(event)) for event in events)) + received = [await actor.next_event() for _ in events] + + assert sorted(received, key=repr) == sorted(events, key=repr) + + +@pytest.mark.asyncio +async def test_writer_is_single_owner_of_websocket_send(): + websocket = FakeWebSocket() + actor = DuplexWebSocketActor(websocket) + writer = asyncio.create_task(actor.writer_loop()) + + await actor.send_json({"type": "one"}) + await actor.send_json({"type": "two"}) + await actor.close_writer() + await writer + + assert websocket.sent == [{"type": "one"}, {"type": "two"}] + + +@pytest.mark.asyncio +async def test_stale_fence_payload_is_dropped_before_websocket_send(): + websocket = FakeWebSocket() + actor = DuplexWebSocketActor(websocket, current_epoch=lambda: 2) + writer = asyncio.create_task(actor.writer_loop()) + + await actor.send_json({"type": "response.audio.delta", "epoch": 1}) + await actor.send_json({"type": "response.audio.delta", "epoch": 2}) + await actor.close_writer() + await writer + + assert websocket.sent == [{"type": "response.audio.delta", "epoch": 2}] + assert actor.stale_output_dropped == 1 + + +@pytest.mark.asyncio +async def test_writer_does_not_revoke_accepted_terminal_after_close_starts(): + websocket = FakeWebSocket() + actor = DuplexWebSocketActor(websocket, current_epoch=lambda: 1) + await actor.send_json({"type": "response.done", "epoch": 1, "response_id": "resp-1"}) + actor.closing = True + await actor.close_writer() + + await actor.writer_loop() + + assert websocket.sent == [{"type": "response.done", "epoch": 1, "response_id": "resp-1"}] + assert actor.stale_output_dropped == 0 diff --git a/tests/fullduplex/test_client.py b/tests/fullduplex/test_client.py new file mode 100644 index 000000000..218050eb1 --- /dev/null +++ b/tests/fullduplex/test_client.py @@ -0,0 +1,240 @@ +import base64 +import wave +from urllib.parse import parse_qs, urlsplit + +import pytest + +from vllm_omni.experimental.fullduplex.client import ( + RealtimeDuplexClient, + RealtimeEventCollector, + build_realtime_url, + read_pcm16_wav, + write_pcm16_wav, +) + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +def test_realtime_client_builds_explicit_native_duplex_url(): + url = build_realtime_url( + "ws://localhost:8099/v1/realtime?custom=1", + "openbmb/MiniCPM-o-4_5", + session_id="session-a", + ) + + query = parse_qs(urlsplit(url).query) + assert query == { + "custom": ["1"], + "duplex": ["1"], + "model": ["openbmb/MiniCPM-o-4_5"], + "minicpmo45_native_duplex": ["1"], + "session_id": ["session-a"], + } + + +def test_realtime_client_builds_resume_only_url_when_autostart_disabled(): + url = build_realtime_url( + "ws://localhost:8099/v1/realtime?duplex=1", + "openbmb/MiniCPM-o-4_5", + autostart=False, + ) + + query = parse_qs(urlsplit(url).query) + assert query["autostart"] == ["0"] + assert query["minicpmo45_native_duplex"] == ["1"] + + +@pytest.mark.asyncio +async def test_realtime_client_configure_omits_ref_audio_by_default(): + class Client(RealtimeDuplexClient): + def __init__(self): + super().__init__("ws://unused") + self.sent = [] + + async def send(self, event): + self.sent.append(event) + self.events.add({"type": "session.created"}) + + client = Client() + + await client.configure("openbmb/MiniCPM-o-4_5", timeout_s=1) + + session = client.sent[0]["session"] + assert "ref_audio" not in session + + +@pytest.mark.asyncio +async def test_realtime_client_configure_sends_explicit_ref_audio(): + class Client(RealtimeDuplexClient): + def __init__(self): + super().__init__("ws://unused") + self.sent = [] + + async def send(self, event): + self.sent.append(event) + self.events.add({"type": "session.created"}) + + client = Client() + + await client.configure( + "openbmb/MiniCPM-o-4_5", + ref_audio="data:audio/wav;base64,AAAA", + timeout_s=1, + ) + + session = client.sent[0]["session"] + assert session["ref_audio"] == "data:audio/wav;base64,AAAA" + + +def test_realtime_event_collector_partitions_audio_by_response(): + collector = RealtimeEventCollector() + collector.add({"type": "response.created", "response": {"id": "resp-a"}}) + collector.add( + { + "type": "response.audio.delta", + "response_id": "resp-a", + "delta": base64.b64encode(b"audio-a").decode("ascii"), + "sample_rate_hz": 16_000, + } + ) + + assert collector.response_ids == ["resp-a"] + assert collector.audio_bytes("resp-a") == b"audio-a" + assert collector.output_sample_rate_hz == 16_000 + assert collector.first_received_at("response.created") is not None + assert collector.last_received_at("response.audio.delta") is not None + + +def test_realtime_event_collector_reports_engine_token_and_audio_intervals(): + collector = RealtimeEventCollector() + collector.add( + {"type": "response.created", "response": {"id": "resp-a"}}, + received_at_s=10.0, + ) + stage_metrics = { + "0": { + "num_tokens_out": 4, + "vllm_ttft_ms": 120.0, + "vllm_tpot_ms": 15.0, + "vllm_itl_ms": 14.0, + "vllm_itls_ms": [10.0, 14.0, 18.0], + } + } + for received_at_s, cumulative_audio_ms in ((10.2, 80), (10.25, 160), (10.36, 240)): + collector.add( + { + "type": "response.audio.delta", + "response_id": "resp-a", + "delta": base64.b64encode(b"audio").decode("ascii"), + "sample_rate_hz": 16_000, + "metadata": { + "audio_duration_ms": cumulative_audio_ms, + "vllm_omni": {"stage_metrics": stage_metrics}, + }, + }, + received_at_s=received_at_s, + ) + + timing = collector.timing_summary( + after_s=10.0, + input_committed_at_s=9.9, + response_id="resp-a", + ) + + assert timing["stage0_tokens"] == { + "source": "engine_stage_metrics", + "output_token_count": 4, + "ttft_ms": 120.0, + "tpot_ms": 15.0, + "inter_token_interval_ms": { + "count": 3, + "mean": 14.0, + "p50": 14.0, + "p95": 18.0, + "max": 18.0, + }, + } + assert timing["audio_output"] == { + "source": "client_monotonic_receive", + "chunk_count": 3, + "response_created_to_first_audio_ms": 200.0, + "commit_to_first_audio_ms": 300.0, + "inter_chunk_interval_ms": { + "count": 2, + "mean": 80.0, + "p50": 50.0, + "p95": 110.0, + "max": 110.0, + }, + "chunk_duration_ms": { + "count": 3, + "mean": 80.0, + "p50": 80.0, + "p95": 80.0, + "max": 80.0, + }, + "max_chunk_gap_ms": 110.0, + } + + +def test_response_timing_ignores_unowned_session_level_metrics(): + collector = RealtimeEventCollector() + collector.add( + {"type": "response.created", "response": {"id": "resp-a"}}, + received_at_s=10.0, + ) + collector.add( + { + "type": "response.audio.delta", + "response_id": "resp-a", + "delta": base64.b64encode(b"audio").decode("ascii"), + "metadata": { + "vllm_omni": { + "stage_metrics": { + "0": { + "num_tokens_out": 20, + "vllm_ttft_ms": 157.0, + "vllm_tpot_ms": 16.0, + "vllm_itls_ms": [15.0, 17.0], + } + } + } + }, + }, + received_at_s=10.2, + ) + collector.add( + { + "type": "response.listen", + "metadata": { + "vllm_omni": { + "stage_metrics": { + "0": { + "num_tokens_out": 2, + "vllm_ttft_ms": 106.0, + "vllm_tpot_ms": 0.0, + "vllm_itls_ms": [], + } + } + } + }, + }, + received_at_s=10.3, + ) + + timing = collector.timing_summary(after_s=10.0, response_id="resp-a") + + assert timing["stage0_tokens"]["output_token_count"] == 20 + assert timing["stage0_tokens"]["ttft_ms"] == 157.0 + + +def test_realtime_client_pcm16_wav_round_trip(tmp_path): + path = tmp_path / "audio.wav" + pcm16 = b"\x01\x00\x02\x00" + + write_pcm16_wav(path, pcm16, sample_rate_hz=16_000) + + with wave.open(str(path), "rb") as wav_file: + assert wav_file.getnchannels() == 1 + assert wav_file.getframerate() == 16_000 + assert read_pcm16_wav(path) == pcm16 diff --git a/tests/model_executor/models/minicpmo_4_5/test_hf_model_path.py b/tests/model_executor/models/minicpmo_4_5/test_hf_model_path.py index dc1599e6b..834b7b0e0 100644 --- a/tests/model_executor/models/minicpmo_4_5/test_hf_model_path.py +++ b/tests/model_executor/models/minicpmo_4_5/test_hf_model_path.py @@ -5,7 +5,6 @@ from types import SimpleNamespace import pytest -import torch from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni_tts import ( MiniCPMO45OmniTTSForConditionalGeneration, @@ -14,10 +13,19 @@ from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni_tts import ( pytestmark = [pytest.mark.core_model, pytest.mark.cpu] +class _FakeMiniCPMTTS: + def __init__(self, config, audio_tokenizer): + self.config = config + self.audio_tokenizer = audio_tokenizer + self.emb_text = object() + self.projector_semantic = object() + + def generate(self): + raise NotImplementedError + + def test_hf_model_id_path_caching(mocker, tmp_path): - """Verify HF model-ID path is resolved via download_weights_from_hf_specific - once during _lazy_init_tts, cached in self._model_path, and reused in generate_speech. - """ + """Verify the resolved HF path is cached and reused for default ref audio.""" resolved_path = str(tmp_path / "resolved_hf_model") assets_dir = tmp_path / "resolved_hf_model" / "assets" assets_dir.mkdir(parents=True, exist_ok=True) @@ -29,11 +37,9 @@ def test_hf_model_id_path_caching(mocker, tmp_path): return_value=resolved_path, ) - # Mock dynamic class loading - fake_tts_class = mocker.MagicMock() mocker.patch( "transformers.dynamic_module_utils.get_class_from_dynamic_module", - return_value=fake_tts_class, + return_value=_FakeMiniCPMTTS, ) # Build fake vllm_config with HF model ID @@ -59,40 +65,10 @@ def test_hf_model_id_path_caching(mocker, tmp_path): tts_model = MiniCPMO45OmniTTSForConditionalGeneration(vllm_config=vllm_config) - # 1. Trigger lazy init + tts_model._lazy_init_tts() tts_model._lazy_init_tts() - # Verify download_weights_from_hf_specific was called with HF model ID and cached mock_download.assert_called_once_with(model_id, None, ["*"]) assert tts_model._model_path == resolved_path assert tts_model._assets_loaded is True - - # 2. Mock TTS object generate method and audio tokenizer for generate_speech - fake_tts_obj = mocker.MagicMock() - fake_tts_obj.emb_text.weight.device = torch.device("cpu") - fake_tts_obj.emb_text.side_effect = lambda t: torch.zeros(t.shape[0], 768) - fake_tts_obj.projector_semantic.side_effect = lambda h, **kw: torch.zeros(h.shape[0], 768) - fake_tts_obj.config = tts_cfg - fake_tts_obj.audio_bos_token_id = 151687 - - fake_output = SimpleNamespace(new_ids=torch.tensor([[[100]]])) - fake_tts_obj.generate.return_value = fake_output - - tts_model.tts_obj = fake_tts_obj - fake_tokenizer = mocker.MagicMock(return_value=b"dummy_wav_bytes") - tts_model.audio_tokenizer = fake_tokenizer - - # Mock sf.read to return fake numpy audio - mocker.patch( - "soundfile.read", - return_value=(torch.zeros(16000).numpy(), 24000), - ) - - # Call generate_speech - tts_token_ids = torch.tensor([1, 2, 3]) - tts_hidden_states = torch.zeros(3, 768) - waveform = tts_model.generate_speech(tts_token_ids, tts_hidden_states) - - # Verify download_weights_from_hf_specific was NOT called a second time - assert mock_download.call_count == 1 - assert waveform is not None + assert tts_model._resolve_prompt_wav_path(None, None) == (str(ref_audio_file), None) diff --git a/tests/model_executor/models/minicpmo_4_5/test_llm2tts.py b/tests/model_executor/models/minicpmo_4_5/test_llm2tts.py index d52bcafa1..7ec8e8114 100644 --- a/tests/model_executor/models/minicpmo_4_5/test_llm2tts.py +++ b/tests/model_executor/models/minicpmo_4_5/test_llm2tts.py @@ -7,11 +7,9 @@ Covers ``vllm_omni.model_executor.stage_input_processors.minicpmo_4_5_omni.llm2t - empty ``source_outputs`` raises - latent fallback to ``hidden_states`` when ``multimodal_output`` is empty - both inputs missing -> raises - - additional_information payload carries the keys the talker expects - (prompt_embeds, prompt_token_ids, llm_output_token_ids, llm_output_text) - - dummy talker prompt ``[BOS, PAD, EOS] = [1, 0, 2]`` (single prefill step) + - structured model_intermediate_buffer carries the thinker ids and text + - scheduler prompt tokens follow the selected TTS region or output tokens - MiniCPM-o 4.5 TTS region detection on 151703 / 151704 tokens - - MiniCPM-o 2.6 fallback detection on 151691 / 151692 when no 4.5 markers - No TTS markers present -> no ``tts_token_ids`` / ``tts_hidden_states`` keys - prompt arg is normalized to a list and ``multi_modal_data`` is gated by ``requires_multimodal_data`` @@ -24,6 +22,9 @@ from types import SimpleNamespace import pytest import torch +from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni import ( + MiniCPMO45OmniForConditionalGeneration, +) from vllm_omni.model_executor.stage_input_processors.minicpmo_4_5_omni import llm2tts pytestmark = [pytest.mark.core_model, pytest.mark.cpu] @@ -32,6 +33,55 @@ pytestmark = [pytest.mark.core_model, pytest.mark.cpu] _HIDDEN_DIM = 4 +def test_native_duplex_talker_emits_empty_text_metadata_to_clear_previous_segment() -> None: + class _Talker: + _ar_last_chunk_flags = [True] + _ar_turn_end_flags = [True] + + def __call__(self, **kwargs): + del kwargs + return None, torch.zeros(0, dtype=torch.float32) + + model = MiniCPMO45OmniForConditionalGeneration.__new__(MiniCPMO45OmniForConditionalGeneration) + torch.nn.Module.__init__(model) + model.model_stage = "tts" + model.config = SimpleNamespace(hidden_size=_HIDDEN_DIM) + model.talker = _Talker() + + output = model.forward( + input_ids=torch.tensor([1]), + positions=torch.tensor([0]), + runtime_additional_information=[ + { + "native_duplex": True, + "duplex": {"turn_id": 1, "epoch": 0}, + "meta": {"native_duplex_segment_text": ""}, + } + ], + ) + + assert output.multimodal_outputs is not None + assert output.multimodal_outputs["meta.llm_output_text_utf8"].numel() == 0 + assert output.multimodal_outputs["meta.audio_text_total_chars"].tolist() == [0] + + +def test_tts_embed_input_ids_uses_active_talker_embedding(monkeypatch) -> None: + model = MiniCPMO45OmniForConditionalGeneration.__new__(MiniCPMO45OmniForConditionalGeneration) + torch.nn.Module.__init__(model) + model.model_stage = "tts" + expected = torch.ones((2, _HIDDEN_DIM)) + + monkeypatch.setattr( + model, + "get_input_embeddings", + lambda input_ids: expected, + ) + + actual = model.embed_input_ids(torch.tensor([11, 12])) + + assert actual is expected + + def _make_thinker_output( *, prompt_token_ids: list[int], @@ -40,6 +90,7 @@ def _make_thinker_output( request_id: str = "req-0", latent: torch.Tensor | None = None, hidden_states: torch.Tensor | None = None, + multimodal_output: dict[str, object] | None = None, ): """Construct a minimal mock of a thinker engine output entry. @@ -48,8 +99,11 @@ def _make_thinker_output( - per-output: ``multimodal_output`` (dict), ``hidden_states`` (opt), ``text``, ``token_ids`` """ + mm_output = dict(multimodal_output or {}) + if latent is not None: + mm_output["latent"] = latent output = SimpleNamespace( - multimodal_output={"latent": latent} if latent is not None else {}, + multimodal_output=mm_output, token_ids=output_token_ids, text=text, ) @@ -90,17 +144,16 @@ class TestBasicShape: ) assert len(out) == 2 - def test_talker_prompt_token_ids_dummy_bos_pad_eos(self) -> None: + def test_talker_scheduler_prompt_uses_output_tokens(self) -> None: hidden = torch.zeros((2, _HIDDEN_DIM)) out = llm2tts( [_make_thinker_output(prompt_token_ids=[10], output_token_ids=[20], hidden_states=hidden)], prompt=None, ) - # The talker AR framework needs *some* tokens to do a single prefill; - # this is the agreed [BOS, PAD, EOS] minimal payload. - assert out[0]["prompt_token_ids"] == [1, 0, 2] + assert out[0]["prompt_token_ids"] == [20] + assert "stream_output" not in out[0]["model_intermediate_buffer"] - def test_additional_information_carries_thinker_outputs(self) -> None: + def test_model_intermediate_buffer_carries_thinker_outputs(self) -> None: prompt_ids = [10, 11, 12] out_ids = [20, 21] hidden = torch.randn(len(prompt_ids) + len(out_ids), _HIDDEN_DIM) @@ -116,21 +169,17 @@ class TestBasicShape: ], prompt=None, ) - ai = result[0]["additional_information"] - assert ai["prompt_token_ids"] == prompt_ids - assert ai["llm_output_token_ids"] == out_ids - assert ai["llm_output_text"] == ["hello"] - # prompt_embeds = float32 view of the prompt-portion of hidden states. - assert ai["prompt_embeds"].dtype == torch.float32 - assert ai["prompt_embeds"].shape == (len(prompt_ids), _HIDDEN_DIM) - assert torch.equal(ai["prompt_embeds"], hidden[: len(prompt_ids)].to(torch.float32)) + buffer = result[0]["model_intermediate_buffer"] + assert buffer["ids"]["prompt"] == prompt_ids + assert buffer["ids"]["output"] == out_ids + assert buffer["llm_output_text"] == ["hello"] def test_latent_in_multimodal_output_takes_precedence(self) -> None: # When both ``multimodal_output["latent"]`` and ``hidden_states`` are # present, the latent payload must win (this is the steady-state path # produced by the thinker stage). prompt_ids = [10, 11] - out_ids = [20] + out_ids = [151703, 30, 151704] latent = torch.ones((len(prompt_ids) + len(out_ids), _HIDDEN_DIM)) hidden = torch.zeros_like(latent) @@ -145,19 +194,17 @@ class TestBasicShape: ], prompt=None, ) - ai = result[0]["additional_information"] + buffer = result[0]["model_intermediate_buffer"] # latent (ones) won over hidden_states (zeros) - assert torch.equal(ai["prompt_embeds"], latent[: len(prompt_ids)].to(torch.float32)) + assert torch.equal(torch.tensor(buffer["hidden_states"]["tts"]), latent[3:4].to(torch.float32)) class TestTtsRegionDetection: - """The bridge auto-detects which MiniCPM-o generation produced the output - by sniffing for 4.5-specific BOS/EOS token ids (151703 / 151704); when - those are absent it falls back to the 2.6 ids (151691 / 151692). + """The bridge detects MiniCPM-o 4.5 BOS/EOS ids (151703 / 151704). This is a regression guard: a single off-by-one or wrong branch - precedence here causes the talker to receive an empty / wrong-region - slice and emit silent audio. + causes the talker to receive an empty / wrong-region slice and emit + silent audio. """ def _run(self, prompt_token_ids, output_token_ids): @@ -173,7 +220,7 @@ class TestTtsRegionDetection: ], prompt=None, ) - return result[0]["additional_information"], hidden + return result[0]["model_intermediate_buffer"], hidden def test_4_5_markers_detected(self) -> None: # prompt: [10, 11] @@ -181,45 +228,26 @@ class TestTtsRegionDetection: # full sequence: [10, 11, 151703, 30, 31, 151704, 40] # 0 1 2 3 4 5 6 # 4.5 BOS at idx 2 -> slice starts at 3; EOS at idx 5 -> slice ends at 5. - ai, hidden = self._run([10, 11], [151703, 30, 31, 151704, 40]) - assert "tts_token_ids" in ai and "tts_hidden_states" in ai - assert ai["tts_token_ids"].tolist() == [30, 31] - assert torch.equal(ai["tts_hidden_states"], hidden[3:5]) - - def test_4_5_takes_precedence_when_both_markers_present(self) -> None: - # If both 2.6 and 4.5 markers appear (shouldn't really happen on a - # real checkpoint, but exercises the loop's break), the 4.5 markers - # win because the source loop pins on the first 4.5 id seen. - # sequence index: 0 1 2 3 4 5 6 7 8 - ai, hidden = self._run([10], [151691, 70, 151692, 151703, 30, 151704, 40]) - assert ai["tts_token_ids"].tolist() == [30] - assert torch.equal(ai["tts_hidden_states"], hidden[5:6]) - - def test_2_6_fallback(self) -> None: - # No 4.5 markers anywhere -> use 2.6 BOS/EOS pair. - # full sequence: [10, 151691, 30, 31, 151692, 40] - # 0 1 2 3 4 5 - # BOS at idx 1 -> slice starts at 2; EOS at idx 4 -> slice ends at 4. - ai, hidden = self._run([10], [151691, 30, 31, 151692, 40]) - assert ai["tts_token_ids"].tolist() == [30, 31] - assert torch.equal(ai["tts_hidden_states"], hidden[2:4]) + buffer, hidden = self._run([10, 11], [151703, 30, 31, 151704, 40]) + assert buffer["ids"]["tts"] == [30, 31] + assert torch.equal(torch.tensor(buffer["hidden_states"]["tts"]), hidden[3:5]) def test_bos_without_eos_runs_to_end(self) -> None: # When BOS is found but EOS is missing (typical for an in-flight or # truncated decode), the slice should extend to the end of the # hidden-state matrix instead of being dropped. # sequence: [10, 11, 151703, 30, 31] - ai, hidden = self._run([10, 11], [151703, 30, 31]) - assert ai["tts_token_ids"].tolist() == [30, 31] - assert torch.equal(ai["tts_hidden_states"], hidden[3:5]) + buffer, hidden = self._run([10, 11], [151703, 30, 31]) + assert buffer["ids"]["tts"] == [30, 31] + assert torch.equal(torch.tensor(buffer["hidden_states"]["tts"]), hidden[3:5]) def test_no_tts_markers_omits_slice_keys(self) -> None: # If neither marker pair is present, the bridge should NOT populate # ``tts_token_ids`` / ``tts_hidden_states`` — the talker should fall # through to the dummy path. - ai, _ = self._run([10, 11], [20, 21, 22]) - assert "tts_token_ids" not in ai - assert "tts_hidden_states" not in ai + buffer, _ = self._run([10, 11], [20, 21, 22]) + assert "tts" not in buffer["ids"] + assert "tts" not in buffer.get("hidden_states", {}) class TestPromptAndMultiModal: @@ -251,14 +279,11 @@ class TestPromptAndMultiModal: ) assert out[0]["multi_modal_data"] == mm - def test_streaming_context_is_accepted_and_ignored(self) -> None: - # The framework passes ``streaming_context`` for streaming-capable - # pipelines; MiniCPM-o 4.5 is turn-taking so the bridge must accept - # but discard it without crashing. + def test_internal_streaming_context_is_accepted(self) -> None: hidden = torch.zeros((2, _HIDDEN_DIM)) out = llm2tts( [_make_thinker_output(prompt_token_ids=[10], output_token_ids=[20], hidden_states=hidden)], prompt=None, - streaming_context=object(), + _streaming_context=object(), ) assert len(out) == 1 diff --git a/tests/model_executor/models/minicpmo_4_5/test_pipeline.py b/tests/model_executor/models/minicpmo_4_5/test_pipeline.py index 48db0d497..b9c7938f0 100644 --- a/tests/model_executor/models/minicpmo_4_5/test_pipeline.py +++ b/tests/model_executor/models/minicpmo_4_5/test_pipeline.py @@ -44,6 +44,20 @@ class TestRegistryDeclaration: assert pipeline.model_type == _PIPELINE_KEY assert pipeline.model_arch == "MiniCPMO45OmniForConditionalGeneration" + def test_native_duplex_control_is_explicit_without_a_fixed_session_cap(self) -> None: + pipeline = OMNI_PIPELINES[_PIPELINE_KEY] + assert pipeline.duplex_control_enabled is True + assert pipeline.duplex_serving_adapter == ( + "vllm_omni.experimental.fullduplex.minicpmo45.serving_adapter.MiniCPMO45ServingRuntimeAdapter" + ) + assert not hasattr(pipeline, "max_native_duplex_sessions") + + def test_ordinary_pipeline_defaults_to_no_duplex_control(self) -> None: + pipeline = PipelineConfig(model_type="ordinary") + assert pipeline.duplex_control_enabled is False + assert pipeline.duplex_serving_adapter is None + assert not hasattr(pipeline, "max_native_duplex_sessions") + class TestPipelineTopology: @pytest.fixture(scope="class") diff --git a/tests/model_executor/stage_input_processors/test_minicpmo_4_5_omni.py b/tests/model_executor/stage_input_processors/test_minicpmo_4_5_omni.py new file mode 100644 index 000000000..453b3a595 --- /dev/null +++ b/tests/model_executor/stage_input_processors/test_minicpmo_4_5_omni.py @@ -0,0 +1,1347 @@ +import base64 +import os +import struct +from types import SimpleNamespace + +import pytest +import torch + +from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni_tts import ( + MiniCPMO45OmniTTSForConditionalGeneration, + _drain_native_duplex_emitted_text, + _queue_native_duplex_segment_text, + _TalkerTurnState, +) +from vllm_omni.model_executor.stage_input_processors.minicpmo_4_5_omni import ( + _extract_first_audio_ref, + _native_duplex_segment_output_ids, + llm2tts, +) + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +def test_native_duplex_transcript_waits_for_buffered_vocoder_audio(): + state = _TalkerTurnState(None, None, turn_id=1) + + _queue_native_duplex_segment_text(state, "好") + assert _drain_native_duplex_emitted_text(state, has_audio=False) == "" + + _queue_native_duplex_segment_text(state, "的,刚") + assert _drain_native_duplex_emitted_text(state, has_audio=True) == "好的,刚" + assert _drain_native_duplex_emitted_text(state, has_audio=True) == "" + + +def test_native_duplex_token2wav_cache_isolated_across_interleaved_sessions(): + model = MiniCPMO45OmniTTSForConditionalGeneration.__new__(MiniCPMO45OmniTTSForConditionalGeneration) + model.audio_tokenizer = SimpleNamespace(stream_cache=None, hift_cache_dict={}) + observed = [] + + def stream_window(token_list, _prompt, *, last_chunk): + del token_list, last_chunk + marker = model.audio_tokenizer.stream_cache["marker"] + observed.append(marker) + prefix = marker[0] + index = int(marker[1:]) + 1 + model.audio_tokenizer.stream_cache = {"marker": f"{prefix}{index}"} + model.audio_tokenizer.hift_cache_dict = {"marker": f"{prefix}{index}"} + return torch.ones(1) + + model._t2w_stream_window = stream_window + a = _TalkerTurnState(None, None) + b = _TalkerTurnState(None, None) + a.stream_cache = {"marker": "A0"} + a.hift_cache_dict = {"marker": "A0"} + a.vocoder_initialized = True + b.stream_cache = {"marker": "B0"} + b.hift_cache_dict = {"marker": "B0"} + b.vocoder_initialized = True + + model._run_vocoder_window(a, [1], last_chunk=False) + model._run_vocoder_window(b, [1], last_chunk=False) + model._run_vocoder_window(a, [1], last_chunk=False) + model._run_vocoder_window(b, [1], last_chunk=True) + + assert observed == ["A0", "B0", "A1", "B1"] + assert a.stream_cache == {"marker": "A2"} + assert b.stream_cache == {"marker": "B2"} + assert model.audio_tokenizer.stream_cache is None + assert model.audio_tokenizer.hift_cache_dict == {} + + +def test_extract_first_audio_ref_accepts_dict_stereo_audio(): + ref = _extract_first_audio_ref( + { + "audio": { + "array": [[1.0, 3.0, 5.0], [2.0, 4.0, 6.0]], + "sampling_rate": 16000, + } + } + ) + + assert ref is not None + waveform, sample_rate = ref + assert sample_rate == 16000 + assert torch.allclose(waveform, torch.tensor([1.5, 3.5, 5.5])) + + +def test_llm2tts_carries_request_ref_audio_to_talker_payload(): + latent = torch.arange(20, dtype=torch.float16).reshape(5, 4) + output = SimpleNamespace( + token_ids=[11, 12, 9002], + text="hello", + multimodal_output={ + "latent": latent, + "meta": { + "tts_bos_token_id": 9001, + "tts_eos_token_id": 9002, + }, + }, + ) + llm_output = SimpleNamespace( + request_id="req-1", + prompt_token_ids=[101, 9001], + outputs=[output], + ) + ref_waveform = torch.tensor([0.1, 0.2, 0.3]) + + converted = llm2tts( + [llm_output], + prompt=[ + { + "multi_modal_data": { + "audio": (ref_waveform, 22050), + } + } + ], + ) + + assert "additional_information" not in converted[0] + info = converted[0]["model_intermediate_buffer"] + assert info["codes"]["ref"] == ref_waveform.tolist() + assert info["meta"]["ref_audio_sr"] == 22050 + assert info["ids"]["tts"] == [11, 12] + assert torch.equal(torch.as_tensor(info["hidden_states"]["tts"]), latent[2:4].float()) + assert "tts_token_ids" not in info + assert "tts_hidden_states" not in info + + +def test_llm2tts_uses_duplex_prompt_token_ids_for_tts_boundary(): + latent = torch.arange(24, dtype=torch.float16).reshape(6, 4) + output = SimpleNamespace( + token_ids=[11, 12, 9002], + text="hello", + multimodal_output={ + "latent": latent, + "duplex_prompt_token_ids": [101, 102, 9001], + "meta": { + "tts_bos_token_id": 9001, + "tts_eos_token_id": 9002, + }, + }, + ) + llm_output = SimpleNamespace( + request_id="duplex-req", + prompt_token_ids=[0, 0, 0], + outputs=[output], + ) + + converted = llm2tts([llm_output], prompt=[{}]) + + assert "additional_information" not in converted[0] + info = converted[0]["model_intermediate_buffer"] + assert info["prompt_token_ids"] == [101, 102, 9001] + assert converted[0]["prompt_token_ids"] == [11, 12] + assert info["ids"]["tts"] == [11, 12] + assert torch.equal(torch.as_tensor(info["hidden_states"]["tts"]), latent[3:5].float()) + assert "tts_token_ids" not in info + assert "tts_hidden_states" not in info + + +def test_llm2tts_accepts_flat_tokenizer_derived_special_token_metadata(): + latent = torch.arange(24, dtype=torch.float16).reshape(6, 4) + output = SimpleNamespace( + token_ids=[11, 12, 9302], + text="hello", + multimodal_output={ + "latent": latent, + "duplex_prompt_token_ids": [101, 102, 9301], + "meta.tts_bos_token_id": torch.tensor([9301]), + "meta.tts_eos_token_id": torch.tensor([9302]), + "meta.listen_token_id": torch.tensor([9303]), + "meta.speak_token_id": torch.tensor([9304]), + "meta.chunk_eos_token_id": torch.tensor([9305]), + "meta.chunk_tts_eos_token_id": torch.tensor([9306]), + "meta.turn_eos_token_id": torch.tensor([9307]), + }, + ) + llm_output = SimpleNamespace( + request_id="duplex-flat-meta", + prompt_token_ids=[0, 0, 0], + outputs=[output], + ) + + converted = llm2tts([llm_output], prompt=[{}]) + + assert "additional_information" not in converted[0] + info = converted[0]["model_intermediate_buffer"] + assert converted[0]["prompt_token_ids"] == [11, 12] + assert info["ids"]["tts"] == [11, 12] + assert info["stream_output"] is True + assert info["native_duplex"] is True + assert torch.equal(torch.as_tensor(info["hidden_states"]["tts"]), latent[3:5].float()) + assert "tts_token_ids" not in info + assert "tts_hidden_states" not in info + + +def test_llm2tts_stops_at_tokenizer_derived_duplex_state_tokens(): + latent = torch.arange(32, dtype=torch.float16).reshape(8, 4) + output = SimpleNamespace( + token_ids=[11, 12, 9303, 13, 9302], + text="hello", + multimodal_output={ + "latent": latent, + "duplex_prompt_token_ids": [101, 102, 9301], + "meta": { + "tts_bos_token_id": 9301, + "tts_eos_token_id": 9302, + "listen_token_id": 9303, + "speak_token_id": 9304, + "tts_pad_token_id": 9305, + "unit_token_id": 9306, + "unit_end_token_id": 9307, + "chunk_eos_token_id": 9308, + "chunk_tts_eos_token_id": 9309, + "turn_eos_token_id": 9310, + }, + }, + ) + llm_output = SimpleNamespace( + request_id="duplex-state-boundary", + prompt_token_ids=[0, 0, 0], + outputs=[output], + ) + + converted = llm2tts([llm_output], prompt=[{}]) + + info = converted[0]["model_intermediate_buffer"] + assert converted[0]["prompt_token_ids"] == [11, 12] + assert info["ids"]["tts"] == [11, 12] + assert torch.equal(torch.as_tensor(info["hidden_states"]["tts"]), latent[3:5].float()) + assert "tts_token_ids" not in info + assert "tts_hidden_states" not in info + + +def test_llm2tts_native_duplex_uses_speak_region_without_tts_bos_prompt(): + latent = torch.arange(36, dtype=torch.float16).reshape(9, 4) + output = SimpleNamespace( + token_ids=[9304, 11, 12, 9308, 13], + text="hello", + multimodal_output={ + "latent": latent, + "duplex_prompt_token_ids": [101, 102, 9306, 9306], + "meta": { + "tts_bos_token_id": 9301, + "tts_eos_token_id": 9302, + "listen_token_id": 9303, + "speak_token_id": 9304, + "tts_pad_token_id": 9305, + "unit_token_id": 9306, + "unit_end_token_id": 9307, + "chunk_eos_token_id": 9308, + "chunk_tts_eos_token_id": 9309, + "turn_eos_token_id": 9310, + }, + }, + ) + llm_output = SimpleNamespace( + request_id="duplex-speak-region", + prompt_token_ids=[0, 0, 0, 0], + outputs=[output], + ) + + converted = llm2tts([llm_output], prompt=[{}]) + + info = converted[0]["model_intermediate_buffer"] + assert converted[0]["prompt_token_ids"] == [11, 12] + assert info["ids"]["tts"] == [11, 12] + assert torch.equal(torch.as_tensor(info["hidden_states"]["tts"]), latent[5:7].float()) + assert "tts_token_ids" not in info + assert "tts_hidden_states" not in info + + +def test_llm2tts_native_duplex_uses_plain_text_region_without_speak_marker(): + latent = torch.arange(32, dtype=torch.float16).reshape(8, 4) + output = SimpleNamespace( + token_ids=[11, 12, 13, 9308], + text="hello", + multimodal_output={ + "latent": latent, + "duplex_prompt_token_ids": [101, 102, 9306, 9306], + "meta": { + "tts_bos_token_id": 9301, + "tts_eos_token_id": 9302, + "listen_token_id": 9303, + "speak_token_id": 9304, + "tts_pad_token_id": 9305, + "unit_token_id": 9306, + "unit_end_token_id": 9307, + "chunk_eos_token_id": 9308, + "chunk_tts_eos_token_id": 9309, + "turn_eos_token_id": 9310, + }, + }, + ) + llm_output = SimpleNamespace( + request_id="duplex-plain-text-region", + prompt_token_ids=[0, 0, 0, 0], + outputs=[output], + ) + + converted = llm2tts([llm_output], prompt=[{}]) + + info = converted[0]["model_intermediate_buffer"] + assert converted[0]["prompt_token_ids"] == [12, 13] + assert info["ids"]["tts"] == [12, 13] + assert torch.equal(torch.as_tensor(info["hidden_states"]["tts"]), latent[5:7].float()) + + +def test_llm2tts_native_duplex_conditions_on_turn_eos_and_midunit_speak(): + latent = torch.arange(36, dtype=torch.float16).reshape(9, 4) + # Official duplex includes mid-unit <|speak|> tokens AND the <|turn_eos|> + # token+hidden in the talker condition (its embedding is the trained + # stop signal); only chunk terminators bound the slice. + output = SimpleNamespace( + token_ids=[9304, 11, 9304, 12, 9310, 9308], + text="hello", + multimodal_output={ + "latent": latent, + "duplex_prompt_token_ids": [101, 102, 9306], + "meta": { + "tts_bos_token_id": 9301, + "tts_eos_token_id": 9302, + "listen_token_id": 9303, + "speak_token_id": 9304, + "tts_pad_token_id": 9305, + "unit_token_id": 9306, + "unit_end_token_id": 9307, + "chunk_eos_token_id": 9308, + "chunk_tts_eos_token_id": 9309, + "turn_eos_token_id": 9310, + }, + }, + ) + llm_output = SimpleNamespace( + request_id="duplex-turn-eos", + prompt_token_ids=[0, 0, 0], + outputs=[output], + ) + + converted = llm2tts([llm_output], prompt=[{}]) + + info = converted[0]["model_intermediate_buffer"] + assert info["ids"]["tts"] == [11, 9304, 12, 9310] + assert torch.equal(torch.as_tensor(info["hidden_states"]["tts"]), latent[4:8].float()) + assert info["meta"]["turn_eos_token_id"] == 9310 + + +def test_llm2tts_native_duplex_ignores_stale_tts_bos_inside_folded_prompt(): + latent = torch.arange(36, dtype=torch.float16).reshape(9, 4) + # A continuation unit: the resumable prompt has an EARLIER reply's + # <|tts_bos|> folded mid-prompt, and the model decided to LISTEN (no new + # speak text). The stale boundary must not re-slice already-spoken text. + output = SimpleNamespace( + token_ids=[9303, 9309], + text="", + multimodal_output={ + "latent": latent, + "duplex_prompt_token_ids": [101, 9301, 41, 42, 9306, 9306], + "meta": { + "tts_bos_token_id": 9301, + "tts_eos_token_id": 9302, + "listen_token_id": 9303, + "speak_token_id": 9304, + "tts_pad_token_id": 9305, + "unit_token_id": 9306, + "unit_end_token_id": 9307, + "chunk_eos_token_id": 9308, + "chunk_tts_eos_token_id": 9309, + "turn_eos_token_id": 9310, + }, + }, + ) + llm_output = SimpleNamespace( + request_id="duplex-stale-bos", + prompt_token_ids=[0, 0, 0, 0, 0, 0], + outputs=[output], + ) + + converted = llm2tts([llm_output], prompt=[{}]) + + assert converted == [] + + +def test_llm2tts_rejects_native_duplex_output_without_special_token_metadata(): + latent = torch.arange(16, dtype=torch.float16).reshape(4, 4) + output = SimpleNamespace( + token_ids=[11, 12], + text="hello", + multimodal_output={ + "latent": latent, + "duplex_prompt_token_ids": [101, 102, 9301], + }, + ) + llm_output = SimpleNamespace( + request_id="duplex-missing-meta", + prompt_token_ids=[0, 0, 0], + outputs=[output], + ) + + with pytest.raises(ValueError, match=r"<\|tts_bos\|>"): + llm2tts([llm_output], prompt=[{}]) + + +def test_minicpmo_talker_normalizes_list_handoff_payload(monkeypatch): + model = MiniCPMO45OmniTTSForConditionalGeneration.__new__(MiniCPMO45OmniTTSForConditionalGeneration) + captured = {} + + monkeypatch.setattr(model, "_should_stream_output", lambda *args, **kwargs: False) + + def fake_generate_speech(tts_token_ids, tts_hidden_states, **kwargs): + captured["tts_token_ids"] = tts_token_ids + captured["tts_hidden_states"] = tts_hidden_states + return torch.tensor([0.0, 0.1], dtype=torch.float32) + + monkeypatch.setattr(model, "generate_speech", fake_generate_speech) + + waveform, mel = model.forward( + additional_information={ + "tts_token_ids": [11, 12], + "tts_hidden_states": [[0.1, 0.2], [0.3, 0.4]], + } + ) + + assert mel is None + assert torch.equal(waveform, torch.tensor([0.0, 0.1], dtype=torch.float32)) + assert torch.equal(captured["tts_token_ids"], torch.tensor([11, 12])) + assert captured["tts_hidden_states"].dtype == torch.float32 + assert captured["tts_hidden_states"].shape == (2, 2) + + +def test_minicpmo_talker_accepts_structured_omni_payload_handoff(monkeypatch): + model = MiniCPMO45OmniTTSForConditionalGeneration.__new__(MiniCPMO45OmniTTSForConditionalGeneration) + captured = {} + + monkeypatch.setattr(model, "_should_stream_output", lambda *args, **kwargs: False) + + def fake_generate_speech(tts_token_ids, tts_hidden_states, **kwargs): + captured["tts_token_ids"] = tts_token_ids + captured["tts_hidden_states"] = tts_hidden_states + return torch.tensor([0.0, 0.1], dtype=torch.float32) + + monkeypatch.setattr(model, "generate_speech", fake_generate_speech) + + waveform, mel = model.forward( + additional_information={ + "ids": {"tts": [11, 12]}, + "hidden_states": {"tts": torch.tensor([[0.1, 0.2], [0.3, 0.4]])}, + } + ) + + assert mel is None + assert torch.equal(waveform, torch.tensor([0.0, 0.1], dtype=torch.float32)) + assert torch.equal(captured["tts_token_ids"], torch.tensor([11, 12])) + assert captured["tts_hidden_states"].dtype == torch.float32 + assert captured["tts_hidden_states"].shape == (2, 2) + + +def test_token2wav_prompt_cache_resets_when_reference_changes(): + model = MiniCPMO45OmniTTSForConditionalGeneration.__new__(MiniCPMO45OmniTTSForConditionalGeneration) + model.audio_tokenizer = SimpleNamespace(cache="old") + + model._reset_token2wav_cache_if_needed("/tmp/ref_a.wav") + assert model.audio_tokenizer.cache is None + + model.audio_tokenizer.cache = "prepared" + model._reset_token2wav_cache_if_needed("/tmp/ref_a.wav") + assert model.audio_tokenizer.cache == "prepared" + + model._reset_token2wav_cache_if_needed("/tmp/ref_b.wav") + assert model.audio_tokenizer.cache is None + + +def test_request_ref_audio_prompt_wav_uses_content_cache(): + model = MiniCPMO45OmniTTSForConditionalGeneration.__new__(MiniCPMO45OmniTTSForConditionalGeneration) + ref_audio = torch.tensor([0.0, 0.1, 0.2], dtype=torch.float32) + + path1 = model._write_ref_audio_prompt_wav(ref_audio, 16000) + path2 = model._write_ref_audio_prompt_wav(ref_audio.clone(), 16000) + + try: + assert path1 == path2 + assert os.path.exists(path1) + assert model._is_cached_ref_audio_prompt_wav(path1) + finally: + for path in set(getattr(model, "_ref_audio_prompt_files", {}).values()): + try: + os.unlink(path) + except OSError: + pass + + +def test_native_duplex_talker_uses_generate_chunk_continuation(monkeypatch): + class _FakeTTS: + audio_bos_token_id = 88 + + def __init__(self): + self.config = SimpleNamespace(num_audio_tokens=128) + self.model = SimpleNamespace(config=SimpleNamespace()) + self.emb_text = torch.nn.Embedding(256, 4) + self.calls = [] + self._responses = [ + (torch.tensor([[[10], [11], [12]]]), "kv1"), + (torch.tensor([[[20]]]), "kv2"), + ] + + def generate_chunk(self, **kwargs): + self.calls.append(kwargs) + return self._responses[len(self.calls) - 1] + + model = MiniCPMO45OmniTTSForConditionalGeneration.__new__(MiniCPMO45OmniTTSForConditionalGeneration) + tts = _FakeTTS() + windows = [] + model.tts_obj = tts + model.audio_tokenizer = SimpleNamespace() + model._talker_turn_states = {} + model._talker_consumed_tokens = {} + + monkeypatch.setattr(model, "_lazy_init_tts", lambda: None) + monkeypatch.setattr( + model, "_build_tts_sampling_params", lambda: SimpleNamespace(temperature=0.8, repetition_penalty=1.05) + ) + monkeypatch.setattr(model, "_tts_runtime_config", lambda: SimpleNamespace(streaming_generator_chunk=2)) + monkeypatch.setattr(model, "_resolve_prompt_wav_path", lambda ref, sr: ("/tmp/ref.wav", None)) + monkeypatch.setattr(model, "_begin_turn_vocoder_cache", lambda prompt, **kwargs: None) + monkeypatch.setattr(model, "_t2w_pre_lookahead", lambda: 1) + monkeypatch.setattr( + model, + "_build_tts_condition_embeds", + lambda ids, hidden: torch.ones((ids.numel(), 4), dtype=torch.float32), + ) + + def fake_stream_window(token_list, prompt_wav_path, *, last_chunk): + windows.append((list(token_list), last_chunk)) + return torch.tensor([float(len(token_list))]) + + monkeypatch.setattr(model, "_t2w_stream_window", fake_stream_window) + + base_info = { + "request_id": "duplex-generate-chunk", + "meta": {"turn_eos_token_id": 9310}, + "ids": {"tts": [21]}, + "hidden_states": {"tts": [[0.1, 0.2, 0.3, 0.4]]}, + "codes": {"ref": None}, + "native_duplex": True, + } + list(model._create_native_duplex_stream_gen(base_info)) + + turn_end_info = { + **base_info, + "ids": {"tts": [21, 9310]}, + "hidden_states": {"tts": [[0.1, 0.2, 0.3, 0.4], [0.5, 0.6, 0.7, 0.8]]}, + } + list(model._create_native_duplex_stream_gen(turn_end_info)) + + assert len(tts.calls) == 2 + first, second = tts.calls + assert first["past_key_values"] is None + assert first["text_start_pos"] == 0 + assert first["min_new_tokens"] == 0 + assert first["max_new_token"] == 3 + assert torch.equal(first["eos_token"], torch.tensor([127])) + assert float(first["temperature"].item()) == pytest.approx(0.8) + assert first["repetition_penalty"] == pytest.approx(1.05) + + assert second["past_key_values"] == "kv1" + assert second["text_start_pos"] == 5 + assert second["min_new_tokens"] == 0 + assert second["max_new_token"] == 3 + assert "duplex-generate-chunk" not in model._talker_turn_states + assert windows[-1] == ([20], True) + + +def test_minicpmo_native_duplex_talker_turn_end_metadata_flushes_tail(monkeypatch): + from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni_tts import ( + MiniCPMO45OmniTTSForConditionalGeneration, + ) + + class _FakeEmbText: + weight = torch.ones((200, 4), dtype=torch.float32) + + def __call__(self, ids): + return torch.ones((ids.numel(), 4), dtype=torch.float32) + + class _FakeTTS: + audio_bos_token_id = 1 + config = SimpleNamespace(num_audio_tokens=128) + emb_text = _FakeEmbText() + model = SimpleNamespace(config=SimpleNamespace(rope_theta=10000.0)) + + def generate_chunk(self, **kwargs): + del kwargs + return torch.tensor([[[33]]]), None + + model = MiniCPMO45OmniTTSForConditionalGeneration.__new__(MiniCPMO45OmniTTSForConditionalGeneration) + model.tts_obj = _FakeTTS() + model.audio_tokenizer = SimpleNamespace() + model._talker_turn_states = {} + model._talker_consumed_tokens = {} + + monkeypatch.setattr(model, "_lazy_init_tts", lambda: None) + monkeypatch.setattr( + model, "_build_tts_sampling_params", lambda: SimpleNamespace(temperature=0.8, repetition_penalty=1.05) + ) + monkeypatch.setattr(model, "_tts_runtime_config", lambda: SimpleNamespace(streaming_generator_chunk=25)) + monkeypatch.setattr(model, "_resolve_prompt_wav_path", lambda ref, sr: ("/tmp/ref.wav", None)) + monkeypatch.setattr(model, "_begin_turn_vocoder_cache", lambda prompt, **kwargs: None) + monkeypatch.setattr(model, "_t2w_pre_lookahead", lambda: 5) + monkeypatch.setattr( + model, + "_build_tts_condition_embeds", + lambda ids, hidden: torch.ones((ids.numel(), 4), dtype=torch.float32), + ) + + windows = [] + + def fake_stream_window(token_list, prompt_wav_path, *, last_chunk): + del prompt_wav_path + windows.append((list(token_list), last_chunk)) + return torch.tensor([float(len(token_list))]) + + monkeypatch.setattr(model, "_t2w_stream_window", fake_stream_window) + + info = { + "request_id": "duplex-metadata-tail-flush", + "meta": {"turn_eos_token_id": 9310}, + "ids": {"tts": [21]}, + "hidden_states": {"tts": [[0.1, 0.2, 0.3, 0.4]]}, + "codes": {"ref": None}, + "end_of_turn": True, + "native_duplex": True, + } + + list(model._create_native_duplex_stream_gen(info)) + + assert windows[-1][1] is True + assert "duplex-metadata-tail-flush" not in model._talker_turn_states + assert "duplex-metadata-tail-flush" not in model._talker_consumed_tokens + + +def test_minicpmo_native_duplex_terminal_only_new_turn_does_not_start_tts(monkeypatch): + from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni_tts import ( + MiniCPMO45OmniTTSForConditionalGeneration, + ) + + class _FakeEmbText: + weight = torch.ones((200, 4), dtype=torch.float32) + + def __call__(self, ids): + return torch.ones((ids.numel(), 4), dtype=torch.float32) + + class _FakeTTS: + audio_bos_token_id = 1 + config = SimpleNamespace(num_audio_tokens=128) + emb_text = _FakeEmbText() + model = SimpleNamespace(config=SimpleNamespace(rope_theta=10000.0)) + + def __init__(self): + self.calls = 0 + + def generate_chunk(self, **kwargs): + del kwargs + self.calls += 1 + return torch.tensor([[[33]]]), None + + model = MiniCPMO45OmniTTSForConditionalGeneration.__new__(MiniCPMO45OmniTTSForConditionalGeneration) + tts = _FakeTTS() + model.tts_obj = tts + model.audio_tokenizer = SimpleNamespace() + model._talker_turn_states = {} + model._talker_consumed_tokens = {} + + monkeypatch.setattr(model, "_lazy_init_tts", lambda: None) + monkeypatch.setattr( + model, "_build_tts_sampling_params", lambda: SimpleNamespace(temperature=0.8, repetition_penalty=1.05) + ) + monkeypatch.setattr(model, "_tts_runtime_config", lambda: SimpleNamespace(streaming_generator_chunk=25)) + monkeypatch.setattr(model, "_resolve_prompt_wav_path", lambda ref, sr: ("/tmp/ref.wav", None)) + monkeypatch.setattr(model, "_begin_turn_vocoder_cache", lambda prompt, **kwargs: None) + monkeypatch.setattr(model, "_t2w_pre_lookahead", lambda: 5) + monkeypatch.setattr( + model, + "_build_tts_condition_embeds", + lambda ids, hidden: torch.ones((ids.numel(), 4), dtype=torch.float32), + ) + monkeypatch.setattr( + model, + "_t2w_stream_window", + lambda token_list, prompt_wav_path, *, last_chunk: torch.tensor([float(len(token_list))]), + ) + + outputs = list( + model._create_native_duplex_stream_gen( + { + "request_id": "duplex-terminal-only", + "duplex": {"epoch": 0, "turn_id": 1}, + "meta": {"turn_eos_token_id": 9310}, + "ids": {"tts": [9310]}, + "hidden_states": {"tts": [[0.1, 0.2, 0.3, 0.4]]}, + "codes": {"ref": None}, + "native_duplex": True, + } + ) + ) + + assert tts.calls == 0 + assert len(outputs) == 1 + assert outputs[0][0].numel() == 0 + assert outputs[0][1] is True + assert model._talker_turn_states == {} + assert model._talker_consumed_tokens == {} + + +def test_minicpmo_native_duplex_talker_finished_request_closes_session_keyed_turn_state(): + from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni_tts import ( + MiniCPMO45OmniTTSForConditionalGeneration, + _TalkerTurnState, + ) + + model = MiniCPMO45OmniTTSForConditionalGeneration.__new__(MiniCPMO45OmniTTSForConditionalGeneration) + model.audio_tokenizer = SimpleNamespace(stream_cache={"stale": True}, hift_cache_dict={"stale": True}) + model._stream_gens = {} + model._talker_turn_states = { + "sid-stage1": _TalkerTurnState(prompt_wav_path=None, temp_prompt_wav_path=None), + } + model._talker_consumed_tokens = {"sid-stage1": 12} + model._talker_request_keys = {"req-stage1": "sid-stage1"} + + model.on_requests_finished({"req-stage1"}) + + assert "sid-stage1" not in model._talker_turn_states + assert "sid-stage1" not in model._talker_consumed_tokens + assert "req-stage1" not in model._talker_request_keys + assert model.audio_tokenizer.stream_cache is None + assert model.audio_tokenizer.hift_cache_dict == {} + + +def test_minicpmo_native_duplex_talker_new_turn_reopens_session_keyed_state(monkeypatch): + from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni_tts import ( + MiniCPMO45OmniTTSForConditionalGeneration, + _TalkerTurnState, + ) + + class _FakeTTS: + audio_bos_token_id = 88 + + def __init__(self): + self.config = SimpleNamespace(num_audio_tokens=128) + self.model = SimpleNamespace(config=SimpleNamespace()) + self.emb_text = torch.nn.Embedding(256, 4) + self.calls = [] + + def generate_chunk(self, **kwargs): + self.calls.append(kwargs) + return torch.tensor([[[10]]]), "new-kv" + + model = MiniCPMO45OmniTTSForConditionalGeneration.__new__(MiniCPMO45OmniTTSForConditionalGeneration) + tts = _FakeTTS() + model.tts_obj = tts + model.audio_tokenizer = SimpleNamespace(stream_cache={"old": True}, hift_cache_dict={"old": True}) + model._talker_turn_states = { + "sid-stage1": _TalkerTurnState( + prompt_wav_path=None, + temp_prompt_wav_path=None, + epoch=0, + turn_id=1, + ), + } + model._talker_turn_states["sid-stage1"].past_key_values = "old-kv" + model._talker_consumed_tokens = {"sid-stage1": 9} + model._talker_request_keys = {} + + monkeypatch.setattr(model, "_lazy_init_tts", lambda: None) + monkeypatch.setattr( + model, "_build_tts_sampling_params", lambda: SimpleNamespace(temperature=0.8, repetition_penalty=1.05) + ) + monkeypatch.setattr(model, "_tts_runtime_config", lambda: SimpleNamespace(streaming_generator_chunk=25)) + monkeypatch.setattr(model, "_resolve_prompt_wav_path", lambda ref, sr: ("/tmp/ref.wav", None)) + monkeypatch.setattr(model, "_begin_turn_vocoder_cache", lambda prompt, **kwargs: None) + monkeypatch.setattr(model, "_t2w_pre_lookahead", lambda: 5) + monkeypatch.setattr( + model, + "_build_tts_condition_embeds", + lambda ids, hidden: torch.ones((ids.numel(), 4), dtype=torch.float32), + ) + monkeypatch.setattr( + model, + "_t2w_stream_window", + lambda token_list, prompt_wav_path, *, last_chunk: torch.tensor([float(len(token_list))]), + ) + + info = { + "request_id": "sid-stage1", + "duplex": {"epoch": 0, "turn_id": 2}, + "meta": {"turn_eos_token_id": 9310}, + "ids": {"tts": [21]}, + "hidden_states": {"tts": [[0.1, 0.2, 0.3, 0.4]]}, + "codes": {"ref": None}, + "native_duplex": True, + } + + list(model._create_native_duplex_stream_gen(info)) + + assert tts.calls[0]["past_key_values"] is None + assert tts.calls[0]["text_start_pos"] == 0 + assert model._talker_consumed_tokens["sid-stage1"] == 1 + assert model._talker_turn_states["sid-stage1"].turn_id == 2 + + +def test_minicpmo_native_duplex_talker_segment_end_preserves_token2wav_stream_until_turn_end(monkeypatch): + from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni_tts import ( + MiniCPMO45OmniTTSForConditionalGeneration, + ) + + class _FakeEmbText: + weight = torch.ones((200, 4), dtype=torch.float32) + + def __call__(self, ids): + return torch.ones((ids.numel(), 4), dtype=torch.float32) + + class _FakeTTS: + audio_bos_token_id = 1 + config = SimpleNamespace(num_audio_tokens=128) + emb_text = _FakeEmbText() + model = SimpleNamespace(config=SimpleNamespace(rope_theta=10000.0)) + + def generate_chunk(self, **kwargs): + del kwargs + return torch.tensor([[[33]]]), "kv" + + model = MiniCPMO45OmniTTSForConditionalGeneration.__new__(MiniCPMO45OmniTTSForConditionalGeneration) + model.tts_obj = _FakeTTS() + model.audio_tokenizer = SimpleNamespace() + model._talker_turn_states = {} + model._talker_consumed_tokens = {} + + monkeypatch.setattr(model, "_lazy_init_tts", lambda: None) + monkeypatch.setattr( + model, "_build_tts_sampling_params", lambda: SimpleNamespace(temperature=0.8, repetition_penalty=1.05) + ) + monkeypatch.setattr(model, "_tts_runtime_config", lambda: SimpleNamespace(streaming_generator_chunk=25)) + monkeypatch.setattr(model, "_resolve_prompt_wav_path", lambda ref, sr: ("/tmp/ref.wav", None)) + monkeypatch.setattr(model, "_begin_turn_vocoder_cache", lambda prompt, **kwargs: None) + monkeypatch.setattr(model, "_t2w_pre_lookahead", lambda: 5) + monkeypatch.setattr( + model, + "_build_tts_condition_embeds", + lambda ids, hidden: torch.ones((ids.numel(), 4), dtype=torch.float32), + ) + + windows = [] + + def fake_stream_window(token_list, prompt_wav_path, *, last_chunk): + del prompt_wav_path + windows.append((list(token_list), last_chunk)) + return torch.tensor([float(len(token_list))]) + + monkeypatch.setattr(model, "_t2w_stream_window", fake_stream_window) + + info = { + "request_id": "duplex-segment-tail-flush", + "meta": {"turn_eos_token_id": 9310, "segment_end": True}, + "ids": {"tts": [21]}, + "hidden_states": {"tts": [[0.1, 0.2, 0.3, 0.4]]}, + "codes": {"ref": None}, + "native_duplex": True, + } + + list(model._create_native_duplex_stream_gen(info)) + + state = model._talker_turn_states["duplex-segment-tail-flush"] + assert windows == [] + assert state.token2wav_buffer == [4218, 4218, 4218, 33] + + turn_end_info = { + **info, + "ids": {"tts": [21, 9310]}, + "hidden_states": { + "tts": [ + [0.1, 0.2, 0.3, 0.4], + [0.5, 0.6, 0.7, 0.8], + ] + }, + "meta": {"turn_eos_token_id": 9310, "turn_end": True}, + } + list(model._create_native_duplex_stream_gen(turn_end_info)) + + assert windows == [([4218, 4218, 4218, 33, 33], True)] + assert "duplex-segment-tail-flush" not in model._talker_turn_states + + +def test_tts_scheduler_eos_uses_tokenizer_im_end_when_config_has_no_eos(): + class _Tokenizer: + unk_token_id = 0 + + def convert_tokens_to_ids(self, token): + return {"<|im_end|>": 77}.get(token, self.unk_token_id) + + def encode(self, text, add_special_tokens=False): + del add_special_tokens + return [77] if text == "<|im_end|>" else [self.unk_token_id] + + model = MiniCPMO45OmniTTSForConditionalGeneration.__new__(MiniCPMO45OmniTTSForConditionalGeneration) + model.config = SimpleNamespace(eos_token_id=None, vocab_size=100) + model.vllm_config = SimpleNamespace(model_config=SimpleNamespace()) + model._text_tokenizer = _Tokenizer() + model._ar_last_chunk_flags = [True] + + logits = model.compute_logits(torch.zeros(1, 4)) + + assert int(torch.argmax(logits[0]).item()) == 77 + + +def test_tts_streaming_non_final_chunk_does_not_mark_extra_rows_eos(): + class _Tokenizer: + unk_token_id = 0 + + def convert_tokens_to_ids(self, token): + return {"<|im_end|>": 77}.get(token, self.unk_token_id) + + def encode(self, text, add_special_tokens=False): + del add_special_tokens + return [77] if text == "<|im_end|>" else [self.unk_token_id] + + model = MiniCPMO45OmniTTSForConditionalGeneration.__new__(MiniCPMO45OmniTTSForConditionalGeneration) + model.config = SimpleNamespace(eos_token_id=None, vocab_size=100) + model.vllm_config = SimpleNamespace(model_config=SimpleNamespace()) + model._text_tokenizer = _Tokenizer() + model._ar_last_chunk_flags = [False] + + logits = model.compute_logits(torch.zeros(3, 4)) + sampled = torch.argmax(logits, dim=-1).tolist() + + assert sampled == [1, 1, 1] + + +def test_native_duplex_turn_end_detection_is_not_segment_end_detection(): + model = MiniCPMO45OmniTTSForConditionalGeneration.__new__(MiniCPMO45OmniTTSForConditionalGeneration) + turn_eos_id = 9310 + + assert model._native_duplex_input_ends_turn( + { + "native_duplex": True, + "ids": {"tts": [9304, 42, turn_eos_id]}, + "meta": {"turn_eos_token_id": turn_eos_id}, + } + ) + assert not model._native_duplex_input_ends_turn( + { + "native_duplex": True, + "ids": {"tts": [9304, 42, 9308]}, + "meta": { + "turn_eos_token_id": turn_eos_id, + "segment_end": True, + }, + } + ) + + +def _native_duplex_meta(): + return { + "tts_bos_token_id": 9301, + "tts_eos_token_id": 9302, + "listen_token_id": 9303, + "speak_token_id": 9304, + "tts_pad_token_id": 9305, + "unit_token_id": 9306, + "unit_end_token_id": 9307, + "chunk_eos_token_id": 9308, + "chunk_tts_eos_token_id": 9309, + "turn_eos_token_id": 9310, + } + + +def _native_duplex_handoff(request_id, prompt_ids, cumulative_output_ids, *, text="hello"): + latent = torch.arange( + (len(prompt_ids) + len(cumulative_output_ids)) * 4, + dtype=torch.float16, + ).reshape(-1, 4) + output = SimpleNamespace( + token_ids=cumulative_output_ids, + text=text, + multimodal_output={ + "latent": latent, + "duplex_prompt_token_ids": list(prompt_ids), + "meta": _native_duplex_meta(), + }, + ) + return SimpleNamespace( + request_id=request_id, + prompt_token_ids=[0, 0, 0], + outputs=[output], + ) + + +def _install_native_duplex_decoder(streaming_context, token_text): + def _decode(token_ids): + return "".join(token_text.get(int(token_id), "") for token_id in token_ids) + + streaming_context.source_token_decoder = _decode + return streaming_context + + +def test_native_duplex_segment_text_comes_from_token_delta_decode(): + streaming_context = _install_native_duplex_decoder( + SimpleNamespace(bridge_states={}), + { + 21: "今早", + 22: "吃东。", + 31: "西了吗?", + }, + ) + + ids1, text1 = _native_duplex_segment_output_ids( + [9304, 21, 22, 9308], + "今早吃东。", + streaming_context, + request_id="duplex-token-text", + ) + assert ids1 == [9304, 21, 22, 9308] + assert text1 == "今早吃东。" + + ids2, text2 = _native_duplex_segment_output_ids( + [9304, 21, 22, 9308, 9304, 31, 9308], + "今早吃东。西了吗?如需陪聊,", + streaming_context, + request_id="duplex-token-text", + ) + assert ids2 == [9304, 31, 9308] + assert text2 == "西了吗?" + + +def test_native_duplex_segment_restart_does_not_reuse_character_cursor(): + streaming_context = _install_native_duplex_decoder( + SimpleNamespace(bridge_states={}), + { + 21: "你好", + 31: "你好", + 32: ",有什么想聊的吗?", + }, + ) + + _native_duplex_segment_output_ids( + [9304, 21, 9310], + "你好,第一轮尾巴", + streaming_context, + request_id="duplex-restart-text", + ) + + ids2, text2 = _native_duplex_segment_output_ids( + [9304, 31, 32, 9310], + "你好,第一轮尾巴你好,有什么想聊的吗?", + streaming_context, + request_id="duplex-restart-text", + ) + assert ids2 == [9304, 31, 32, 9310] + assert text2 == "你好,有什么想聊的吗?" + + +def test_native_duplex_segment_turn_id_change_keeps_cumulative_token_cursor(): + streaming_context = _install_native_duplex_decoder( + SimpleNamespace(bridge_states={"duplex": {"turn_id": 1}}), + { + 21: "你好", + 32: ",有什么想聊的吗?", + }, + ) + + ids1, text1 = _native_duplex_segment_output_ids( + [9304, 21], + "你好", + streaming_context, + request_id="duplex-turn-id-reset", + ) + assert ids1 == [9304, 21] + assert text1 == "你好" + + streaming_context.bridge_states["duplex"]["turn_id"] = 2 + ids2, text2 = _native_duplex_segment_output_ids( + [9304, 21, 32], + "你好,有什么想聊的吗?", + streaming_context, + request_id="duplex-turn-id-reset", + ) + assert ids2 == [32] + assert text2 == ",有什么想聊的吗?" + + +def test_llm2tts_native_duplex_hands_off_segment_deltas(): + """Each duplex handoff must carry only the current segment's tokens. + + The thinker's resumable request reports cumulative output ids while + earlier segments are already folded into the prompt, so re-forwarding + the full list misaligns the hidden states and grows the talker prompt + without bound. + """ + streaming_context = SimpleNamespace(bridge_states={}) + + # Segment 1: speak + two text tokens + chunk_eos. + seg1_output = [9304, 21, 22, 9308] + handoff1 = _native_duplex_handoff("duplex-delta", [101, 102], list(seg1_output)) + converted1 = llm2tts([handoff1], prompt=[{}], _streaming_context=streaming_context) + info1 = converted1[0]["model_intermediate_buffer"] + assert info1["ids"]["output"] == seg1_output + assert info1["llm_output_text"] == ["hello"] + assert info1["meta"]["native_duplex_segment_text"] == "hello" + assert "llm_output_text" in info1["meta"]["override_keys"] + assert ["meta", "native_duplex_segment_text"] in info1["meta"]["override_keys"] + assert converted1[0]["prompt_token_ids"] == [21, 22] + assert info1["meta"]["segment_end"] is True + + # Segment 2: prompt now contains the folded segment-1 tokens; the + # cumulative output list still carries segment 1 ahead of segment 2. + seg2_output = [9304, 31, 9308] + prompt2 = [101, 102, *seg1_output, 555] + handoff2 = _native_duplex_handoff("duplex-delta", prompt2, [*seg1_output, *seg2_output]) + converted2 = llm2tts([handoff2], prompt=[{}], _streaming_context=streaming_context) + info2 = converted2[0]["model_intermediate_buffer"] + assert info2["ids"]["output"] == seg2_output + # Cumulative text "hello" was fully delivered with segment 1; only the + # delta (empty here) rides along with segment 2. + assert info2["llm_output_text"] == [""] + assert info2["meta"]["native_duplex_segment_text"] == "" + assert converted2[0]["prompt_token_ids"] == [31] + assert info2["meta"]["segment_end"] is True + + +def test_llm2tts_native_duplex_marks_talker_handoff_as_data_plane(): + streaming_context = SimpleNamespace( + bridge_states={ + "duplex": { + "session_id": "sid-stage1-stop", + "incarnation": 2, + "epoch": 0, + "turn_id": 7, + "session_config": { + "extra_body": { + "duplex_stage_sampling_params": { + "1": {"stop_token_ids": [151645]}, + }, + }, + }, + }, + }, + ) + handoff = _native_duplex_handoff("duplex-stage1-stop", [101, 102], [9304, 21, 9308]) + + converted = llm2tts([handoff], prompt=[{}], _streaming_context=streaming_context) + + duplex = converted[0]["model_intermediate_buffer"]["duplex"] + assert duplex["data_plane"] is True + assert duplex["session_id"] == "sid-stage1-stop" + assert duplex["incarnation"] == 2 + assert duplex["epoch"] == 0 + assert duplex["turn_id"] == 7 + assert duplex["session_config"]["extra_body"]["duplex_stage_sampling_params"]["1"]["stop_token_ids"] == [151645] + + +def test_llm2tts_native_duplex_carries_runtime_ref_audio_to_talker_payload(): + ref_audio = base64.b64encode(struct.pack("<3f", 0.1, 0.2, 0.3)).decode("ascii") + streaming_context = SimpleNamespace( + bridge_states={ + "duplex": { + "session_id": "sid-runtime-ref", + "runtime_config": { + "ref_audio_data": ref_audio, + "ref_audio_format": "pcm_f32le", + "ref_audio_sample_rate_hz": 16000, + }, + }, + }, + ) + handoff = _native_duplex_handoff("duplex-runtime-ref", [101, 102], [9304, 21, 9308]) + + converted = llm2tts([handoff], prompt=[{}], _streaming_context=streaming_context) + + info = converted[0]["model_intermediate_buffer"] + assert torch.allclose(torch.tensor(info["codes"]["ref"]), torch.tensor([0.1, 0.2, 0.3])) + assert info["meta"]["ref_audio_sr"] == 16000 + assert info["duplex"]["runtime_config"]["ref_audio_data"] == ref_audio + + +def test_llm2tts_native_duplex_advances_model_turn_only_after_turn_eos(): + streaming_context = SimpleNamespace( + bridge_states={ + "duplex": { + "session_id": "sid-model-turn", + "epoch": 0, + "turn_id": 9, + "model_turn_id": 0, + }, + } + ) + + first = _native_duplex_handoff( + "duplex-model-turn", + [101], + [9304, 21, 9310], + text="first", + ) + converted_first = llm2tts([first], prompt=[{}], _streaming_context=streaming_context) + + first_duplex = converted_first[0]["model_intermediate_buffer"]["duplex"] + assert first_duplex["turn_id"] == 0 + assert streaming_context.bridge_states["duplex"]["model_turn_id"] == 1 + + second = _native_duplex_handoff( + "duplex-model-turn", + [101, 9304, 21, 9310, 555], + [9304, 21, 9310, 9304, 22, 9308], + text="second", + ) + converted_second = llm2tts([second], prompt=[{}], _streaming_context=streaming_context) + + second_duplex = converted_second[0]["model_intermediate_buffer"]["duplex"] + assert second_duplex["turn_id"] == 1 + assert streaming_context.bridge_states["duplex"]["model_turn_id"] == 1 + + +def test_llm2tts_native_duplex_accumulates_tts_condition_across_handoffs(): + """Every handoff must carry the FULL accumulated tts condition. + + The runner's resume-prefill path REPLACES the streaming buffer (only + in-place updates merge), so per-segment tts payloads were silently lost + for alternating segments; the talker then vocalized text it never saw. + Handing the complete history per handoff makes replacement lossless. + """ + streaming_context = SimpleNamespace(bridge_states={}) + + seg1_output = [9304, 21, 22, 9308] + handoff1 = _native_duplex_handoff("duplex-acc", [101, 102], list(seg1_output)) + converted1 = llm2tts([handoff1], prompt=[{}], _streaming_context=streaming_context) + info1 = converted1[0]["model_intermediate_buffer"] + assert info1["ids"]["tts"] == [21, 22] + assert len(info1["hidden_states"]["tts"]) == 2 + + seg2_output = [9304, 31, 9308] + prompt2 = [101, 102, *seg1_output, 555] + handoff2 = _native_duplex_handoff("duplex-acc", prompt2, [*seg1_output, *seg2_output]) + converted2 = llm2tts([handoff2], prompt=[{}], _streaming_context=streaming_context) + info2 = converted2[0]["model_intermediate_buffer"] + assert info2["ids"]["tts"] == [21, 22, 31] + assert len(info2["hidden_states"]["tts"]) == 3 + + +def test_llm2tts_native_duplex_preserves_text_cursor_across_turn_reset(): + streaming_context = _install_native_duplex_decoder( + SimpleNamespace(bridge_states={}), + { + 21: "hello", + 31: "hello", + }, + ) + + seg1_output = [9304, 21, 9310] + handoff1 = _native_duplex_handoff("duplex-turn-text", [101, 102], seg1_output, text="hello") + converted1 = llm2tts([handoff1], prompt=[{}], _streaming_context=streaming_context) + info1 = converted1[0]["model_intermediate_buffer"] + assert info1["meta"]["native_duplex_segment_text"] == "hello" + + seg2_output = [9304, 31, 9310] + handoff2 = _native_duplex_handoff( + "duplex-turn-text", + [101, 102, *seg1_output], + seg2_output, + text="hellohello", + ) + converted2 = llm2tts([handoff2], prompt=[{}], _streaming_context=streaming_context) + info2 = converted2[0]["model_intermediate_buffer"] + assert info2["ids"]["tts"] == [31, 9310] + assert info2["meta"]["native_duplex_segment_text"] == "hello" + + +def test_llm2tts_native_duplex_preserves_output_cursor_across_turn_reset(): + streaming_context = _install_native_duplex_decoder( + SimpleNamespace(bridge_states={}), + { + 21: "hello", + 31: "hello", + }, + ) + + seg1_output = [9304, 21, 9310] + handoff1 = _native_duplex_handoff("duplex-turn-output", [101, 102], seg1_output, text="hello") + converted1 = llm2tts([handoff1], prompt=[{}], _streaming_context=streaming_context) + info1 = converted1[0]["model_intermediate_buffer"] + assert info1["ids"]["output"] == seg1_output + assert info1["ids"]["tts"] == [21, 9310] + + seg2_output = [9304, 31, 9310] + handoff2 = _native_duplex_handoff( + "duplex-turn-output", + [101, 102, *seg1_output], + [*seg1_output, *seg2_output], + text="hellohello", + ) + converted2 = llm2tts([handoff2], prompt=[{}], _streaming_context=streaming_context) + info2 = converted2[0]["model_intermediate_buffer"] + assert info2["ids"]["output"] == seg2_output + assert info2["ids"]["tts"] == [31, 9310] + assert info2["meta"]["native_duplex_segment_text"] == "hello" + + +def test_llm2tts_native_duplex_resets_tts_condition_on_new_turn_without_resetting_output_cursor(): + streaming_context = _install_native_duplex_decoder( + SimpleNamespace(bridge_states={"duplex": {"turn_id": 0}}), + { + 21: "first", + 31: "second", + }, + ) + + seg1_output = [9304, 21, 9308] + handoff1 = _native_duplex_handoff("duplex-turn-condition", [101, 102], seg1_output, text="first") + converted1 = llm2tts([handoff1], prompt=[{}], _streaming_context=streaming_context) + info1 = converted1[0]["model_intermediate_buffer"] + assert info1["ids"]["output"] == seg1_output + assert info1["ids"]["tts"] == [21] + + streaming_context.bridge_states["duplex"]["turn_id"] = 1 + seg2_output = [9304, 31, 9308] + handoff2 = _native_duplex_handoff( + "duplex-turn-condition", + [101, 102, *seg1_output], + [*seg1_output, *seg2_output], + text="firstsecond", + ) + converted2 = llm2tts([handoff2], prompt=[{}], _streaming_context=streaming_context) + info2 = converted2[0]["model_intermediate_buffer"] + assert info2["ids"]["output"] == seg2_output + assert info2["ids"]["tts"] == [31] + assert info2["meta"]["native_duplex_segment_text"] == "second" + + +def test_llm2tts_never_aliases_thinker_token_list(): + """The talker prompt must never be the thinker's live token list object. + + CompletionOutput.token_ids can alias the upstream detokenizer's internal + list; forwarding that object lets the stage-1 streaming update extend the + list with itself, doubling the recorded output every segment. + """ + live_token_list = [9304, 21, 22, 9308] + handoff = _native_duplex_handoff("duplex-alias", [101, 102], live_token_list) + converted = llm2tts([handoff], prompt=[{}]) + scheduler_prompt = converted[0]["prompt_token_ids"] + assert scheduler_prompt is not live_token_list + scheduler_prompt.extend(scheduler_prompt) + assert live_token_list == [9304, 21, 22, 9308] diff --git a/tests/test_config_factory.py b/tests/test_config_factory.py index c70b8c487..13a491896 100644 --- a/tests/test_config_factory.py +++ b/tests/test_config_factory.py @@ -131,6 +131,18 @@ class TestStageConfig: assert "max_num_seqs" not in omega_config.engine_args # Legacy field name for backward compatibility assert omega_config.engine_input_source == [] + assert omega_config.session_mode == "turn" + + def test_to_omegaconf_duplex_session_mode(self): + """Test that session mode is preserved as stage metadata.""" + config = StageConfig( + stage_id=0, + model_stage="thinker", + session_mode="duplex", + ) + omega_config = config.to_omegaconf() + + assert omega_config.session_mode == "duplex" def test_to_omegaconf_with_runtime_overrides(self): """Test that runtime overrides are applied to OmegaConf output.""" @@ -1107,6 +1119,57 @@ class TestResolveScheduler: class TestDeployConfigLoading: + def test_load_minicpmo_duplex_deploy_config(self): + deploy_path = Path(__file__).parent.parent / "vllm_omni" / "deploy" / "minicpmo_4_5_duplex.yaml" + + deploy = load_deploy_config(deploy_path) + pipeline = resolve_pipeline_config("minicpmo_4_5") + assert isinstance(pipeline, PipelineConfig) + + stages = merge_pipeline_deploy(pipeline, deploy) + + assert deploy.session_mode == "duplex" + assert deploy.async_chunk is False + assert deploy.active_stream_window == 1 + assert [stage.session_mode for stage in stages] == ["duplex", "duplex"] + assert [stage.to_omegaconf().session_mode for stage in stages] == ["duplex", "duplex"] + assert [stage.yaml_engine_args["async_scheduling"] for stage in stages] == [False, False] + assert all("Async" not in (stage.scheduler_cls or "") for stage in stages) + assert deploy.stages[1].enforce_eager is False + assert deploy.stages[1].compilation_config == {"cudagraph_mode": "PIECEWISE"} + assert stages[1].yaml_extras["default_sampling_params"]["max_tokens"] == 4096 + assert stages[1].yaml_extras["default_sampling_params"]["stop_token_ids"] == [151645] + + @pytest.mark.parametrize( + ("filename", "stage0_devices", "stage1_devices", "stage1_replicas"), + [ + ("minicpmo_4_5_3gpu_stage1_replicas.yaml", "0", "1,2", 2), + ("minicpmo_4_5_4gpu_stage1_replicas.yaml", "0", "1,2,3", 3), + ("minicpmo_4_5_8x4090_stage1_replicas.yaml", "0,1,2,3", "4,5,6,7", 4), + ], + ) + def test_load_minicpmo_replica_deploy_configs( + self, + filename: str, + stage0_devices: str, + stage1_devices: str, + stage1_replicas: int, + ): + deploy_path = Path(__file__).parent.parent / "vllm_omni" / "deploy" / filename + + deploy = load_deploy_config(deploy_path) + pipeline = resolve_pipeline_config("minicpmo_4_5") + assert isinstance(pipeline, PipelineConfig) + + stages = merge_pipeline_deploy(pipeline, deploy) + + assert deploy.session_mode == "duplex" + assert [stage.yaml_runtime["devices"] for stage in stages] == [stage0_devices, stage1_devices] + assert stages[0].yaml_runtime["num_replicas"] == 1 + assert stages[1].yaml_runtime["num_replicas"] == stage1_replicas + assert all(stage.yaml_engine_args["async_scheduling"] is False for stage in stages) + assert all("Async" not in (stage.scheduler_cls or "") for stage in stages) + def test_custom_voice_dir_is_pipeline_wide_engine_arg(self, tmp_path): deploy_path = tmp_path / "qwen3_tts_custom_voice.yaml" custom_voice_dir = tmp_path / "voices" diff --git a/tests/test_data_entry_keys.py b/tests/test_data_entry_keys.py index 73dcc82c2..4720cd8ea 100644 --- a/tests/test_data_entry_keys.py +++ b/tests/test_data_entry_keys.py @@ -21,6 +21,8 @@ from vllm_omni.data_entry_keys import ( ) from vllm_omni.engine import AdditionalInformationPayload +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + class TestOmniPayloadStruct: """Runtime-validated mirror of OmniPayload (msgspec.Struct).""" @@ -43,6 +45,16 @@ class TestOmniPayloadStruct: with pytest.raises(msgspec.ValidationError, match="unknown field"): to_struct({"meta": {"finisheed": True}}) + def test_to_struct_rejects_minicpmo_duplex_runtime_handoff_fields(self): + with pytest.raises(msgspec.ValidationError, match="unknown field"): + to_struct({"hidden_states": {"tts": torch.zeros(1)}}) + with pytest.raises(msgspec.ValidationError, match="unknown field"): + to_struct({"ids": {"tts": [1]}}) + with pytest.raises(msgspec.ValidationError, match="unknown field"): + to_struct({"meta": {"ref_audio_sr": 16000}}) + with pytest.raises(msgspec.ValidationError, match="unknown field"): + to_struct({"meta": {"listen_token_id": 151685}}) + def test_to_struct_rejects_wrong_type(self): with pytest.raises(msgspec.ValidationError, match="Expected"): to_struct({"meta": {"left_context_size": "not_an_int"}}) @@ -66,10 +78,10 @@ class TestOmniPayloadStruct: def test_struct_with_all_categories(self): d = { - "hidden_states": {"output": torch.zeros(1)}, + "hidden_states": {"output": torch.zeros(1), "last": torch.ones(2, 4)}, "embed": {"prefill": torch.zeros(1), "tts_bos": torch.zeros(1)}, - "ids": {"all": [1, 2], "prompt": [1]}, - "codes": {"audio": torch.zeros(1)}, + "ids": {"all": [1, 2], "prompt": [1], "speech_token": [3, 4]}, + "codes": {"audio": torch.zeros(1), "ref": torch.ones(4)}, "meta": {"left_context_size": 3, "num_processed_tokens": 7}, } s = to_struct(d) @@ -79,7 +91,10 @@ class TestOmniPayloadStruct: assert isinstance(s.codes, CodesStruct) assert isinstance(s.meta, MetaStruct) assert s.ids.all == [1, 2] + assert s.ids.speech_token == [3, 4] + assert torch.equal(s.hidden_states.last, torch.ones(2, 4)) assert s.meta.num_processed_tokens == 7 + assert torch.equal(s.codes.ref, torch.ones(4)) class TestValidatePayload: @@ -332,7 +347,7 @@ class TestSerializeDeserializePayload: "hidden_states": {"output": torch.tensor([1.0, 2.0])}, "ids": {"all": [1, 2, 3]}, "meta": {"finished": torch.tensor(False, dtype=torch.bool), "ar_width": 4}, - "codes": {"audio": torch.tensor([3.0])}, + "codes": {"audio": torch.tensor([3.0]), "ref": torch.tensor([0.1, 0.2])}, } wire = serialize_payload(original) restored = deserialize_payload(wire) @@ -341,6 +356,7 @@ class TestSerializeDeserializePayload: assert restored["meta"]["finished"].item() is False assert restored["meta"]["ar_width"] == 4 assert torch.equal(restored["codes"]["audio"], original["codes"]["audio"]) + assert torch.equal(restored["codes"]["ref"], original["codes"]["ref"]) def test_hidden_states_layers_round_trip(self): original = { diff --git a/tests/test_omni_request.py b/tests/test_omni_request.py index 09f893a03..4f4718f09 100644 --- a/tests/test_omni_request.py +++ b/tests/test_omni_request.py @@ -20,7 +20,7 @@ from vllm.sampling_params import SamplingParams from vllm.v1.request import Request from vllm_omni.engine import PromptEmbedsPayload -from vllm_omni.request import OmniRequest +from vllm_omni.request import OmniRequest, OmniStreamingUpdate pytestmark = [pytest.mark.core_model, pytest.mark.cpu] @@ -32,7 +32,12 @@ def test_omni_params_are_keyword_only(): came first positionally and broke positional ``Request(...)`` construction. """ params = inspect.signature(OmniRequest.__init__).parameters - for name in ("prompt_embeds", "external_req_id", "additional_information"): + for name in ( + "prompt_embeds", + "external_req_id", + "additional_information", + "model_intermediate_buffer", + ): assert params[name].kind is inspect.Parameter.KEYWORD_ONLY, name @@ -87,3 +92,21 @@ def test_keyword_omni_params_round_trip(): # The serialized payload is retained and decoded into a tensor on the base. assert req.prompt_embeds_payload is payload assert torch.equal(req.prompt_embeds, torch.from_numpy(arr)) + + +def test_model_intermediate_buffer_round_trips_to_streaming_update(): + buffer = {"duplex": {"turn_id": 3}} + request = OmniRequest( + request_id="req-model-buffer", + prompt_token_ids=[1, 2], + sampling_params=SamplingParams(max_tokens=16), + pooling_params=None, + resumable=True, + model_intermediate_buffer=buffer, + ) + + update = OmniStreamingUpdate.from_request(request) + + assert update is not None + assert request.model_intermediate_buffer is buffer + assert update.model_intermediate_buffer is buffer diff --git a/tests/utils/test_mm_outputs_partition.py b/tests/utils/test_mm_outputs_partition.py index f9affa471..d70b95a79 100644 --- a/tests/utils/test_mm_outputs_partition.py +++ b/tests/utils/test_mm_outputs_partition.py @@ -1,7 +1,10 @@ +import pytest import torch from vllm_omni.utils.mm_outputs import partition_flat_payload, partition_payload_list +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + def test_partition_thinker_latent_payload(): payload = { @@ -42,3 +45,27 @@ def test_partition_payload_list_preserves_request_alignment(): inter_list, client_list = partition_payload_list(payloads) assert inter_list == [payloads[0], None] assert client_list == [None, payloads[1]] + + +def test_partition_duplex_audio_transcript_metadata_to_client_mm(): + payload = { + "model_outputs": torch.zeros(1, 2400), + "meta.duplex_epoch": torch.tensor([3], dtype=torch.int32), + "meta.duplex_turn_id": torch.tensor([2], dtype=torch.int32), + "meta.llm_output_text_utf8": torch.tensor([104, 105], dtype=torch.uint8), + "meta.audio_text_total_chars": torch.tensor([2], dtype=torch.int32), + "meta.tts_is_last_chunk": torch.tensor([1], dtype=torch.int32), + "meta.turn_end": torch.tensor([1], dtype=torch.int32), + "meta.native_duplex_segment_text": "hi", + } + + inter, client = partition_flat_payload(payload) + + assert "meta.llm_output_text_utf8" in client + assert "meta.audio_text_total_chars" in client + assert "meta.duplex_epoch" in client + assert "meta.duplex_turn_id" in client + assert "meta.tts_is_last_chunk" in client + assert "meta.turn_end" in client + assert "meta.native_duplex_segment_text" not in client + assert "meta.native_duplex_segment_text" in inter diff --git a/tests/worker/test_native_duplex_hooks.py b/tests/worker/test_native_duplex_hooks.py new file mode 100644 index 000000000..a5a81a1de --- /dev/null +++ b/tests/worker/test_native_duplex_hooks.py @@ -0,0 +1,2576 @@ +from __future__ import annotations + +import inspect +import os +import threading +from types import SimpleNamespace + +import numpy as np +import pytest +import torch + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +def _minicpmo_duplex_policy_case( + state: SimpleNamespace, + payload: dict[str, object], +): + from vllm_omni.experimental.fullduplex.model_executor import DuplexSamplingRow + from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni import ( + MiniCPMO45OmniForConditionalGeneration, + ) + + session_key = ("sid-policy", 1) + model = MiniCPMO45OmniForConditionalGeneration.__new__(MiniCPMO45OmniForConditionalGeneration) + model.model_stage = "llm" + model._minicpmo45_native_duplex_token_ids_cache = { + "listen_token_id": 7, + "tts_bos_token_id": 8, + "turn_eos_token_id": 9, + } + model._minicpmo45_duplex_data_plane_helper = SimpleNamespace(sessions={session_key: state}) + row = DuplexSamplingRow( + row_idx=0, + request_id="req-policy", + session_id=session_key[0], + incarnation=session_key[1], + seq=3, + payload=payload, + max_tokens=20, + ) + return model, row + + +def test_minicpmo_model_hook_owns_duplex_sampling_rows_and_force_listen(): + from vllm_omni.experimental.fullduplex.model_executor import DuplexSamplingRow + from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni import ( + MiniCPMO45OmniForConditionalGeneration, + ) + + listen_id = 7 + state = SimpleNamespace( + current_turn_ended=True, + last_terminator_token=None, + pending_terminator_token=None, + ) + model = MiniCPMO45OmniForConditionalGeneration.__new__(MiniCPMO45OmniForConditionalGeneration) + model.model_stage = "llm" + model._minicpmo45_native_duplex_token_ids_cache = { + "listen_token_id": listen_id, + "tts_bos_token_id": 8, + "turn_eos_token_id": 9, + } + model._minicpmo45_duplex_data_plane_helper = SimpleNamespace(sessions={("sid-hook", 2): state}) + logits = torch.zeros((1, 16), dtype=torch.float32) + row = DuplexSamplingRow( + row_idx=0, + request_id="req-hook", + session_id="sid-hook", + incarnation=2, + seq=3, + payload={"force_listen": True, "is_speech": True}, + max_tokens=20, + ) + + model.prepare_duplex_sampling(logits, SimpleNamespace(), (row,)) + + assert model._minicpmo45_active_duplex_rows == [0] + assert model._minicpmo45_duplex_row_sessions == {0: ("sid-hook", 2)} + assert model._minicpmo45_duplex_row_payloads == {0: row.payload} + assert model._minicpmo45_duplex_row_max_tokens == {0: 20} + assert logits[0, listen_id].item() == 0.0 + assert torch.isneginf(logits[0, :listen_id]).all() + assert torch.isneginf(logits[0, listen_id + 1 :]).all() + + +def test_minicpmo_model_hook_turn_end_latch_forces_silence_to_listen(): + state = SimpleNamespace( + current_turn_ended=True, + last_terminator_token=9, + pending_terminator_token=None, + ) + model, row = _minicpmo_duplex_policy_case(state, {"is_speech": False}) + logits = torch.zeros((1, 16), dtype=torch.float32) + logits[0, 10] = 20.0 + + model.prepare_duplex_sampling(logits, SimpleNamespace(), (row,)) + + assert logits[0, 7].item() == 0.0 + assert torch.isneginf(logits[0, :7]).all() + assert torch.isneginf(logits[0, 8:]).all() + + +def test_minicpmo_model_hook_pending_speech_after_turn_eos_allows_silence_sampling(): + state = SimpleNamespace( + current_turn_ended=True, + last_terminator_token=9, + pending_terminator_token=None, + pending_speech_context=True, + ) + model, row = _minicpmo_duplex_policy_case(state, {"is_speech": False}) + logits = torch.zeros((1, 16), dtype=torch.float32) + logits[0, 7] = 30.0 + logits[0, 10] = 20.0 + original_logits = logits.clone() + + model.prepare_duplex_sampling(logits, SimpleNamespace(), (row,)) + + assert torch.equal(logits, original_logits) + assert state.pending_speech_context is True + + +def test_minicpmo_model_hook_speech_row_does_not_create_pending_context(): + state = SimpleNamespace( + current_turn_ended=True, + last_terminator_token=9, + pending_terminator_token=None, + pending_speech_context=False, + ) + model, row = _minicpmo_duplex_policy_case(state, {"is_speech": True}) + logits = torch.zeros((1, 16), dtype=torch.float32) + logits[0, 7] = 30.0 + logits[0, 8] = -2.0 + logits[0, 10] = 20.0 + original_logits = logits.clone() + + model.prepare_duplex_sampling(logits, SimpleNamespace(), (row,)) + + assert state.current_turn_ended is True + assert state.last_terminator_token is None + assert state.pending_speech_context is False + assert torch.equal(logits, original_logits) + + +def test_minicpmo_model_hook_old_response_output_does_not_clear_pending_speech_context(): + state = SimpleNamespace( + current_turn_ended=False, + last_terminator_token=None, + pending_terminator_token=None, + pending_speech_context=True, + ) + model, row = _minicpmo_duplex_policy_case(state, {"is_speech": True}) + model._minicpmo45_duplex_row_sessions = {0: (row.session_id, row.incarnation)} + model._minicpmo45_duplex_row_payloads = {0: row.payload} + + model._record_minicpmo45_duplex_terminator( + 0, + 10, + {"listen_token_id": 7, "chunk_eos_token_id": -1, "chunk_tts_eos_token_id": -1, "turn_eos_token_id": 9}, + ) + + assert state.pending_speech_context is True + assert state.current_turn_ended is False + + +def test_minicpmo_model_hook_new_response_output_clears_pending_speech_context(): + state = SimpleNamespace( + current_turn_ended=True, + last_terminator_token=7, + pending_terminator_token=7, + pending_speech_context=True, + ) + model, row = _minicpmo_duplex_policy_case(state, {"is_speech": False}) + model._minicpmo45_duplex_row_sessions = {0: (row.session_id, row.incarnation)} + model._minicpmo45_duplex_row_payloads = {0: row.payload} + + model._record_minicpmo45_duplex_terminator( + 0, + 8, + {"listen_token_id": 7, "chunk_eos_token_id": -1, "chunk_tts_eos_token_id": -1, "turn_eos_token_id": 9}, + ) + + assert state.pending_speech_context is False + assert state.current_turn_ended is False + + +def test_minicpmo_model_hook_empty_speak_envelope_preserves_pending_speech_context(): + state = SimpleNamespace( + current_turn_ended=True, + last_terminator_token=9, + pending_terminator_token=None, + pending_speech_context=True, + pending_speech_response_open=False, + ) + model, row = _minicpmo_duplex_policy_case(state, {"is_speech": False}) + model._minicpmo45_duplex_row_sessions = {0: (row.session_id, row.incarnation)} + model._minicpmo45_duplex_row_payloads = {0: row.payload} + token_ids = { + "listen_token_id": 7, + "tts_bos_token_id": 8, + "chunk_eos_token_id": -1, + "chunk_tts_eos_token_id": -1, + "turn_eos_token_id": 9, + } + + model._record_minicpmo45_duplex_terminator(0, 8, token_ids) + model._record_minicpmo45_duplex_terminator(0, 9, token_ids) + + assert state.current_turn_ended is True + assert state.pending_speech_context is True + assert state.pending_speech_response_open is False + + logits = torch.zeros((1, 16), dtype=torch.float32) + logits[0, 7] = 30.0 + logits[0, 10] = 20.0 + original_logits = logits.clone() + + model.prepare_duplex_sampling(logits, SimpleNamespace(), (row,)) + + assert torch.equal(logits, original_logits) + + +def test_minicpmo_model_hook_second_new_response_step_is_not_forced_to_listen(): + state = SimpleNamespace( + current_turn_ended=True, + last_terminator_token=9, + pending_terminator_token=None, + pending_speech_context=True, + ) + model, row = _minicpmo_duplex_policy_case(state, {"is_speech": False}) + logits = torch.zeros((1, 16), dtype=torch.float32) + logits[0, 7] = 30.0 + logits[0, 10] = 20.0 + original_logits = logits.clone() + + model.prepare_duplex_sampling(logits, SimpleNamespace(), (row,)) + model._minicpmo45_duplex_row_sessions = {0: (row.session_id, row.incarnation)} + model._minicpmo45_duplex_row_payloads = {0: row.payload} + model._record_minicpmo45_duplex_terminator( + 0, + 8, + {"listen_token_id": 7, "chunk_eos_token_id": -1, "chunk_tts_eos_token_id": -1, "turn_eos_token_id": 9}, + ) + next_logits = torch.zeros((1, 16), dtype=torch.float32) + next_logits[0, 7] = 30.0 + next_logits[0, 10] = 20.0 + next_original_logits = next_logits.clone() + + model.prepare_duplex_sampling(next_logits, SimpleNamespace(), (row,)) + + assert torch.equal(logits, original_logits) + assert torch.equal(next_logits, next_original_logits) + assert state.pending_speech_context is False + assert state.current_turn_ended is False + + +def test_minicpmo_model_hook_same_speech_row_second_step_does_not_rearm_pending_context(): + state = SimpleNamespace( + current_turn_ended=True, + last_terminator_token=9, + pending_terminator_token=None, + pending_speech_context=True, + ) + model, row = _minicpmo_duplex_policy_case(state, {"is_speech": True}) + logits = torch.zeros((1, 16), dtype=torch.float32) + logits[0, 7] = 30.0 + logits[0, 10] = 20.0 + + model.prepare_duplex_sampling(logits, SimpleNamespace(), (row,)) + model._minicpmo45_duplex_row_sessions = {0: (row.session_id, row.incarnation)} + model._minicpmo45_duplex_row_payloads = {0: row.payload} + model._record_minicpmo45_duplex_terminator( + 0, + 10, + {"listen_token_id": 7, "chunk_eos_token_id": -1, "chunk_tts_eos_token_id": -1, "turn_eos_token_id": 9}, + ) + next_logits = torch.zeros((1, 16), dtype=torch.float32) + next_logits[0, 7] = 30.0 + next_logits[0, 8] = -2.0 + next_logits[0, 10] = 20.0 + + model.prepare_duplex_sampling(next_logits, SimpleNamespace(), (row,)) + + assert state.pending_speech_context is False + assert state.current_turn_ended is False + assert torch.isneginf(next_logits[0, 7]) + assert next_logits[0, 8].item() == 30.0 + + +def test_minicpmo_model_hook_mid_turn_speech_redirects_listen_to_tts_bos(): + state = SimpleNamespace( + current_turn_ended=False, + last_terminator_token=None, + pending_terminator_token=None, + ) + model, row = _minicpmo_duplex_policy_case(state, {"is_speech": True}) + logits = torch.zeros((1, 16), dtype=torch.float32) + logits[0, 7] = 30.0 + logits[0, 8] = -2.0 + logits[0, 10] = 20.0 + + model.prepare_duplex_sampling(logits, SimpleNamespace(), (row,)) + + assert torch.isneginf(logits[0, 7]) + assert logits[0, 8].item() == 30.0 + assert logits[0, 10].item() == 20.0 + + +def test_minicpmo_model_hook_ignores_serving_new_user_turn_marker(): + state = SimpleNamespace( + current_turn_ended=False, + last_terminator_token=8, + pending_terminator_token=8, + ) + model, row = _minicpmo_duplex_policy_case( + state, + {"is_speech": True, "new_user_turn": True, "force_speak": True}, + ) + logits = torch.zeros((1, 16), dtype=torch.float32) + logits[0, 7] = 5.0 + logits[0, 10] = 20.0 + model.prepare_duplex_sampling(logits, SimpleNamespace(), (row,)) + + assert state.current_turn_ended is False + assert state.last_terminator_token is None + assert torch.isneginf(logits[0, 7]) + assert logits[0, 8].item() == 5.0 + + +def test_generic_ar_runner_builds_typed_duplex_sampling_rows(): + from vllm_omni.experimental.fullduplex.model_executor import DuplexSamplingHelper + from vllm_omni.worker.gpu_ar_model_runner import GPUARModelRunner + + runner = GPUARModelRunner.__new__(GPUARModelRunner) + runner.input_batch = SimpleNamespace(req_ids=["req-duplex", "req-plain"]) + runner.model_intermediate_buffer = { + "req-duplex": { + "duplex": { + "data_plane": True, + "session_id": "sid-runner-hook", + "incarnation": 4, + "seq": 4, + "payload": {"is_speech": True}, + } + } + } + runner.requests = { + "req-duplex": SimpleNamespace( + sampling_params=SimpleNamespace(max_tokens=32), + ) + } + helper = DuplexSamplingHelper() + helper.active_request_ids = {"req-duplex"} + + rows = helper.rows(runner) + + assert len(rows) == 1 + assert rows[0].row_idx == 0 + assert rows[0].request_id == "req-duplex" + assert rows[0].session_id == "sid-runner-hook" + assert rows[0].incarnation == 4 + assert rows[0].seq == 4 + assert rows[0].payload == {"is_speech": True} + assert rows[0].max_tokens == 32 + + +def test_generic_ar_runner_skips_duplex_rows_without_model_hook(): + from vllm_omni.worker.gpu_ar_model_runner import GPUARModelRunner + + runner = GPUARModelRunner.__new__(GPUARModelRunner) + runner.input_batch = SimpleNamespace( + sampling_metadata=object(), + update_async_output_token_ids=lambda: None, + ) + runner.model = SimpleNamespace(sample=lambda *_args, **_kwargs: None, prefer_model_sampler=False) + runner.sampler = lambda **_kwargs: "standard-sampler" + runner._duplex_sampling_helper = SimpleNamespace( + active_request_ids={"req-duplex"}, + rows=lambda *_args: pytest.fail("duplex rows built without a model hook"), + ) + + assert runner._sample(torch.zeros((1, 4)), spec_decode_metadata=None) == "standard-sampler" + + +def test_plain_model_hook_resolution_does_not_allocate_duplex_tracking(): + from vllm_omni.worker.gpu_ar_model_runner import GPUARModelRunner + + runner = GPUARModelRunner.__new__(GPUARModelRunner) + runner.model = SimpleNamespace() + runner._duplex_sampling_hook = None + runner._duplex_sampling_hook_resolved = False + + assert runner._resolve_duplex_sampling_hook() is None + assert not hasattr(runner, "_duplex_sampling_helper") + + +def test_minicpmo_non_duplex_sample_skips_duplex_row_scan(): + from vllm_omni.worker.gpu_ar_model_runner import GPUARModelRunner + + calls = [] + runner = GPUARModelRunner.__new__(GPUARModelRunner) + runner.input_batch = SimpleNamespace( + sampling_metadata=SimpleNamespace(output_token_ids=[]), + update_async_output_token_ids=lambda: None, + ) + runner.model = SimpleNamespace( + sample=lambda *_args, **_kwargs: "model-sampler", + prefer_model_sampler=True, + skips_model_sampler_output_token_history=True, + prepare_duplex_sampling=lambda *_args: calls.append("prepare"), + ) + runner.sampler = SimpleNamespace() + runner._duplex_sampling_helper = SimpleNamespace( + active_request_ids=set(), + hook_active=False, + rows=lambda *_args: pytest.fail("non-duplex MiniCPM scanned request rows"), + ) + + assert runner._sample(torch.zeros((1, 4)), spec_decode_metadata=None) == "model-sampler" + assert calls == [] + + +def test_minicpmo_duplex_sample_clears_stale_rows_once_without_scanning(): + from vllm_omni.worker.gpu_ar_model_runner import GPUARModelRunner + + calls = [] + runner = GPUARModelRunner.__new__(GPUARModelRunner) + runner.input_batch = SimpleNamespace( + sampling_metadata=SimpleNamespace(output_token_ids=[]), + update_async_output_token_ids=lambda: None, + ) + runner.model = SimpleNamespace( + sample=lambda *_args, **_kwargs: "model-sampler", + prefer_model_sampler=True, + skips_model_sampler_output_token_history=True, + prepare_duplex_sampling=lambda _logits, _metadata, rows: calls.append(rows), + ) + runner.sampler = SimpleNamespace() + runner._duplex_sampling_helper = SimpleNamespace( + active_request_ids=set(), + hook_active=True, + rows=lambda *_args: pytest.fail("duplex cleanup scanned request rows"), + ) + + assert runner._sample(torch.zeros((1, 4)), spec_decode_metadata=None) == "model-sampler" + assert runner._sample(torch.zeros((1, 4)), spec_decode_metadata=None) == "model-sampler" + assert calls == [()] + + +def test_generic_ar_runner_has_no_minicpmo_sampler_state_or_typeerror_probe(): + from vllm_omni.worker.gpu_ar_model_runner import GPUARModelRunner + + source = inspect.getsource(GPUARModelRunner) + + assert "_minicpmo45_duplex_row" not in source + assert "_minicpmo45_native_duplex_token_ids" not in source + assert 'if "duplex_rows" not in str(exc)' not in source + + +def test_minicpmo_model_cleans_incarnation_state_when_request_finishes(): + from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni import ( + MiniCPMO45OmniForConditionalGeneration, + ) + + request_id = "duplex-sid-cleanup-i3-e0-stage0" + session_key = ("sid-cleanup", 3) + model = MiniCPMO45OmniForConditionalGeneration.__new__(MiniCPMO45OmniForConditionalGeneration) + model.model = SimpleNamespace() + model._minicpmo45_duplex_data_plane_helper = SimpleNamespace(sessions={session_key: object()}) + model._minicpmo45_duplex_request_sessions = {request_id: session_key} + model._minicpmo45_force_listen_applied_segments = { + (request_id, 1), + ("duplex-sid-other-e0-stage0", 2), + } + + model.on_requests_finished({request_id}) + + assert model._minicpmo45_duplex_data_plane_helper.sessions == {} + assert model._minicpmo45_duplex_request_sessions == {} + assert model._minicpmo45_force_listen_applied_segments == { + ("duplex-sid-other-e0-stage0", 2), + } + + +def test_minicpmo_stage0_routes_duplex_metadata_per_batched_request(): + from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni import ( + MiniCPMO45OmniForConditionalGeneration, + ) + from vllm_omni.utils.mm_outputs import to_payload_element + + class _Thinker(torch.nn.Module): + def __init__(self): + super().__init__() + self.register_buffer("device_anchor", torch.zeros(1)) + + def forward(self, *, input_ids, **kwargs): + del kwargs + return torch.arange(input_ids.numel() * 4, dtype=torch.float32).reshape(input_ids.numel(), 4) + + model = MiniCPMO45OmniForConditionalGeneration.__new__(MiniCPMO45OmniForConditionalGeneration) + torch.nn.Module.__init__(model) + model.model_stage = "llm" + model.thinker = _Thinker() + output = model.forward( + input_ids=torch.tensor([1, 2, 3, 4]), + positions=torch.arange(4), + runtime_additional_information=[ + { + "global_request_id": ["req-a"], + "duplex": { + "duplex_prompt_token_ids": [101, 102], + "special_token_ids": {"listen_token_id": 701}, + }, + }, + { + "global_request_id": ["req-b"], + "duplex": { + "duplex_prompt_token_ids": [201, 202, 203], + "special_token_ids": {"listen_token_id": 702}, + }, + }, + ], + ) + + prompt_rows = output.multimodal_outputs["duplex_prompt_token_ids"] + listen_rows = output.multimodal_outputs["meta"]["listen_token_id"] + assert to_payload_element(prompt_rows, 0, 0, 2) == [101, 102] + assert to_payload_element(prompt_rows, 1, 2, 4) == [201, 202, 203] + assert int(to_payload_element(listen_rows, 0, 0, 2).reshape(-1)[0]) == 701 + assert int(to_payload_element(listen_rows, 1, 2, 4).reshape(-1)[0]) == 702 + + +def test_minicpmo_stage0_rejects_invalid_resolved_ref_audio(): + from vllm_omni.experimental.fullduplex.minicpmo45.stage0 import ( + MiniCPMO45Stage0DuplexRuntime, + ) + + with pytest.raises(ValueError, match="invalid native duplex ref_audio_data"): + MiniCPMO45Stage0DuplexRuntime._decode_ref_audio_from_session_config( + { + "ref_audio_data": "a", + "ref_audio_format": "pcm_f32le", + } + ) + + +def test_minicpmo_tts_native_duplex_exports_segment_text_not_accumulated_condition_text(): + from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni import ( + MiniCPMO45OmniForConditionalGeneration, + ) + + class _Talker: + _ar_last_chunk_flags = [False] + _ar_last_emitted_text = "你好,你有什莫想聊的吗?" + + def __call__(self, **kwargs): + del kwargs + return None, torch.zeros(8, dtype=torch.float32) + + model = MiniCPMO45OmniForConditionalGeneration.__new__(MiniCPMO45OmniForConditionalGeneration) + model.model_stage = "tts" + model.config = SimpleNamespace(hidden_size=4) + model.talker = _Talker() + + output = model.forward( + input_ids=torch.zeros(1, dtype=torch.long), + positions=torch.zeros(1, dtype=torch.long), + runtime_additional_information=[ + { + "native_duplex": True, + "llm_output_text": ["你好,你有什莫想聊的吗?你好,你有什莫想聊的吗?"], + "meta": { + "native_duplex_segment_text": "你好,你有什莫想聊的吗?", + }, + } + ], + ) + + text_bytes = output.multimodal_outputs["meta.llm_output_text_utf8"].detach().cpu().tolist() + assert bytes(text_bytes).decode("utf-8") == "你好,你有什莫想聊的吗?" + assert int(output.multimodal_outputs["meta.audio_text_total_chars"].item()) == len("你好,你有什莫想聊的吗?") + + +def test_minicpmo_tts_native_duplex_exports_model_turn_end_metadata(): + from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni import ( + MiniCPMO45OmniForConditionalGeneration, + ) + + class _Talker: + _ar_last_chunk_flags = [True] + _ar_turn_end_flags = [True] + + def __call__(self, **kwargs): + del kwargs + return None, torch.zeros(0, dtype=torch.float32) + + model = MiniCPMO45OmniForConditionalGeneration.__new__(MiniCPMO45OmniForConditionalGeneration) + model.model_stage = "tts" + model.config = SimpleNamespace(hidden_size=4) + model.talker = _Talker() + + output = model.forward( + input_ids=torch.zeros(1, dtype=torch.long), + positions=torch.zeros(1, dtype=torch.long), + runtime_additional_information=[ + { + "native_duplex": True, + "meta": {"native_duplex_segment_text": ""}, + } + ], + ) + + assert int(output.multimodal_outputs["meta.turn_end"].item()) == 1 + + +def test_minicpmo_tts_routes_batched_requests_and_terminal_flags_independently(): + from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni import ( + MiniCPMO45OmniForConditionalGeneration, + ) + + class _Talker: + def __init__(self): + self.calls = [] + self._ar_last_chunk_flags = [True] + self._ar_turn_end_flags = [False] + self._ar_last_emitted_text = "" + + def __call__(self, **kwargs): + info = kwargs["additional_information"] + request_id = info["global_request_id"][0] + self.calls.append(request_id) + is_last = request_id == "req-b" + self._ar_last_chunk_flags = [is_last] + self._ar_turn_end_flags = [is_last] + self._ar_last_emitted_text = request_id + value = 1.0 if request_id == "req-a" else 2.0 + return None, torch.full((2,), value, dtype=torch.float32) + + model = MiniCPMO45OmniForConditionalGeneration.__new__(MiniCPMO45OmniForConditionalGeneration) + model.model_stage = "tts" + model.config = SimpleNamespace(hidden_size=4) + model.talker = _Talker() + + output = model.forward( + input_ids=torch.zeros(3, dtype=torch.long), + positions=torch.zeros(3, dtype=torch.long), + runtime_additional_information=[ + { + "global_request_id": ["req-a"], + "native_duplex": True, + "duplex": {"epoch": 3, "turn_id": 7}, + "meta": {"native_duplex_segment_text": "first"}, + }, + { + "global_request_id": ["req-idle"], + "native_duplex": True, + "duplex": {"epoch": 3, "turn_id": 9}, + "meta": {"native_duplex_segment_text": "idle"}, + }, + { + "global_request_id": ["req-b"], + "native_duplex": True, + "duplex": {"epoch": 4, "turn_id": 8}, + "meta": {"native_duplex_segment_text": "second"}, + }, + ], + request_token_spans=[(0, 2), (2, 2), (2, 3)], + ) + + assert model.talker.calls == ["req-a", "req-b"] + assert model.talker._ar_last_chunk_flags == [False, False, True] + assert model.talker._ar_turn_end_flags == [False, False, True] + assert output.multimodal_outputs["meta.req_id"] == ["req-a", "req-b"] + assert output.multimodal_outputs["meta.sparse_audio"] == ["1"] + assert [waveform.tolist() for waveform in output.multimodal_outputs["model_outputs"]] == [ + [1.0, 1.0], + [2.0, 2.0], + ] + assert [int(value.item()) for value in output.multimodal_outputs["meta.tts_is_last_chunk"]] == [0, 1] + assert [int(value.item()) for value in output.multimodal_outputs["meta.turn_end"]] == [0, 1] + assert [int(value.item()) for value in output.multimodal_outputs["meta.duplex_epoch"]] == [3, 4] + assert [int(value.item()) for value in output.multimodal_outputs["meta.duplex_turn_id"]] == [7, 8] + + +@pytest.mark.parametrize( + ("runtime_info", "request_token_spans", "expected_rows", "expected_ids"), + [ + ( + [ + { + "global_request_id": ["req-a"], + "native_duplex": True, + "duplex": {"epoch": 3, "turn_id": 7}, + "meta": {"native_duplex_segment_text": "first"}, + } + ], + [(0, 3)], + [3], + ["req-a"], + ), + ( + [ + { + "global_request_id": ["req-a"], + "native_duplex": True, + "duplex": {"epoch": 3, "turn_id": 7}, + "meta": {"native_duplex_segment_text": "first"}, + }, + { + "global_request_id": ["req-b"], + "native_duplex": True, + "duplex": {"epoch": 4, "turn_id": 8}, + "meta": {"native_duplex_segment_text": "second"}, + }, + ], + [(0, 2), (2, 3)], + [2, 1], + ["req-a", "req-b"], + ), + ], + ids=["single-request", "batched-requests"], +) +def test_minicpmo_tts_ignores_cuda_graph_padding_after_request_spans( + runtime_info, + request_token_spans, + expected_rows, + expected_ids, +): + from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni import ( + MiniCPMO45OmniForConditionalGeneration, + ) + + class _Talker: + def __init__(self): + self.input_rows = [] + self._ar_last_chunk_flags = [True] + self._ar_turn_end_flags = [False] + self._ar_last_emitted_text = "" + + def __call__(self, **kwargs): + self.input_rows.append(int(kwargs["input_ids"].shape[0])) + return None, torch.ones(2, dtype=torch.float32) + + model = MiniCPMO45OmniForConditionalGeneration.__new__(MiniCPMO45OmniForConditionalGeneration) + model.model_stage = "tts" + model.config = SimpleNamespace(hidden_size=4) + model.talker = _Talker() + + output = model.forward( + input_ids=torch.zeros(4, dtype=torch.long), + positions=torch.zeros(4, dtype=torch.long), + runtime_additional_information=runtime_info, + request_token_spans=request_token_spans, + ) + + assert model.talker.input_rows == expected_rows + assert output.text_hidden_states.shape == (4, 4) + assert output.multimodal_outputs["meta.req_id"] == expected_ids + + +def test_minicpmo_stage0_special_token_ids_are_tokenizer_derived(): + from vllm_omni.experimental.fullduplex.minicpmo45.stage0 import ( + MiniCPMO45Stage0DuplexRuntime, + ) + + class _Tokenizer: + unk_token_id = 0 + ids = { + "": 101, + "": 102, + "<|listen|>": 103, + "<|speak|>": 104, + "<|tts_bos|>": 105, + "<|tts_eos|>": 106, + "<|tts_pad|>": 107, + "<|chunk_eos|>": 108, + "<|chunk_tts_eos|>": 109, + "<|turn_eos|>": 110, + } + + def convert_tokens_to_ids(self, token): + return self.ids.get(token, self.unk_token_id) + + def encode(self, text, add_special_tokens=False): + del add_special_tokens + return [self.ids[text]] if text in self.ids else [201, self.ids["<|tts_bos|>"]] + + runtime = MiniCPMO45Stage0DuplexRuntime.__new__(MiniCPMO45Stage0DuplexRuntime) + runtime.tokenizer = _Tokenizer() + runtime._init_token_ids() + + runtime._require_special_token_ids() + assert runtime.tts_bos_token_id == 105 + assert runtime.stage_padding_token_id() == 102 + assert runtime._special_token_ids()["chunk_tts_eos_token_id"] == 109 + + +def test_minicpmo_stage0_rejects_unknown_special_token_fallbacks(): + from vllm_omni.experimental.fullduplex.minicpmo45.stage0 import ( + MiniCPMO45Stage0DuplexRuntime, + ) + + class _Tokenizer: + unk_token_id = 0 + ids = { + "": 101, + "": 102, + "<|listen|>": 103, + "<|speak|>": 104, + "<|tts_eos|>": 106, + "<|tts_pad|>": 107, + "<|chunk_eos|>": 108, + "<|chunk_tts_eos|>": 109, + "<|turn_eos|>": 110, + } + + def convert_tokens_to_ids(self, token): + return self.ids.get(token, self.unk_token_id) + + def encode(self, text, add_special_tokens=False): + del text, add_special_tokens + return [self.unk_token_id] + + runtime = MiniCPMO45Stage0DuplexRuntime.__new__(MiniCPMO45Stage0DuplexRuntime) + runtime.tokenizer = _Tokenizer() + runtime._init_token_ids() + + with pytest.raises(ValueError, match=r"<\|tts_bos\|>"): + runtime._require_special_token_ids() + + +def test_minicpmo_stage0_data_plane_prefill_matches_official_unit_format(): + import torch + + from vllm_omni.experimental.fullduplex.minicpmo45.stage0 import ( + MiniCPMO45Stage0DuplexRuntime, + _MiniCPMO45Stage0SessionState, + ) + + class _StageModel(torch.nn.Module): + def __init__(self): + super().__init__() + self.embed = torch.nn.Embedding(256, 2) + + def get_input_embeddings(self): + return self.embed + + def get_audio_hidden_states(self, _data): + return [torch.tensor([[0.5, 0.5]], dtype=torch.float32)] + + runtime = MiniCPMO45Stage0DuplexRuntime.__new__(MiniCPMO45Stage0DuplexRuntime) + runtime.stage_model = _StageModel() + runtime.thinker = runtime.stage_model + runtime.tokenizer = SimpleNamespace( + unk_token_id=0, + convert_tokens_to_ids=lambda token: { + "": 1, + "": 2, + "<|listen|>": 3, + "<|speak|>": 4, + "<|tts_bos|>": 5, + "<|tts_eos|>": 6, + "<|tts_pad|>": 7, + "<|chunk_eos|>": 8, + "<|chunk_tts_eos|>": 9, + "<|turn_eos|>": 10, + "<|audio|>": 11, + }.get(token, 0), + encode=lambda text, add_special_tokens=False: [201, 5], + ) + runtime.processor = SimpleNamespace(get_streaming_chunk_size=lambda: 4) + runtime.device = "cpu" + runtime._init_token_ids() + state = _MiniCPMO45Stage0SessionState(session_id="sid-data-plane-prefill") + + # Official duplex format: each unit is + audio embeddings with no + # per-chunk assistant header or <|tts_bos|> boundary. Decoding starts right + # after the audio so the first sampled token is the listen/speak decision. + result = runtime._stage_prefill_embeddings_only(state, np.zeros(4, dtype=np.float32), seq=1) + + assert result["success"] is True + assert result["input_token_ids"] == [1, 11] + assert result["prompt_suffix_len"] == 0 + + # Subsequent units must close the previous unit with first, + # mirroring the official finalize_unit() feed. + result = runtime._stage_prefill_embeddings_only(state, np.zeros(4, dtype=np.float32), seq=2) + + assert result["success"] is True + assert result["input_token_ids"] == [2, 1, 11] + assert result["prompt_suffix_len"] == 0 + + +def test_minicpmo_stage0_speech_append_sets_pending_context_once(): + import torch + + from vllm_omni.experimental.fullduplex.minicpmo45.stage0 import ( + MiniCPMO45Stage0DuplexRuntime, + _MiniCPMO45Stage0SessionState, + ) + + class _StageModel(torch.nn.Module): + def __init__(self): + super().__init__() + self.embed = torch.nn.Embedding(256, 2) + + def get_input_embeddings(self): + return self.embed + + def get_audio_hidden_states(self, _data): + return [torch.tensor([[0.5, 0.5]], dtype=torch.float32)] + + runtime = MiniCPMO45Stage0DuplexRuntime.__new__(MiniCPMO45Stage0DuplexRuntime) + runtime.stage_model = _StageModel() + runtime.thinker = runtime.stage_model + runtime.tokenizer = SimpleNamespace( + unk_token_id=0, + convert_tokens_to_ids=lambda token: { + "": 1, + "": 2, + "<|listen|>": 3, + "<|speak|>": 4, + "<|tts_bos|>": 5, + "<|tts_eos|>": 6, + "<|tts_pad|>": 7, + "<|chunk_eos|>": 8, + "<|chunk_tts_eos|>": 9, + "<|turn_eos|>": 10, + "<|audio|>": 11, + }.get(token, 0), + encode=lambda text, add_special_tokens=False: [], + ) + runtime.processor = SimpleNamespace(get_streaming_chunk_size=lambda: 4) + runtime.device = "cpu" + runtime._init_token_ids() + state = _MiniCPMO45Stage0SessionState(session_id="sid-speech-append") + + result = runtime._stage_prefill_embeddings_only( + state, + np.zeros(4, dtype=np.float32), + epoch=0, + seq=1, + is_speech=True, + ) + + assert result["success"] is True + assert state.pending_speech_context is True + assert state.pending_speech_append_identity == (0, 1) + + state.pending_speech_context = False + cached = runtime._stage_prefill_embeddings_only( + state, + np.zeros(4, dtype=np.float32), + epoch=0, + seq=1, + is_speech=True, + ) + + assert cached["success"] is True + assert state.pending_speech_context is False + assert state.pending_speech_append_identity == (0, 1) + + next_seq = runtime._stage_prefill_embeddings_only( + state, + np.zeros(4, dtype=np.float32), + epoch=0, + seq=2, + is_speech=True, + ) + + assert next_seq["success"] is True + assert state.pending_speech_context is True + assert state.pending_speech_append_identity == (0, 2) + + state.pending_speech_context = False + next_epoch = runtime._stage_prefill_embeddings_only( + state, + np.zeros(4, dtype=np.float32), + epoch=1, + seq=1, + is_speech=True, + ) + + assert next_epoch["success"] is True + assert state.pending_speech_context is True + assert state.pending_speech_append_identity == (1, 1) + + state.pending_speech_context = False + silence = runtime._stage_prefill_embeddings_only( + state, + np.zeros(4, dtype=np.float32), + epoch=1, + seq=2, + is_speech=False, + ) + + assert silence["success"] is True + assert state.pending_speech_context is False + assert state.pending_speech_append_identity == (1, 1) + + +def test_minicpmo_stage0_failed_append_does_not_set_pending_speech_context(): + import torch + + from vllm_omni.experimental.fullduplex.minicpmo45.stage0 import ( + MiniCPMO45Stage0DuplexRuntime, + _MiniCPMO45Stage0SessionState, + ) + + class _StageModel(torch.nn.Module): + def __init__(self): + super().__init__() + self.embed = torch.nn.Embedding(256, 2) + + def get_input_embeddings(self): + return self.embed + + def get_audio_hidden_states(self, _data): + return [torch.tensor([[0.5, 0.5]], dtype=torch.float32)] + + runtime = MiniCPMO45Stage0DuplexRuntime.__new__(MiniCPMO45Stage0DuplexRuntime) + runtime.stage_model = _StageModel() + runtime.thinker = runtime.stage_model + runtime.tokenizer = SimpleNamespace( + unk_token_id=0, + convert_tokens_to_ids=lambda token: { + "": 1, + "": 2, + "<|listen|>": 3, + "<|speak|>": 4, + "<|tts_bos|>": 5, + "<|tts_eos|>": 6, + "<|tts_pad|>": 7, + "<|chunk_eos|>": 8, + "<|chunk_tts_eos|>": 9, + "<|turn_eos|>": 10, + "<|audio|>": 11, + }.get(token, 0), + encode=lambda text, add_special_tokens=False: [], + ) + runtime.processor = SimpleNamespace(get_streaming_chunk_size=lambda: 4) + runtime.device = "cpu" + runtime._init_token_ids() + state = _MiniCPMO45Stage0SessionState(session_id="sid-failed-speech") + + result = runtime._stage_prefill_embeddings_only( + state, + np.zeros(0, dtype=np.float32), + epoch=0, + seq=1, + is_speech=True, + ) + + assert result["success"] is False + assert state.pending_speech_context is False + assert state.pending_speech_append_identity is None + + +def test_minicpmo_stage0_streaming_processor_is_isolated_per_session(): + from vllm_omni.experimental.fullduplex.minicpmo45.stage0 import ( + MiniCPMO45Stage0DuplexRuntime, + _MiniCPMO45Stage0SessionState, + ) + + class _Mel: + def __init__(self): + self.counter = 0 + + class _Processor: + def __init__(self): + self._streaming_mel_processor = _Mel() + + def set_streaming_mode(self, **_kwargs): + return None + + def reset_streaming(self): + self._streaming_mel_processor.counter = 0 + + def process_audio_streaming(self, _audio, **_kwargs): + self._streaming_mel_processor.counter += 1 + return self._streaming_mel_processor.counter + + runtime = MiniCPMO45Stage0DuplexRuntime.__new__(MiniCPMO45Stage0DuplexRuntime) + runtime.processor = _Processor() + runtime.stage_model = SimpleNamespace() + runtime.thinker = runtime.stage_model + a = _MiniCPMO45Stage0SessionState(session_id="a") + b = _MiniCPMO45Stage0SessionState(session_id="b") + processor_a = runtime._configure_streaming_processor(a) + processor_b = runtime._configure_streaming_processor(b) + + observed = [ + runtime._process_streaming_audio([], 0, processor=processor_a), + runtime._process_streaming_audio([], 0, processor=processor_b), + runtime._process_streaming_audio([], 1, processor=processor_a), + runtime._process_streaming_audio([], 1, processor=processor_b), + ] + + assert observed == [1, 1, 2, 2] + assert processor_a is not processor_b + assert runtime.processor._streaming_mel_processor.counter == 0 + + +def test_minicpmo_stage0_data_plane_next_append_reinjects_previous_listen(): + import torch + + from vllm_omni.experimental.fullduplex.minicpmo45.stage0 import ( + MiniCPMO45Stage0DuplexRuntime, + _MiniCPMO45Stage0SessionState, + ) + + class _StageModel(torch.nn.Module): + def __init__(self): + super().__init__() + self.embed = torch.nn.Embedding(256, 2) + + def get_input_embeddings(self): + return self.embed + + def get_audio_hidden_states(self, data): + return [torch.tensor([[0.5, 0.5]], dtype=torch.float32)] + + runtime = MiniCPMO45Stage0DuplexRuntime.__new__(MiniCPMO45Stage0DuplexRuntime) + runtime.stage_model = _StageModel() + runtime.thinker = runtime.stage_model + runtime.tokenizer = SimpleNamespace( + unk_token_id=0, + convert_tokens_to_ids=lambda token: { + "": 1, + "": 2, + "<|listen|>": 3, + "<|speak|>": 4, + "<|tts_bos|>": 5, + "<|tts_eos|>": 6, + "<|tts_pad|>": 7, + "<|chunk_eos|>": 8, + "<|chunk_tts_eos|>": 9, + "<|turn_eos|>": 10, + "<|audio|>": 11, + }.get(token, 0), + encode=lambda text, add_special_tokens=False: [], + ) + runtime.processor = SimpleNamespace(get_streaming_chunk_size=lambda: 4) + runtime.device = "cpu" + runtime._init_token_ids() + state = _MiniCPMO45Stage0SessionState( + session_id="sid-new-speech-prefill", + audio_chunk_idx=1, + pending_terminator_token=3, + last_terminator_token=3, + current_turn_ended=True, + ) + + result = runtime._stage_prefill_embeddings_only( + state, + np.zeros(4, dtype=np.float32), + seq=2, + ) + + assert result["success"] is True + assert result["input_token_ids"] == [3, 2, 1, 11] + assert state.pending_terminator_token is None + assert state.last_terminator_token == 3 + assert state.current_turn_ended is True + + +def test_minicpmo_stage0_data_plane_turn_eos_closes_previous_unit(): + import torch + + from vllm_omni.experimental.fullduplex.minicpmo45.stage0 import ( + MiniCPMO45Stage0DuplexRuntime, + _MiniCPMO45Stage0SessionState, + ) + + class _StageModel(torch.nn.Module): + def __init__(self): + super().__init__() + self.embed = torch.nn.Embedding(256, 2) + + def get_input_embeddings(self): + return self.embed + + def get_audio_hidden_states(self, data): + return [torch.tensor([[0.5, 0.5]], dtype=torch.float32)] + + runtime = MiniCPMO45Stage0DuplexRuntime.__new__(MiniCPMO45Stage0DuplexRuntime) + runtime.stage_model = _StageModel() + runtime.thinker = runtime.stage_model + runtime.tokenizer = SimpleNamespace( + unk_token_id=0, + convert_tokens_to_ids=lambda token: { + "": 1, + "": 2, + "<|listen|>": 3, + "<|speak|>": 4, + "<|tts_bos|>": 5, + "<|tts_eos|>": 6, + "<|tts_pad|>": 7, + "<|chunk_eos|>": 8, + "<|chunk_tts_eos|>": 9, + "<|turn_eos|>": 10, + "<|audio|>": 11, + }.get(token, 0), + encode=lambda text, add_special_tokens=False: [], + ) + runtime.processor = SimpleNamespace(get_streaming_chunk_size=lambda: 4) + runtime.device = "cpu" + runtime._init_token_ids() + state = _MiniCPMO45Stage0SessionState( + session_id="sid-new-user-turn-prefill", + audio_chunk_idx=1, + pending_terminator_token=10, + last_terminator_token=10, + current_turn_ended=True, + ) + + result = runtime._stage_prefill_embeddings_only( + state, + np.zeros(4, dtype=np.float32), + seq=2, + ) + + assert result["success"] is True + assert result["input_token_ids"] == [10, 2, 1, 11] + assert state.pending_terminator_token is None + assert state.last_terminator_token == 10 + assert state.current_turn_ended is True + + +def test_minicpmo_stage0_data_plane_model_owned_turn_boundary_preserves_audio_cache(): + import torch + + from vllm_omni.experimental.fullduplex.minicpmo45.stage0 import ( + MiniCPMO45Stage0DuplexRuntime, + _MiniCPMO45Stage0SessionState, + ) + + stale_cache = object() + + class _StageModel(torch.nn.Module): + def __init__(self): + super().__init__() + self.embed = torch.nn.Embedding(256, 2) + self.audio_past_key_values = stale_cache + + def get_input_embeddings(self): + return self.embed + + def get_audio_hidden_states(self, data): + return [torch.tensor([[0.5, 0.5]], dtype=torch.float32)] + + runtime = MiniCPMO45Stage0DuplexRuntime.__new__(MiniCPMO45Stage0DuplexRuntime) + runtime.stage_model = _StageModel() + runtime.thinker = runtime.stage_model + runtime.tokenizer = SimpleNamespace( + unk_token_id=0, + convert_tokens_to_ids=lambda token: { + "": 1, + "": 2, + "<|listen|>": 3, + "<|speak|>": 4, + "<|tts_bos|>": 5, + "<|tts_eos|>": 6, + "<|tts_pad|>": 7, + "<|chunk_eos|>": 8, + "<|chunk_tts_eos|>": 9, + "<|turn_eos|>": 10, + "<|audio|>": 11, + }.get(token, 0), + encode=lambda text, add_special_tokens=False: [], + ) + runtime.processor = SimpleNamespace(get_streaming_chunk_size=lambda: 4) + runtime.device = "cpu" + runtime._init_token_ids() + state = _MiniCPMO45Stage0SessionState( + session_id="sid-new-user-turn-audio-cache", + audio_chunk_idx=1, + audio_past_key_values=stale_cache, + pending_terminator_token=8, + last_terminator_token=8, + current_turn_ended=True, + ) + + result = runtime._stage_prefill_embeddings_only( + state, + np.zeros(4, dtype=np.float32), + seq=2, + ) + + assert result["success"] is True + assert state.audio_past_key_values is stale_cache + assert runtime.thinker.audio_past_key_values is stale_cache + + +def test_minicpmo_stage0_data_plane_final_first_chunk_does_not_add_silence_unit(): + import torch + + from vllm_omni.experimental.fullduplex.minicpmo45.stage0 import ( + MiniCPMO45Stage0DuplexRuntime, + _MiniCPMO45Stage0SessionState, + ) + + class _StageModel(torch.nn.Module): + first_chunk_ms = 10 + sample_rate = 1000 + + def __init__(self): + super().__init__() + self.seen_audio = None + self.embed = torch.nn.Embedding(256, 2) + + def get_input_embeddings(self): + return self.embed + + def get_audio_hidden_states(self, data): + self.seen_audio = np.asarray(data["audio_features"], dtype=np.float32) + return [torch.tensor([[0.5, 0.5]], dtype=torch.float32)] + + class _MelProcessor: + sample_rate = 1000 + + def get_config(self): + return {"effective_first_chunk_ms": 10} + + runtime = MiniCPMO45Stage0DuplexRuntime.__new__(MiniCPMO45Stage0DuplexRuntime) + runtime.stage_model = _StageModel() + runtime.thinker = runtime.stage_model + runtime.tokenizer = SimpleNamespace( + unk_token_id=0, + convert_tokens_to_ids=lambda token: { + "": 1, + "": 2, + "<|listen|>": 3, + "<|speak|>": 4, + "<|tts_bos|>": 5, + "<|tts_eos|>": 6, + "<|tts_pad|>": 7, + "<|chunk_eos|>": 8, + "<|chunk_tts_eos|>": 9, + "<|turn_eos|>": 10, + "<|audio|>": 11, + }.get(token, 0), + encode=lambda text, add_special_tokens=False: [], + ) + runtime.processor = SimpleNamespace( + _streaming_mel_processor=_MelProcessor(), + get_streaming_chunk_size=lambda: 10, + ) + runtime.device = "cpu" + runtime._init_token_ids() + state = _MiniCPMO45Stage0SessionState(session_id="sid-first-chunk-padding") + + result = runtime._stage_prefill_embeddings_only( + state, + np.arange(8, dtype=np.float32), + seq=1, + final=True, + ) + + assert result["success"] is True + assert result["input_token_ids"] == [1, 11] + assert runtime.stage_model.seen_audio is not None + np.testing.assert_allclose( + runtime.stage_model.seen_audio.reshape(-1), + np.array([0, 0, 0, 1, 2, 3, 4, 5, 6, 7], dtype=np.float32), + ) + + +def test_minicpmo_stage0_runtime_uses_loaded_vllm_embed_tokens_when_get_input_embeddings_is_broken(): + import torch + + from vllm_omni.experimental.fullduplex.minicpmo45.stage0 import ( + MiniCPMO45Stage0DuplexRuntime, + ) + + class _Embed(torch.nn.Module): + def __init__(self): + super().__init__() + self.weight = torch.nn.Parameter(torch.ones(128, 2)) + self.calls = [] + + def forward(self, input_ids): + ids = input_ids.reshape(-1).tolist() + self.calls.append(ids) + return torch.tensor([[float(i), 0.0] for i in ids], dtype=torch.float32) + + class _Thinker: + def __init__(self): + self.llm = SimpleNamespace(model=SimpleNamespace(embed_tokens=_Embed())) + + def get_input_embeddings(self, input_ids, multimodal_embeddings=None): + raise AttributeError("'Qwen3ForCausalLM' object has no attribute 'get_input_embeddings'") + + thinker = _Thinker() + stage_model = SimpleNamespace(model_stage="llm", thinker=thinker, processor=None) + runtime = MiniCPMO45Stage0DuplexRuntime.__new__(MiniCPMO45Stage0DuplexRuntime) + runtime.stage_model = stage_model + runtime.thinker = thinker + runtime.device = "cpu" + + embeds = runtime._embed_token(11) + + assert embeds.shape == (1, 2) + assert thinker.llm.model.embed_tokens.calls == [[11]] + + +def test_minicpmo_remote_config_patch_handles_nested_and_dict_configs(): + from vllm_omni.experimental.fullduplex.minicpmo45.compat import ( + patch_minicpmo_remote_config, + ) + + nested = SimpleNamespace(base_model_tp_plan=None) + config = SimpleNamespace( + base_model_tp_plan=None, + text_config=nested, + tts_config={}, + ) + + patch_minicpmo_remote_config(config) + + assert config.base_model_tp_plan == {} + assert nested.base_model_tp_plan == {} + assert config.tts_config["top_p"] == 0.8 + assert config.tts_config["top_k"] == 100 + assert config.tts_config["temperature"] == 0.8 + assert config.tts_config["repetition_penalty"] == 1.05 + + +def test_minicpmo_stage0_short_audio_buffers_without_context_mutation(): + from vllm_omni.experimental.fullduplex.minicpmo45.stage0 import ( + MiniCPMO45Stage0DuplexRuntime, + _MiniCPMO45Stage0SessionState, + ) + + class _Processor: + def get_streaming_chunk_size(self): + return 16000 + + runtime = MiniCPMO45Stage0DuplexRuntime.__new__(MiniCPMO45Stage0DuplexRuntime) + runtime.stage_model = SimpleNamespace() + runtime.thinker = SimpleNamespace() + runtime.processor = _Processor() + runtime._require_special_token_ids = lambda: None + state = _MiniCPMO45Stage0SessionState(session_id="sid") + + result = runtime._stage_prefill_embeddings_only(state, np.zeros(1600, dtype=np.float32)) + + assert result["success"] is False + assert result["reason"] + assert len(state.audio_buffer) >= 1600 + assert state.context_embeds == [] + + +def test_minicpmo_stage0_native_sampler_penalizes_repeated_text_token(): + from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni import ( + MiniCPMO45OmniForConditionalGeneration, + ) + + class _Tokenizer: + eos_token_id = 151705 + unk_token_id = -1 + bad_token_ids = [] + all_special_ids = [] + + def convert_tokens_to_ids(self, token): + return { + "": 151683, + "": 151684, + "<|listen|>": 151705, + "<|speak|>": 151706, + "<|tts_bos|>": 151703, + "<|tts_eos|>": 151704, + "<|tts_pad|>": 151722, + "<|chunk_eos|>": 151718, + "<|chunk_tts_eos|>": 151721, + "<|turn_eos|>": 151717, + }.get(token, -1) + + model = MiniCPMO45OmniForConditionalGeneration.__new__(MiniCPMO45OmniForConditionalGeneration) + model.model_stage = "llm" + model.thinker = SimpleNamespace(get_tokenizer=lambda: _Tokenizer()) + vocab_size = 151723 + repeated = 198 + alternative = 1234 + logits = torch.full((1, vocab_size), -100.0) + logits[0, repeated] = 20.0 + logits[0, alternative] = 19.5 + sampling_metadata = SimpleNamespace( + all_greedy=False, + all_random=True, + temperature=torch.tensor([1.0]), + top_k=torch.tensor([1]), + top_p=torch.tensor([1.0]), + generators={}, + prompt_token_ids=torch.tensor([[151683] * 16]), + output_token_ids=[[repeated] * 8], + ) + + sampled = model.sample(logits, sampling_metadata) + + assert sampled is not None + assert sampled.sampled_token_ids.tolist() == [[alternative]] + + +def test_minicpmo_stage0_native_sampler_penalizes_text_from_prior_chunk(): + from vllm_omni.experimental.fullduplex.minicpmo45.stage0 import ( + _MiniCPMO45Stage0SessionState, + ) + from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni import ( + MiniCPMO45OmniForConditionalGeneration, + ) + + class _Tokenizer: + eos_token_id = 151705 + unk_token_id = -1 + bad_token_ids = [] + all_special_ids = [] + + def convert_tokens_to_ids(self, token): + return { + "": 151683, + "": 151684, + "<|listen|>": 151705, + "<|speak|>": 151706, + "<|tts_bos|>": 151703, + "<|tts_eos|>": 151704, + "<|tts_pad|>": 151722, + "<|chunk_eos|>": 151718, + "<|chunk_tts_eos|>": 151721, + "<|turn_eos|>": 151717, + }.get(token, -1) + + model = MiniCPMO45OmniForConditionalGeneration.__new__(MiniCPMO45OmniForConditionalGeneration) + model.model_stage = "llm" + model.thinker = SimpleNamespace(get_tokenizer=lambda: _Tokenizer()) + session_key = ("sid-cross-chunk-repetition", 0) + state = _MiniCPMO45Stage0SessionState(session_id=session_key[0]) + state.generated_text_tokens = [198] * 8 + model._minicpmo45_duplex_data_plane_helper = SimpleNamespace(sessions={session_key: state}) + model._minicpmo45_duplex_row_sessions = {0: session_key} + + vocab_size = 151723 + repeated = 198 + alternative = 1234 + logits = torch.full((1, vocab_size), -100.0) + logits[0, repeated] = 20.0 + logits[0, alternative] = 19.5 + sampling_metadata = SimpleNamespace( + all_greedy=False, + all_random=True, + temperature=torch.tensor([1.0]), + top_k=torch.tensor([1]), + top_p=torch.tensor([1.0]), + generators={}, + prompt_token_ids=torch.tensor([[151683] * 16]), + output_token_ids=[[]], + ) + + sampled = model.sample(logits, sampling_metadata) + + assert sampled is not None + assert sampled.sampled_token_ids.tolist() == [[alternative]] + + +def test_minicpmo_stage0_native_sampler_matches_official_negative_logit_penalty(): + from vllm_omni.experimental.fullduplex.minicpmo45.stage0 import ( + _MiniCPMO45Stage0SessionState, + ) + from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni import ( + MiniCPMO45OmniForConditionalGeneration, + ) + + class _Tokenizer: + eos_token_id = 151705 + unk_token_id = -1 + bad_token_ids = [] + all_special_ids = [] + + def convert_tokens_to_ids(self, token): + return { + "": 151683, + "": 151684, + "<|listen|>": 151705, + "<|speak|>": 151706, + "<|tts_bos|>": 151703, + "<|tts_eos|>": 151704, + "<|tts_pad|>": 151722, + "<|chunk_eos|>": 151718, + "<|chunk_tts_eos|>": 151721, + "<|turn_eos|>": 151717, + }.get(token, -1) + + model = MiniCPMO45OmniForConditionalGeneration.__new__(MiniCPMO45OmniForConditionalGeneration) + model.model_stage = "llm" + model.thinker = SimpleNamespace(get_tokenizer=lambda: _Tokenizer()) + session_key = ("sid-negative-logit-repetition", 0) + state = _MiniCPMO45Stage0SessionState(session_id=session_key[0]) + repeated = 198 + alternative = 1234 + state.generated_text_tokens = [repeated] + model._minicpmo45_duplex_data_plane_helper = SimpleNamespace(sessions={session_key: state}) + model._minicpmo45_duplex_row_sessions = {0: session_key} + + logits = torch.full((1, 151723), -100.0) + logits[0, repeated] = -1.0 + logits[0, alternative] = -0.97 + sampling_metadata = SimpleNamespace( + all_greedy=False, + all_random=True, + temperature=torch.tensor([1.0]), + top_k=torch.tensor([1]), + top_p=torch.tensor([1.0]), + generators={}, + prompt_token_ids=torch.tensor([[151683] * 16]), + output_token_ids=[[]], + ) + + sampled = model.sample(logits, sampling_metadata) + + assert sampled is not None + assert sampled.sampled_token_ids.tolist() == [[repeated]] + + +def test_minicpmo_stage0_records_only_bounded_text_repetition_history(): + from vllm_omni.experimental.fullduplex.minicpmo45.stage0 import ( + _MiniCPMO45Stage0SessionState, + ) + from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni import ( + MiniCPMO45OmniForConditionalGeneration, + ) + + session_key = ("sid-text-history", 0) + state = _MiniCPMO45Stage0SessionState(session_id=session_key[0]) + model = MiniCPMO45OmniForConditionalGeneration.__new__(MiniCPMO45OmniForConditionalGeneration) + model._minicpmo45_duplex_data_plane_helper = SimpleNamespace(sessions={session_key: state}) + model._minicpmo45_duplex_row_sessions = {0: session_key} + token_ids = { + "unit_token_id": 1, + "unit_end_token_id": 2, + "listen_token_id": 3, + "speak_token_id": 4, + "tts_bos_token_id": 5, + "tts_eos_token_id": 6, + "tts_pad_token_id": 7, + "chunk_eos_token_id": 8, + "chunk_tts_eos_token_id": 9, + "turn_eos_token_id": 10, + } + + for sampled in range(1000, 1520): + model._record_minicpmo45_duplex_terminator(0, sampled, token_ids) + + expected_history = list(range(1008, 1520)) + assert state.generated_text_tokens == expected_history + + for sampled in token_ids.values(): + model._record_minicpmo45_duplex_terminator(0, sampled, token_ids) + + assert state.generated_text_tokens == expected_history + + +def test_minicpmo_stage0_native_sampler_does_not_override_model_at_punctuation(): + from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni import ( + MiniCPMO45OmniForConditionalGeneration, + ) + + class _Tokenizer: + eos_token_id = 151705 + unk_token_id = -1 + bad_token_ids = [] + all_special_ids = [] + + text = { + 200: "我", + 201: "喜", + 202: "欢", + 203: "。", + } + + def convert_tokens_to_ids(self, token): + return { + "": 151683, + "": 151684, + "<|listen|>": 151705, + "<|speak|>": 151706, + "<|tts_bos|>": 151703, + "<|tts_eos|>": 151704, + "<|tts_pad|>": 151722, + "<|chunk_eos|>": 151718, + "<|chunk_tts_eos|>": 151721, + "<|turn_eos|>": 151717, + }.get(token, -1) + + def decode(self, ids, skip_special_tokens=True): + del skip_special_tokens + return "".join(self.text.get(int(token_id), "") for token_id in ids) + + model = MiniCPMO45OmniForConditionalGeneration.__new__(MiniCPMO45OmniForConditionalGeneration) + model.model_stage = "llm" + model.thinker = SimpleNamespace(get_tokenizer=lambda: _Tokenizer()) + model.min_new_speak_tokens_before_chunk_boundary = 4 + model.max_new_speak_tokens_per_chunk = 64 + vocab_size = 151723 + alternative = 1234 + logits = torch.full((1, vocab_size), -100.0) + logits[0, alternative] = 20.0 + sampling_metadata = SimpleNamespace( + all_greedy=False, + all_random=True, + temperature=torch.tensor([1.0]), + top_k=torch.tensor([1]), + top_p=torch.tensor([1.0]), + generators={}, + prompt_token_ids=torch.tensor([[151683] * 16]), + output_token_ids=[[151706, 200, 201, 202, 203]], + ) + + sampled = model.sample(logits, sampling_metadata) + + assert sampled is not None + assert sampled.sampled_token_ids.tolist() == [[alternative]] + + +def test_minicpmo_stage0_native_sampler_does_not_cut_before_natural_boundary_minimum(): + from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni import ( + MiniCPMO45OmniForConditionalGeneration, + ) + + class _Tokenizer: + eos_token_id = 151705 + unk_token_id = -1 + bad_token_ids = [] + all_special_ids = [] + + def convert_tokens_to_ids(self, token): + return { + "": 151683, + "": 151684, + "<|listen|>": 151705, + "<|speak|>": 151706, + "<|tts_bos|>": 151703, + "<|tts_eos|>": 151704, + "<|tts_pad|>": 151722, + "<|chunk_eos|>": 151718, + "<|chunk_tts_eos|>": 151721, + "<|turn_eos|>": 151717, + }.get(token, -1) + + def decode(self, ids, skip_special_tokens=True): + del ids, skip_special_tokens + return "。" + + model = MiniCPMO45OmniForConditionalGeneration.__new__(MiniCPMO45OmniForConditionalGeneration) + model.model_stage = "llm" + model.thinker = SimpleNamespace(get_tokenizer=lambda: _Tokenizer()) + model.min_new_speak_tokens_before_chunk_boundary = 4 + model.max_new_speak_tokens_per_chunk = 64 + vocab_size = 151723 + alternative = 1234 + logits = torch.full((1, vocab_size), -100.0) + logits[0, alternative] = 20.0 + sampling_metadata = SimpleNamespace( + all_greedy=False, + all_random=True, + temperature=torch.tensor([1.0]), + top_k=torch.tensor([1]), + top_p=torch.tensor([1.0]), + generators={}, + prompt_token_ids=torch.tensor([[151683] * 16]), + output_token_ids=[[151706, 200, 201, 202]], + ) + + sampled = model.sample(logits, sampling_metadata) + + assert sampled is not None + assert sampled.sampled_token_ids.tolist() == [[alternative]] + + +def test_minicpmo_stage0_native_sampler_does_not_rewrite_model_punctuation(): + from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni import ( + MiniCPMO45OmniForConditionalGeneration, + ) + + class _Tokenizer: + eos_token_id = 151705 + unk_token_id = -1 + bad_token_ids = [] + all_special_ids = [] + + text = { + 108386: "你", + 104256: "好", + 3837: ",", + 100644: "今", + 99172: "天", + 100281: "想", + 27442: "聊", + 99217: "什", + 1773: "。", + 99218: "么", + } + + def convert_tokens_to_ids(self, token): + return { + "": 151683, + "": 151684, + "<|listen|>": 151705, + "<|speak|>": 151706, + "<|tts_bos|>": 151703, + "<|tts_eos|>": 151704, + "<|tts_pad|>": 151722, + "<|chunk_eos|>": 151718, + "<|chunk_tts_eos|>": 151721, + "<|turn_eos|>": 151717, + }.get(token, -1) + + def encode(self, text, add_special_tokens=False): + del add_special_tokens + return {"。": [1773]}.get(text, []) + + def decode(self, ids, skip_special_tokens=True): + del skip_special_tokens + return "".join(self.text.get(int(token_id), "") for token_id in ids) + + model = MiniCPMO45OmniForConditionalGeneration.__new__(MiniCPMO45OmniForConditionalGeneration) + model.model_stage = "llm" + model.thinker = SimpleNamespace(get_tokenizer=lambda: _Tokenizer()) + model.min_new_speak_tokens_before_chunk_boundary = 4 + model.max_new_speak_tokens_per_chunk = 64 + vocab_size = 151723 + period = 1773 + continuation = 99218 + logits = torch.full((1, vocab_size), -100.0) + logits[0, period] = 30.0 + logits[0, continuation] = 20.0 + sampling_metadata = SimpleNamespace( + all_greedy=False, + all_random=True, + temperature=torch.tensor([1.0]), + top_k=torch.tensor([1]), + top_p=torch.tensor([1.0]), + generators={}, + prompt_token_ids=torch.tensor([[151683] * 16]), + output_token_ids=[[151706, 108386, 104256, 3837, 100644, 99172, 100281, 27442, 99217]], + ) + + sampled = model.sample(logits, sampling_metadata) + + assert sampled is not None + assert sampled.sampled_token_ids.tolist() == [[period]] + + +def test_minicpmo_stage0_native_sampler_preserves_model_chunk_eos_decision(): + from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni import ( + MiniCPMO45OmniForConditionalGeneration, + ) + + class _Tokenizer: + eos_token_id = 151705 + unk_token_id = -1 + bad_token_ids = [] + all_special_ids = [] + + def convert_tokens_to_ids(self, token): + return { + "": 151683, + "": 151684, + "<|listen|>": 151705, + "<|speak|>": 151706, + "<|tts_bos|>": 151703, + "<|tts_eos|>": 151704, + "<|tts_pad|>": 151722, + "<|chunk_eos|>": 151718, + "<|chunk_tts_eos|>": 151721, + "<|turn_eos|>": 151717, + }.get(token, -1) + + model = MiniCPMO45OmniForConditionalGeneration.__new__(MiniCPMO45OmniForConditionalGeneration) + model.model_stage = "llm" + model.thinker = SimpleNamespace(get_tokenizer=lambda: _Tokenizer()) + logits = torch.full((1, 151723), -100.0) + logits[0, 151718] = 30.0 + logits[0, 1234] = 20.0 + sampling_metadata = SimpleNamespace( + all_greedy=True, + all_random=False, + temperature=torch.tensor([0.0]), + top_k=torch.tensor([1]), + top_p=torch.tensor([1.0]), + generators={}, + prompt_token_ids=torch.tensor([[151683] * 16]), + output_token_ids=[[151706, 200, 201]], + ) + + sampled = model.sample(logits, sampling_metadata) + + assert sampled is not None + assert sampled.sampled_token_ids.tolist() == [[151718]] + + +def test_minicpmo_stage0_native_sampler_preserves_early_model_turn_eos_decision(): + from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni import ( + MiniCPMO45OmniForConditionalGeneration, + ) + + class _Tokenizer: + eos_token_id = 151705 + unk_token_id = -1 + bad_token_ids = [] + all_special_ids = [] + + def convert_tokens_to_ids(self, token): + return { + "": 151683, + "": 151684, + "<|listen|>": 151705, + "<|speak|>": 151706, + "<|tts_bos|>": 151703, + "<|tts_eos|>": 151704, + "<|tts_pad|>": 151722, + "<|chunk_eos|>": 151718, + "<|chunk_tts_eos|>": 151721, + "<|turn_eos|>": 151717, + }.get(token, -1) + + model = MiniCPMO45OmniForConditionalGeneration.__new__(MiniCPMO45OmniForConditionalGeneration) + model.model_stage = "llm" + model.thinker = SimpleNamespace(get_tokenizer=lambda: _Tokenizer()) + model.min_new_speak_tokens_before_chunk_boundary = 8 + logits = torch.full((1, 151723), -100.0) + logits[0, 151717] = 30.0 + logits[0, 1234] = 20.0 + sampling_metadata = SimpleNamespace( + all_greedy=True, + all_random=False, + temperature=torch.tensor([0.0]), + top_k=torch.tensor([1]), + top_p=torch.tensor([1.0]), + generators={}, + prompt_token_ids=torch.tensor([[151683] * 16]), + output_token_ids=[[151706, 200, 201]], + ) + + sampled = model.sample(logits, sampling_metadata) + + assert sampled is not None + assert sampled.sampled_token_ids.tolist() == [[151717]] + + +def test_minicpmo_stage0_native_sampler_char_cap_does_not_override_special_token(): + from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni import ( + MiniCPMO45OmniForConditionalGeneration, + ) + + class _Tokenizer: + eos_token_id = 151705 + unk_token_id = -1 + bad_token_ids = [] + all_special_ids = [151706, 151717] + + def convert_tokens_to_ids(self, token): + return { + "": 151683, + "": 151684, + "<|listen|>": 151705, + "<|speak|>": 151706, + "<|tts_bos|>": 151703, + "<|tts_eos|>": 151704, + "<|tts_pad|>": 151722, + "<|chunk_eos|>": 151718, + "<|chunk_tts_eos|>": 151721, + "<|turn_eos|>": 151717, + }.get(token, -1) + + def decode(self, ids, skip_special_tokens=True): + del skip_special_tokens + return "已经超过二十八个字符的当前语音分段文本" + + model = MiniCPMO45OmniForConditionalGeneration.__new__(MiniCPMO45OmniForConditionalGeneration) + model.model_stage = "llm" + model.thinker = SimpleNamespace(get_tokenizer=lambda: _Tokenizer()) + logits = torch.full((1, 151723), -100.0) + logits[0, 151717] = 30.0 + sampling_metadata = SimpleNamespace( + all_greedy=True, + all_random=False, + temperature=torch.tensor([0.0]), + top_k=torch.tensor([1]), + top_p=torch.tensor([1.0]), + generators={}, + prompt_token_ids=torch.tensor([[151683] * 16]), + output_token_ids=[[151706, 200, 201]], + ) + + sampled = model.sample(logits, sampling_metadata) + + assert sampled is not None + assert sampled.sampled_token_ids.tolist() == [[151717]] + + +def test_minicpmo_stage0_native_sampler_preserves_model_chunk_eos(): + from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni import ( + MiniCPMO45OmniForConditionalGeneration, + ) + + class _Tokenizer: + eos_token_id = 151705 + unk_token_id = -1 + bad_token_ids = [] + all_special_ids = [] + + def convert_tokens_to_ids(self, token): + return { + "": 151683, + "": 151684, + "<|listen|>": 151705, + "<|speak|>": 151706, + "<|tts_bos|>": 151703, + "<|tts_eos|>": 151704, + "<|tts_pad|>": 151722, + "<|chunk_eos|>": 151718, + "<|chunk_tts_eos|>": 151721, + "<|turn_eos|>": 151717, + }.get(token, -1) + + model = MiniCPMO45OmniForConditionalGeneration.__new__(MiniCPMO45OmniForConditionalGeneration) + model.model_stage = "llm" + model.thinker = SimpleNamespace(get_tokenizer=lambda: _Tokenizer()) + model.min_new_speak_tokens_before_chunk_boundary = 8 + model.max_new_speak_tokens_per_chunk = 64 + vocab_size = 151723 + logits = torch.full((1, vocab_size), -100.0) + logits[0, 151718] = 30.0 + logits[0, 1234] = 20.0 + sampling_metadata = SimpleNamespace( + all_greedy=False, + all_random=True, + temperature=torch.tensor([1.0]), + top_k=torch.tensor([1]), + top_p=torch.tensor([1.0]), + generators={}, + prompt_token_ids=torch.tensor([[151683] * 16]), + output_token_ids=[[151706, 200, 201, 202]], + ) + + sampled = model.sample(logits, sampling_metadata) + + assert sampled is not None + assert sampled.sampled_token_ids.tolist() == [[151718]] + + +def test_minicpmo_stage0_native_sampler_keeps_hard_chunk_cap(): + from vllm_omni.experimental.fullduplex.minicpmo45.policy import ( + MiniCPMO45DuplexPolicy, + ) + from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni import ( + MiniCPMO45OmniForConditionalGeneration, + ) + + class _Tokenizer: + eos_token_id = 151705 + unk_token_id = -1 + bad_token_ids = [] + all_special_ids = [] + + def convert_tokens_to_ids(self, token): + return { + "": 151683, + "": 151684, + "<|listen|>": 151705, + "<|speak|>": 151706, + "<|tts_bos|>": 151703, + "<|tts_eos|>": 151704, + "<|tts_pad|>": 151722, + "<|chunk_eos|>": 151718, + "<|chunk_tts_eos|>": 151721, + "<|turn_eos|>": 151717, + }.get(token, -1) + + def decode(self, ids, skip_special_tokens=True): + del ids, skip_special_tokens + return "没有自然边界" + + model = MiniCPMO45OmniForConditionalGeneration.__new__(MiniCPMO45OmniForConditionalGeneration) + model.model_stage = "llm" + model.thinker = SimpleNamespace(get_tokenizer=lambda: _Tokenizer()) + vocab_size = 151723 + alternative = 1234 + logits = torch.full((1, vocab_size), -100.0) + logits[0, alternative] = 20.0 + sampling_metadata = SimpleNamespace( + all_greedy=False, + all_random=True, + temperature=torch.tensor([1.0]), + top_k=torch.tensor([1]), + top_p=torch.tensor([1.0]), + generators={}, + prompt_token_ids=torch.tensor([[151683] * 16]), + output_token_ids=[[200] * (MiniCPMO45DuplexPolicy.DEFAULT_MAX_NEW_SPEAK_TOKENS_PER_CHUNK - 1)], + ) + + sampled = model.sample(logits, sampling_metadata) + + assert MiniCPMO45DuplexPolicy.DEFAULT_MAX_NEW_SPEAK_TOKENS_PER_CHUNK == 20 + assert sampled is not None + assert sampled.sampled_token_ids.tolist() == [[151718]] + + +def test_minicpmo_stage0_native_sampler_cuts_before_request_length_cap(): + from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni import ( + MiniCPMO45OmniForConditionalGeneration, + ) + + class _Tokenizer: + eos_token_id = 151705 + unk_token_id = -1 + bad_token_ids = [] + all_special_ids = [] + + def convert_tokens_to_ids(self, token): + return { + "": 151683, + "": 151684, + "<|listen|>": 151705, + "<|speak|>": 151706, + "<|tts_bos|>": 151703, + "<|tts_eos|>": 151704, + "<|tts_pad|>": 151722, + "<|chunk_eos|>": 151718, + "<|chunk_tts_eos|>": 151721, + "<|turn_eos|>": 151717, + }.get(token, -1) + + def decode(self, ids, skip_special_tokens=True): + del ids, skip_special_tokens + return "没有自然边界" + + model = MiniCPMO45OmniForConditionalGeneration.__new__(MiniCPMO45OmniForConditionalGeneration) + model.model_stage = "llm" + model.thinker = SimpleNamespace(get_tokenizer=lambda: _Tokenizer()) + model.max_new_speak_tokens_per_chunk = 64 + model._minicpmo45_duplex_row_max_tokens = {0: 20} + vocab_size = 151723 + alternative = 1234 + logits = torch.full((1, vocab_size), -100.0) + logits[0, alternative] = 20.0 + sampling_metadata = SimpleNamespace( + all_greedy=False, + all_random=True, + temperature=torch.tensor([1.0]), + top_k=torch.tensor([1]), + top_p=torch.tensor([1.0]), + generators={}, + prompt_token_ids=torch.tensor([[151683] * 16]), + output_token_ids=[[151706] + [200] * 18], + ) + + sampled = model.sample(logits, sampling_metadata) + + assert sampled is not None + assert sampled.sampled_token_ids.tolist() == [[151718]] + + +def test_minicpmo_stage0_native_sampler_cuts_on_decoded_text_length(): + from vllm_omni.experimental.fullduplex.minicpmo45.policy import ( + MiniCPMO45DuplexPolicy, + ) + from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni import ( + MiniCPMO45OmniForConditionalGeneration, + ) + + class _Tokenizer: + eos_token_id = 151705 + unk_token_id = -1 + bad_token_ids = [] + all_special_ids = [151706, 151718, 151717, 151705] + + def convert_tokens_to_ids(self, token): + return { + "": 151683, + "": 151684, + "<|listen|>": 151705, + "<|speak|>": 151706, + "<|tts_bos|>": 151703, + "<|tts_eos|>": 151704, + "<|tts_pad|>": 151722, + "<|chunk_eos|>": 151718, + "<|chunk_tts_eos|>": 151721, + "<|turn_eos|>": 151717, + }.get(token, -1) + + def decode(self, ids, skip_special_tokens=True): + token_text = { + 200: "一二三四五六七八九十", + 201: "十一十二十三十四十五", + 202: "十六十七十八十九", + } + special = set(self.all_special_ids) if skip_special_tokens else set() + return "".join(token_text.get(int(token_id), "") for token_id in ids if int(token_id) not in special) + + model = MiniCPMO45OmniForConditionalGeneration.__new__(MiniCPMO45OmniForConditionalGeneration) + model.model_stage = "llm" + model.thinker = SimpleNamespace(get_tokenizer=lambda: _Tokenizer()) + vocab_size = 151723 + candidate = 202 + logits = torch.full((1, vocab_size), -100.0) + logits[0, candidate] = 20.0 + sampling_metadata = SimpleNamespace( + all_greedy=True, + all_random=False, + temperature=torch.tensor([0.0]), + top_k=torch.tensor([1]), + top_p=torch.tensor([1.0]), + generators={}, + prompt_token_ids=torch.tensor([[151683] * 16]), + output_token_ids=[[151706, 200, 201]], + ) + + sampled = model.sample(logits, sampling_metadata) + + assert MiniCPMO45DuplexPolicy.DEFAULT_MAX_SPEAK_CHARS_PER_CHUNK == 28 + assert sampled is not None + assert sampled.sampled_token_ids.tolist() == [[151718]] + + +def test_minicpmo_stage0_native_sampler_ignores_pending_placeholders(): + from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni import ( + MiniCPMO45OmniForConditionalGeneration, + ) + + class _Tokenizer: + eos_token_id = 151705 + unk_token_id = -1 + bad_token_ids = [] + all_special_ids = [] + + def convert_tokens_to_ids(self, token): + return { + "": 151683, + "": 151684, + "<|listen|>": 151705, + "<|speak|>": 151706, + "<|tts_bos|>": 151703, + "<|tts_eos|>": 151704, + "<|tts_pad|>": 151722, + "<|chunk_eos|>": 151718, + "<|chunk_tts_eos|>": 151721, + "<|turn_eos|>": 151717, + }.get(token, -1) + + model = MiniCPMO45OmniForConditionalGeneration.__new__(MiniCPMO45OmniForConditionalGeneration) + model.model_stage = "llm" + model.thinker = SimpleNamespace(get_tokenizer=lambda: _Tokenizer()) + vocab_size = 151723 + newline = 198 + alternative = 1234 + logits = torch.full((1, vocab_size), -100.0) + logits[0, newline] = 20.0 + logits[0, alternative] = 19.5 + sampling_metadata = SimpleNamespace( + all_greedy=False, + all_random=True, + temperature=torch.tensor([1.0]), + top_k=torch.tensor([1]), + top_p=torch.tensor([1.0]), + generators={}, + prompt_token_ids=torch.tensor([[151683] * 16]), + output_token_ids=[[-1, -1, -1]], + ) + + sampled = model.sample(logits, sampling_metadata) + + assert sampled is not None + assert sampled.sampled_token_ids.tolist() == [[newline]] + + +def test_minicpmo_stage0_native_sampler_converts_mid_turn_listen_to_tts_bos(): + from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni import ( + MiniCPMO45OmniForConditionalGeneration, + ) + + class _Tokenizer: + eos_token_id = 151705 + unk_token_id = -1 + bad_token_ids = [] + all_special_ids = [] + + def convert_tokens_to_ids(self, token): + return { + "": 151683, + "": 151684, + "<|listen|>": 151705, + "<|speak|>": 151706, + "<|tts_bos|>": 151703, + "<|tts_eos|>": 151704, + "<|tts_pad|>": 151722, + "<|chunk_eos|>": 151718, + "<|chunk_tts_eos|>": 151721, + "<|turn_eos|>": 151717, + }.get(token, -1) + + state = SimpleNamespace(current_turn_ended=False) + model = MiniCPMO45OmniForConditionalGeneration.__new__(MiniCPMO45OmniForConditionalGeneration) + model.model_stage = "llm" + model.thinker = SimpleNamespace(get_tokenizer=lambda: _Tokenizer()) + model._minicpmo45_duplex_row_sessions = {0: ("sid-native", 0)} + model._minicpmo45_duplex_data_plane_helper = SimpleNamespace(sessions={("sid-native", 0): state}) + vocab_size = 151723 + logits = torch.full((1, vocab_size), -100.0) + logits[0, 151705] = 30.0 + sampling_metadata = SimpleNamespace( + all_greedy=True, + all_random=False, + temperature=torch.tensor([0.0]), + top_k=torch.tensor([1]), + top_p=torch.tensor([1.0]), + generators={}, + prompt_token_ids=torch.tensor([[151683] * 16]), + output_token_ids=[[]], + ) + + sampled = model.sample(logits, sampling_metadata) + + assert sampled is not None + assert sampled.sampled_token_ids.tolist() == [[151703]] + assert state.current_turn_ended is False + + +def test_minicpmo_stage0_native_sampler_forced_listen_yields_floor(): + from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni import ( + MiniCPMO45OmniForConditionalGeneration, + ) + + class _Tokenizer: + eos_token_id = 151705 + unk_token_id = -1 + bad_token_ids = [] + all_special_ids = [] + + def convert_tokens_to_ids(self, token): + return { + "": 151683, + "": 151684, + "<|listen|>": 151705, + "<|speak|>": 151706, + "<|tts_bos|>": 151703, + "<|tts_eos|>": 151704, + "<|tts_pad|>": 151722, + "<|chunk_eos|>": 151718, + "<|chunk_tts_eos|>": 151721, + "<|turn_eos|>": 151717, + }.get(token, -1) + + state = SimpleNamespace(current_turn_ended=False) + model = MiniCPMO45OmniForConditionalGeneration.__new__(MiniCPMO45OmniForConditionalGeneration) + model.model_stage = "llm" + model.thinker = SimpleNamespace(get_tokenizer=lambda: _Tokenizer()) + model._minicpmo45_duplex_row_sessions = {0: ("sid-native", 0)} + model._minicpmo45_duplex_row_payloads = {0: {"force_listen": True}} + model._minicpmo45_duplex_data_plane_helper = SimpleNamespace(sessions={("sid-native", 0): state}) + vocab_size = 151723 + logits = torch.full((1, vocab_size), -100.0) + logits[0, 151705] = 30.0 + sampling_metadata = SimpleNamespace( + all_greedy=True, + all_random=False, + temperature=torch.tensor([0.0]), + top_k=torch.tensor([1]), + top_p=torch.tensor([1.0]), + generators={}, + prompt_token_ids=torch.tensor([[151683] * 16]), + output_token_ids=[[]], + ) + + sampled = model.sample(logits, sampling_metadata) + + assert sampled is not None + assert sampled.sampled_token_ids.tolist() == [[151705]] + assert state.current_turn_ended is True + + +def test_minicpmo_stage0_native_sampler_uses_runner_duplex_rows(): + from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni import ( + MiniCPMO45OmniForConditionalGeneration, + ) + + model = MiniCPMO45OmniForConditionalGeneration.__new__(MiniCPMO45OmniForConditionalGeneration) + metadata = SimpleNamespace( + prompt_token_ids=torch.tensor([[1, 2, 3]]), + ) + + rows = model._minicpmo45_native_duplex_prompt_rows( + metadata, + unit_id=151683, + batch_size=1, + duplex_rows=[0], + ) + + assert rows == [0] + + +def test_minicpmo_stage0_session_context_includes_resolved_ref_audio(): + from vllm_omni.experimental.fullduplex.minicpmo45.stage0 import ( + MiniCPMO45Stage0DuplexRuntime, + _MiniCPMO45Stage0SessionState, + ) + + runtime = MiniCPMO45Stage0DuplexRuntime.__new__(MiniCPMO45Stage0DuplexRuntime) + runtime.unit_token_id = 151683 + runtime.processor = SimpleNamespace() + runtime.stage_model = SimpleNamespace() + runtime.thinker = SimpleNamespace() + runtime.device = "cpu" + token_map = { + "<|im_start|>system\nUse speech.\n<|audio_start|>": [1, 2, 3], + "<|audio_end|><|im_end|>": [4, 5], + } + runtime._stage_runtime_ready = lambda: True + runtime._require_special_token_ids = lambda: None + runtime._decode_ref_audio_from_session_config = lambda _config: np.array([0.1, -0.1], dtype=np.float32) + runtime._encode_text = lambda text: token_map[text] + runtime._embed_token = lambda token_id: torch.full((1, 2), float(token_id)) + runtime._stage_ref_audio_embeddings = lambda ref_audio, state=None: torch.tensor([[10.0, 11.0], [12.0, 13.0]]) + + state = _MiniCPMO45Stage0SessionState(session_id="sid-ref") + runtime._prepare_session_context(state, {"instructions": "Use speech.", "extra_body": {"ref_audio_data": "x"}}) + + assert state.context_token_ids == [1, 2, 3, 151683, 151683, 4, 5] + assert len(state.context_embeds) == 6 + + +def test_minicpmo_tts_ref_audio_lru_evicts_matching_vocoder_base_cache(tmp_path): + from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni_tts import ( + MiniCPMO45OmniTTSForConditionalGeneration, + MiniCPMO45TTSRuntimeConfig, + ) + + model = MiniCPMO45OmniTTSForConditionalGeneration.__new__(MiniCPMO45OmniTTSForConditionalGeneration) + model._runtime_config = MiniCPMO45TTSRuntimeConfig(ref_audio_file_cache_size=1) + model._t2w_base_caches = {} + model._token2wav_state_lock = threading.RLock() + first_path = model._write_ref_audio_prompt_wav(np.array([0.1, -0.1], dtype=np.float32), 16_000) + assert first_path is not None + model._t2w_base_caches[first_path] = (torch.tensor([1]), {"cache": torch.tensor([2])}) + + second_path = model._write_ref_audio_prompt_wav(np.array([0.2, -0.2], dtype=np.float32), 16_000) + + assert second_path is not None + assert second_path != first_path + assert first_path not in model._t2w_base_caches + assert not os.path.exists(first_path) + + +def test_minicpmo_tts_prompt_wav_path_does_not_default_to_model_asset(tmp_path): + from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni_tts import ( + MiniCPMO45OmniTTSForConditionalGeneration, + ) + + model_dir = tmp_path / "MiniCPM-o-4_5" + assets_dir = model_dir / "assets" + assets_dir.mkdir(parents=True) + (assets_dir / "HT_ref_audio.wav").write_bytes(b"placeholder") + + model = MiniCPMO45OmniTTSForConditionalGeneration.__new__(MiniCPMO45OmniTTSForConditionalGeneration) + model.vllm_config = SimpleNamespace(model_config=SimpleNamespace(model=str(model_dir))) + model._write_ref_audio_prompt_wav = lambda ref_audio, ref_audio_sr: None + + assert model._resolve_prompt_wav_path(None, None) == (None, None) + + +def test_minicpmo_tts_stream_turn_id_prefers_duplex_identity(): + from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni_tts import ( + MiniCPMO45OmniTTSForConditionalGeneration, + ) + + model = MiniCPMO45OmniTTSForConditionalGeneration.__new__(MiniCPMO45OmniTTSForConditionalGeneration) + + assert ( + model._stream_turn_id( + { + "duplex": {"turn_id": 0, "model_turn_id": 4}, + "meta": {"turn_id": 3}, + } + ) + == 4 + ) + assert ( + model._stream_turn_id( + { + "duplex": {"turn_id": 0}, + "meta": {"turn_id": 3}, + } + ) + == 0 + ) + assert model._stream_turn_id({"meta": {"turn_id": 3}, "turn_id": 2}) == 3 + assert model._stream_turn_id({"turn_id": 2}) == 2 + + +def test_minicpmo_tts_no_ref_audio_initializes_empty_vocoder_cache(): + from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni_tts import ( + MiniCPMO45OmniTTSForConditionalGeneration, + _TalkerTurnState, + ) + + class AudioTokenizer: + def set_stream_cache(self, prompt_wav_path): + raise AssertionError(f"unexpected reference-audio cache load: {prompt_wav_path!r}") + + model = MiniCPMO45OmniTTSForConditionalGeneration.__new__(MiniCPMO45OmniTTSForConditionalGeneration) + model._t2w_base_caches = {} + model._token2wav_state_lock = threading.RLock() + model.audio_tokenizer = AudioTokenizer() + state = _TalkerTurnState(None, None) + + model._begin_turn_vocoder_cache(None, state=state) + + assert state.stream_cache is None + assert state.hift_cache_dict == {} + assert state.vocoder_initialized is True + assert model._t2w_base_caches[""] == (None, {}) + + +def test_minicpmo_tts_native_duplex_merges_and_left_pads_nonfinal_unit_audio(): + from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni_tts import ( + _native_duplex_unit_waveform, + ) + + waveform = _native_duplex_unit_waveform( + [torch.tensor([1.0, 2.0]), torch.tensor([3.0])], + turn_end=False, + target_samples=6, + ) + + assert waveform is not None + assert waveform.tolist() == [0.0, 0.0, 0.0, 1.0, 2.0, 3.0] + + +def test_minicpmo_tts_native_duplex_does_not_pad_final_unit_audio(): + from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni_tts import ( + _native_duplex_unit_waveform, + ) + + waveform = _native_duplex_unit_waveform( + [torch.tensor([1.0, 2.0]), torch.tensor([3.0])], + turn_end=True, + target_samples=6, + ) + + assert waveform is not None + assert waveform.tolist() == [1.0, 2.0, 3.0] + + +def test_minicpmo_tts_turn_cleanup_removes_deleted_ref_audio_vocoder_cache(tmp_path): + from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni_tts import ( + MiniCPMO45OmniTTSForConditionalGeneration, + _TalkerTurnState, + ) + + prompt_path = tmp_path / "session-ref.wav" + prompt_path.write_bytes(b"wav") + model = MiniCPMO45OmniTTSForConditionalGeneration.__new__(MiniCPMO45OmniTTSForConditionalGeneration) + model._talker_turn_states = { + "request": _TalkerTurnState(str(prompt_path), str(prompt_path)), + } + model._talker_consumed_tokens = {"request": 1} + model._talker_request_keys = {} + model._t2w_base_caches = { + str(prompt_path): (torch.tensor([1]), {"cache": torch.tensor([2])}), + } + model._token2wav_state_lock = threading.RLock() + model.audio_tokenizer = None + + assert model._close_turn_state("request") is True + assert str(prompt_path) not in model._t2w_base_caches + assert not prompt_path.exists() diff --git a/tests/worker/test_omni_gpu_model_runner.py b/tests/worker/test_omni_gpu_model_runner.py index 48c0bc6de..cff36b9f6 100644 --- a/tests/worker/test_omni_gpu_model_runner.py +++ b/tests/worker/test_omni_gpu_model_runner.py @@ -196,8 +196,7 @@ def _make_runner_for_mimo(req_id="r_mimo"): def test_talker_mtp_forward_cpu_updates_inputs_and_info(monkeypatch): - # `_talker_mtp_forward` calls `current_omni_platform.set_forward_context`, - # which would otherwise dispatch to the real device implementation. + # `_talker_mtp_forward` dispatches through the active platform. import vllm_omni.worker.gpu_model_runner as mod # Must be the same module that defines OmniGPUModelRunner monkeypatch.setattr(mod.current_omni_platform, "set_forward_context", _noop_forward_context) @@ -459,28 +458,33 @@ def test_update_intermediate_buffer_skips_unknown_req_id(): assert "unknown_req" not in runner.model_intermediate_buffer -def test_maybe_run_batch_preprocess_calls_model_hook(): - runner = object.__new__(OmniGPUModelRunner) - runner.model_intermediate_buffer = {"r1": {"text": ["hello"]}} - calls = [] +def test_streaming_input_update_merges_model_intermediate_buffer(): + runner = _make_runner(req_ids=("r1",), hidden_size=4) + runner.model_intermediate_buffer["r1"] = { + "duplex": { + "session_id": "sid", + "seq": 1, + } + } + runner.requests["r1"].additional_information_cpu = runner.model_intermediate_buffer["r1"] + new_req_data = SimpleNamespace( + model_intermediate_buffer={ + "duplex": { + "session_id": "sid", + "seq": 2, + "payload": {"type": "audio"}, + } + }, + additional_information=None, + ) - class DummyModel: - def preprocess_batch(self, *, req_ids, model_intermediate_buffer, device): - calls.append((req_ids, model_intermediate_buffer, device)) + OmniGPUModelRunner._update_streaming_input_additional_info(runner, new_req_data, "r1") - runner.model = DummyModel() - - OmniGPUModelRunner._maybe_run_batch_preprocess(runner, ["r1"], torch.device("cpu")) - - assert calls == [(["r1"], runner.model_intermediate_buffer, torch.device("cpu"))] - - -def test_maybe_run_batch_preprocess_skips_missing_hook(): - runner = object.__new__(OmniGPUModelRunner) - runner.model_intermediate_buffer = {} - runner.model = object() - - OmniGPUModelRunner._maybe_run_batch_preprocess(runner, ["r1"], torch.device("cpu")) + info = runner.model_intermediate_buffer["r1"] + assert info["duplex"]["session_id"] == "sid" + assert info["duplex"]["seq"] == 2 + assert info["duplex"]["payload"] == {"type": "audio"} + assert runner.requests["r1"].additional_information_cpu is info def _make_full_payload_accumulation_runner( @@ -501,8 +505,6 @@ def _make_full_payload_accumulation_runner( runner._custom_process_func = object() runner._pending_full_payload_send = {} runner._stage_id = 1 - # Non-None sentinel: the gate short-circuits to False when no connector - # is configured at all (terminal stages in pipelines with no connector). runner._omni_connector = object() return runner @@ -519,7 +521,7 @@ def test_accumulate_full_payload_output_preserves_aligned_all_zero_qwen3_omni_co def test_accumulate_full_payload_output_keeps_misaligned_all_zero_qwen3_omni_codec_rows(): - # After removing the sender-side zero filter, the full-payload accumulator keeps every + # After removing the sender-side zero filter, the accumulator keeps every # codec row including misaligned all-zero rows. The downstream consumer # (_extract_qwen3_full_payload_codec_rows) is the authoritative crop and # filters by output_token_ids. @@ -566,45 +568,25 @@ def test_accumulate_full_payload_output_keeps_all_zero_qwen3_omni_prefill_placeh def test_full_payload_output_accumulation_hook_matrix(): - """Producer-side gate: fires iff an explicit next-stage payload hook is loaded. - - A derived `*_full_payload` helper from `custom_process_input_func` is not - enough: terminal/input-only consumer stages must not enqueue orphan - downstream payloads. - """ - # Thinker / talker producer stages: explicit next-stage payload hook -> gate fires. assert _make_full_payload_accumulation_runner(model_stage="thinker")._should_accumulate_full_payload_output() assert _make_full_payload_accumulation_runner(model_stage="talker")._should_accumulate_full_payload_output() - - # Terminal stage: even if _load_custom_func derived a builder from - # custom_process_input_func, final output stages are not producers. - runner = _make_full_payload_accumulation_runner(model_stage="code2wav", final_output=True) - assert not runner._should_accumulate_full_payload_output() - - # Input-only consumer stage without an explicit producer hook must not - # accumulate/send just because a same-module *_full_payload helper exists. - runner = _make_full_payload_accumulation_runner( + assert not _make_full_payload_accumulation_runner( + model_stage="code2wav", final_output=True + )._should_accumulate_full_payload_output() + assert not _make_full_payload_accumulation_runner( model_stage="token2audio", custom_process_next_stage_input_func=None, - ) - assert not runner._should_accumulate_full_payload_output() - - # async_chunk mode -> gate off. + )._should_accumulate_full_payload_output() assert not _make_full_payload_accumulation_runner( model_stage="talker", async_chunk=True )._should_accumulate_full_payload_output() - - # Non-qwen3 arches: gate is arch-agnostic, but if the fixture's arch - # does not configure a connector payload builder, its runtime - # `_custom_process_func` is None. Emulate that. - runner = _make_full_payload_accumulation_runner(model_arch="Qwen3TTSForConditionalGeneration") - runner._custom_process_func = None - runner._should_accumulate_full_payload_output_cached = None - assert not runner._should_accumulate_full_payload_output() - runner = _make_full_payload_accumulation_runner(model_arch="Qwen2_5OmniForConditionalGeneration") - runner._custom_process_func = None - runner._should_accumulate_full_payload_output_cached = None - assert not runner._should_accumulate_full_payload_output() + for model_arch in ( + "Qwen3TTSForConditionalGeneration", + "Qwen2_5OmniForConditionalGeneration", + ): + runner = _make_full_payload_accumulation_runner(model_arch=model_arch) + runner._custom_process_func = None + assert not runner._should_accumulate_full_payload_output() def test_sync_local_stage_payloads_retains_payload_until_request_is_active(): diff --git a/vllm_omni/config/stage_config.py b/vllm_omni/config/stage_config.py index 367e1b317..5e16140b8 100644 --- a/vllm_omni/config/stage_config.py +++ b/vllm_omni/config/stage_config.py @@ -270,6 +270,15 @@ class PipelineConfig: # multi-component repos (e.g. GLM-Image). ``None`` = not a diffusers model. diffusers_class_name: str | None = None endpoint_restrictions: tuple[EndpointRestriction, ...] = () + # Optional model-owned duplex planner loaded by the stable engine runtime. + duplex_runtime_extension: str | None = None + # Optional model-owned Serving adapter loaded only when the Realtime duplex + # endpoint is enabled. Generic OpenAI modules must not select a model. + duplex_serving_adapter: str | None = None + # Explicitly enable the stable duplex control mechanism. This is separate + # from the optional model extension because turn-commit-only deployments + # do not require a model planner. + duplex_control_enabled: bool = False # Bundled deploy defaults for this concrete pipeline topology. The file is # loaded from vllm_omni/deploy; None uses DeployConfig defaults. default_deploy_config_name: str | None = None @@ -406,6 +415,38 @@ class StageDeployConfig: engine_extras: dict[str, Any] = field(default_factory=dict) +@dataclass(frozen=True) +class DuplexSessionRuntimeConfig: + """Server-owned lifecycle and per-session buffering limits.""" + + idle_ttl_s: float | None = 300.0 + disconnect_grace_s: float = 30.0 + reaper_interval_s: float = 5.0 + resume_replay_ttl_s: float = 60.0 + resume_replay_max_bytes_per_session: int = 8 * 1024 * 1024 + max_pending_input_bytes_per_session: int = 16 * 1024 * 1024 + max_pending_turns_per_session: int = 4 + max_sessions: int = 1 + completed_append_cache_size: int = 256 + + def __post_init__(self) -> None: + positive = { + "disconnect_grace_s": self.disconnect_grace_s, + "reaper_interval_s": self.reaper_interval_s, + "resume_replay_ttl_s": self.resume_replay_ttl_s, + "resume_replay_max_bytes_per_session": self.resume_replay_max_bytes_per_session, + "max_pending_input_bytes_per_session": self.max_pending_input_bytes_per_session, + "max_pending_turns_per_session": self.max_pending_turns_per_session, + "max_sessions": self.max_sessions, + "completed_append_cache_size": self.completed_append_cache_size, + } + if self.idle_ttl_s is not None and self.idle_ttl_s <= 0: + raise ValueError("duplex_session.idle_ttl_s must be positive or null") + for name, value in positive.items(): + if value <= 0: + raise ValueError(f"duplex_session.{name} must be positive") + + @dataclass class DeployConfig: """Loaded from deploy/.yaml — the only config file users edit. @@ -419,8 +460,10 @@ class DeployConfig: """ async_chunk: bool = True + session_mode: str = "turn" # Stage-1 active stream slots; 0 preserves legacy all-stream cycling. active_stream_window: int = 0 + duplex_session: DuplexSessionRuntimeConfig = field(default_factory=DuplexSessionRuntimeConfig) connectors: dict[str, Any] | None = None edges: list[dict[str, Any]] | None = None stages: list[StageDeployConfig] = field(default_factory=list) @@ -610,7 +653,9 @@ def load_deploy_config(path: str | Path) -> DeployConfig: kwargs: dict[str, Any] = { "async_chunk": raw_dict.get("async_chunk", True), + "session_mode": raw_dict.get("session_mode", "turn"), "active_stream_window": int(raw_dict.get("active_stream_window", 0) or 0), + "duplex_session": DuplexSessionRuntimeConfig(**(raw_dict.get("duplex_session") or {})), "connectors": raw_dict.get("connectors", None), "edges": raw_dict.get("edges", None), "stages": stages, @@ -896,6 +941,7 @@ def merge_pipeline_deploy( StageConfig( stage_id=ps.stage_id, model_stage=ps.model_stage, + session_mode=deploy.session_mode, stage_type=stage_type, input_sources=list(ps.input_sources), custom_process_input_func=input_proc, @@ -922,6 +968,7 @@ class StageConfig: stage_id: int model_stage: str + session_mode: str = "turn" stage_type: StageType = StageType.LLM input_sources: list[int] = field(default_factory=list) custom_process_input_func: str | None = None @@ -984,6 +1031,7 @@ class StageConfig: config_dict: dict[str, Any] = { "stage_id": self.stage_id, "stage_type": StageType(self.stage_type).value, + "session_mode": self.session_mode, "engine_args": create_config(engine_args), "runtime": create_config(runtime), "engine_input_source": self.input_sources, # Legacy field name diff --git a/vllm_omni/core/sched/__init__.py b/vllm_omni/core/sched/__init__.py index 0f80769d3..ce968893d 100644 --- a/vllm_omni/core/sched/__init__.py +++ b/vllm_omni/core/sched/__init__.py @@ -2,9 +2,28 @@ Scheduling components for vLLM-Omni. """ -from .omni_ar_scheduler import OmniARAsyncScheduler, OmniARScheduler -from .omni_generation_scheduler import OmniGenerationScheduler -from .output import OmniNewRequestData +from __future__ import annotations + +from importlib import import_module +from typing import Any + +_LAZY_EXPORTS = { + "OmniARAsyncScheduler": (".omni_ar_scheduler", "OmniARAsyncScheduler"), + "OmniARScheduler": (".omni_ar_scheduler", "OmniARScheduler"), + "OmniGenerationScheduler": (".omni_generation_scheduler", "OmniGenerationScheduler"), + "OmniNewRequestData": (".output", "OmniNewRequestData"), +} + + +def __getattr__(name: str) -> Any: + target = _LAZY_EXPORTS.get(name) + if target is None: + raise AttributeError(name) + module_name, attribute_name = target + value = getattr(import_module(module_name, __name__), attribute_name) + globals()[name] = value + return value + __all__ = [ "OmniARAsyncScheduler", diff --git a/vllm_omni/core/sched/omni_ar_scheduler.py b/vllm_omni/core/sched/omni_ar_scheduler.py index 912b67bd7..e88730353 100644 --- a/vllm_omni/core/sched/omni_ar_scheduler.py +++ b/vllm_omni/core/sched/omni_ar_scheduler.py @@ -259,6 +259,9 @@ class OmniARScheduler(OmniSchedulerMixin, VLLMScheduler): prompt_embeds=(getattr(request, "prompt_embeds", None) if request else None), prompt_is_token_ids=nr.prompt_is_token_ids, additional_information=(getattr(request, "additional_information", None) if request else None), + model_intermediate_buffer=( + getattr(request, "model_intermediate_buffer", None) if request else None + ), ) new_list.append(omni_nr) @@ -718,6 +721,8 @@ class OmniARScheduler(OmniSchedulerMixin, VLLMScheduler): session.record_event(EngineCoreEventType.QUEUED) return super()._update_request_as_session(session, update) + if hasattr(update, "model_intermediate_buffer"): + session.model_intermediate_buffer = update.model_intermediate_buffer def _free_request(self, request: Request, delay_free_blocks: bool = False) -> dict[str, Any] | None: # TODO(wzliu)! for offline mode, we should not end process until all data is transferred diff --git a/vllm_omni/core/sched/omni_generation_scheduler.py b/vllm_omni/core/sched/omni_generation_scheduler.py index 77a07be7d..c6f82ec23 100644 --- a/vllm_omni/core/sched/omni_generation_scheduler.py +++ b/vllm_omni/core/sched/omni_generation_scheduler.py @@ -319,6 +319,9 @@ class OmniGenerationScheduler(OmniSchedulerMixin, VLLMScheduler): prompt_embeds=(getattr(request, "prompt_embeds", None) if request else None), prompt_is_token_ids=nr.prompt_is_token_ids, additional_information=(getattr(request, "additional_information", None) if request else None), + model_intermediate_buffer=( + getattr(request, "model_intermediate_buffer", None) if request else None + ), ) new_list.append(omni_nr) diff --git a/vllm_omni/core/sched/omni_scheduler_mixin.py b/vllm_omni/core/sched/omni_scheduler_mixin.py index fc7e1b626..4eaa07cbb 100644 --- a/vllm_omni/core/sched/omni_scheduler_mixin.py +++ b/vllm_omni/core/sched/omni_scheduler_mixin.py @@ -40,16 +40,16 @@ except ValueError: class OmniSchedulerMixin: """Shared scheduler helpers for omni-specific request handling.""" + # ------------------------------------------------------------------ # + # Shared scheduler/output helpers (lift the AR / generation duplicates) + # ------------------------------------------------------------------ # + def _free_input_coordinator_request(self, request_id: str) -> None: """Prune full-payload coordinator state for a completed request.""" input_coordinator = getattr(self, "input_coordinator", None) if input_coordinator is not None: input_coordinator.free_finished_request(request_id) - # ------------------------------------------------------------------ # - # Shared scheduler/output helpers (lift the AR / generation duplicates) - # ------------------------------------------------------------------ # - def _consume_pending_connector_output(self, model_mode: str) -> None: """Drain ``self._latest_omni_connector_output`` into the coordinator. diff --git a/vllm_omni/core/sched/output.py b/vllm_omni/core/sched/output.py index ec017b11c..be9c867b1 100644 --- a/vllm_omni/core/sched/output.py +++ b/vllm_omni/core/sched/output.py @@ -20,10 +20,13 @@ class OmniNewRequestData(NewRequestData): external_req_id: Optional external request ID for tracking additional_information: Optional serialized additional information dictionary containing tensors or lists + model_intermediate_buffer: Optional runner-owned payload for + GPUModelRunner.model_intermediate_buffer """ external_req_id: str | None = None additional_information: AdditionalInformationPayload | None = None + model_intermediate_buffer: dict[str, object] | None = None @classmethod def from_request( @@ -56,6 +59,7 @@ class OmniNewRequestData(NewRequestData): prompt_is_token_ids=getattr(request, "prompt_is_token_ids", None), prefill_token_ids=prefill_token_ids, additional_information=getattr(request, "additional_information", None), + model_intermediate_buffer=getattr(request, "model_intermediate_buffer", None), ) diff --git a/vllm_omni/deploy/minicpmo_4_5_3gpu_stage1_replicas.yaml b/vllm_omni/deploy/minicpmo_4_5_3gpu_stage1_replicas.yaml new file mode 100644 index 000000000..17cd014e9 --- /dev/null +++ b/vllm_omni/deploy/minicpmo_4_5_3gpu_stage1_replicas.yaml @@ -0,0 +1,17 @@ +# Experimental validation profile: one thinker and two Stage1 replicas. +# This is not a recommended production entrypoint. +# +# Stage 0: GPU 0 +# Stage 1: replicas on GPU 1 and 2 +base_config: minicpmo_4_5.yaml +pipeline: minicpmo_4_5 +session_mode: duplex + +stages: + - stage_id: 0 + async_scheduling: false + + - stage_id: 1 + devices: "1,2" + num_replicas: 2 + async_scheduling: false diff --git a/vllm_omni/deploy/minicpmo_4_5_4gpu_stage1_replicas.yaml b/vllm_omni/deploy/minicpmo_4_5_4gpu_stage1_replicas.yaml new file mode 100644 index 000000000..e4fe45922 --- /dev/null +++ b/vllm_omni/deploy/minicpmo_4_5_4gpu_stage1_replicas.yaml @@ -0,0 +1,17 @@ +# Experimental validation profile: one thinker and three Stage1 replicas. +# This is not a recommended production entrypoint. +# +# Stage 0: GPU 0 +# Stage 1: replicas on GPU 1, 2, and 3 +base_config: minicpmo_4_5.yaml +pipeline: minicpmo_4_5 +session_mode: duplex + +stages: + - stage_id: 0 + async_scheduling: false + + - stage_id: 1 + devices: "1,2,3" + num_replicas: 3 + async_scheduling: false diff --git a/vllm_omni/deploy/minicpmo_4_5_8x4090_stage1_replicas.yaml b/vllm_omni/deploy/minicpmo_4_5_8x4090_stage1_replicas.yaml new file mode 100644 index 000000000..f4ad2c6b4 --- /dev/null +++ b/vllm_omni/deploy/minicpmo_4_5_8x4090_stage1_replicas.yaml @@ -0,0 +1,20 @@ +# Experimental validation profile: 4-way thinker TP and four Stage1 replicas. +# This is not a recommended production entrypoint. +# +# Stage 0: GPU 0, 1, 2, and 3 +# Stage 1: replicas on GPU 4, 5, 6, and 7 +base_config: minicpmo_4_5_8x4090.yaml +pipeline: minicpmo_4_5 +session_mode: duplex + +stages: + - stage_id: 0 + async_scheduling: false + gpu_memory_utilization: 0.40 + max_num_batched_tokens: 16384 + + - stage_id: 1 + devices: "4,5,6,7" + num_replicas: 4 + async_scheduling: false + gpu_memory_utilization: 0.8 diff --git a/vllm_omni/deploy/minicpmo_4_5_duplex.yaml b/vllm_omni/deploy/minicpmo_4_5_duplex.yaml new file mode 100644 index 000000000..c227f2817 --- /dev/null +++ b/vllm_omni/deploy/minicpmo_4_5_duplex.yaml @@ -0,0 +1,36 @@ +# MiniCPM-o 4.5 native duplex deployment. +# +# The registered minicpmo_4_5 pipeline owns the fixed Stage0 -> Stage1 +# topology and llm2tts bridge. This overlay only changes session lifetime and +# the Stage1 token budget needed to drain streaming audio segments. +base_config: minicpmo_4_5.yaml +pipeline: minicpmo_4_5 +session_mode: duplex +active_stream_window: 1 +duplex_session: + idle_ttl_s: 300 + disconnect_grace_s: 30 + reaper_interval_s: 5 + resume_replay_ttl_s: 60 + resume_replay_max_bytes_per_session: 8388608 + max_pending_input_bytes_per_session: 16777216 + max_pending_turns_per_session: 4 + # Matches the configured Stage0/Stage1 sequence capacity below. + max_sessions: 2 + completed_append_cache_size: 256 + +stages: + - stage_id: 0 + max_num_seqs: 2 + async_scheduling: false + + - stage_id: 1 + max_num_seqs: 2 + async_scheduling: false + # The streaming talker advances Python-owned per-request generators on + # every step, so only fixed-topology model regions are graph-capturable. + compilation_config: + cudagraph_mode: PIECEWISE + default_sampling_params: + max_tokens: 4096 + stop_token_ids: [151645] diff --git a/vllm_omni/distributed/omni_connectors/transfer_adapter/chunk_transfer_adapter.py b/vllm_omni/distributed/omni_connectors/transfer_adapter/chunk_transfer_adapter.py index 04368842c..26be84012 100644 --- a/vllm_omni/distributed/omni_connectors/transfer_adapter/chunk_transfer_adapter.py +++ b/vllm_omni/distributed/omni_connectors/transfer_adapter/chunk_transfer_adapter.py @@ -771,8 +771,12 @@ class OmniChunkTransferAdapter(OmniTransferAdapterBase): self.waiting_for_chunk_running_requests = deque( request for request in self.waiting_for_chunk_running_requests if request.request_id not in request_ids ) + self._held_non_active = deque( + request for request in self._held_non_active if request.request_id not in request_ids + ) for req_id in request_ids: + self._active_streams.pop(req_id, None) self.requests_with_ready_chunks.discard(req_id) self.finished_requests.discard(req_id) self._finished_load_reqs.discard(req_id) diff --git a/vllm_omni/engine/__init__.py b/vllm_omni/engine/__init__.py index 89bb41908..e72ce9fed 100644 --- a/vllm_omni/engine/__init__.py +++ b/vllm_omni/engine/__init__.py @@ -75,6 +75,10 @@ class OmniEngineCoreRequest(EngineCoreRequest): # Optional additional information dictionary (serialized) additional_information: AdditionalInformationPayload | None = None + # Runner-owned runtime payload. This is materialized directly into + # GPUModelRunner.model_intermediate_buffer instead of using the deprecated + # additional_information request transport. + model_intermediate_buffer: dict[str, Any] | None = None @classmethod def from_request( @@ -83,6 +87,7 @@ class OmniEngineCoreRequest(EngineCoreRequest): *, prompt_embeds: torch.Tensor | None = None, additional_information: AdditionalInformationPayload | None = None, + model_intermediate_buffer: dict[str, Any] | None = None, ) -> "OmniEngineCoreRequest": """Clone an EngineCoreRequest into an OmniEngineCoreRequest with optional payload overrides.""" @@ -90,6 +95,8 @@ class OmniEngineCoreRequest(EngineCoreRequest): prompt_embeds = request.prompt_embeds if additional_information is None: additional_information = getattr(request, "additional_information", None) + if model_intermediate_buffer is None: + model_intermediate_buffer = getattr(request, "model_intermediate_buffer", None) return cls( request_id=request.request_id, @@ -113,6 +120,7 @@ class OmniEngineCoreRequest(EngineCoreRequest): reasoning_parser_kwargs=request.reasoning_parser_kwargs, abort_immediately=request.abort_immediately, additional_information=additional_information, + model_intermediate_buffer=model_intermediate_buffer, ) diff --git a/vllm_omni/engine/async_engine_utils.py b/vllm_omni/engine/async_engine_utils.py new file mode 100644 index 000000000..58f8351b0 --- /dev/null +++ b/vllm_omni/engine/async_engine_utils.py @@ -0,0 +1,160 @@ +"""Stateless request and shutdown helpers for :mod:`async_omni_engine`.""" + +from __future__ import annotations + +import threading +from typing import Any + +import janus +import torch +from vllm.logger import init_logger +from vllm.v1.engine import EngineCoreRequest + +from vllm_omni.engine import OmniEngineCoreRequest +from vllm_omni.engine.messages import EngineQueueMessage, ShutdownRequestMessage +from vllm_omni.engine.rpc_result_router import CorrelatedRpcClient +from vllm_omni.engine.serialization import ( + deserialize_additional_information, + serialize_additional_information, +) +from vllm_omni.engine.stage_runtime import StageRuntime + +logger = init_logger(__name__) + +SHUTDOWN_ENQUEUE_TIMEOUT_S = 1.0 +SHUTDOWN_JOIN_TIMEOUT_S = 30.0 +_WEAK_SHUTDOWN_JOIN_TIMEOUT_S = 1.0 + + +def inject_global_id(target: Any, request_id: str) -> None: + """Inject ``global_request_id`` into a prompt's additional information.""" + if isinstance(target, dict): + if "additional_information" not in target: + target["additional_information"] = {} + if target["additional_information"] is None: + target["additional_information"] = {} + if isinstance(target["additional_information"], dict): + target["additional_information"]["global_request_id"] = [str(request_id)] + + +def upgrade_to_omni_request( + request: EngineCoreRequest, + raw_prompt: Any, +) -> EngineCoreRequest: + """Restore omni-only fields omitted by the upstream input processor.""" + prompt_embeds = request.prompt_embeds + additional_information = None + wire_payload = None + model_intermediate_buffer = None + + if isinstance(raw_prompt, dict): + if prompt_embeds is None: + raw_prompt_embeds = raw_prompt.get("prompt_embeds") + if isinstance(raw_prompt_embeds, torch.Tensor): + prompt_embeds = raw_prompt_embeds + raw_info = raw_prompt.get("additional_information") + raw_buffer = raw_prompt.get("model_intermediate_buffer") + if isinstance(raw_info, dict): + wire_payload = dict(raw_info) + if isinstance(raw_buffer, dict): + model_intermediate_buffer = raw_buffer + additional_information = serialize_additional_information( + wire_payload, + log_prefix="AsyncOmniEngine", + ) + + if prompt_embeds is None and additional_information is None and model_intermediate_buffer is None: + return request + + return OmniEngineCoreRequest.from_request( + request, + prompt_embeds=prompt_embeds, + additional_information=additional_information, + model_intermediate_buffer=model_intermediate_buffer, + ) + + +def apply_omni_final_stage_metadata( + request: EngineCoreRequest, + final_stage_id: int, +) -> EngineCoreRequest: + """Tag a request so the AR scheduler can skip DiT KV for stage zero.""" + merged: dict[str, Any] = {} + if isinstance(request, OmniEngineCoreRequest) and request.additional_information is not None: + merged = deserialize_additional_information(request.additional_information) + merged["omni_final_stage_id"] = final_stage_id + payload = serialize_additional_information(merged) + return OmniEngineCoreRequest.from_request( + request, + additional_information=payload, + ) + + +def weak_shutdown_async_omni_engine( + orchestrator_thread: threading.Thread | None, + request_queue: janus.Queue[EngineQueueMessage] | None, + output_queue: janus.Queue[EngineQueueMessage] | None, + rpc_output_queue: janus.Queue[EngineQueueMessage] | None, + rpc_client: CorrelatedRpcClient | None, +) -> None: + """Best-effort orchestrator cleanup for garbage-collection finalization.""" + request_queue_closed = False + shutdown_enqueued = enqueue_orchestrator_shutdown( + request_queue, + timeout=SHUTDOWN_ENQUEUE_TIMEOUT_S, + ) + if request_queue is not None and not shutdown_enqueued: + try: + request_queue.close() + request_queue_closed = True + except Exception: + pass + + if rpc_client is not None: + rpc_client.close() + + try: + if orchestrator_thread is not None and orchestrator_thread.is_alive(): + orchestrator_thread.join(timeout=_WEAK_SHUTDOWN_JOIN_TIMEOUT_S) + except Exception: + pass + + for q in (request_queue, output_queue, rpc_output_queue): + try: + if q is not None and not (q is request_queue and request_queue_closed): + q.close() + except Exception: + pass + + +def enqueue_orchestrator_shutdown( + request_queue: janus.Queue[EngineQueueMessage] | None, + *, + timeout: float, +) -> bool: + """Deliver shutdown without waiting forever behind a full request queue.""" + if request_queue is None: + return False + try: + request_queue.sync_q.put(ShutdownRequestMessage(), timeout=timeout) + except Exception: + return False + return True + + +def shutdown_runtime_after_orchestrator( + orchestrator_thread: threading.Thread, + runtime: StageRuntime, +) -> None: + """Release a stage runtime once its orchestrator can no longer use it.""" + try: + orchestrator_thread.join() + except Exception: + logger.exception("[AsyncOmniEngine] Failed while awaiting deferred Orchestrator shutdown") + if orchestrator_thread.is_alive(): + logger.error("[AsyncOmniEngine] Orchestrator is still alive; StageRuntime cleanup remains deferred") + return + try: + runtime.shutdown() + except Exception: + logger.exception("[AsyncOmniEngine] Failed to shutdown deferred StageRuntime") diff --git a/vllm_omni/engine/async_omni_engine.py b/vllm_omni/engine/async_omni_engine.py index 459057486..dbebd7e6d 100644 --- a/vllm_omni/engine/async_omni_engine.py +++ b/vllm_omni/engine/async_omni_engine.py @@ -18,7 +18,7 @@ import uuid import weakref from collections.abc import Mapping, Sequence from dataclasses import asdict -from typing import Any, Literal, cast +from typing import TYPE_CHECKING, Any, Literal, cast import janus import torch @@ -31,10 +31,23 @@ from vllm.v1.engine import EngineCoreRequest from vllm.v1.engine.input_processor import InputProcessor from vllm_omni.config.config_factory import StageConfigFactory -from vllm_omni.config.stage_config import strip_parent_engine_args +from vllm_omni.config.stage_config import ( + DuplexSessionRuntimeConfig, + load_deploy_config, + strip_parent_engine_args, +) from vllm_omni.diffusion.data import DiffusionParallelConfig, parse_attention_config from vllm_omni.diffusion.diffusion_engine import supports_audio_output -from vllm_omni.engine import OmniEngineCoreRequest +from vllm_omni.engine.async_engine_utils import ( + SHUTDOWN_ENQUEUE_TIMEOUT_S, + SHUTDOWN_JOIN_TIMEOUT_S, + apply_omni_final_stage_metadata, + enqueue_orchestrator_shutdown, + inject_global_id, + shutdown_runtime_after_orchestrator, + upgrade_to_omni_request, + weak_shutdown_async_omni_engine, +) from vllm_omni.engine.messages import ( AbortRequestMessage, AddCompanionRequestMessage, @@ -42,14 +55,10 @@ from vllm_omni.engine.messages import ( CollectiveRPCResultMessage, EngineQueueMessage, ErrorMessage, - ShutdownRequestMessage, StageSubmissionMessage, ) from vllm_omni.engine.orchestrator import Orchestrator -from vllm_omni.engine.serialization import ( - deserialize_additional_information, - serialize_additional_information, -) +from vllm_omni.engine.rpc_result_router import CorrelatedRpcClient from vllm_omni.engine.stage_client import StageClient from vllm_omni.engine.stage_init_utils import build_stage0_input_processor from vllm_omni.engine.stage_pool import StagePool @@ -67,10 +76,13 @@ from vllm_omni.metrics.prometheus import OmniRequestCounter logger = init_logger(__name__) +if TYPE_CHECKING: + from vllm_omni.experimental.fullduplex.engine.duplex_control_client import DuplexControlClient + from vllm_omni.experimental.fullduplex.engine.lease import DuplexLeaseActivity + from vllm_omni.experimental.fullduplex.engine.messages import DuplexFence + _STARTUP_POLL_INTERVAL_S = 1.0 _REQUEST_QUEUE_MAXSIZE = 256 - - # ============================================================================ # Parent-EngineArgs field-routing contracts (consumed by # AsyncOmniEngine._strip_parent_engine_args when ``stage_configs_path`` is set). @@ -106,88 +118,6 @@ _PARENT_ARGS_STRIP: frozenset[str] = frozenset({"stage_configs_path"}) _PARENT_ARGS_NO_WARN: frozenset[str] = frozenset({"model"}) -def _inject_global_id(target: Any, request_id: str) -> None: - """Inject global_request_id into a prompt dict's additional_information.""" - if isinstance(target, dict): - if "additional_information" not in target: - target["additional_information"] = {} - if target["additional_information"] is None: - target["additional_information"] = {} - if isinstance(target["additional_information"], dict): - target["additional_information"]["global_request_id"] = [str(request_id)] - - -def _upgrade_to_omni_request( - request: EngineCoreRequest, - raw_prompt: Any, -) -> EngineCoreRequest: - """Restore omni-only fields omitted by upstream InputProcessor.""" - prompt_embeds = request.prompt_embeds - additional_information = None - - if isinstance(raw_prompt, dict): - if prompt_embeds is None: - raw_prompt_embeds = raw_prompt.get("prompt_embeds") - if isinstance(raw_prompt_embeds, torch.Tensor): - prompt_embeds = raw_prompt_embeds - additional_information = serialize_additional_information( - raw_prompt.get("additional_information"), - log_prefix="AsyncOmniEngine", - ) - - if prompt_embeds is None and additional_information is None: - return request - - return OmniEngineCoreRequest.from_request( - request, - prompt_embeds=prompt_embeds, - additional_information=additional_information, - ) - - -def _apply_omni_final_stage_metadata( - request: EngineCoreRequest, - final_stage_id: int, -) -> EngineCoreRequest: - """Tag EngineCoreRequest so OmniARScheduler can skip DiT KV when final_stage_id is 0.""" - merged: dict[str, Any] = {} - if isinstance(request, OmniEngineCoreRequest) and request.additional_information is not None: - merged = deserialize_additional_information(request.additional_information) - merged["omni_final_stage_id"] = final_stage_id - payload = serialize_additional_information(merged) - return OmniEngineCoreRequest.from_request( - request, - additional_information=payload, - ) - - -def _weak_shutdown_async_omni_engine( - orchestrator_thread: threading.Thread | None, - request_queue: janus.Queue[EngineQueueMessage] | None, - output_queue: janus.Queue[EngineQueueMessage] | None, - rpc_output_queue: janus.Queue[EngineQueueMessage] | None, -) -> None: - """Best-effort orchestrator cleanup for GC finalization.""" - try: - if request_queue is not None: - request_queue.sync_q.put_nowait(ShutdownRequestMessage()) - except Exception: - pass - - try: - if orchestrator_thread is not None and orchestrator_thread.is_alive(): - orchestrator_thread.join() - except Exception: - pass - - for q in (request_queue, output_queue, rpc_output_queue): - try: - if q is not None: - q.close() - except Exception: - pass - - class AsyncOmniEngine: """Thin proxy that launches an Orchestrator in a background thread. @@ -282,11 +212,22 @@ class AsyncOmniEngine: # restriction beforehand. TODO (Alex) make this cleaner and refactor # stage config resolution to remove kwargs hacks. deploy_config_path = kwargs.get("deploy_config") - self.endpoint_restrictions = StageConfigFactory.get_pipeline_endpoint_restrictions( + pipeline_config = StageConfigFactory.get_pipeline_config( model=model, trust_remote_code=trust_remote_code, deploy_config_path=deploy_config_path, ) + self.endpoint_restrictions = pipeline_config.endpoint_restrictions if pipeline_config is not None else () + self._duplex_runtime_extension_path = ( + pipeline_config.duplex_runtime_extension if pipeline_config is not None else None + ) + self.duplex_serving_adapter_path = ( + pipeline_config.duplex_serving_adapter if pipeline_config is not None else None + ) + self._duplex_control_enabled = bool(pipeline_config and pipeline_config.duplex_control_enabled) + self.duplex_session_config = DuplexSessionRuntimeConfig() + if deploy_config_path is not None: + self.duplex_session_config = load_deploy_config(deploy_config_path).duplex_session kwargs["trust_remote_code"] = trust_remote_code self.config_path, self.stage_configs = self._resolve_stage_configs( @@ -315,7 +256,8 @@ class AsyncOmniEngine: self.rpc_output_queue: janus.Queue[EngineQueueMessage] = janus.Queue() self._shutdown_called = False self._weak_finalizer: weakref.finalize | None = None - self._rpc_lock = threading.Lock() + self._correlated_rpc_client: CorrelatedRpcClient | None = None + self._duplex_control_client: DuplexControlClient | None = None self._running_counter = OmniRequestCounter() logger.info(f"[AsyncOmniEngine] Launching Orchestrator thread with {self.num_stages} stages") @@ -334,15 +276,20 @@ class AsyncOmniEngine: ) self.orchestrator_thread.start() self._wait_for_orchestrator_init(startup_future, startup_timeout) + self._correlated_rpc_client = CorrelatedRpcClient( + self.request_queue.sync_q, + self.rpc_output_queue.sync_q, + ) # Stage runtime fields are assigned directly on self by the bootstrap thread. self._weak_finalizer = weakref.finalize( self, - _weak_shutdown_async_omni_engine, + weak_shutdown_async_omni_engine, self.orchestrator_thread, self.request_queue, self.output_queue, self.rpc_output_queue, + self._correlated_rpc_client, ) logger.info(f"[AsyncOmniEngine] Orchestrator ready with {self.num_stages} stages") @@ -435,6 +382,21 @@ class AsyncOmniEngine: pd_config = self._detect_pd_config() membership_controller = self._runtime.create_membership_controller() + duplex_runtime_extension = None + if self._duplex_control_enabled: + from vllm_omni.experimental.fullduplex.engine.duplex_runtime import ( + load_duplex_runtime_extension, + validate_duplex_runtime_extension, + ) + + duplex_runtime_extension = load_duplex_runtime_extension( + getattr(self, "_duplex_runtime_extension_path", None) + ) + if duplex_runtime_extension is not None: + validate_duplex_runtime_extension( + duplex_runtime_extension, + sampling_defaults=tuple(pool.stage_client.default_sampling_params for pool in self.stage_pools), + ) orchestrator = Orchestrator( request_async_queue=self.request_queue.async_q, @@ -448,6 +410,9 @@ class AsyncOmniEngine: transfer_emitter=self._transfer_emitter, log_stats=self._log_stats, enable_orch_monitor=self._enable_orch_monitor, + duplex_runtime_extension=duplex_runtime_extension, + enable_duplex_control=self._duplex_control_enabled, + duplex_session_config=self.duplex_session_config, ) if not startup_future.done(): startup_future.set_result(asyncio.get_running_loop()) @@ -718,10 +683,10 @@ class AsyncOmniEngine: if stage_type != "diffusion" and not isinstance(prompt, EngineCoreRequest): # Inject global_request_id into the raw prompt. if isinstance(prompt, dict): - _inject_global_id(prompt, request_id) + inject_global_id(prompt, request_id) elif isinstance(prompt, list): for item in prompt: - _inject_global_id(item, request_id) + inject_global_id(item, request_id) preselected_stage0_replica = self._scope_stage0_multimodal_cache_to_replica( request_id, @@ -751,7 +716,7 @@ class AsyncOmniEngine: _preprocess_ms = (time.perf_counter() - _t_preprocess) * 1000.0 # TODO (Peiqi): add this for Qwen3-TTS only. Other models don't have # additional_information field in the prompt. - request = _upgrade_to_omni_request(request, prompt) + request = upgrade_to_omni_request(request, prompt) if reasoning_ended is not None: request.reasoning_ended = reasoning_ended @@ -763,7 +728,7 @@ class AsyncOmniEngine: # to match the key used in Orchestrator.request_states so that # output routing (output.request_id lookup) can find the req_state. request.external_req_id = request_id - request = _apply_omni_final_stage_metadata(request, final_stage_id) + request = apply_omni_final_stage_metadata(request, final_stage_id) # Registration with stage 0's output processor is deferred to the # orchestrator thread (see Orchestrator._handle_add_request), which @@ -811,7 +776,7 @@ class AsyncOmniEngine: companion_params, companion_spl = ep.apply_overrides(stage0_params, sampling_params_list) if isinstance(companion_prompt, dict): - _inject_global_id(companion_prompt, cid) + inject_global_id(companion_prompt, cid) request = self.input_processor.process_inputs( request_id=cid, @@ -1434,6 +1399,268 @@ class AsyncOmniEngine: resumable=resumable, ) + def open_duplex_session( + self, + session_id: str, + *, + session_mode: str = "duplex", + capabilities: dict[str, object] | None = None, + session_config: dict[str, object] | None = None, + runtime_config: dict[str, object] | None = None, + fence: DuplexFence, + timeout: float | None = 10.0, + ) -> dict[str, object]: + """Open an engine-level duplex session.""" + return self._get_duplex_control_client().open( + session_id, + session_mode=session_mode, + capabilities=capabilities, + session_config=session_config, + runtime_config=runtime_config, + fence=fence, + timeout=timeout, + ) + + async def open_duplex_session_async( + self, + session_id: str, + *, + session_mode: str = "duplex", + capabilities: dict[str, object] | None = None, + session_config: dict[str, object] | None = None, + runtime_config: dict[str, object] | None = None, + fence: DuplexFence, + timeout: float | None = 10.0, + ) -> dict[str, object]: + """Async wrapper for opening an engine-level duplex session.""" + loop = asyncio.get_running_loop() + return await loop.run_in_executor( + None, + lambda: self.open_duplex_session( + session_id, + session_mode=session_mode, + capabilities=capabilities, + session_config=session_config, + runtime_config=runtime_config, + fence=fence, + timeout=timeout, + ), + ) + + def append_duplex_input( + self, + session_id: str, + *, + mode: str, + payload: object, + operation_id: str | None = None, + final: bool = False, + expected_epoch: int | None = None, + fence: DuplexFence, + timeout: float | None = 10.0, + ) -> dict[str, object]: + """Append input to an engine-level duplex session.""" + return self._get_duplex_control_client().append( + session_id, + mode=mode, + payload=payload, + operation_id=operation_id, + final=final, + expected_epoch=expected_epoch, + fence=fence, + timeout=timeout, + ) + + async def append_duplex_input_async( + self, + session_id: str, + *, + mode: str, + payload: object, + operation_id: str | None = None, + final: bool = False, + expected_epoch: int | None = None, + fence: DuplexFence, + timeout: float | None = 10.0, + ) -> dict[str, object]: + """Async wrapper for appending duplex input.""" + loop = asyncio.get_running_loop() + return await loop.run_in_executor( + None, + lambda: self.append_duplex_input( + session_id, + mode=mode, + payload=payload, + operation_id=operation_id, + final=final, + expected_epoch=expected_epoch, + fence=fence, + timeout=timeout, + ), + ) + + def signal_duplex_turn( + self, + session_id: str, + *, + event: str, + fence: DuplexFence, + next_fence: DuplexFence | None = None, + session_config: dict[str, object] | None = None, + runtime_config: dict[str, object] | None = None, + timeout: float | None = 10.0, + ) -> dict[str, object]: + """Signal an engine-level duplex turn.""" + return self._get_duplex_control_client().signal( + session_id, + event=event, + fence=fence, + next_fence=next_fence, + session_config=session_config, + runtime_config=runtime_config, + timeout=timeout, + ) + + async def signal_duplex_turn_async( + self, + session_id: str, + *, + event: str, + fence: DuplexFence, + next_fence: DuplexFence | None = None, + session_config: dict[str, object] | None = None, + runtime_config: dict[str, object] | None = None, + timeout: float | None = 10.0, + ) -> dict[str, object]: + """Async wrapper for signaling a duplex turn.""" + loop = asyncio.get_running_loop() + return await loop.run_in_executor( + None, + lambda: self.signal_duplex_turn( + session_id, + event=event, + fence=fence, + next_fence=next_fence, + session_config=session_config, + runtime_config=runtime_config, + timeout=timeout, + ), + ) + + def close_duplex_session( + self, + session_id: str, + *, + reason: str = "client_close", + fence: DuplexFence, + timeout: float | None = 10.0, + ) -> dict[str, object]: + """Close an engine-level duplex session.""" + return self._get_duplex_control_client().close( + session_id, + reason=reason, + fence=fence, + timeout=timeout, + ) + + def touch_duplex_session( + self, + session_id: str, + *, + fence: DuplexFence, + activity: DuplexLeaseActivity, + timeout: float | None = 10.0, + ) -> dict[str, object]: + return self._get_duplex_control_client().touch( + session_id, + fence=fence, + activity=activity, + timeout=timeout, + ) + + async def touch_duplex_session_async( + self, + session_id: str, + *, + fence: DuplexFence, + activity: DuplexLeaseActivity, + timeout: float | None = 10.0, + ) -> dict[str, object]: + loop = asyncio.get_running_loop() + return await loop.run_in_executor( + None, + lambda: self.touch_duplex_session( + session_id, + fence=fence, + activity=activity, + timeout=timeout, + ), + ) + + def resume_duplex_session( + self, + session_id: str, + *, + fence: DuplexFence, + expected_lease_generation: int, + timeout: float | None = 10.0, + ) -> dict[str, object]: + return self._get_duplex_control_client().resume( + session_id, + fence=fence, + expected_lease_generation=expected_lease_generation, + timeout=timeout, + ) + + async def resume_duplex_session_async( + self, + session_id: str, + *, + fence: DuplexFence, + expected_lease_generation: int, + timeout: float | None = 10.0, + ) -> dict[str, object]: + loop = asyncio.get_running_loop() + return await loop.run_in_executor( + None, + lambda: self.resume_duplex_session( + session_id, + fence=fence, + expected_lease_generation=expected_lease_generation, + timeout=timeout, + ), + ) + + def _get_duplex_control_client(self) -> DuplexControlClient: + from vllm_omni.experimental.fullduplex.engine.duplex_control_client import DuplexControlClient + + client = getattr(self, "_duplex_control_client", None) + if client is None: + transport = getattr(self, "_correlated_rpc_client", None) + if transport is None: + raise RuntimeError("correlated RPC client is not initialized") + client = DuplexControlClient( + transport, + control_id_factory=lambda: uuid.uuid4().hex, + ) + self._duplex_control_client = client + return client + + async def close_duplex_session_async( + self, + session_id: str, + *, + reason: str = "client_close", + fence: DuplexFence, + timeout: float | None = 10.0, + ) -> dict[str, object]: + """Async wrapper for closing an engine-level duplex session.""" + loop = asyncio.get_running_loop() + return await loop.run_in_executor( + None, + lambda: self.close_duplex_session(session_id, reason=reason, fence=fence, timeout=timeout), + ) + def try_get_output(self, timeout: float = 0.001) -> EngineQueueMessage | None: """Read one output message from the Orchestrator output queue.""" try: @@ -1489,36 +1716,19 @@ class AsyncOmniEngine: stage_ids=stage_ids, ) - with self._rpc_lock: - self.request_queue.sync_q.put(msg) - deadline = None if timeout is None else time.monotonic() + timeout - - while True: - remaining = None if deadline is None else max(0.0, deadline - time.monotonic()) - try: - result_msg = self.rpc_output_queue.sync_q.get(timeout=remaining) - except queue.Empty as exc: - raise TimeoutError(f"collective_rpc timed out after {timeout} seconds") from exc - - if isinstance(result_msg, ErrorMessage): - raise RuntimeError(result_msg.error) - - if not isinstance(result_msg, CollectiveRPCResultMessage): - logger.warning( - "[AsyncOmniEngine] Dropping unexpected rpc queue message type=%s", - getattr(result_msg, "type", type(result_msg).__name__), - ) - continue - - if result_msg.rpc_id != rpc_id: - logger.warning( - "[AsyncOmniEngine] Dropping mismatched rpc result rpc_id=%s expected=%s", - result_msg.rpc_id, - rpc_id, - ) - continue - - return list(result_msg.results) + transport = self._correlated_rpc_client + if transport is None: + raise RuntimeError("correlated RPC client is not initialized") + result_msg = transport.execute( + ("collective", rpc_id), + msg, + timeout=timeout, + timeout_message=f"collective_rpc timed out after {timeout} seconds", + block_on_submit=True, + ) + if not isinstance(result_msg, CollectiveRPCResultMessage): + raise RuntimeError(f"unexpected collective RPC result type: {type(result_msg).__name__}") + return list(result_msg.results) async def collective_rpc_async( self, @@ -1555,22 +1765,61 @@ class AsyncOmniEngine: finalizer.detach() logger.info("[AsyncOmniEngine] Shutting down Orchestrator") - if self.request_queue is not None: - self.request_queue.sync_q.put_nowait(ShutdownRequestMessage()) - if self.is_alive(): - self.orchestrator_thread.join() + request_queue_closed = False + shutdown_enqueued = enqueue_orchestrator_shutdown( + self.request_queue, + timeout=SHUTDOWN_ENQUEUE_TIMEOUT_S, + ) + if self.request_queue is not None and not shutdown_enqueued: + logger.error( + "[AsyncOmniEngine] Failed to enqueue orchestrator shutdown; " + "closing the request queue to wake the request handler" + ) + try: + self.request_queue.close() + request_queue_closed = True + except Exception: + logger.exception("[AsyncOmniEngine] Failed to close the request queue") + + if self._correlated_rpc_client is not None: + try: + self._correlated_rpc_client.close() + except Exception: + logger.exception("[AsyncOmniEngine] Failed to close correlated RPC client") + + orchestrator_stopped = False + try: + if self.is_alive(): + self.orchestrator_thread.join(timeout=SHUTDOWN_JOIN_TIMEOUT_S) + orchestrator_stopped = not self.is_alive() + if not orchestrator_stopped: + logger.error( + "[AsyncOmniEngine] Orchestrator did not stop within %.1f seconds; continuing cleanup", + SHUTDOWN_JOIN_TIMEOUT_S, + ) + except Exception: + logger.exception("[AsyncOmniEngine] Failed to join Orchestrator thread") for q in (self.request_queue, self.output_queue, self.rpc_output_queue): try: - q.close() + if not (q is self.request_queue and request_queue_closed): + q.close() except Exception: pass - if hasattr(self, "_runtime") and self._runtime is not None: + if hasattr(self, "_runtime") and self._runtime is not None and orchestrator_stopped: try: self._runtime.shutdown() except Exception: logger.exception("[AsyncOmniEngine] Failed to shutdown StageRuntime") + elif hasattr(self, "_runtime") and self._runtime is not None: + logger.warning("[AsyncOmniEngine] Deferring StageRuntime shutdown until the Orchestrator exits") + threading.Thread( + target=shutdown_runtime_after_orchestrator, + args=(self.orchestrator_thread, self._runtime), + daemon=True, + name="omni-stage-runtime-shutdown", + ).start() # ── Release CuMem allocator memory pool ────────────────────────────── # When enable_sleep_mode is in use, the CuMem (CUDA Virtual Memory diff --git a/vllm_omni/engine/membership_controller.py b/vllm_omni/engine/membership_controller.py index 48b6fe559..a7676de54 100644 --- a/vllm_omni/engine/membership_controller.py +++ b/vllm_omni/engine/membership_controller.py @@ -157,7 +157,7 @@ class MembershipController: input_addr = StagePool._client_input_addr(client) if input_addr is None: raise RuntimeError(f"remote replica factory for stage {stage_id} produced a client without input address") - pool.add_client(input_addr, client) + pool.add_client(input_addr, client, replica_id=replica_id) logger.info( "[MembershipController] attached remote replica stage=%d replica=%d addr=%s", stage_id, diff --git a/vllm_omni/engine/messages.py b/vllm_omni/engine/messages.py index bb73f74a3..384d8a055 100644 --- a/vllm_omni/engine/messages.py +++ b/vllm_omni/engine/messages.py @@ -106,3 +106,7 @@ class CollectiveRPCResultMessage(EngineQueueMessage, kw_only=True): method: str stage_ids: list[int] results: list[object] + + @property + def rpc_correlation_key(self) -> tuple[str, str]: + return ("collective", self.rpc_id) diff --git a/vllm_omni/engine/orchestrator.py b/vllm_omni/engine/orchestrator.py index bdd678f27..9372fee28 100644 --- a/vllm_omni/engine/orchestrator.py +++ b/vllm_omni/engine/orchestrator.py @@ -13,8 +13,9 @@ from __future__ import annotations import asyncio import time as _time +from collections.abc import Callable from dataclasses import dataclass, field -from typing import Any +from typing import TYPE_CHECKING, Any import janus import torch @@ -22,11 +23,12 @@ from vllm.config import ModelConfig from vllm.logger import init_logger from vllm.outputs import CompletionOutput, RequestOutput from vllm.pooling_params import PoolingParams -from vllm.sampling_params import SamplingParams +from vllm.sampling_params import RequestOutputKind, SamplingParams from vllm.v1.engine import EngineCoreOutputs from vllm.v1.engine.exceptions import EngineDeadError from vllm.v1.metrics.stats import IterationStats +from vllm_omni.config.stage_config import DuplexSessionRuntimeConfig from vllm_omni.engine import OmniEngineCoreRequest from vllm_omni.engine.cfg_companion_tracker import CfgCompanionTracker from vllm_omni.engine.membership_controller import MembershipController @@ -53,6 +55,23 @@ from vllm_omni.outputs import OmniRequestOutput logger = init_logger(__name__) +if TYPE_CHECKING: + from vllm_omni.experimental.fullduplex.engine.contracts import ( + DuplexControlPlanePort, + DuplexOutputContext, + DuplexOutputDecision, + DuplexRequestIdentity, + DuplexRuntimeExtension, + DuplexStageRequestContext, + DuplexStageSubmission, + DuplexStageSubmissionResult, + ) + from vllm_omni.experimental.fullduplex.engine.duplex_session import ( + DuplexSessionRuntimeManager, + DuplexSessionRuntimeState, + ) + from vllm_omni.experimental.fullduplex.engine.messages import DuplexFence + def _build_terminal_empty_output( request_id: str, @@ -141,8 +160,13 @@ def build_engine_core_request_from_tokens( pooling_params = params.clone() prompt_embeds: torch.Tensor | None = prompt.get("prompt_embeds") + raw_additional_information = prompt.get("additional_information") + model_intermediate_buffer = prompt.get("model_intermediate_buffer") + wire_payload: dict[str, Any] | None = None + if isinstance(raw_additional_information, dict): + wire_payload = dict(raw_additional_information) additional_info_payload = serialize_additional_information( - prompt.get("additional_information"), + wire_payload, log_prefix=f"build_engine_core_request_from_tokens req={request_id}", ) @@ -159,6 +183,7 @@ def build_engine_core_request_from_tokens( prompt_embeds=prompt_embeds, resumable=resumable, additional_information=additional_info_payload, + model_intermediate_buffer=model_intermediate_buffer if isinstance(model_intermediate_buffer, dict) else None, ) @@ -186,6 +211,10 @@ class OrchestratorRequestState: # Per-request pipeline timing accumulator (milliseconds) pipeline_timings: dict[str, float] = field(default_factory=dict) + duplex_identity: DuplexRequestIdentity | None = None + duplex_stage_fences: dict[int, DuplexFence] = field(default_factory=dict) + duplex_config_generation: int = -1 + running_counter_registered: bool = False @dataclass @@ -194,10 +223,124 @@ class StreamingInputState: enabled: bool = False # Flag of segment of streaming input finished segment_finished: bool = False + # Tokens from the current raw segment boundary. The vLLM output processor + # does not guarantee that EngineCoreOutput.new_token_ids survives on the + # processed RequestOutput used by the routing layer. + segment_token_ids: list[int] = field(default_factory=list) + segment_output_metadata: dict[str, Any] = field(default_factory=dict) # Streaming update prompt length new_prompt_len_snapshot: int | None = None # Model/bridge-specific runtime states (e.g., thinker->talker) bridge_states: dict[str, Any] = field(default_factory=dict) + # Synchronous stage-transition capability installed by the orchestrator + # while the downstream input processor consumes upstream token output. + source_token_decoder: Callable[..., str] | None = None + + +class _OrchestratorDuplexStagePort: + """Adapts generic stage pools to the model-neutral duplex control plane.""" + + def __init__( + self, + *, + stage_pools: list[StagePool], + request_states: dict[str, OrchestratorRequestState], + running_counter: OmniRequestCounter | None, + cleanup_request_ids: Callable[..., Any], + ) -> None: + self._stage_pools = stage_pools + self._request_states = request_states + self._running_counter = running_counter + self._cleanup_request_ids = cleanup_request_ids + + @property + def stage_count(self) -> int: + return len(self._stage_pools) + + def sampling_defaults(self) -> tuple[object, ...]: + return tuple(pool.stage_client.default_sampling_params for pool in self._stage_pools) + + @staticmethod + def _sync_bridge_state( + request_state: OrchestratorRequestState, + context: DuplexStageRequestContext, + ) -> None: + duplex_state = request_state.streaming.bridge_states.setdefault("duplex", {}) + if not isinstance(duplex_state, dict): + duplex_state = {} + request_state.streaming.bridge_states["duplex"] = duplex_state + previous_epoch = duplex_state.get("epoch") + if not isinstance(duplex_state.get("model_turn_id"), int) or previous_epoch != context.fence.epoch: + duplex_state["model_turn_id"] = context.fence.turn_id + duplex_state.update( + { + "session_id": context.session_id, + "fence": context.fence, + "incarnation": context.fence.incarnation, + "epoch": context.fence.epoch, + "turn_id": context.fence.turn_id, + "response_seq": context.fence.response_seq, + "session_config": dict(context.session_config), + "runtime_config": dict(context.runtime_config), + } + ) + + def ensure_request(self, context: DuplexStageRequestContext) -> None: + from vllm_omni.experimental.fullduplex.engine.contracts import DuplexRequestIdentity + + request_state = self._request_states.get(context.request_id) + if request_state is None: + request_state = OrchestratorRequestState( + request_id=context.request_id, + prompt=None, + sampling_params_list=list(context.sampling_params), + final_stage_id=context.final_stage_id, + duplex_config_generation=context.config_generation, + ) + request_state.streaming.enabled = True + self._request_states[context.request_id] = request_state + elif request_state.duplex_config_generation != context.config_generation: + request_state.sampling_params_list = list(context.sampling_params) + request_state.duplex_config_generation = context.config_generation + request_state.duplex_identity = DuplexRequestIdentity( + session_id=context.session_id, + fence=context.fence, + ) + self._sync_bridge_state(request_state, context) + + async def submit(self, submission: DuplexStageSubmission) -> DuplexStageSubmissionResult: + from vllm_omni.experimental.fullduplex.engine.contracts import DuplexStageSubmissionResult + + context = submission.context + request_state = self._request_states.get(context.request_id) + if request_state is None: + raise RuntimeError(f"duplex request was not preregistered: {context.request_id}") + request = build_engine_core_request_from_tokens( + request_id=context.request_id, + prompt=dict(submission.prompt), + params=context.stage_sampling_params, + model_config=self._stage_pools[context.stage_id].stage_vllm_config.model_config, + resumable=True, + ) + request.external_req_id = request.request_id + pool = self._stage_pools[context.stage_id] + if submission.already_submitted: + replica_id = await pool.submit_update(context.request_id, request_state, request) + else: + replica_id = await pool.submit_initial(context.request_id, request_state, request, prompt_text=None) + request_state.duplex_stage_fences[context.stage_id] = context.fence + request_state.stage_submit_ts[context.stage_id] = _time.time() + if not request_state.running_counter_registered and self._running_counter is not None: + self._running_counter.increment() + request_state.running_counter_registered = True + return DuplexStageSubmissionResult( + request_id=context.request_id, + stage_id=context.stage_id, + replica_id=replica_id, + ) + + async def cleanup(self, request_ids: list[str], *, abort: bool = False) -> None: + await self._cleanup_request_ids(request_ids, abort=abort) class Orchestrator: @@ -208,6 +351,7 @@ class Orchestrator: _running_counter: OmniRequestCounter | None = None _transfer_emitter: Any = None _stat_logger: OmniPrometheusStatLogger | None = None + duplex_control_plane: DuplexControlPlanePort | None = None def __init__( self, @@ -223,6 +367,9 @@ class Orchestrator: transfer_emitter: Any = None, log_stats: bool = False, enable_orch_monitor: bool = False, + duplex_runtime_extension: DuplexRuntimeExtension | None = None, + enable_duplex_control: bool = False, + duplex_session_config: DuplexSessionRuntimeConfig | None = None, ) -> None: self.request_async_queue = request_async_queue self.output_async_queue = output_async_queue @@ -236,7 +383,7 @@ class Orchestrator: replica_sampler=self._sample_replica_metrics, ) for stage_id, pool in enumerate(self.stage_pools): - for replica_id in pool.live_replica_ids(): + for replica_id in pool.available_replica_ids(): self._orch_monitor.register_replica(stage_id, replica_id) # PD disaggregation state @@ -254,6 +401,33 @@ class Orchestrator: self._cfg_tracker = CfgCompanionTracker() self._stage_input_processors: dict[int, Any] = {} + self.duplex_control_plane: DuplexControlPlanePort | None = None + self._duplex_reaper_interval_s = 1.0 + if enable_duplex_control: + from vllm_omni.experimental.fullduplex.engine.duplex_control_plane import DuplexControlPlane + from vllm_omni.experimental.fullduplex.engine.lease import DuplexLeaseConfig + + runtime_session_config = duplex_session_config or DuplexSessionRuntimeConfig() + self._duplex_reaper_interval_s = runtime_session_config.reaper_interval_s + + self.duplex_control_plane = DuplexControlPlane( + extension=duplex_runtime_extension, + stage_port=_OrchestratorDuplexStagePort( + stage_pools=self.stage_pools, + request_states=self.request_states, + running_counter=self._running_counter, + cleanup_request_ids=self._cleanup_request_ids, + ), + result_sink=self.rpc_async_queue, + lifecycle_sink=self.output_async_queue, + lease_config=DuplexLeaseConfig( + idle_ttl_s=runtime_session_config.idle_ttl_s, + disconnect_grace_s=runtime_session_config.disconnect_grace_s, + ), + max_sessions=runtime_session_config.max_sessions, + completed_append_limit=runtime_session_config.completed_append_cache_size, + ) + self._shutdown_event = asyncio.Event() self._stages_shutdown = False self._fatal_error: str | None = None @@ -327,6 +501,17 @@ class Orchestrator: logger.exception("[Orchestrator] OmniPrometheusStatLogger init failed; metrics wrap disabled") self._stat_logger = None + @property + def duplex_sessions(self) -> DuplexSessionRuntimeManager: + if self.duplex_control_plane is None: + raise RuntimeError("duplex control plane is disabled") + return self.duplex_control_plane.sessions + + def _require_duplex_control_plane(self) -> DuplexControlPlanePort: + if self.duplex_control_plane is None: + raise RuntimeError("duplex control plane is disabled") + return self.duplex_control_plane + async def run(self) -> None: """Main entry point for the Orchestrator event loop.""" logger.info("[Orchestrator] Starting event loop") @@ -347,6 +532,8 @@ class Orchestrator: membership_watcher = self._membership.start() tasks = [request_task, output_task] + if self.duplex_control_plane is not None: + tasks.append(asyncio.create_task(self._duplex_reaper_loop(), name="orchestrator-duplex-reaper")) if membership_watcher is not None: tasks.append(membership_watcher) @@ -361,6 +548,15 @@ class Orchestrator: # During teardown this is expected; the finally block handles # proper cleanup. Do not re-raise. logger.info("[Orchestrator] Engine dead during shutdown: %s", e) + if self._fatal_error is None: + self._fatal_error = str(e) or "Stage engine died" + await self.rpc_async_queue.put( + ErrorMessage( + error=self._fatal_error or str(e), + fatal=True, + stage_id=self._fatal_error_stage_id, + ) + ) except Exception: logger.exception("[Orchestrator] Fatal error in orchestrator tasks") raise @@ -373,6 +569,8 @@ class Orchestrator: await asyncio.gather(*tasks, return_exceptions=True) except Exception: pass + if self.duplex_control_plane is not None: + await self.duplex_control_plane.shutdown() if self._fatal_error is not None: await self._drain_pending_requests_on_fatal() @@ -405,6 +603,8 @@ class Orchestrator: await self._handle_streaming_update(msg) elif msg_type == "add_companion_request": await self._handle_add_companion(msg) + elif self.duplex_control_plane is not None and self.duplex_control_plane.accepts(msg): + self.duplex_control_plane.dispatch(msg) elif msg_type == "abort": await self._handle_abort(msg) elif msg_type == "collective_rpc": @@ -432,6 +632,21 @@ class Orchestrator: else: logger.warning("[Orchestrator] Unknown message type: %s", msg_type) + async def _duplex_reaper_loop(self) -> None: + while not self._shutdown_event.is_set(): + try: + await asyncio.wait_for( + self._shutdown_event.wait(), + timeout=self._duplex_reaper_interval_s, + ) + except TimeoutError: + plane = self.duplex_control_plane + if plane is not None: + try: + await plane.reap_expired() + except Exception: + logger.exception("[Orchestrator] Duplex expiry cleanup failed; retrying on next tick") + async def _handle_add_request(self, msg: StageSubmissionMessage) -> None: """Handle an add_request message from the main thread.""" stage_id = 0 @@ -466,8 +681,7 @@ class Orchestrator: mm_features=getattr(prompt, "mm_features", None), ) self.request_states[request_id] = req_state - if self._running_counter is not None: - self._running_counter.increment() + self._register_running_request(req_state) req_state.streaming.enabled = bool(getattr(prompt, "resumable", False)) req_state.stage_submit_ts[stage_id] = _time.time() enqueue_ts = msg.enqueue_ts @@ -590,6 +804,7 @@ class Orchestrator: return for pool in self.stage_pools: await pool.abort_requests(request_ids) + pool.release_bindings(request_ids) def _release_request_bindings(self, request_ids: list[str]) -> None: """Release all stage-local route bindings for the given request ids.""" @@ -662,7 +877,7 @@ class Orchestrator: idle = True for stage_id in range(self.num_stages): pool = self.stage_pools[stage_id] - for replica_id in pool.live_replica_ids(): + for replica_id in pool.available_replica_ids(): if self._shutdown_event.is_set(): return @@ -686,6 +901,17 @@ class Orchestrator: if req_state is None or not req_state.streaming.enabled: continue req_state.streaming.segment_finished = bool(getattr(eco, "is_segment_finished", False)) + req_state.streaming.segment_token_ids = ( + self._coerce_int_list(getattr(eco, "new_token_ids", None)) + if req_state.streaming.segment_finished + else [] + ) + raw_mm = self._completion_multimodal_output(eco, None) + req_state.streaming.segment_output_metadata = ( + dict(raw_mm) + if req_state.streaming.segment_finished and isinstance(raw_mm, dict) + else {} + ) req_state.streaming.new_prompt_len_snapshot = getattr( eco, "new_prompt_len_snapshot", @@ -715,32 +941,62 @@ class Orchestrator: raise except EngineDeadError as e: logger.error( - "[Orchestrator] Stage-%s is dead: %s", + "[Orchestrator] Stage-%s replica-%s is dead: %s", stage_id, + replica_id, e, ) - # TODO: Fault handling is intentionally fail-stop at - # the orchestrator level today. If one replica in a - # logical stage dies, we promote it to `_fatal_error`, - # notify requests already admitted to that stage, and - # re-raise so `run()` shuts down all stages. This is - # conservative but means a single unhealthy replica in - # a multi-replica deployment can take down otherwise - # healthy replicas in other stages. Revisit this when - # adding per-replica fault isolation / eviction. - self._fatal_error = str(e) - self._fatal_error_stage_id = stage_id - for req_id, req_state in list(self.request_states.items()): - if stage_id in req_state.stage_submit_ts: + affected_request_ids = pool.mark_replica_unavailable(replica_id) + closed_sessions = ( + self.duplex_control_plane.close_sessions_for_request_ids( + affected_request_ids, + abort=False, + ) + if self.duplex_control_plane is not None + else {} + ) + for session_id, stale_request_ids in closed_sessions.items(): + affected_request_ids.extend(stale_request_ids) + logger.warning( + "[Orchestrator] closed duplex session %s after stage-%s replica-%s died; " + "stale_request_ids=%s", + session_id, + stage_id, + replica_id, + stale_request_ids, + ) + affected_request_ids = list(dict.fromkeys(affected_request_ids)) + if pool.available_replica_ids(): + for req_id in affected_request_ids: await self.output_async_queue.put( ErrorMessage( error=str(e), - fatal=True, + fatal=False, request_id=req_id, stage_id=stage_id, ) ) - self.request_states.pop(req_id, None) + await self._cleanup_request_ids( + affected_request_ids, + close_duplex_sessions=True, + ) + continue + + self._fatal_error = str(e) + self._fatal_error_stage_id = stage_id + for req_id in affected_request_ids: + await self.output_async_queue.put( + ErrorMessage( + error=str(e), + fatal=True, + request_id=req_id, + stage_id=stage_id, + ) + ) + await self._cleanup_request_ids( + affected_request_ids, + close_duplex_sessions=True, + ) self._shutdown_event.set() raise except Exception: @@ -781,7 +1037,8 @@ class Orchestrator: continue stage_metrics = None - if output.finished: + segment_finished = req_state.streaming.enabled and req_state.streaming.segment_finished + if output.finished or segment_finished: stage_metrics = pool.build_stage_metrics( [output], submit_ts=req_state.stage_submit_ts.get(stage_id, _time.time()), @@ -811,22 +1068,56 @@ class Orchestrator: await self._cleanup_request_ids( [parent_id, *self._cfg_tracker.cleanup_parent(parent_id)], abort=True, + close_duplex_sessions=True, ) # ---- Shared helpers ---- - async def _cleanup_request_ids(self, request_ids: list[str], *, abort: bool = False) -> None: + async def _cleanup_request_ids( + self, + request_ids: list[str], + *, + abort: bool = False, + close_duplex_sessions: bool = False, + ) -> None: """Release pool bindings and logical request state for the given ids.""" if not request_ids: return - if abort: - await self._abort_request_ids(request_ids) - self._release_request_bindings(request_ids) - for request_id in request_ids: - self._pd_kv_params.pop(request_id, None) - if self.request_states.pop(request_id, None) is not None and self._running_counter is not None: - self._running_counter.decrement() + cleanup_ids = list(dict.fromkeys(request_ids)) + closing_session_ids: list[str] = [] + if close_duplex_sessions and self.duplex_control_plane is not None: + closed_sessions = self.duplex_control_plane.close_sessions_for_request_ids( + cleanup_ids, + abort=abort, + cleanup_in_progress=True, + ) + closing_session_ids.extend(closed_sessions) + for session_id, stale_request_ids in closed_sessions.items(): + logger.info( + "[Orchestrator] closed duplex session %s while cleaning failed request ids %s", + session_id, + stale_request_ids, + ) + cleanup_ids.extend(stale_request_ids) + cleanup_ids = list(dict.fromkeys(cleanup_ids)) + + try: + if abort: + await self._abort_request_ids(cleanup_ids) + self._release_request_bindings(cleanup_ids) + for request_id in cleanup_ids: + self._pd_kv_params.pop(request_id, None) + req_state = self.request_states.pop(request_id, None) + if req_state is not None and req_state.running_counter_registered and self._running_counter is not None: + self._running_counter.decrement() + req_state.running_counter_registered = False + except BaseException: + if closing_session_ids and self.duplex_control_plane is not None: + self.duplex_control_plane.defer_request_cleanups(closing_session_ids) + raise + if closing_session_ids and self.duplex_control_plane is not None: + self.duplex_control_plane.finalize_closed_sessions(closing_session_ids) async def _apply_raw_terminal_stage_finish( self, @@ -873,6 +1164,34 @@ class Orchestrator: params.cfg_kv_request_ids = companion_request_ids return params + def _duplex_session_for_req_state(self, req_state: OrchestratorRequestState) -> DuplexSessionRuntimeState | None: + if self.duplex_control_plane is None: + return None + return self.duplex_control_plane.session_for_identity(req_state.duplex_identity) + + def _record_duplex_stage_submission( + self, + stage_id: int, + request_id: str, + replica_id: int, + req_state: OrchestratorRequestState, + ) -> None: + del replica_id + identity = req_state.duplex_identity + session = self._duplex_session_for_req_state(req_state) + if identity is None or session is None: + return + req_state.duplex_stage_fences[stage_id] = identity.fence + session.bind_stage_request(stage_id, request_id, fence=identity.fence) + req_state.stage_submit_ts[stage_id] = _time.time() + self._register_running_request(req_state) + + def _register_running_request(self, req_state: OrchestratorRequestState) -> None: + if req_state.running_counter_registered or self._running_counter is None: + return + self._running_counter.increment() + req_state.running_counter_registered = True + async def _route_output( self, stage_id: int, @@ -885,7 +1204,6 @@ class Orchestrator: req_id = output.request_id finished = output.finished submit_ts = req_state.stage_submit_ts.get(stage_id) - # CFG companion: stash output so parent can bundle [parent, *companions] # into source_outputs for the bridge (e.g. thinker2imagegen). if finished and self._cfg_tracker.is_companion(req_id): @@ -899,7 +1217,17 @@ class Orchestrator: req_state.finished_final_output_stage_ids.add(stage_id) final_output_stage_ids = req_state.final_output_stage_ids or {req_state.final_stage_id} request_finished = final_output_stage_ids.issubset(req_state.finished_final_output_stage_ids) - if self.stage_pools[stage_id].final_output: + # Duplex stage-0 segment boundaries are not client-visible outputs: + # direct decisions are emitted by the model runtime extension below, + # while spoken content flows through the next stage. Forwarding + # the raw stage-0 output as well injects one cumulative-text, + # no-audio message per unit that every downstream consumer must + # filter out again (the official implementation returns exactly one + # result per audio chunk). + is_duplex_stage0_segment = ( + stage_id == 0 and self._is_duplex_session_request(req_state) and req_state.streaming.segment_finished + ) + if self.stage_pools[stage_id].final_output and not is_duplex_stage0_segment: await self.output_async_queue.put( OutputMessage( request_id=req_id, @@ -907,7 +1235,10 @@ class Orchestrator: replica_id=replica_id, engine_outputs=output, metrics=stage_metrics, - finished=request_finished, + finished=( + request_finished + or (self._is_duplex_session_request(req_state) and req_state.streaming.segment_finished) + ), stage_submit_ts=submit_ts, ) ) @@ -928,6 +1259,18 @@ class Orchestrator: self._pd_kv_params[req_id] = kv_params if isinstance(kv_params, dict) else dict(kv_params) req_state.pd_prefill_multimodal_output = getattr(output, "multimodal_output", None) + duplex_output_decision = self._duplex_output_decision(stage_id, output, req_state) + if duplex_output_decision is not None: + await self._emit_duplex_direct_output( + stage_id, + req_id, + output, + duplex_output_decision, + stage_metrics, + submit_ts, + ) + return + if ( (finished or (req_state.streaming.enabled and req_state.streaming.segment_finished)) and stage_id < req_state.final_stage_id @@ -941,6 +1284,12 @@ class Orchestrator: ): self._cfg_tracker.defer_parent(req_id, output, stage_id) else: + stage_params = req_state.sampling_params_list[stage_id] + final_only_finished = ( + req_state.streaming.enabled + and finished + and getattr(stage_params, "output_kind", None) == RequestOutputKind.FINAL_ONLY + ) await self._forward_to_next_stage( req_id, stage_id, @@ -948,9 +1297,14 @@ class Orchestrator: req_state, src_replica_id=replica_id, is_streaming_session=req_state.streaming.enabled, - is_final_update=False, + is_final_update=final_only_finished, ) - if req_state.streaming.enabled and finished: + if ( + req_state.streaming.enabled + and finished + and not final_only_finished + and not self._is_duplex_session_request(req_state) + ): # For streaming sessions, send the terminal (resumable=False) update only on a finish await self._forward_to_next_stage( req_id, @@ -962,7 +1316,7 @@ class Orchestrator: is_final_update=True, ) - if request_finished: + if request_finished and not self._is_duplex_session_request(req_state): await self._cleanup_request_ids([req_id, *self._cfg_tracker.cleanup_parent(req_id)]) def _next_stage_already_submitted(self, stage_id: int, req_state: OrchestratorRequestState) -> bool: @@ -1039,6 +1393,137 @@ class Orchestrator: request.external_req_id = req_id return request + @staticmethod + def _duplex_output_context( + req_state: OrchestratorRequestState, + *, + stage_id: int | None = None, + ) -> DuplexOutputContext | None: + identity = req_state.duplex_identity + if identity is None: + return None + from vllm_omni.experimental.fullduplex.engine.contracts import ( + DuplexOutputContext, + DuplexRequestIdentity, + ) + + fence = req_state.duplex_stage_fences.get(stage_id, identity.fence) if stage_id is not None else identity.fence + return DuplexOutputContext( + identity=DuplexRequestIdentity( + session_id=identity.session_id, + fence=fence, + ), + final_stage_id=req_state.final_stage_id, + segment_finished=req_state.streaming.enabled and req_state.streaming.segment_finished, + segment_token_ids=tuple(req_state.streaming.segment_token_ids), + segment_output_metadata=req_state.streaming.segment_output_metadata, + ) + + @staticmethod + def _is_duplex_session_request(req_state: OrchestratorRequestState) -> bool: + return req_state.duplex_identity is not None + + @staticmethod + def _duplex_fence_for_req_state( + req_state: OrchestratorRequestState, + *, + stage_id: int | None = None, + ) -> DuplexFence | None: + context = Orchestrator._duplex_output_context(req_state, stage_id=stage_id) + return context.identity.fence if context is not None else None + + def _duplex_output_decision( + self, + stage_id: int, + output: Any, + req_state: OrchestratorRequestState, + ) -> DuplexOutputDecision | None: + if self.duplex_control_plane is None: + return None + context = self._duplex_output_context(req_state, stage_id=stage_id) + decision = self.duplex_control_plane.decide_output( + stage_id, + output, + context, + ) + return decision + + async def _emit_duplex_direct_output( + self, + stage_id: int, + req_id: str, + output: Any, + decision: DuplexOutputDecision, + stage_metrics: Any, + submit_ts: float | None, + ) -> None: + action = getattr(decision.action, "value", decision.action) + if action != "direct_response": + raise ValueError(f"Unsupported duplex output action: {action}") + from vllm_omni.experimental.fullduplex.output import attach_duplex_output_decision + + engine_output = attach_duplex_output_decision( + OmniRequestOutput( + request_id=req_id, + finished=True, + stage_id=stage_id, + final_output_type=decision.final_output_type, + request_output=output, + ), + decision, + ) + await self.output_async_queue.put( + OutputMessage( + request_id=req_id, + stage_id=stage_id, + engine_outputs=engine_output, + metrics=stage_metrics, + finished=True, + stage_submit_ts=submit_ts, + ) + ) + + @staticmethod + def _completion_multimodal_output(output: Any, completion: Any) -> dict[str, Any]: + mm_output = getattr(output, "multimodal_output", None) + if isinstance(mm_output, dict): + return mm_output + mm_output = getattr(completion, "multimodal_output", None) if completion is not None else None + return mm_output if isinstance(mm_output, dict) else {} + + @classmethod + def _coerce_int_list(cls, value: Any) -> list[int]: + if value is None: + return [] + if hasattr(value, "detach"): + try: + value = value.detach().cpu().reshape(-1).tolist() + except Exception: + return [] + if not isinstance(value, (list, tuple)): + return [] + out: list[int] = [] + for item in value: + token_id = cls._coerce_int(item) + if token_id is not None: + out.append(token_id) + return out + + @staticmethod + def _coerce_int(value: Any) -> int | None: + if hasattr(value, "detach"): + try: + value = value.detach().cpu().reshape(-1) + if value.numel() == 0: + return None + value = value[0].item() + except Exception: + return None + try: + return int(value) + except (TypeError, ValueError): + return None + async def _handle_cfg_companion_ready(self, req_id: str) -> None: """Mark a CFG companion as done; if all companions are done, flush deferred parent.""" parent_id = self._cfg_tracker.on_companion_completed(req_id) @@ -1293,9 +1778,9 @@ class Orchestrator: diffusion_prompt = req_state.prompt if already_submitted: - await next_pool.submit_update(req_id, req_state, diffusion_prompt) + replica_id = await next_pool.submit_update(req_id, req_state, diffusion_prompt) else: - await next_pool.submit_initial( + replica_id = await next_pool.submit_initial( req_id, req_state, diffusion_prompt, @@ -1307,6 +1792,12 @@ class Orchestrator: }, params_override=self._maybe_clone_diffusion_params_for_cfg(req_id, params), ) + self._record_duplex_stage_submission( + next_logical, + req_id, + replica_id, + req_state, + ) req_state.stage_submit_ts[next_logical] = _time.time() _tx_ms = (_time.perf_counter() - _t_submit_start) * 1000.0 self._emit_tx_edge( @@ -1351,9 +1842,15 @@ class Orchestrator: ) request.external_req_id = request.request_id if already_submitted: - await next_pool.submit_update(req_id, req_state, request) + replica_id = await next_pool.submit_update(req_id, req_state, request) else: - await next_pool.submit_initial(req_id, req_state, request, prompt_text=None) + replica_id = await next_pool.submit_initial(req_id, req_state, request, prompt_text=None) + self._record_duplex_stage_submission( + next_logical, + req_id, + replica_id, + req_state, + ) req_state.stage_submit_ts[next_logical] = _time.time() _tx_ms = (_time.perf_counter() - _t_submit_start) * 1000.0 @@ -1373,6 +1870,13 @@ class Orchestrator: {}, )[req_id] = req_state.pd_prefill_multimodal_output + previous_decoder = req_state.streaming.source_token_decoder + source_processor = self.stage_pools[src_stage_id].output_processor + tokenizer = getattr(source_processor, "tokenizer", None) + decode = getattr(tokenizer, "decode", None) + if callable(decode): + req_state.streaming.source_token_decoder = decode + try: next_inputs = next_client.process_engine_inputs( source_outputs, @@ -1386,6 +1890,8 @@ class Orchestrator: next_logical, ) raise + finally: + req_state.streaming.source_token_decoder = previous_decoder if not next_inputs: if not getattr(output, "finished", False): @@ -1446,9 +1952,15 @@ class Orchestrator: ) if already_submitted: - await next_pool.submit_update(req_id, req_state, request) + replica_id = await next_pool.submit_update(req_id, req_state, request) else: - await next_pool.submit_initial(req_id, req_state, request, prompt_text=None) + replica_id = await next_pool.submit_initial(req_id, req_state, request, prompt_text=None) + self._record_duplex_stage_submission( + next_logical, + req_id, + replica_id, + req_state, + ) req_state.stage_submit_ts[next_logical] = _time.time() _tx_ms = (_time.perf_counter() - _t_submit_start) * 1000.0 diff --git a/vllm_omni/engine/rpc_result_router.py b/vllm_omni/engine/rpc_result_router.py new file mode 100644 index 000000000..516b36f16 --- /dev/null +++ b/vllm_omni/engine/rpc_result_router.py @@ -0,0 +1,161 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from __future__ import annotations + +import queue +import threading +from typing import TypeAlias + +from vllm.logger import init_logger + +from vllm_omni.engine.messages import ( + EngineQueueMessage, + ErrorMessage, +) + +logger = init_logger(__name__) + +RpcCorrelationKey: TypeAlias = tuple[str, str] +RpcWaiter: TypeAlias = queue.Queue[EngineQueueMessage] + + +class RpcResultRouter: + """Single consumer that dispatches shared RPC results by correlation ID.""" + + def __init__(self, source_queue) -> None: + self._source_queue = source_queue + self._pending: dict[RpcCorrelationKey, RpcWaiter] = {} + self._lock = threading.Lock() + self._stopped = threading.Event() + self._terminal_error: ErrorMessage | None = None + self._thread = threading.Thread( + target=self._run, + daemon=True, + name="omni-rpc-result-router", + ) + self._thread.start() + + def register(self, key: RpcCorrelationKey) -> RpcWaiter: + waiter: RpcWaiter = queue.Queue(maxsize=1) + with self._lock: + if self._stopped.is_set(): + raise RuntimeError("RPC result router is closed") + if self._terminal_error is not None: + raise RuntimeError(self._terminal_error.error) + if key in self._pending: + raise RuntimeError(f"duplicate pending RPC correlation key: {key!r}") + self._pending[key] = waiter + return waiter + + def unregister(self, key: RpcCorrelationKey, waiter: RpcWaiter) -> None: + with self._lock: + if self._pending.get(key) is waiter: + self._pending.pop(key, None) + + def close(self) -> None: + if self._stopped.is_set(): + return + self._stopped.set() + self._broadcast_error(ErrorMessage(error="RPC result router closed", fatal=True)) + self._thread.join(timeout=1.0) + + def _run(self) -> None: + while not self._stopped.is_set(): + try: + message = self._source_queue.get(timeout=0.1) + except queue.Empty: + continue + except Exception as exc: + if not self._stopped.is_set(): + logger.exception("RPC result router source queue failed") + self._broadcast_error(ErrorMessage(error=str(exc), fatal=True)) + return + + if isinstance(message, ErrorMessage): + if message.fatal: + self._broadcast_error(message) + else: + logger.warning( + "Dropping uncorrelated non-fatal RPC error request_id=%s stage_id=%s: %s", + message.request_id, + message.stage_id, + message.error, + ) + continue + key = self._correlation_key(message) + if key is None: + logger.warning( + "Dropping unexpected RPC result message type=%s", + getattr(message, "type", type(message).__name__), + ) + continue + with self._lock: + waiter = self._pending.pop(key, None) + if waiter is None: + logger.warning("Dropping late or unknown RPC result correlation_key=%s", key) + continue + waiter.put_nowait(message) + + def _broadcast_error(self, message: ErrorMessage) -> None: + with self._lock: + if message.fatal: + self._terminal_error = message + waiters = list(self._pending.values()) + self._pending.clear() + for waiter in waiters: + try: + waiter.put_nowait(message) + except queue.Full: + pass + + @staticmethod + def _correlation_key(message: EngineQueueMessage) -> RpcCorrelationKey | None: + key = getattr(message, "rpc_correlation_key", None) + if not isinstance(key, tuple) or len(key) != 2 or not all(isinstance(part, str) and part for part in key): + return None + return key + + +class CorrelatedRpcClient: + """Own request submission and correlated result waiting as one lifecycle.""" + + def __init__(self, request_queue, result_queue) -> None: + self._request_queue = request_queue + self._router = RpcResultRouter(result_queue) + + def execute( + self, + key: RpcCorrelationKey, + message: EngineQueueMessage, + *, + timeout: float | None, + timeout_message: str, + block_on_submit: bool = False, + ) -> EngineQueueMessage: + waiter = self._router.register(key) + try: + if block_on_submit: + self._request_queue.put(message) + else: + self._request_queue.put_nowait(message) + try: + result = waiter.get(timeout=timeout) + except queue.Empty as exc: + raise TimeoutError(timeout_message) from exc + if isinstance(result, ErrorMessage): + raise RuntimeError(result.error) + return result + finally: + self._router.unregister(key, waiter) + + def close(self) -> None: + self._router.close() + + +__all__ = [ + "CorrelatedRpcClient", + "RpcCorrelationKey", + "RpcResultRouter", + "RpcWaiter", +] diff --git a/vllm_omni/engine/stage_pool.py b/vllm_omni/engine/stage_pool.py index 34af459ab..4c4d3b7ee 100644 --- a/vllm_omni/engine/stage_pool.py +++ b/vllm_omni/engine/stage_pool.py @@ -97,6 +97,7 @@ class StagePool: self._stage_vllm_config = stage_vllm_config self._next_replica_id = 0 self._request_bindings: dict[str, int] = {} + self._unavailable_replicas: set[int] = set() self._replica_metrics: list[_ReplicaMetrics] = [_ReplicaMetrics() for _ in self.clients] self._output_timestamps_by_request: dict[str, list[float]] = {} self._non_empty_first_output_timestamps_by_request: dict[str, float] = {} @@ -206,27 +207,41 @@ class StagePool: return addr return None - def add_client(self, input_addr: str, client: Any) -> int: + def add_client(self, input_addr: str, client: Any, *, replica_id: int | None = None) -> int: """Register a head-side client for ``input_addr``. Returns the assigned ``replica_id`` (index into :attr:`clients`). - If the address is already known, replaces the existing client and - returns its existing id (this should not happen in practice — the - master server assigns unique slots — but the contract is idempotent - to keep the dispatch layer robust). + A coordinator-provided ``replica_id`` restores that exact slot after + unregister/re-register. Without one, a known address keeps its slot + and a new address takes the first vacant slot. """ if not input_addr: raise ValueError("input_addr must be a non-empty string") + if replica_id is not None and replica_id < 0: + raise ValueError("replica_id must be non-negative") existing = self._addr_to_replica_id.get(input_addr) - if existing is not None: - self.clients[existing] = client - return existing + if replica_id is not None and existing is not None and existing != replica_id: + raise ValueError( + f"input address {input_addr!r} is already assigned to replica {existing}, not {replica_id}" + ) + if replica_id is None: + replica_id = existing + if replica_id is None: + replica_id = next( + (index for index, existing_client in enumerate(self.clients) if existing_client is None), + len(self.clients), + ) - replica_id = len(self.clients) - self.clients.append(client) + while len(self.clients) <= replica_id: + self.clients.append(None) + self._replica_metrics.append(_ReplicaMetrics()) + for known_addr, known_replica_id in list(self._addr_to_replica_id.items()): + if known_replica_id == replica_id and known_addr != input_addr: + self._addr_to_replica_id.pop(known_addr, None) + self.clients[replica_id] = client self._addr_to_replica_id[input_addr] = replica_id - self._replica_metrics.append(_ReplicaMetrics()) + self._unavailable_replicas.discard(replica_id) return replica_id def remove_client(self, input_addr: str) -> Any | None: @@ -234,7 +249,7 @@ class StagePool: Slot is marked ``None`` to preserve indices for outstanding bindings. """ - replica_id = self._addr_to_replica_id.pop(input_addr, None) + replica_id = self._addr_to_replica_id.get(input_addr) if replica_id is None: return None client = self.clients[replica_id] @@ -475,6 +490,39 @@ class StagePool: for request_id in request_ids: self.release_binding(request_id) + def release_replica_bindings(self, replica_id: int) -> list[str]: + """Drop all route/session bindings owned by one physical replica.""" + released_request_ids = [ + request_id + for request_id, bound_replica_id in list(self._request_bindings.items()) + if bound_replica_id == replica_id + ] + released_request_ids.extend( + request_id + for request_id, input_addr in list(self._affinity.items()) + if self._addr_to_replica_id.get(input_addr) == replica_id + ) + released_request_ids = list(dict.fromkeys(released_request_ids)) + for request_id in released_request_ids: + self.release_binding(request_id) + return released_request_ids + + def mark_replica_unavailable(self, replica_id: int) -> list[str]: + """Evict a failed replica from admission and release its bindings.""" + if 0 <= replica_id < self.num_replicas: + self._unavailable_replicas.add(replica_id) + return self.release_replica_bindings(replica_id) + + def is_replica_available(self, replica_id: int) -> bool: + return ( + 0 <= replica_id < self.num_replicas + and self.clients[replica_id] is not None + and replica_id not in self._unavailable_replicas + ) + + def available_replica_ids(self) -> list[int]: + return [replica_id for replica_id in range(self.num_replicas) if self.is_replica_available(replica_id)] + def select_replica_id( self, request_id: str, @@ -483,23 +531,27 @@ class StagePool: ) -> int: """Pick a replica id for *request_id* and cache the choice (legacy path).""" cached = self.get_bound_replica_id(request_id) - if cached is not None and self.clients[cached] is not None: + if cached is not None and self.clients[cached] is not None and self.is_replica_available(cached): return cached + if cached is not None: + # Cached replica is dead/unavailable: drop the stale binding. + self.release_binding(request_id) chosen: int | None = None if affinity_request_id is not None: parent = self.get_bound_replica_id(affinity_request_id) - if parent is not None and self.clients[parent] is not None: + if parent is not None and self.clients[parent] is not None and self.is_replica_available(parent): chosen = parent if chosen is None: - live = self.live_replica_ids() + # Prefer replicas that are both live (client up) and available. + live = [r for r in self.live_replica_ids() if self.is_replica_available(r)] if not live: raise RuntimeError(f"stage {self.stage_id} has no live replicas") if len(live) == 1: chosen = live[0] else: - # Round-robin over live replicas only. + # Round-robin over live, available replicas only. start = self._next_replica_id % len(live) chosen = live[start] self._next_replica_id = (self._next_replica_id + 1) % len(live) @@ -539,18 +591,23 @@ class StagePool: non_empty_first_output_ts = ( self._non_empty_first_output_timestamps_by_request.pop(request_id, None) if request_id else None ) - num_tokens_out = count_tokens_from_outputs(request_outputs) + native_text_metrics = {} + if request_id: + pop_native_text_metrics = getattr(self.output_processor, "pop_native_text_metrics", None) + if callable(pop_native_text_metrics): + native_text_metrics = pop_native_text_metrics(request_id) + native_generation_tokens = native_text_metrics.get("num_generation_tokens") + num_tokens_out = ( + max(int(native_generation_tokens), 0) + if isinstance(native_generation_tokens, int) and not isinstance(native_generation_tokens, bool) + else count_tokens_from_outputs(request_outputs) + ) output_unit_type = self._infer_output_unit_type(request_outputs, token_count=num_tokens_out) output_unit_count = self._count_output_units( request_outputs, unit_type=output_unit_type, fallback_token_count=num_tokens_out, ) - native_text_metrics = {} - if request_id: - pop_native_text_metrics = getattr(self.output_processor, "pop_native_text_metrics", None) - if callable(pop_native_text_metrics): - native_text_metrics = pop_native_text_metrics(request_id) current_audio_frames, current_audio_sample_rate, _ = self._collect_audio_metrics(request_outputs) accumulated_audio_frames = self._audio_frames_by_request.pop(request_id, 0) if request_id else 0 accumulated_audio_sample_rate = self._audio_sample_rate_by_request.pop(request_id, 0) if request_id else 0 @@ -1029,14 +1086,28 @@ class StagePool: # Refresh the shared output-processor state before yielding to the # stage client so streaming segments are merged against the latest # prompt/token metadata. - self.output_processor.add_request( - request=request, - prompt=prompt_text, - parent_req=None, - request_index=0, - queue=None, - ) - await self._llm_client(replica_id).add_request_async(request) + try: + self.output_processor.add_request( + request=request, + prompt=prompt_text, + parent_req=None, + request_index=0, + queue=None, + ) + await self._llm_client(replica_id).add_request_async(request) + except Exception: + rollback = getattr(self.output_processor, "remove_request", None) + if callable(rollback): + try: + rollback(request_id) + except Exception as rollback_error: + logger.warning( + "[StagePool] Failed to rollback output processor update for req=%s stage-%s: %s", + request_id, + self.stage_id, + rollback_error, + ) + raise return replica_id async def _pick_or_select( @@ -1097,6 +1168,8 @@ class StagePool: timeout_s: float = 0.001, ) -> EngineCoreOutputs | None: """Poll raw EngineCore outputs from one LLM replica once.""" + if not self.is_replica_available(replica_id): + return None raw_client = self.clients[replica_id] if raw_client is None: return None @@ -1120,6 +1193,8 @@ class StagePool: def poll_diffusion_output(self, replica_id: int) -> Any | None: """Drain one ready diffusion output from the given replica if present.""" + if not self.is_replica_available(replica_id): + return None raw_client = self.clients[replica_id] if raw_client is None: return None diff --git a/vllm_omni/engine/stage_runtime.py b/vllm_omni/engine/stage_runtime.py index 8897043ca..4ee4ccc61 100644 --- a/vllm_omni/engine/stage_runtime.py +++ b/vllm_omni/engine/stage_runtime.py @@ -396,7 +396,6 @@ class StageRuntime: replica_metadata.replica_id = replica_id if launch_mode == "remote" and replica_metadata.stage_type != "diffusion": replica_metadata.runtime_cfg = None - replicas.append( ReplicaInitPlan( replica_id=replica_id, diff --git a/vllm_omni/entrypoints/async_omni.py b/vllm_omni/entrypoints/async_omni.py index 74911af9e..34f5f0a91 100644 --- a/vllm_omni/entrypoints/async_omni.py +++ b/vllm_omni/entrypoints/async_omni.py @@ -44,6 +44,12 @@ if TYPE_CHECKING: from vllm.tokenizers import TokenizerLike from vllm.v1.engine import PauseMode + from vllm_omni.experimental.fullduplex.engine.lease import DuplexLeaseActivity + from vllm_omni.experimental.fullduplex.engine.messages import ( + DuplexFence, + DuplexSessionLifecycleMessage, + ) + from vllm_omni.experimental.fullduplex.request_client import DuplexRequestClient from vllm_omni.inputs.data import OmniPromptType logger = init_logger(__name__) @@ -143,12 +149,16 @@ class AsyncOmni(EngineClient, OmniBase): self._paused: bool = False self._sleeping_tags: set[str] = set() self._level2_sleeping: bool = False + self._duplex_request_client: DuplexRequestClient | None = None + self.duplex_lifecycle_events: asyncio.Queue[DuplexSessionLifecycleMessage] = asyncio.Queue() self.final_output_task: asyncio.Task | None = None self.event_resolver = AsyncEventResolver(orchestrator=self) self.config_path = self.engine.config_path self.tts_max_instructions_length = kwargs.get("tts_max_instructions_length", None) self.input_processor = self.engine.input_processor self.endpoint_restrictions = self.engine.endpoint_restrictions + self.duplex_session_config = self.engine.duplex_session_config + self.duplex_serving_adapter_path = self.engine.duplex_serving_adapter_path stage_index = self._get_comprehension_stage_index() if stage_index is None: @@ -254,6 +264,191 @@ class AsyncOmni(EngineClient, OmniBase): prefix = "" if not external_request_id else f"{external_request_id}-" return f"{prefix}{uuid:.8}" + async def open_duplex_session_async( + self, + session_id: str, + *, + session_mode: str = "duplex", + capabilities: dict[str, object] | None = None, + session_config: dict[str, object] | None = None, + runtime_config: dict[str, object] | None = None, + fence: DuplexFence, + timeout: float | None = 10.0, + ) -> dict[str, object]: + """Open an engine-level duplex session when the backend supports it.""" + return await self._get_duplex_request_client().open( + session_id, + session_mode=session_mode, + capabilities=capabilities, + session_config=session_config, + runtime_config=runtime_config, + fence=fence, + timeout=timeout, + ) + + async def append_duplex_input_async( + self, + session_id: str, + *, + mode: str, + payload: object, + operation_id: str | None = None, + final: bool = False, + expected_epoch: int | None = None, + fence: DuplexFence, + timeout: float | None = 10.0, + collect_outputs: bool = True, + ) -> dict[str, object]: + """Append input to an engine-level duplex session.""" + return await self._get_duplex_request_client().append( + session_id, + mode=mode, + payload=payload, + operation_id=operation_id, + final=final, + expected_epoch=expected_epoch, + fence=fence, + timeout=timeout, + collect_outputs=collect_outputs, + ) + + async def collect_duplex_data_plane_outputs_async( + self, + request_id: str, + *, + response_stage_id: int | None = None, + timeout: float | None = 10.0, + ) -> list[OmniRequestOutput]: + """Collect the next duplex data-plane output batch for a live request.""" + return await self._get_duplex_request_client().collect_registered_outputs( + request_id, + response_stage_id=response_stage_id, + timeout=timeout, + ) + + async def signal_duplex_turn_async( + self, + session_id: str, + *, + event: str, + fence: DuplexFence, + next_fence: DuplexFence | None = None, + session_config: dict[str, object] | None = None, + runtime_config: dict[str, object] | None = None, + timeout: float | None = 10.0, + ) -> dict[str, object]: + """Send a turn/control signal to an engine-level duplex session.""" + return await self._get_duplex_request_client().signal( + session_id, + event=event, + fence=fence, + next_fence=next_fence, + session_config=session_config, + runtime_config=runtime_config, + timeout=timeout, + ) + + async def close_duplex_session_async( + self, + session_id: str, + *, + reason: str = "client_close", + fence: DuplexFence, + timeout: float | None = 10.0, + ) -> dict[str, object]: + """Close an engine-level duplex session.""" + return await self._get_duplex_request_client().close( + session_id, + reason=reason, + fence=fence, + timeout=timeout, + ) + + async def touch_duplex_session_async( + self, + session_id: str, + *, + fence: DuplexFence, + activity: DuplexLeaseActivity, + timeout: float | None = 10.0, + ) -> dict[str, object]: + return await self._get_duplex_request_client().touch( + session_id, + fence=fence, + activity=activity, + timeout=timeout, + ) + + async def resume_duplex_session_async( + self, + session_id: str, + *, + fence: DuplexFence, + expected_lease_generation: int, + timeout: float | None = 10.0, + ) -> dict[str, object]: + return await self._get_duplex_request_client().resume( + session_id, + fence=fence, + expected_lease_generation=expected_lease_generation, + timeout=timeout, + ) + + def _get_duplex_request_client(self) -> DuplexRequestClient: + from vllm_omni.experimental.fullduplex.request_client import ( + DuplexRequestClient, + DuplexRequestOutputPort, + ) + + client = getattr(self, "_duplex_request_client", None) + if client is None: + engine = getattr(self, "engine", None) + client = DuplexRequestClient( + engine, + DuplexRequestOutputPort( + request_states=getattr(self, "request_states", {}), + num_stages=getattr(engine, "num_stages", 1), + log_stats=getattr(self, "log_stats", False), + start_output_handler=self._final_output_handler, + process_single_result=self._process_single_result, + ), + ) + self._duplex_request_client = client + return client + + @staticmethod + def _duplex_data_plane_request_info(result: dict[str, object]) -> tuple[str | None, int | None]: + from vllm_omni.experimental.fullduplex.request_client import DuplexRequestClient + + return DuplexRequestClient.request_info(result) + + async def _collect_duplex_data_plane_outputs( + self, + request_id: str, + req_state: ClientRequestState, + *, + response_stage_id: int | None, + timeout: float | None, + ) -> list[OmniRequestOutput]: + return await self._get_duplex_request_client().collect_outputs( + request_id, + req_state, + response_stage_id=response_stage_id, + timeout=timeout, + ) + + @classmethod + def _is_direct_duplex_data_plane_response(cls, output: object) -> bool: + from vllm_omni.experimental.fullduplex.request_client import DuplexRequestClient + + return DuplexRequestClient.is_direct_response(output) + + @classmethod + def _duplex_multimodal_output(cls, output: object) -> dict[str, object]: + from vllm_omni.experimental.fullduplex.request_client import DuplexRequestClient + + return DuplexRequestClient.multimodal_output(output) + # ==================== Generate Method ==================== async def generate( @@ -693,6 +888,10 @@ class AsyncOmni(EngineClient, OmniBase): await self.event_resolver.resolve(msg) continue + if getattr(msg, "type", None) == "duplex_session_lifecycle": + await self.duplex_lifecycle_events.put(msg) + continue + if isinstance(msg, ErrorMessage) and not msg.fatal: req_state = self.request_states.get(msg.request_id) if req_state is not None: diff --git a/vllm_omni/entrypoints/client_request_state.py b/vllm_omni/entrypoints/client_request_state.py index 316d4c2aa..9526df36a 100644 --- a/vllm_omni/entrypoints/client_request_state.py +++ b/vllm_omni/entrypoints/client_request_state.py @@ -4,7 +4,7 @@ from vllm_omni.metrics import OrchestratorAggregator class ClientRequestState: - """Tracks the state of an individual request in the orchestrator.""" + """Tracks one entrypoint request and its output queue.""" def __init__( self, diff --git a/vllm_omni/entrypoints/omni_base.py b/vllm_omni/entrypoints/omni_base.py index d8b2c0291..44db512b2 100644 --- a/vllm_omni/entrypoints/omni_base.py +++ b/vllm_omni/entrypoints/omni_base.py @@ -473,6 +473,7 @@ class OmniBase(PDDisaggregationMixin): req_start_ts: dict[str, float], wall_start_ts: float, final_stage_id_for_e2e: int, + stage_event_cursor: int = 0, ) -> OmniRequestOutput | None: req_id = result.request_id engine_outputs = result.engine_outputs @@ -481,7 +482,7 @@ class OmniBase(PDDisaggregationMixin): # Merge AR stage timing from OrchestratorAggregator.stage_events if self._enable_ar_profiler: - ar_events = metrics.stage_events.get(str(req_id), []) + ar_events = metrics.stage_events.get(str(req_id), [])[stage_event_cursor:] for evt in ar_events: if evt.stage_id != stage_id: stage_durations[f"ar_stage_{evt.stage_id}"] = evt.stage_gen_time_ms / 1000.0 @@ -494,7 +495,7 @@ class OmniBase(PDDisaggregationMixin): stage_durations[key] = value # Merge per-stage gen times into stage_durations - for evt in metrics.stage_events.get(str(req_id), []): + for evt in metrics.stage_events.get(str(req_id), [])[stage_event_cursor:]: key = f"stage_{evt.stage_id}_gen_ms" if key not in stage_durations: stage_durations[key] = evt.stage_gen_time_ms @@ -546,7 +547,7 @@ class OmniBase(PDDisaggregationMixin): # Token counters — aggregate across all stages for this request. _prompt_tok = 0 _gen_tok = 0 - for evt in metrics.stage_events.get(rid_key, []): + for evt in metrics.stage_events.get(rid_key, [])[stage_event_cursor:]: if evt.stage_id == 0: _prompt_tok += int(evt.num_tokens_in) _gen_tok += int(evt.num_tokens_out) @@ -581,7 +582,7 @@ class OmniBase(PDDisaggregationMixin): response_metrics: dict[str, Any] = {} stage_metrics: dict[str, dict[str, Any]] = {} rid_key = str(req_id) - for evt in metrics.stage_events.get(rid_key, []): + for evt in metrics.stage_events.get(rid_key, [])[stage_event_cursor:]: if evt.stage_id is None: continue sid = int(evt.stage_id) @@ -604,6 +605,7 @@ class OmniBase(PDDisaggregationMixin): response_metrics["num_tokens_out"] = current_stage_metrics["num_tokens_out"] return OmniRequestOutput( request_id=req_id or "", + finished=finished, stage_id=stage_id, final_output_type=output_type, request_output=engine_outputs, @@ -616,7 +618,6 @@ class OmniBase(PDDisaggregationMixin): metrics=response_metrics, stage_durations=stage_durations, peak_memory_mb=peak_memory_mb, - finished=finished, ) def shutdown(self, timeout: float | None = None) -> None: diff --git a/vllm_omni/entrypoints/openai/api_server.py b/vllm_omni/entrypoints/openai/api_server.py index c9ddab2e9..56c7ccf46 100755 --- a/vllm_omni/entrypoints/openai/api_server.py +++ b/vllm_omni/entrypoints/openai/api_server.py @@ -94,6 +94,7 @@ from vllm.v1.engine.exceptions import EngineDeadError, EngineGenerateError from vllm_omni.config.endpoint_policy import shutdown_unsupported_routes from vllm_omni.diffusion.models.interface import ReferenceVideoDecodeSpec from vllm_omni.entrypoints.async_omni import AsyncOmni +from vllm_omni.entrypoints.openai.duplex_capability import should_enable_duplex_endpoint from vllm_omni.entrypoints.openai.errors import InvalidInputReferenceError from vllm_omni.entrypoints.openai.image_api_utils import ( SUPPORTED_LAYERED_RESOLUTIONS, @@ -793,6 +794,7 @@ async def omni_init_app_state( model_name=model_name, stage_configs=diffusion_stage_configs, ) + state.openai_serving_duplex = None state.openai_streaming_speech = None state.openai_streaming_video = None state.openai_serving_realtime_robot = ServingRealtimeRobotOpenPI.create_policy_server( @@ -1112,6 +1114,18 @@ async def omni_init_app_state( if state.openai_serving_chat is not None else None ) + state.openai_serving_duplex = None + if state.openai_serving_chat is not None and should_enable_duplex_endpoint( + state.stage_configs, + config_path=getattr(args, "stage_configs_path", None) or getattr(args, "deploy_config", None), + ): + from vllm_omni.experimental.fullduplex.openai.serving import OmniDuplexSessionHandler + + state.openai_serving_duplex = OmniDuplexSessionHandler( + chat_service=state.openai_serving_chat, + duplex_session_config=getattr(engine_client, "duplex_session_config", None), + serving_runtime_adapter_path=getattr(engine_client, "duplex_serving_adapter_path", None), + ) state.openai_serving_realtime = OpenAIServingRealtime( engine_client=engine_client, models=state.openai_serving_models, @@ -1612,6 +1626,31 @@ async def streaming_video_output(websocket: WebSocket): @router.websocket("/v1/realtime") async def realtime_websocket(websocket: WebSocket): """WebSocket endpoint for OpenAI-style realtime interactions.""" + duplex_handler = getattr(websocket.app.state, "openai_serving_duplex", None) + duplex_query = websocket.query_params.get("duplex") + use_duplex_realtime = ( + duplex_handler is not None and isinstance(duplex_query, str) and duplex_query.lower() in {"1", "true", "on"} + ) + if use_duplex_realtime and duplex_handler is not None: + await duplex_handler.handle_realtime_session(websocket) + return + + engine_client = getattr(websocket.app.state, "engine_client", None) + if engine_client is not None and getattr(engine_client, "async_chunk", False): + await websocket.accept() + await websocket.send_json( + { + "type": "error", + "error": ( + "The /v1/realtime API is not supported when async_chunk is enabled on the server. " + "Use a stage configuration with async_chunk disabled and restart the server before using " + "this endpoint." + ), + "code": "unsupported", + } + ) + await websocket.close() + return serving = getattr(websocket.app.state, "openai_serving_realtime", None) if serving is None: await websocket.accept() @@ -1639,6 +1678,18 @@ async def realtime_robot_openpi(websocket: WebSocket): await connection.handle_connection() +@router.websocket("/v1/duplex") +async def duplex_websocket(websocket: WebSocket): + """WebSocket endpoint for vLLM-Omni duplex session control.""" + handler = getattr(websocket.app.state, "openai_serving_duplex", None) + if handler is None: + await websocket.accept() + await websocket.send_json({"type": "error", "error": "Duplex API is not available", "code": "unsupported"}) + await websocket.close() + return + await handler.handle_session(websocket) + + # Health and Model endpoints for diffusion mode diff --git a/vllm_omni/entrypoints/openai/duplex_capability.py b/vllm_omni/entrypoints/openai/duplex_capability.py new file mode 100644 index 000000000..c3e4e3e38 --- /dev/null +++ b/vllm_omni/entrypoints/openai/duplex_capability.py @@ -0,0 +1,35 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from vllm.logger import init_logger + +logger = init_logger(__name__) + + +def should_enable_duplex_endpoint( + stage_configs: list | None, + *, + config_path: str | None = None, +) -> bool: + """Enable duplex routes only for deployments that explicitly opt in.""" + if stage_configs: + for stage in stage_configs: + session_mode = ( + stage.get("session_mode") if isinstance(stage, dict) else getattr(stage, "session_mode", None) + ) + if session_mode == "duplex": + return True + if config_path: + try: + from omegaconf import OmegaConf + + raw_config = OmegaConf.load(config_path) + session_mode = raw_config.get("session_mode") if hasattr(raw_config, "get") else None + if session_mode == "duplex": + return True + except Exception as exc: + logger.warning("Failed to inspect duplex session_mode from %s: %s", config_path, exc) + return False + + +__all__ = ["should_enable_duplex_endpoint"] diff --git a/vllm_omni/entrypoints/openai/serving_chat.py b/vllm_omni/entrypoints/openai/serving_chat.py index f819c7b57..93b73ad66 100755 --- a/vllm_omni/entrypoints/openai/serving_chat.py +++ b/vllm_omni/entrypoints/openai/serving_chat.py @@ -79,7 +79,7 @@ from vllm.outputs import RequestOutput from vllm.reasoning import ReasoningParser from vllm.renderers import BaseRenderer, merge_kwargs from vllm.renderers.inputs import TokPrompt -from vllm.sampling_params import SamplingParams +from vllm.sampling_params import RequestOutputKind, SamplingParams from vllm.tokenizers import TokenizerLike from vllm.tokenizers import TokenizerLike as AnyTokenizer from vllm.tokenizers.mistral import ( @@ -187,6 +187,63 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin): tp_cls = self.parser_cls.tool_parser_cls if self.parser_cls is not None else None return request.tools and tp_cls is not None and self.enable_auto_tools and request.tool_choice in ["auto", None] + @staticmethod + def _stage_get(obj: Any, key: str, default: Any = None) -> Any: + if obj is None: + return default + if hasattr(obj, "get"): + return obj.get(key, default) + return getattr(obj, key, default) + + def _has_minicpmo45_stage(self) -> bool: + for stage in getattr(self.engine_client, "stage_configs", []) or []: + engine_args = self._stage_get(stage, "engine_args") + model_arch = self._stage_get(engine_args, "model_arch") + if model_arch == "MiniCPMO45OmniForConditionalGeneration": + return True + return False + + def _fix_minicpmo45_audio_stream_output_kinds( + self, + sampling_params_list: list[Any], + output_modalities: list[str] | tuple[str, ...] | None, + ) -> list[Any]: + """Keep MiniCPM-o 4.5's thinker final-only during audio streaming. + + Chat ``stream=true`` normally coerces every AR stage to DELTA output. + That is correct for token-level text streaming, but MiniCPM-o 4.5's + Talker consumes the completed TTS span plus aligned thinker hidden + states. If Stage0 is DELTA, llm2tts receives one generated token at a + time and repeatedly starts tiny, independent TTS streams. The API then + never represents one coherent audio response. + + For this model, stream the audio stage, not the thinker-to-talker + boundary. + """ + if not output_modalities or "audio" not in output_modalities: + return sampling_params_list + if not self._has_minicpmo45_stage(): + return sampling_params_list + + stage_configs = getattr(self.engine_client, "stage_configs", []) or [] + for idx, stage in enumerate(stage_configs): + if idx >= len(sampling_params_list): + break + sp = sampling_params_list[idx] + if not hasattr(sp, "output_kind"): + continue + + engine_args = self._stage_get(stage, "engine_args") + if self._stage_get(engine_args, "model_arch") != "MiniCPMO45OmniForConditionalGeneration": + continue + + model_stage = self._stage_get(engine_args, "model_stage") + if model_stage == "llm": + sp.output_kind = RequestOutputKind.FINAL_ONLY + elif model_stage == "tts": + sp.output_kind = RequestOutputKind.DELTA + return sampling_params_list + @classmethod def for_diffusion( cls, @@ -605,6 +662,11 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin): # to delta to ensure emitted outputs are correctly drained. Otherwise # convert cumulative to Final Only to ensure the output is correct. sampling_params_list = coerce_param_message_types(sampling_params_list, request.stream) + if request.stream: + sampling_params_list = self._fix_minicpmo45_audio_stream_output_kinds( + sampling_params_list, + output_modalities, + ) # Apply user-specified overrides to diffusion stage(s) for image generation for idx, sp in enumerate(sampling_params_list): diff --git a/vllm_omni/entrypoints/openai/video_stream_base.py b/vllm_omni/entrypoints/openai/video_stream_base.py index 460a66d7c..1694704f3 100644 --- a/vllm_omni/entrypoints/openai/video_stream_base.py +++ b/vllm_omni/entrypoints/openai/video_stream_base.py @@ -10,12 +10,14 @@ via :class:`VideoStreamPipelineHooks`. Protocol: Client -> Server: {"type": "session.config", ...} # Session config (sent once) - {"type": "video.frame", "data": "..."} # base64 JPEG/PNG frame + {"type": "video.frame", "data": "...", "frame_id": "...", "pts_ms": 0} {"type": "audio.chunk", "data": "..."} # base64 PCM16 16kHz mono {"type": "video.query", "text": "..."} # Submit query about buffered frames {"type": "video.done"} # End of session Server -> Client: + {"type": "video.frame.ack", ...} # when frame_id is provided + {"type": "video.frames.consumed", ...} # after first engine output {"type": "response.start"} {"type": "response.text.delta", "delta": "..."} {"type": "response.text.done", "text": "..."} @@ -215,6 +217,7 @@ class OmniStreamingVideoHandler: return frame_buffer: list[str] = [] # base64-encoded JPEG frames + frame_metadata: list[dict[str, Any]] = [] # Per-frame PIL cache + uuid for mm_hash reuse. Aligned with frame_buffer by index. frame_pil_cache: dict[str, tuple[Any, str] | object] = {} # b64 -> (PIL.Image, uuid) or _BAD_FRAME frame_filter = ( @@ -259,6 +262,8 @@ class OmniStreamingVideoHandler: if msg_type.startswith("_internal."): await self._send_error(websocket, f"Unknown type: {msg_type}") continue + if msg_type == "video.frame": + msg["_receiver_received_ts_ms"] = _time.monotonic() * 1000 await msg_queue.put(msg) if msg.get("type") == "video.done": @@ -308,6 +313,7 @@ class OmniStreamingVideoHandler: active_request_id = request_id interrupt_event.clear() query_frames = list(frame_buffer) + query_frame_metadata = list(frame_metadata) query_audio_buffer = bytearray(audio_buffer) audio_buffer.clear() query_prewarmed_frames = dict(frame_pil_cache) @@ -315,6 +321,9 @@ class OmniStreamingVideoHandler: async def _run_query() -> None: nonlocal active_request_id, prev_request_id try: + process_kwargs: dict[str, Any] = {} + if any(metadata.get("frame_id") for metadata in query_frame_metadata): + process_kwargs["frame_metadata"] = query_frame_metadata await self._process_query( websocket, config, @@ -325,6 +334,7 @@ class OmniStreamingVideoHandler: request_id, interrupt_event, query_prewarmed_frames, + **process_kwargs, ) finally: if active_request_id == request_id: @@ -349,7 +359,11 @@ class OmniStreamingVideoHandler: frame_data = msg.get("b64", "") removed = frame_data in frame_buffer if removed: - frame_buffer[:] = [f for f in frame_buffer if f != frame_data] + retained_indices = [ + index for index, frame in enumerate(frame_buffer) if frame != frame_data + ] + frame_buffer[:] = [frame_buffer[index] for index in retained_indices] + frame_metadata[:] = [frame_metadata[index] for index in retained_indices] if frame_pil_cache.get(frame_data) is _BAD_FRAME: frame_pil_cache.pop(frame_data, None) if removed: @@ -370,16 +384,43 @@ class OmniStreamingVideoHandler: if frame_filter is not None: try: if not frame_filter.should_retain(raw_bytes): + await self._send_frame_ack( + websocket, + msg, + accepted=False, + buffered_frames=len(frame_buffer), + reason="filtered", + ) continue except Exception: await self._send_error(websocket, "Invalid image data") continue max_buf = config.max_frames + dropped_frame_id: str | None = None if len(frame_buffer) >= max_buf: dropped = frame_buffer.pop(0) + dropped_metadata = frame_metadata.pop(0) + dropped_frame_id = dropped_metadata.get("frame_id") frame_pil_cache.pop(dropped, None) frame_buffer.append(frame_data) + frame_metadata.append( + { + "frame_id": msg.get("frame_id"), + "pts_ms": msg.get("pts_ms"), + "source_pts_ms": msg.get("source_pts_ms"), + "quality_profile": msg.get("quality_profile"), + "capture_ts_ms": msg.get("capture_ts_ms"), + "receiver_received_ts_ms": msg.get("_receiver_received_ts_ms"), + } + ) self.on_frame_buffered(raw_bytes, frame_data, message_history, config) + await self._send_frame_ack( + websocket, + msg, + accepted=True, + buffered_frames=len(frame_buffer), + dropped_frame_id=dropped_frame_id, + ) # Prewarm: decode PIL off the event loop so query-time chat_template # can skip base64+Image.open. uuid=md5 lets mm_cache dedupe identical frames. if frame_data not in frame_pil_cache: @@ -537,6 +578,7 @@ class OmniStreamingVideoHandler: request_id: str, interrupt_event: asyncio.Event, prewarmed_frames: dict[str, tuple[Any, str]], + frame_metadata: list[dict[str, Any]] | None = None, ) -> None: """Build prompt, run inference, stream text + audio response.""" @@ -544,6 +586,9 @@ class OmniStreamingVideoHandler: await self._send_error(websocket, "Streaming video requires an engine client") return + engine_kwargs: dict[str, Any] = {} + if frame_metadata: + engine_kwargs["frame_metadata"] = frame_metadata await self._process_query_engine( websocket, config, @@ -554,6 +599,7 @@ class OmniStreamingVideoHandler: request_id, interrupt_event, prewarmed_frames, + **engine_kwargs, ) # ------------------------------------------------------------------ @@ -571,6 +617,7 @@ class OmniStreamingVideoHandler: request_id: str, interrupt_event: asyncio.Event, prewarmed_frames: dict[str, tuple[Any, str]], + frame_metadata: list[dict[str, Any]] | None = None, ) -> None: """Direct engine_client.generate() path for async_chunk audio.""" from vllm.entrypoints.openai.chat_completion.protocol import ( @@ -613,6 +660,9 @@ class OmniStreamingVideoHandler: except Exception as e: await self._send_error(websocket, f"Preprocess failed: {e}") return + decoded_ready_ts_ms = _time.monotonic() * 1000 + selected_metadata = self._sample_frame_metadata(frame_metadata or [], config.num_frames) + model_selected_ts_ms = _time.monotonic() * 1000 await websocket.send_json({"type": "response.start"}) text_parts: list[str] = [] @@ -623,6 +673,7 @@ class OmniStreamingVideoHandler: audio_chunks_drained = 0 previous_text = "" interrupted = False + frames_consumed_sent = False t_start = _time.monotonic() t_first_text = None t_first_audio = None @@ -652,6 +703,33 @@ class OmniStreamingVideoHandler: if not isinstance(output, OmniRequestOutput): continue + if not frames_consumed_sent and frame_metadata: + await websocket.send_json( + { + "type": "video.frames.consumed", + "request_id": request_id, + "model_selected_ts_ms": model_selected_ts_ms, + "frame_ids": [ + metadata["frame_id"] + for metadata in selected_metadata + if isinstance(metadata.get("frame_id"), str) + ], + "frames": [ + { + "frame_id": metadata.get("frame_id"), + "pts_ms": metadata.get("pts_ms"), + "source_pts_ms": metadata.get("source_pts_ms"), + "quality_profile": metadata.get("quality_profile"), + "receiver_received_ts_ms": metadata.get("receiver_received_ts_ms"), + "decoded_ready_ts_ms": decoded_ready_ts_ms, + } + for metadata in selected_metadata + ], + "latest_pts_ms": selected_metadata[-1].get("pts_ms") if selected_metadata else None, + } + ) + frames_consumed_sent = True + out_type = getattr(output, "final_output_type", "text") if out_type == "audio": @@ -971,3 +1049,42 @@ class OmniStreamingVideoHandler: await websocket.send_json({"type": "error", "message": message}) except Exception: pass + + @staticmethod + def _sample_frame_metadata( + frame_metadata: list[dict[str, Any]], + num_frames: int, + ) -> list[dict[str, Any]]: + if len(frame_metadata) <= num_frames: + return list(frame_metadata) + stride = max(1, len(frame_metadata) // num_frames) + indices = [index * stride for index in range(num_frames - 1)] + [len(frame_metadata) - 1] + return [frame_metadata[index] for index in indices] + + @staticmethod + async def _send_frame_ack( + websocket: WebSocket, + message: Mapping[str, Any], + *, + accepted: bool, + buffered_frames: int, + reason: str | None = None, + dropped_frame_id: str | None = None, + ) -> None: + frame_id = message.get("frame_id") + if not isinstance(frame_id, str) or not frame_id: + return + ack: dict[str, Any] = { + "type": "video.frame.ack", + "frame_id": frame_id, + "pts_ms": message.get("pts_ms"), + "capture_ts_ms": message.get("capture_ts_ms"), + "accepted": accepted, + "buffered_frames": buffered_frames, + "server_receive_ts_ms": message.get("_receiver_received_ts_ms", _time.monotonic() * 1000), + } + if reason is not None: + ack["reason"] = reason + if dropped_frame_id is not None: + ack["dropped_frame_id"] = dropped_frame_id + await websocket.send_json(ack) diff --git a/vllm_omni/experimental/fullduplex/DESIGN.md b/vllm_omni/experimental/fullduplex/DESIGN.md new file mode 100644 index 000000000..256214299 --- /dev/null +++ b/vllm_omni/experimental/fullduplex/DESIGN.md @@ -0,0 +1,1029 @@ +# MiniCPM-o 4.5 Full-Duplex Runtime Architecture + +## Purpose + +This document records the runtime architecture implemented by PR #3907, the +cleanup applied after published commit `e011d936`, and the remaining work that +must not be advertised as complete. Implemented contracts and future +architecture work are called out separately below. + +The checkpoint preserves the active runtime path already exercised on H20. It does +not introduce another reducer, controller, worker provider, or shadow runtime. + +## Review Snapshot + +- PR: `vllm-project/vllm-omni#3907` +- Published head before this refactor: `e011d936` +- Published base snapshot: `62589203` +- Published diff: 93 files, approximately `+29.2k/-0.6k` +- Local runtime and pytest validation: intentionally not used +- Required validation environment: an NVIDIA H20 CUDA host +- PR head reviewed for the current cleanup: + `f4f78fa555af4d65e46b003022d352ca2ea9d703` +- Validated tree: the uncommitted refactor synchronized file-for-file to the + isolated H20 worktree on 2026-07-20 + +The current tree has received fresh H20 focused and E2E validation against vLLM +0.25.0. The final affected matrix includes the pre-response continuation and +Stage1 CUDA Graph padding regressions. The checked-in MiniCPM profile admits +two concurrent sessions through an engine-owned limit; a third +session was rejected and capacity returned after one session closed. The two +accepted sessions independently completed their audio, transcript, response, +playback, and close lifecycles. This is isolation and admission evidence for the +tested two-session deployment, not a production fairness, arbitrary-capacity, +or failure-recovery claim. The validated paths include the experimental runtime +extension, typed resumable policy, extracted control plane and clients, request +preregistration, typed direct-output decision, engine-managed two-session +capacity in the checked-in deploy profile, separate public/runtime configuration +channels, private Session ledgers, ordered `session.update`, the MiniCPM runner +fast path, continuous +browser input, and the final Realtime input-lifecycle fixes. Exact evidence and +scope limits are recorded below. + +## Scope + +The checkpoint keeps these verified contracts: + +- MiniCPM Stage0 conversation KV continuity; +- Stage1 TTS and Token2Wav continuity; +- model-owned listen/speak decisions on the normal auto-response path; +- continuous browser PCM upload during assistant playback, without browser VAD + or browser-generated input commits; +- segment EOS and turn EOS as different boundaries; +- transcript/audio cursors scoped to a response and turn; +- playback acknowledgement and history commit; +- Stage0 request metadata routed per batch row rather than inherited from the + first request in the batch; +- Stage0 token timing snapshots and client-observed audio chunk cadence; +- scheduler data-plane append over a resumable request; +- stale epoch/turn/response fencing; +- `/v1/duplex` and OpenAI Realtime projection; +- existing JoyVL behavior. + +The checkpoint does not claim: + +- scheduler-native KV append; +- deterministic VAD-triggered interruption (the browser intentionally does not + run VAD; MiniCPM owns listen/speak decisions at model-unit boundaries); +- production multi-session admission, fairness, capacity, or failure recovery; +- bounded long-session KV; +- video input or audio/video synchronization. + +## Active Runtime Path + +The actual path is: + +```text +WebSocket + -> OmniDuplexSessionHandler + -> DuplexSession + MiniCPMO45ServingSessionState + -> DuplexRequestClient + -> AsyncOmni thin open/append/signal/close proxies + -> DuplexControlClient over the engine-owned correlated RPC transport + -> DuplexControlPlane + DuplexSessionRuntimeManager + -> experimental DuplexRuntimeExtension + engine session/stage bindings + -> StagePool + -> resumable scheduler request + -> MiniCPM Stage0 + -> Stage1 TTS / Token2Wav + -> output processor + -> Realtime protocol projection + -> WebSocket writer +``` + +The removed typed reducer/facade and worker-provider runtimes were not in this +path. This refactor continues to modify only the path above. + +## State Ownership + +The architecture allows several state objects only when their facts are +orthogonal. + +| Owner | Authoritative facts | +| --- | --- | +| `DuplexSession` | session state, epoch, turn, active/last response, pending input, overlap duration, playback cursor, history | +| `MiniCPMO45ServingSessionState` | MiniCPM PCM buffer, deferred overlap payload, data-plane task, continuation effect counter | +| `DuplexWebSocketActor` | inbound mailbox, outbound queue, writer, transport close state, effect task handles | +| `NativeRealtimeSessionProtocol` | wire-only response/item projection and input-buffer projection | +| `DuplexSessionRuntimeManager` | fence snapshot, append reservation, completed-operation cache, resource reservations, stage bindings | +| `DuplexControlPlane` | control-message validation, transactional open/append/update/close, extension invocation, typed segment/output decisions | + +Important distinctions: + +- Realtime response IDs are projection caches, not domain decision sources. +- Engine fences duplicate identity across a process boundary by design; they + are immutable validation snapshots. +- Stage bindings are engine resources, not serving response state. +- `continuation_owner_id` scopes an effect counter either to a visible response + or, before `response.created`, to an explicit model turn. A pre-response + continuation also revalidates request, epoch, turn state, and model-turn + identity immediately before append; it cannot survive turn completion. +- runtime open/close acknowledgements are handler-local effect bookkeeping, + not Session lifecycle. + +## Implemented Concurrency Contracts + +### Single inbound mailbox + +The Actor now uses one: + +```python +asyncio.Queue[event] +``` + +This replaces competing control/input/event `Queue.get()` tasks and the +deferred-event list while preserving WebSocket arrival order. A later close, +cancel, or clear must not overtake earlier append, playback ACK, or +session-update events. Slow runtime appends remain background tasks. Cancelling +their asyncio task does not stop a synchronous callable already running in the +default executor, so cancellation correctness is provided by the engine fence, +not by `Task.cancel()`. + +Properties covered by focused tests: + +- every enqueued event is delivered exactly once; +- terminal control preserves wire order while cancelling background work; +- clear and cancel never scan or delete later mailbox events; +- one writer owns all WebSocket sends. + +### Correlated RPC result router + +`AsyncOmniEngine` now has one consumer for `rpc_output_queue`. Results route by: + +```text +("duplex", control_id) +("collective", rpc_id) +``` + +The router replaces the global RPC lock. It supports out-of-order replies, +timeout unregister, late-result rejection, fatal-error broadcast, and close +unblocking. + +Additional lifecycle rules: + +- uncorrelated non-fatal errors are not broadcast to unrelated waiters; +- a latched fatal rejects new calls before they enter the request queue; +- `EngineDeadError` is broadcast directly to the RPC queue even though the + orchestrator consumes that exception during teardown; +- shutdown closes the router before joining the orchestrator; +- failure to enqueue shutdown closes the request queue and still attempts a + bounded orchestrator join; +- `StageRuntime` cleanup runs only after the orchestrator has stopped, avoiding + concurrent client teardown. + +The router removes wait-side head-of-line blocking. The orchestrator request +handler remains ordered, but executor submission order is not itself a session +ordering guarantee. + +### Extracted control ownership + +The generic engine mechanism and the experimental duplex plugin are split by +responsibility: + +| Module | Responsibility | +| --- | --- | +| `engine.messages` | ordinary engine queue messages and collective RPC envelopes | +| `engine.rpc_result_router` | generic correlation routing through each result message's `rpc_correlation_key` | +| `experimental.fullduplex.engine.contracts` | immutable duplex DTOs, extension/stage protocols, typed decisions, request-ID codec | +| `experimental.fullduplex.engine.lease` | duplex lease activity, TTL/grace configuration, and expiry records | +| `experimental.fullduplex.engine.messages` | identity fence and duplex control message/result envelopes | +| `experimental.fullduplex.engine.duplex_runtime` | extension loading, validation, and compatibility exports | +| `experimental.fullduplex.engine.duplex_session` | session resources, fences, append reservations, idempotency cache, binding cleanup | +| `experimental.fullduplex.engine.duplex_control_plane` | control algorithms and the narrow `DuplexStagePort` used to submit and clean up stage requests | +| `experimental.fullduplex.engine.duplex_control_client` | typed control-message construction and calls through the engine-owned correlated RPC transport | +| `experimental.fullduplex.request_client` | deterministic request identity, client-state preregistration, acknowledgement validation, data-plane collection, rollback | + +`Orchestrator`, `AsyncOmniEngine`, and `AsyncOmni` retain explicit wiring and +compatibility proxies. They no longer own the corresponding duplex algorithms. +The ControlClient does not own waiters or consume the result queue; +`RpcResultRouter` remains the single correlation owner for both duplex and +collective RPC. The ControlPlane receives a narrow stage port rather than the +Orchestrator object. + +Control-plane enablement is independent from extension presence. +`PipelineConfig.duplex_control_enabled` determines whether an Orchestrator +constructs a ControlPlane. This preserves extension-free +`TURN_COMMIT_ONLY` deployments while allowing ordinary deployments to use a +single `None` fast-path check. The MiniCPM pipeline explicitly enables control +and separately selects its runtime extension. + +### Transactional open and append + +Opening a native session registers its engine state and reserves the Stage0 +request resource as one externally atomic operation. If capability, sampling, +extension, or request-state initialization fails, the ControlPlane removes the +session, reserved request resources, and any preregistered request state before +returning the error. Cleanup distinguishes a reserved resource from a submitted +request, so it does not decrement the running counter for unrelated work. +Closing a session that never appended also removes the preregistered resource. + +Expiry and request-triggered cleanup use a two-step close. The runtime manager +first marks the session closing but retains it in the admission set. Only a +successful stage cleanup finalizes and removes the session. A blocked or failed +cleanup therefore continues to consume its server-owned slot and is retried; +replacement admission cannot overlap request or KV resources that are still +owned by the closing session. + +Append uses prepare/submit/commit ordering. `prepare_append()` computes the +next sequence values without mutating the session. Prompt planning and stage +submission use that reservation, and only a successful submit commits +`input_seq`, `input_turn_seq`, and the accepted fence. Serving uses a matching +PCM reservation and consumes bytes only after the append acknowledgement; +failure or a closed predecessor rolls the bytes back. An `operation_id` caches +completed append results so a retry cannot submit the same physical append +twice. + +### Ordered append and configuration effects + +Native append effects form a per-session tail. Each append awaits its +predecessor before entering the engine, and `session.update` awaits that same +tail before changing either the serving aggregate or engine configuration. +Consequently, an append received before an update uses the old immutable +sampling snapshot, while the first accepted append after the update uses +the new generation. + +The engine increments `config_generation` when it accepts the replacement +configuration. At the next append boundary, the control plane rebuilds sampling +parameters from stage defaults through the model extension. It never mutates +the snapshot of a segment already in progress. + +### Irreversible cancellation fence + +Every cancel control carries the cancelled fence and the next session fence. +The `AsyncOmni` facade preserves both values across the serving, +entrypoint, engine-message, and orchestrator boundaries. The orchestrator then +atomically: + +1. advances the runtime state to the next fence; +2. releases and aborts bindings owned by the cancelled fence; +3. permanently rejects an append that arrives later with the old fence. + +Missing `next_fence` is rejected without releasing bindings; the engine never +advances independently from serving. A late cancel cannot move a session +backward when the runtime is already on a newer fence. Idle playback clear does +not send an engine cancel because no runtime resource is being cancelled. + +`model_native_duplex` also validates its engine-client contract before opening +the session. The open, append, signal, and close methods must exist and declare +the required `fence` arguments; signal must additionally accept `next_fence`. +An older or custom client that cannot preserve this boundary fails with +`runtime_contract_invalid` instead of silently running an unfenced native +session. The generic serving-session adapter retains its compatibility behavior. + +`DuplexFence` also carries a server-side session incarnation. Reusing a public +session ID after close increments that incarnation, and resource request IDs +include it after the first incarnation. A synchronous append from a closed +session therefore cannot be accepted by a newly opened session with the same +public ID. Stale close validates the full fence before removing the runtime +registry entry or releasing bindings. + +## Implemented Serving Ownership + +### Session-owned response and overlap state + +The Actor no longer stores: + +- a Session reference; +- lifecycle mirror state; +- active or last response ID; +- overlap duration; +- response-in-progress or playback business predicates; +- runtime open/close acknowledgements. + +`DuplexSession.begin_response()` owns both active and last response identity. +The Session owns overlap accumulation/reset and playback state. + +One `native_response_in_progress()` predicate now covers: + +- active response ID; +- active engine request ID; +- uncommitted ACK-only playback; +- active generic response task; +- response-bound append tasks; +- MiniCPM data-plane task; +- assistant-generating turn state. + +The write-only `MiniCPMO45ServingSessionState.response_emitted` field and a dead +data-plane task mirror were removed. + +### Response-scoped options + +`response.create` is represented by an immutable `ResponseCreateOptions` value. +The Session reserves it for one response, applies it to a response-local copy +when that response begins, and restores the base session configuration on +completion, listen, cancellation, or failure. Instructions, voice, modalities, +temperature, token limit, format, tools, and conversation metadata therefore do +not mutate the permanent session configuration or leak into a later response. + +### Explicit event effects and pure writer + +Business paths call `emit_event()`. It serializes domain effects and outbound +queue insertion with one session-local lock. Deferred append starts outside the +lock so recursive event emission cannot deadlock. + +The writer only: + +1. dequeues an event; +2. applies late stale filtering to streaming model output; +3. projects the event to the selected wire protocol; +4. sends JSON. + +The writer does not promote overlap, clear input, advance Session state, call +the engine, or commit history. + +### Single terminal acceptance point + +Terminal events are accepted or rejected before domain effects run: + +```text +response.done +response.listen +audio.cancelled +input.cancelled +session.closed +``` + +For a terminal carrying an epoch, that epoch must match the current Session. +Stale cancellation therefore cannot reset overlap belonging to a later turn. +Once a terminal is accepted and its effect is applied, the writer cannot revoke +it because close or epoch state changed while it waited in the output queue. + +Streaming audio/text deltas still receive late stale filtering at send time, +which prevents queued old audio from leaking after a cancel or epoch bump. + +## Runtime Type Boundary + +The stable kernel is deliberately limited to generic engine and scheduler +mechanisms: + +```text +vllm_omni.engine.messages # ordinary queue and collective RPC envelopes +vllm_omni.engine.rpc_result_router # generic correlation routing +vllm_omni.engine.async_engine_utils # ordinary engine lifecycle helpers +``` + +The full-duplex implementation remains experimental: + +```text +vllm_omni.experimental.fullduplex.engine.duplex_runtime + # extension loading and validation +vllm_omni.experimental.fullduplex.engine.contracts + # immutable DTOs and narrow protocols +vllm_omni.experimental.fullduplex.engine.lease + # duplex lease primitives +vllm_omni.experimental.fullduplex.engine.messages + # duplex identity and queue envelopes +vllm_omni.experimental.fullduplex.engine.duplex_session + # session transaction implementation +vllm_omni.experimental.fullduplex.engine.duplex_control_plane + # engine-side control algorithms +vllm_omni.experimental.fullduplex.engine.duplex_control_client + # correlated control client +vllm_omni.experimental.fullduplex.request_client + # entrypoint request/output lifecycle +``` + +Stable engine and entrypoint modules import only the kernel at module load. +They load the experimental ControlPlane, ControlClient, request client, and +runtime extension inside duplex-only construction or call paths. Importing +`Orchestrator`, `AsyncOmniEngine`, or `AsyncOmni` for an ordinary deployment +therefore does not import `vllm_omni.experimental.fullduplex`. Compatibility +exports remain lazy and are not part of ordinary startup. + +`PipelineConfig.duplex_control_enabled` enables the generic mechanism, while +`PipelineConfig.duplex_runtime_extension` separately selects the model adapter. +The MiniCPM pipeline opts into both and installs +`MiniCPMO45DuplexRuntimeExtension`; ordinary pipelines construct neither a +ControlPlane nor a ControlClient. The model-neutral protocol exposes +sampling-parameter configuration, append planning, and a typed stage-output +decision. MiniCPM owns +stage-specific overrides, PCM-to-token budgeting, force-listen payload policy, +special-token interpretation, and `listen` response metadata. The generic +Orchestrator does not parse MiniCPM session keys, `listen_token_id`, or construct +MiniCPM metadata, and it does not import the adapter. + +Extension loading is fail-fast. Startup validates every required callable and +the number and type of returned stage sampling parameters. Invalid custom +extensions fail before a session can open. + +Direct listen/speak outcomes use `DuplexOutputDecision` on +`OmniRequestOutput`. That typed envelope is authoritative over inner completion +metadata for native projection, so a processed output cannot hide a listen +decision merely because it retains unrelated multimodal metadata. + +Generic multimodal chunk accumulation lives in +`vllm_omni.outputs.multimodal_accumulation`. It owns snapshot replacement, +DELTA draining, and audio-finality interpretation; `OmniRequestState` only +invokes that policy while assembling request output. Model output processors no +longer need to add chunk-retention rules to the main output orchestration file. + +The strict chunk-transfer schema still contains `hidden_states.tts` as the +generic AR-stage-to-TTS handoff and several MiniCPM streaming token IDs as +compatibility wire fields. The latter are not a preferred generic model +contract. Moving them behind a model-namespaced metadata envelope requires a +versioned producer/consumer migration because unknown msgspec fields are +rejected; silently deleting or renaming them in this checkpoint would break +existing Stage0-to-Stage1 transfer. + +### Public and runtime configuration channels + +Control messages carry two independent snapshots: + +- `session_config` contains the public Realtime/session values used by serving + and model prompt policy; +- `runtime_config` contains server-derived MiniCPM sampling, scheduler, context + reserve, prefix-budget, and resolved reference-audio values. + +The MiniCPM serving adapter derives `runtime_config` from deploy/model defaults +and validated public options. Clients cannot provide private runtime keys in +`session.create` or `session.update`; response-scoped `extra_body` drops those +keys rather than turning them into engine policy. The ControlPlane constructs +sampling parameters only from `runtime_config`, +while append planning receives both snapshots explicitly. Raw Realtime +`extra_body` is not used as the engine's sampling-policy channel. + +`session.update` prepares both candidates, validates the runtime candidate in +the engine, and replaces the serving snapshots only after the engine ACK. A +rejected candidate therefore leaves both the public configuration and runtime +sampling generation unchanged. + +`DeployConfig.duplex_session` is an immutable, server-owned +`DuplexSessionRuntimeConfig`. These limits are not read from client +`extra_body` or session payloads: + +| Field | Default | Owner | Lifecycle or resource effect | +|---|---:|---|---| +| `idle_ttl_s` | `300.0` | Engine lease manager | Expires an attached or detached session after inactivity; `null` disables only idle expiry. | +| `disconnect_grace_s` | `30.0` | Engine lease manager and API attachment registry | Bounds how long a detached session may retain engine resources before cleanup. | +| `reaper_interval_s` | `5.0` | Orchestrator | Sets the cadence for lease expiry and pending cleanup retries. | +| `resume_replay_ttl_s` | `60.0` | API attachment registry | Expires replay events retained for resume or takeover. | +| `resume_replay_max_bytes_per_session` | `8388608` | API attachment registry | Bounds each session's replay buffer. | +| `max_pending_input_bytes_per_session` | `16777216` | API session input ledger | Applies per-session byte backpressure before input is admitted. | +| `max_pending_turns_per_session` | `4` | API session input ledger | Bounds queued turns that have not completed runtime processing. | +| `max_sessions` | `1` | Engine session manager | Atomically enforces admission; closing sessions retain capacity until cleanup finalizes. | +| `completed_append_cache_size` | `256` | Engine session manager | Bounds completed append idempotency records per session. | + +Deploy profiles may lower or raise these values to match scheduler capacity, +but clients cannot override them. The engine remains authoritative for +admission and cleanup ownership even when the API performs an early check. + +## Generic-Path Cleanup + +Model-specific `MINICPMO45_PROFILE_LOGS` probes were removed from: + +- the generic AR scheduler; +- orchestrator; +- AsyncOmni; +- the generic GPU AR runner. + +The cleanup removes the temporary serving and generic-runtime probes. The +generic AR runner builds `DuplexSamplingRow` values only when the model exposes +the optional `prepare_duplex_sampling()` hook, then invokes that hook exactly +once before the normal sampler. Ordinary models do not scan request rows. +MiniCPM owns force-listen logits, turn-ended state, and its row-local sampling +policy behind that hook; the runner no longer reads `_minicpmo45_*` attributes +or retries sampling after matching `TypeError` text. + +The scheduler no longer parses serving `session_config`, `extra_body`, or +`duplex_stage_sampling_params`. Stage-specific overrides are materialized as +ordinary `SamplingParams` by the model runtime extension before scheduler +admission. Existing vLLM stop-token, EOS, and max-token handling defines each +segment boundary; `resumable=True` parks the request for the next streaming +update. The scheduler does not inspect a duplex dictionary, Realtime +configuration, or MiniCPM sampling fields. + +The generic runner's unused `_duplex_force_listen_applied_segments` cleanup and +the obsolete `streaming_accumulated_keys` accumulation hook were removed. The +latter had no model producer after its upstream Qwen removal; streaming input +still performs the ordinary replacement merge needed by active models. + +Local `turn.signal` transitions such as `user_started` and +`assistant_started` no longer perform an engine RPC. Only cancellation and +`session.update` cross the engine boundary because those operations change +runtime resource identity or runtime configuration. + +During a stage transition, `StreamingInputState.source_token_decoder` exposes a +short-lived, model-neutral decoder sourced from the upstream output processor. +The MiniCPM Stage1 input processor consumes that capability for token-faithful +delta transcripts. The generic orchestrator no longer installs or names a +MiniCPM-specific bridge-state key. + +The stable API server inspects the deployment `session_mode` before importing or +constructing the experimental duplex handler. Ordinary deployments do not load +the full-duplex serving package or create its registries and background state. +For an enabled deployment, the client still selects the Realtime duplex route +with `?duplex=1` (or an equivalent explicit true value). Model-name matching is +not used for routing or native-runtime activation. MiniCPM clients explicitly +set `extra_body.minicpmo45_native_duplex=true`; repository demos do so in their +session payloads. + +`DuplexRequestClient` derives the data-plane request identity from the accepted +fence and preregisters a resumable `ClientRequestState` before the append crosses +the engine boundary. The acknowledgement must return the same identity or the +operation fails. Unregistered outputs, including names that merely use the old +`duplex-` prefix, are dropped. Timeout, cancel, and close remove the +preregistered state. + +## Why Scheduler Changes Remain + +Serving cannot preserve model KV after a segment ends. The scheduler must keep +one resumable request in a waiting state and accept a later update: + +```text +RUNNING + -> segment stop +WAITING_FOR_STREAMING_REQ (KV retained) + -> next append +RUNNING + -> session close +FINISHED (KV released) +``` + +Scheduler responsibilities are limited to resumable request state, runtime +context update, stop/boundary handling, and final release. Scheduler must not +own response IDs, playback, overlap, Realtime events, or model policy. + +## Serving and Compatibility Boundaries + +### Normative Realtime event contract + +The public contract in this section applies to `/v1/realtime?duplex=1`. Internal +duplex event names such as `response.output_audio.delta` are projection inputs, +not additional public aliases. Events for one attachment preserve mailbox +order; replay after resume preserves the original event order and `event_id`. + +Accepted compatibility aliases are normalized before mailbox admission: + +| Canonical input event | Accepted aliases | Contract | +| --- | --- | --- | +| `input.text.append` | `input_text.append`, `push_text` | Appends text to the open input item. | +| `input_audio_buffer.append` | `input.audio.append`, `push_chunk` | Appends audio; legacy aliases default to WAV when no format is supplied. | +| `turn.signal` | `signal_turn` | Sends an explicit turn signal; it does not imply response creation. | +| `playback.ack` | `audio.playback_ack` | Advances playback/history only for the identified response and attachment. | +| `session.close` | `close_session`, `close` | Closes the session after previously admitted mailbox events. | + +The Realtime projector emits the following response lifecycle. Cardinality is +scoped to one `response_id` unless stated otherwise. + +| Public event | Trigger and ordering | Cardinality | +| --- | --- | --- | +| `response.created` | First event for a visible assistant response; precedes its output-item, content-part, speak, text, and audio events. | Exactly once for every visible response. | +| `response.speak` | The model selected speak; emitted after `response.created` and no later than the first `response.audio.delta`. It carries decision metadata, not transcript text. | At most once. | +| `response.audio.delta` | Carries one ordered audio chunk. Every emitted audio delta is followed by a `response.audio_transcript.delta` for the same chunk. | Zero or more. | +| `response.audio_transcript.delta` | Append-only transcript contribution paired with an audio delta; it may be empty for a text-less audio unit and must not repeat or overlap earlier text. | One per audio delta. | +| `response.audio.done` | Closes the audio stream after its final delta and before response terminal events. It is omitted for a response with no audio or transcript. | At most once. | +| `response.audio_transcript.done` | Contains the exact concatenation of all non-overlapping transcript deltas for the response. | At most once, and only for a non-empty transcript. | +| `response.output_item.done` / `conversation.item.done` | Finalize the assistant item after content-part terminal events and before `response.done`. | At most once each. | +| `response.done` | Terminal event for a created response; follows audio, transcript, content-part, and item terminal events. | Exactly once for every created response that reaches a terminal state. | +| `rate_limits.updated` | Compatibility event emitted immediately after `response.done`; the current payload has an empty rate-limit list. | Once per emitted `response.done`. | +| `response.listen` | The model selected listen at a model-unit boundary. It may be emitted without `response.created`; clients must not infer a missing or pending response from it. | Zero or more per session, at most once per model-unit decision. | + +Session lifecycle events (`session.created`, `session.updated`, +`session.resumed`, `session.replaced`, `session.expired`, and +`session.resync_required`) are session-scoped rather than response-scoped. +`session.created` is emitted before held response output on a new attachment. +Cancellation and truncation use terminal status/details on the affected +response and never reuse that response ID for later model turns. + +Compatibility changes must update this table and its protocol contract tests in +the same change. The golden transcript test requires joined +`response.audio_transcript.delta` values to equal +`response.audio_transcript.done.transcript`; response tests separately enforce +at-most-once `response.speak` and terminal event cardinality. + +### Session internal ledgers + +`DuplexSession` remains the serving aggregate root and now composes +`InputBufferState`, `ResponseState`, `PlaybackLedger`, and +`ConversationHistory`. The ledgers are private. Read-only properties return +tuples, snapshots, or mapping proxies, and serving modules mutate them through +Session transition methods for request, response-turn, lifecycle, playback, +capability replacement, and history. The Realtime session-update parser still +normalizes individual fields before atomically retaining or rolling back the +aggregate configuration. Further work may narrow the read-only compatibility +surface, but it must not introduce another reducer or a second session state +machine. + +### Realtime input ownership + +The browser is a continuous PCM producer. While the session is open and the +microphone is unmuted, it sends `input_audio_buffer.append` every 200 ms, +including while assistant audio is being generated or played. It does not run +VAD and does not send `input_audio_buffer.commit`. Native Stage0 consumes the +stream in approximately one-second model units; MiniCPM listen, speak, and turn +EOS decisions advance the model conversation. + +Explicit Realtime clients may still use `input_audio_buffer.commit` to create a +conversation item. The translator validates that wire input before producing a +commit carrying `realtime_item_id`. Native auto-response may already have +streamed those PCM samples into the runtime, so the runner accepts a validated +commit even when no runtime-side chunk remains. A truly empty explicit buffer +continues to return `input_audio_buffer_empty`. + +The commit-to-response contract is: + +| Session state at `input_audio_buffer.commit` | Runtime action | Required event outcome | +|---|---|---| +| Auto-response disabled, valid idle input | Commit and retain the input item. | Emit `input_audio_buffer.committed`; emit no response until an explicit `response.create`. | +| Auto-response enabled, valid idle speech | Submit the final native append for the current model turn. | Emit `input_audio_buffer.committed` before model output. Create a response only on the first visible text/audio output; a model listen decision may end with `response.listen` and no response. | +| Auto-response enabled, short overlap during an active response | Discard the short overlap as an acknowledgement. | Emit the committed/no-response acknowledgement; do not create or defer a response. | +| Auto-response enabled, meaningful overlap during an active response | Retain one deferred payload and promote it after the active response terminates. | Emit `input_audio_buffer.committed` with deferred metadata. A later response follows the same visible-output rule and never reuses the prior response ID. | +| Explicit silence/noise (`is_speech=false`) | Clear the physical input without a runtime append. | Emit the committed/no-response acknowledgement followed by `response.listen`. | +| Empty or invalid input | Reject without changing response ownership. | Emit one typed `error`; do not emit `response.created`. | +| Previous response cancelled or truncated and cleanup complete | Apply the corresponding idle rule above. | The prior response remains terminal; any later visible output receives a new response ID. | + +Append tasks, explicit commits, cancellation, and deferred promotion are +serialized by the session actor. Tests cover post-response commits, active +playback deferral, truncated-response ownership, and two-turn +response-required execution; the H20 validation record below is the golden +multi-turn runtime evidence. + +During auto-response overlap, `preserve_realtime_input` distinguishes "do not +append this silent chunk to the native buffer" from "clear the open Realtime +item". Silent overlap no longer discards earlier user PCM. This is an input +ownership correction. It does not add a VAD policy: overlap is admitted to +Stage0 and the model decides whether to listen or speak. + +The first chunk of one overlapping input item also reserves its target model +turn. A later Realtime commit uses that reserved identity even if response EOS +has already advanced the Session turn; clear, cancel, close, and successful +promotion release the reservation. + +### Physical input and model-turn identity + +A Realtime input item, an optional `input_audio_buffer.commit`, and a MiniCPM +model turn are intentionally different identities. The browser normally keeps +one input stream open without commits. Native Stage0 evaluates that stream in +approximately one-second model units. A sampled `<|turn_eos|>` closes the +current model turn and allows a later streamed unit to start another turn. +Explicitly committed input may also produce only model-listen decisions and no +spoken response. + +Therefore neither `model_turn_id` nor response cardinality is derived from the +number of physical inputs. `model_turn_id` advances at model EOS. The +`response-required` fixture may require exactly one response per requested +turn, but `model-policy` validation must accept additional model-owned responses +and cannot assert cross-turn transcript independence through physical-input +attribution. +Before closing, the model-policy demo drains every created response through +`response.done`, acknowledges every completed audio playback, and requires a +bounded quiet interval with no newly created response. This validates lifecycle +completion without suppressing or overriding the model's listen/speak policy. + +### Non-cancel signal semantics + +`turn.signal` remains a public compatibility surface for local Session and +Realtime transitions. Serving forwards only cancellation and `session.update` +to the engine. The engine rejects arbitrary non-cancel signal names instead of +returning a misleading supported acknowledgement. `session.update` remains a +real engine operation because it replaces the configuration consumed by later +append plans. + +## Remaining Architecture Work + +The following work is deliberately not hidden inside this checkpoint. + +### Serving composition + +The generic runner and runtime bridge now depend on the model-neutral +`ServingRuntimeAdapter` protocol and no longer import MiniCPM modules. The +`MiniCPMO45ServingRuntimeAdapter` owns MiniCPM serving state, PCM preparation, +data-plane projection, client/runtime configuration validation, prefix policy, +and capability projection. `PipelineConfig.duplex_serving_adapter` explicitly +selects its import path; `AsyncOmniEngine` and `AsyncOmni` carry that path to the +API server, which passes it into the generic handler. The handler has no model +default and fails startup when the duplex endpoint is enabled without an +adapter. Tests may inject an adapter instance directly. Import-boundary tests +verify that loading the generic runner, bridge, or handler does not load MiniCPM +modules. + +Behavior is still assembled through `DuplexSessionRunnerMixin`, +`NativeRuntimeBridgeMixin`, and `ChatFallbackProjectorMixin`, so explicit +adapter composition is only the model-policy boundary, not a completed removal +of host-method coupling. A later refactor may replace the remaining mixins with +explicit protocol and effect-runner dependencies while preserving one actor +mailbox and one `DuplexSession` transition owner. Splitting those components +must not create a second reducer or concurrent state machine. + +The four private Session ledgers are data partitions, not independent owners. +`DuplexSession` intentionally remains the aggregate root. Further encapsulation +may narrow compatibility properties, but serving code must continue to perform +state changes through Session transitions. + +### Plugin descriptor and typed payloads + +The pipeline currently selects the MiniCPM engine extension and Serving adapter +through two explicit fields, but there is not yet one versioned plugin descriptor +that binds those identities in an open handshake. Realtime client payloads do +not select either implementation. Adding a second native model should first +introduce that descriptor and reject serving/engine plugin mismatches explicitly. + +The stable boundary has typed fences, plans, policies, decisions, and control +messages, but `session_config`, `runtime_config`, and append payloads still use +generic mappings or objects at the extension boundary. Model-neutral typed +session and input-chunk contracts remain follow-up work, especially before +adding video. + +### Output modality policy + +The current Realtime duplex contract supports text and audio output. Unknown +output modalities fail closed with `unsupported_response_modality`; they are +never silently projected as text. This checkpoint deliberately does not add a +modality registry because there is only one native text/audio projection path +to register. + +A registry becomes justified when a second non-audio/non-text implementation +uses the same validation, encoding, lifecycle, and projection contract. At that +point the abstraction must be extracted from two working adapters and retain +the same fail-closed behavior. A speculative registry before that condition +would add selection state without removing any existing branch. + +### Performance measurement contract + +Stage0 token timing comes only from the engine's per-segment metric snapshot: +`vllm_ttft_ms`, `vllm_tpot_ms`, and `vllm_itls_ms`. A resumable Stage0 request +exports its segment-local `num_generation_tokens`; ordinary requests retain the +existing output-token fallback. Realtime response state clears its metric +accumulator at `response.created`, combines only model units owned by that +response, and ignores session-level events with no matching `response_id`. +Realtime projection carries the result under +`metadata.vllm_omni.stage_metrics`; clients do not estimate TPOT from transcript +deltas or subtract cumulative snapshots. + +Audio timing uses the client's monotonic receive clock and is partitioned by +`response_id`. `response_created_to_first_audio_ms` measures projection startup +after `response.created`; `commit_to_first_audio_ms` measures end-to-end output +from an explicit input commit. Neither field is called TTFP without naming its +origin. Reports also include inter-chunk and chunk-duration +count/mean/p50/p95/max plus maximum chunk gap. Engine and client measurements +have different clock origins and must not be merged into one timeline. + +### Stage1 CUDA Graph boundary + +Native Stage1 streaming advances Python-owned request generators and turn +state on every scheduler step. A full-model CUDA Graph would execute that +Python control flow only while capturing and then replay stale chunk and turn +metadata, even if its host-to-device tensor construction were made +capture-safe. The duplex deploy profile therefore keeps `enforce_eager=false` +but selects `cudagraph_mode=PIECEWISE` for Stage1. Python orchestration runs on +every step while fixed-topology compiled regions remain eligible for CUDA +Graph capture. This is a model execution constraint, not an E2E-only override. + +### Resource capabilities + +The checked-in MiniCPM duplex deploy profile sets Stage0 and Stage1 +`max_num_seqs` to two and configures the engine runtime manager with +`max_sessions=2`. The Realtime capability response therefore advertises +multi-session and same-replica multi-session support with +`session_admission_mode="engine_managed"`. Client session fields cannot raise +this server-owned limit. Fresh E2E evidence covers two concurrent sessions in +both model-policy and response-required modes, rejection of a third session, +and admission of a replacement after one accepted session closes. It does not +establish behavior beyond that configured limit. + +Session leases, lease TTL, resumable attachment, orphan reaping, and bounded +Serving input/backpressure are implemented. A production admission controller, +KV-aware capacity budgeting, fairness, starvation metrics, and multi-session +worker-failure recovery remain follow-up work. Capability claims must stay +limited to the mechanisms and two-session execution shape validated here. + +## Validation Evidence + +All pytest and runtime evidence for this branch must run on the remote H20. + +### Current synchronized tree + +The current dirty tree was synchronized to the isolated H20 worktree at +`/home/admin/workspace/aop_lab/model_runner_v2/vllm-omni-worktrees/pr3907-boundary-cg-0721`. + +- Final affected matrix: 661 passed, one network-dependent config test + deselected, 20 warnings. It includes the complete MiniCPM native-duplex hook + file and all control/session/handler/Orchestrator/import-boundary suites. The + run is task `be221f67` (`/tmp/remote_gpu_logs/be221f67.log`). The deselected + `test_resolve_when_autodetect_resolves_none` attempted a Hugging Face model + lookup and hung in the isolated environment; the duplex deploy-config test + passed independently. +- Stage1 PIECEWISE CUDA Graph validation: the deploy-config assertion first + failed with no Stage1 compilation config, then passed in task `3ed6091d`. + Server task `95827027` started with `enforce_eager=False`, logged + `cudagraph_mode=PIECEWISE`, and captured batch sizes 1, 2, and 4 without the + CPU-to-CUDA capture error. The first concurrent E2E exposed graph padding as + `request_token_spans` covering three valid rows in a four-row capture. A + focused RED/GREEN now verifies both single-request and batched-request + padding, and the full MiniCPM hook file passed 47 tests in task `e89dba88`. + After the fix, two concurrent response-required sessions each completed two + spoken responses, playback acknowledgements, and history lifecycles with no + error or stale event. Artifacts are under + `/tmp/pr3907_piecewise_response_required_2x2_fixed_20260721`. +- An exploratory two-session, two-turn model-policy run still left one session + waiting after its second input commit while the other session completed both + turns. The same liveness symptom was reproduced with Stage1 eager mode before + the padding fix, and no Stage1 fatal occurred after the fix. This run is not + counted as PIECEWISE acceptance evidence; the remaining Stage0 resumable + reactivation path requires separate diagnosis. Artifacts are under + `/tmp/pr3907_piecewise_model_policy_2x2_fixed_20260721`. +- Latest broad affected matrix before the final pre-response continuation fix: + 608 passed, 19 warnings. This includes the + Stage0 row-routing regression, cleanup-held admission regressions, + segment-local token accounting, response-owned metric filtering, the generic + output-processor suite, direct E2E script execution, and continuous-response + overlap ownership. The run is task `e2146318` + (`/tmp/remote_gpu_logs/e2146318.log`). +- The pre-response continuation regression group passed 5 focused tests. The + response-required two-turn scenario then passed 5/5 repeated runs in task + two-session semantic isolation plus resume, takeover, and + admission passed in task `49225794`; model-unit overlap passed in task + `3888387d`. The final affected matrix above supersedes the earlier pending + rerun requirement. +- Serving plugin selection was first reproduced as three RED tests: the generic + handler silently selected MiniCPM and the pipeline had no explicit Serving + adapter field. The final pipeline/import/fail-fast group passed 9 tests in task + `4db6d959` (`/tmp/remote_gpu_logs/4db6d959.log`). The token-metric and client + audio-cadence producer-to-consumer group passed 8 tests in task `cf80f6b2` + (`/tmp/remote_gpu_logs/cf80f6b2.log`). +- Stable-import isolation was first reproduced as a RED that loaded twelve + `experimental.fullduplex` modules. After extracting the kernel and making + implementation loading conditional, the isolated import regression passed. + The focused ControlPlane/ControlClient/session/runtime/Orchestrator/AsyncOmni + suite then passed 139 tests in task `fad613ea` + (`/tmp/remote_gpu_logs/fad613ea.log`). + After the final contract-import cleanup, the import-boundary, runtime, and + lease suites passed 45 tests in task `bc138887` + (`/tmp/remote_gpu_logs/bc138887.log`). +- Concurrency and failure-path hardening now has direct regressions for + single-flight cancel/close cleanup, cleanup retry without dropping request + resources, failed-replica affinity cleanup and same-address reattachment, + append ordering across resume, takeover ownership of pending-turn + reservations, and a closed-transport send immediately after resumable + credentials are delivered. The final focused resume/takeover group passed 5 + tests in task `dc2ac96e` (`/tmp/remote_gpu_logs/dc2ac96e.log`). +- Stage1 batching regression: the focused test first failed because only the + first request in `runtime_additional_information` reached the talker. After + routing every request by `request_token_spans`, preserving per-row terminal + flags, and emitting request-keyed sparse audio, the complete MiniCPM duplex + hook file passed 42 tests. +- Stage0 batching regression: two requests in one batch now retain different + `duplex_prompt_token_ids` and `special_token_ids`. The model emits row-aligned + payload elements, so Stage0 no longer copies request zero's duplex metadata to + later sessions. +- Admission finalization regressions cover `max_sessions=1` with blocked and + failed expiry cleanup plus request-triggered cleanup. Every replacement is + rejected with `resource_exhausted` until stage resources are cleaned and the + old session is explicitly finalized. +- Resumable token accounting exports each model unit's native generation-token + count instead of recounting the request's cumulative token IDs. Realtime + response state resets and accumulates those segment snapshots, and the client + ignores unowned session-level metrics while summarizing a response. +- Engine-owned admission and two-session model-policy E2E: two random session + IDs were admitted and remained isolated, a third was rejected with + `resource_exhausted`, closing one accepted session returned capacity, and a + random replacement session was admitted. The capability projection reported + `configured_limit=2`, `advertised_multi_session=true`, and + `session_admission_mode=engine_managed`. In the same run one session selected + speak and the other selected listen without identity leakage. Detach/resume + advanced attachment generation from one to two and rotated its token; active + takeover rejected all four writes from the replaced connection. The run is + task `1d9dd1ed` (`/tmp/remote_gpu_logs/1d9dd1ed.log`), with artifacts under + `/tmp/pr3907_resume_takeover_native_metrics`. +- Concurrent two-session, two-turn response-required E2E: both sessions produced + two complete spoken responses, committed two playback acknowledgements and + history records, and emitted no error, cancel, truncate, or stale events. The + semantically distinct inputs produced isolated transcripts containing + "当时" and "一加一" respectively. Per-response Stage0 output counts were 20, + 38, 32, and 33 tokens rather than cumulative request totals. The run is task + `a16f1467` (`/tmp/remote_gpu_logs/a16f1467.log`), with artifacts under + `/tmp/pr3907_semantic_2turn_native_metrics`. +- Disconnect-grace E2E: a detached session expired after the configured + 30-second engine grace, and a later resume was rejected with + `session_resume_expired`. The run is task `9872d50c`, with artifacts under + `/tmp/pr3907_0719_disconnect_grace_expiry`. +- A separate black-box takeover client exercised the current backend with two + semantically distinct concurrent inputs. Active resume replayed 17 events + including five audio deltas and committed playback history; takeover rejected + all four stale-attachment sends, emitted `session.replaced`, and preserved + attachment/session isolation. Both transcripts contained only their expected + semantic token. The client task is `6773bd34` + (`/tmp/remote_gpu_logs/6773bd34.log`), with artifacts under + `/tmp/pr3907_gaohan_final_takeover`. The backend task was `5abcd1ee`. +- Across the final response-required, overlap, and resume/takeover runs, engine + Stage0 TTFT ranged from 31.17 to 310.18 ms, TPOT from 15.18 to 21.18 ms, and + reported ITL p95 from 16.35 to 32.83 ms. Explicit + `commit_to_first_audio_ms` ranged from 386.80 to 1033.25 ms, while the + client-monotonic maximum audio chunk gap ranged from 388.30 to 1316.37 ms. + These are response-scoped observations from a small, warm-state-sensitive + sample, not an SLO claim. The slowest response still does not satisfy a 200 ms + end-to-end first-output target. +- Post-merge focused handler/demo/Web/Stage0/runner/video suite: 324 passed, + 19 warnings. The run is task `f78cb7a8` + (`/tmp/remote_gpu_logs/f78cb7a8.log`). The Web contract explicitly rejects + browser VAD markers and browser-generated input commits while the merged + video path retains camera-frame admission. +- No-response listen and residual accounting: the focused handler regressions + passed after first reproducing both failures. A model-listen unit without a + Realtime response no longer defers the next committed input, and three + consecutive 1.2-second `pcm_f32le` inputs each returned pending input bytes + to zero. +- Model-unit overlap E2E: the second physical input streamed while the first + response remained active. Seven overlap decisions had + `defer_runtime_append=false`; the one-second model-unit boundary occurred + before the first `response.done`, and no barge-in, cancel, truncate, stale, or + error event occurred. The model-policy contract accepts a later unit ending + the continuous response instead of requiring a second physical-input-owned + response. The two response-owned Stage0 snapshots independently reported 20 + tokens / 152.8 ms TTFT and 19 tokens / 43.3 ms TTFT. The run is task + `a4977622` (`/tmp/remote_gpu_logs/a4977622.log`), with artifacts under + `/tmp/pr3907_overlap_native_metrics_scoped`. +- The split multi-session E2E runner is directly executable from the repository + even when another installed package owns the top-level `tests` name. Its + script-import and continuous-terminal regressions passed in task `f11b106c`. +- Pinned response-required E2E: the first 1400 ms of the pinned fixture produced + one complete spoken response, five audio/transcript deltas, and one playback + history commit, with symmetric created/audio-done/done lifecycle and no + duplicate `response.speak`. The client task is `a76dec52` + (`/tmp/remote_gpu_logs/a76dec52.log`). +- Continuous-input model-policy E2E: 36 browser-like 200 ms chunks produced + seven model-listen decisions with zero input commits and no + error/cancel/truncate/stale events. The client task is `fb256595`; artifacts + are in `/tmp/minicpmo_pr3907_continuous_model_policy_20260718`. +- Three-response continuous-input E2E: three consecutive 9469 ms streams, + composed of the original fixture plus four seconds of silence, produced three + spoken responses in one session. All three completed `response.done`, audio + transcript, playback ACK, and history commit; the run had 22 audio/transcript + deltas, zero input commits, and no error/cancel/truncate/stale events. This is + lifecycle evidence, not semantic cross-input independence, because the same + fixture was repeated. The client task is `2a5f29f0`; artifacts are in + `/tmp/minicpmo_pr3907_continuous_three_response_20260718`. +- The fresh validation backend was stopped after the E2E runs and GPU0/1 returned + to baseline. Audio-quality and ASR evaluation were not rerun for these + lifecycle-only changes. Other GPU workloads were not modified. + +### Regression lineage + +Earlier synchronized checkpoints passed broader affected matrices before the +extension, policy, and ledger consolidation. Those runs were useful while +developing the refactor but are superseded by the current file-identical H20 +matrix and E2E evidence above; they are intentionally omitted to avoid mixing +different trees in the acceptance argument. + +### Focused suites + +- RPC router and AsyncOmni shutdown/error routing; +- orchestrator fatal and duplex control paths; +- WebSocket Actor mailbox and writer; +- duplex protocol and Session ownership; +- Realtime handler, terminal ordering, overlap, playback ACK; +- scheduler resumable boundary; +- MiniCPM runner and Stage0/Stage1 lifecycle. + +### E2E contract + +Run two deliberately separate fixture contracts: + +- model-policy uses three distinct inputs, no transcript hints, 200 ms + real-time pacing, and continuous input with no browser commit. It allows + natural listen or speak decisions and assumes no 1:1 mapping between + physical inputs, model turns, and responses; +- response-required uses `minicpmo_pr3907_jiayan_16k.wav` pinned to SHA-256 + `2e5fd4eb3ee434ce107ee3a0591fa624a33f7683c7462f45fe651c443c9af941`, + sends its first 1400 ms, and requires one complete spoken response. + It is an output-lifecycle fixture, not evidence that arbitrary no-hint, + real-time input must select speak; +- late and old-response playback ACK handling in the affected test matrix; +- final model-policy response drain before `session.close`; +- close/disconnect with no post-terminal audio; +- transcript delta/final consistency; +- generated WAV validity and external ASR sanity. + +Distinct-input fixtures must also be semantically distinct. Different files or +waveform hashes containing the same greeting can legitimately produce the same +reply and are not sufficient evidence for cross-turn transcript independence. + +For each run record the exact SHA/worktree, command, server log, client log, +WAV path, and cleanup result. + +## Acceptance Criteria + +This checkpoint is ready to publish only when: + +- one mailbox passes exactly-once and wire-order tests; +- cancel advances the engine fence even when the old append continues in an + executor, and the next-epoch append remains accepted; +- request completion removes MiniCPM helper and model-owned force-listen state; +- RPC fatal/close wakes every waiter and no new work enters after terminal; +- Session is the only serving response/overlap authority; +- stale terminal events cannot mutate a newer epoch; +- an accepted terminal cannot be dropped by the writer; +- no duplicate `response.speak` or transcript delta appears; +- Stage0 KV continuity and Stage1 turn reset pass; +- the real H20 multi-turn audio E2E passes; +- ordinary non-duplex paths in the affected matrix still pass; +- no local path, proxy token, temporary profiling probe, or test-only switch is present. + +Passing this checkpoint supports the statement: + +> Single-session, model-owned MiniCPM-o 4.5 native duplex is reviewable on the +> validated H20 configuration. + +It does not support claims for deterministic VAD-triggered interruption, +multi-session production concurrency, bounded long-session KV, +scheduler-native append, or video input. diff --git a/vllm_omni/experimental/fullduplex/README.md b/vllm_omni/experimental/fullduplex/README.md index 7f9e22fa6..5359edf87 100644 --- a/vllm_omni/experimental/fullduplex/README.md +++ b/vllm_omni/experimental/fullduplex/README.md @@ -1,80 +1,28 @@ -# Full-duplex interaction framework +# Experimental Full-Duplex Runtime -A model-agnostic framework for real-time, full-duplex (streaming-in / streaming-out) -model interaction, plus the JoyVL implementation built on it. +This package contains two experimental integrations: -To **run the JoyVL model**, see the recipe: +- the existing JoyVL framework and example integration; +- the MiniCPM-o 4.5 native audio path used by `/v1/duplex` and + `/v1/realtime?duplex=1`. + +For the MiniCPM active runtime path, lifecycle invariants, capability boundary, +Realtime response contract, and validation scope, see [`DESIGN.md`](DESIGN.md). + +To run JoyVL, see [`recipes/JD/JoyAI-VL-Interaction.md`](../../../recipes/JD/JoyAI-VL-Interaction.md). -This README covers the framework itself and how to add a new model. -## Layout +## Package boundaries -``` -vllm_omni/experimental/fullduplex/ - core/ generic full-duplex framework (model-agnostic): DuplexRuntime (event - loop + epoch barge-in), DuplexSession, DuplexAdapter (ABC), protocol - joyvl/ JoyVL implementation (model-specific): - adapter.py JoyVLDuplexAdapter (implements core.DuplexAdapter) - decision/ policy + output_parser + prompts (speak/silence/delegate) - memory/ InteractionBrain — 3-tier summary memory (async) - serving/ OpenAI-compatible HTTP orchestrator - bridges/ model backend + delegation +```text +core/ existing JoyVL framework and experimental compatibility exports +engine/ AsyncOmni/orchestrator scheduler data-plane adapter +openai/ WebSocket transport, Realtime projection, and audio codecs +minicpmo45/ MiniCPM input framing, policy, compatibility, and Stage0 state +joyvl/ JoyVL model-specific integration ``` -`core/` is the only part shared across models; data planes differ by model and are -intentionally not shared. - -## Scope - -The runnable serving path is `joyvl/serving/` driving `decision/` + `memory/` directly. -`joyvl/adapter.py` + `core/` are a **demonstration** of how a model plugs into the -generic full-duplex framework (exercised by `tests/fullduplex/`), not the serving path -the HTTP orchestrator currently uses — a fused-audio model (e.g. MiniCPM-o) is the case -`core/` is built for. - -## Adding a full-duplex model - -The seam is `core.DuplexAdapter`. `core/` owns the session lifecycle, epoch-based -barge-in, playback cursor, and the event protocol — you implement only model policy. - -1. Create a sibling package `vllm_omni/experimental/fullduplex//` next to `joyvl/`; keep - model-specific code there and do not touch `core/`. -2. Implement one `DuplexAdapter` (three required methods; the rest have defaults): - - ```python - from collections.abc import AsyncIterator - from vllm_omni.experimental.fullduplex.core.adapter import DuplexAdapter, DuplexCapability, OutputChunk - from vllm_omni.experimental.fullduplex.core.session import DuplexSession - - class MyModelAdapter(DuplexAdapter): - def capabilities(self) -> DuplexCapability: - return DuplexCapability( - input_modalities=frozenset({"audio", "text"}), - output_modalities=frozenset({"audio", "text"}), - proactive=True, # speak without being asked? - ) - - async def on_input(self, session: DuplexSession, modality: str, data) -> None: - ... # buffer/route an incoming chunk - - async def respond(self, session: DuplexSession) -> AsyncIterator[OutputChunk]: - async for piece in self._model_stream(session): - yield OutputChunk(modality="audio", data=piece) # runtime drops stale - # chunks after a barge-in - - # optional: should_respond / on_barge_in / on_playback_ack - ``` - -3. Run it through the shared runtime — no new control-plane code: - - ```python - from vllm_omni.experimental.fullduplex.core.runtime import DuplexRuntime - from vllm_omni.experimental.fullduplex.core.session import DuplexSession, DuplexSessionConfig - - rt = DuplexRuntime(DuplexSession("sid", DuplexSessionConfig()), MyModelAdapter()) - await rt.run(input_events, emit) - ``` - -`joyvl/adapter.py` is the worked demonstration (currently exercised by tests, not the -HTTP serving path). Promote a helper from a model package up into `core/` only once a -second model actually needs it. +MiniCPM does not run through the removed experimental `core.DuplexRuntime` +facade. Its active path uses the `openai` session controller, the experimental +engine contracts, the standard scheduler/model runners, and an injected +MiniCPM-specific runtime extension from `minicpmo45/runtime.py`. diff --git a/vllm_omni/experimental/fullduplex/client.py b/vllm_omni/experimental/fullduplex/client.py new file mode 100644 index 000000000..57cc2f700 --- /dev/null +++ b/vllm_omni/experimental/fullduplex/client.py @@ -0,0 +1,417 @@ +"""Reusable Realtime WebSocket, PCM, and event helpers for MiniCPM-o demos.""" + +from __future__ import annotations + +import asyncio +import base64 +import json +import math +import time +import wave +from collections.abc import Callable +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any +from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit + +try: + import websockets + from websockets.exceptions import ConnectionClosed +except ImportError as exc: # pragma: no cover - example dependency + raise SystemExit("Install websockets first: pip install websockets") from exc + +PCM16_SAMPLE_RATE = 16_000 +PCM16_BYTES_PER_SAMPLE = 2 + + +def _rounded_ms(value: float) -> float: + return round(float(value), 3) + + +def _interval_summary(values: list[float]) -> dict[str, float | int]: + clean = sorted(_rounded_ms(value) for value in values if math.isfinite(value) and value >= 0) + if not clean: + return {"count": 0, "mean": 0.0, "p50": 0.0, "p95": 0.0, "max": 0.0} + + def nearest_rank(percentile: float) -> float: + index = max(0, math.ceil(percentile * len(clean)) - 1) + return clean[min(index, len(clean) - 1)] + + return { + "count": len(clean), + "mean": _rounded_ms(sum(clean) / len(clean)), + "p50": nearest_rank(0.50), + "p95": nearest_rank(0.95), + "max": clean[-1], + } + + +def _event_stage_metrics(event: dict[str, object]) -> dict[str, object] | None: + candidates: list[object] = [event.get("vllm_omni")] + metadata = event.get("metadata") + if isinstance(metadata, dict): + candidates.extend((metadata, metadata.get("vllm_omni"))) + response = event.get("response") + if isinstance(response, dict): + response_metadata = response.get("metadata") + if isinstance(response_metadata, dict): + candidates.extend((response_metadata, response_metadata.get("vllm_omni"))) + for candidate in candidates: + if not isinstance(candidate, dict): + continue + stage_metrics = candidate.get("stage_metrics") + if isinstance(stage_metrics, dict): + return stage_metrics + return None + + +def build_realtime_url( + url: str, + model: str, + *, + autostart: bool | None = None, + session_id: str | None = None, +) -> str: + """Add the explicit native-duplex query parameters to a Realtime URL.""" + parts = urlsplit(url) + query = dict(parse_qsl(parts.query, keep_blank_values=True)) + query.setdefault("duplex", "1") + query.setdefault("model", model) + query.setdefault("minicpmo45_native_duplex", "1") + if autostart is not None: + query.setdefault("autostart", "1" if autostart else "0") + if session_id: + query.setdefault("session_id", session_id) + return urlunsplit((parts.scheme, parts.netloc, parts.path, urlencode(query), parts.fragment)) + + +def read_pcm16_wav(path: Path) -> bytes: + """Read a mono, uncompressed, 16 kHz PCM16 WAV file.""" + with wave.open(str(path), "rb") as wav_file: + if wav_file.getnchannels() != 1: + raise ValueError("input WAV must be mono") + if wav_file.getsampwidth() != PCM16_BYTES_PER_SAMPLE: + raise ValueError("input WAV must be 16-bit PCM") + if wav_file.getframerate() != PCM16_SAMPLE_RATE: + raise ValueError("input WAV must be 16 kHz") + if wav_file.getcomptype() != "NONE": + raise ValueError("input WAV must be uncompressed PCM") + return wav_file.readframes(wav_file.getnframes()) + + +def write_pcm16_wav(path: Path, pcm16: bytes, *, sample_rate_hz: int) -> None: + """Write mono PCM16 bytes as a WAV artifact.""" + with wave.open(str(path), "wb") as wav_file: + wav_file.setnchannels(1) + wav_file.setsampwidth(PCM16_BYTES_PER_SAMPLE) + wav_file.setframerate(sample_rate_hz) + wav_file.writeframes(pcm16) + + +async def wait_for( + predicate: Callable[[], bool], + *, + timeout_s: float, + label: str, +) -> None: + """Wait for a collector predicate without coupling to a scenario runner.""" + deadline = time.monotonic() + timeout_s + while time.monotonic() < deadline: + if predicate(): + return + await asyncio.sleep(0.02) + raise TimeoutError(f"Timed out waiting for {label}") + + +@dataclass +class RealtimeEventCollector: + """Collect server events and decode response audio by response identity.""" + + events: list[dict[str, object]] = field(default_factory=list) + event_received_at_s: list[float] = field(default_factory=list) + response_audio: dict[str, list[bytes]] = field(default_factory=dict) + response_ids: list[str] = field(default_factory=list) + output_sample_rate_hz: int = 24_000 + + @staticmethod + def response_id(event: dict[str, object]) -> str | None: + response_id = event.get("response_id") + if isinstance(response_id, str): + return response_id + response = event.get("response") + if isinstance(response, dict): + response_id = response.get("id") + if isinstance(response_id, str): + return response_id + return None + + def add(self, event: dict[str, object], *, received_at_s: float | None = None) -> None: + received_at = time.monotonic() if received_at_s is None else float(received_at_s) + stored_event = dict(event) + stored_event.setdefault("_client_received_at_s", received_at) + self.events.append(stored_event) + self.event_received_at_s.append(received_at) + response_id = self.response_id(stored_event) + event_type = stored_event.get("type") + if event_type == "response.created" and response_id and response_id not in self.response_ids: + self.response_ids.append(response_id) + if event_type == "response.audio.delta": + delta = stored_event.get("delta") or stored_event.get("audio") + if isinstance(delta, str) and response_id: + try: + self.response_audio.setdefault(response_id, []).append(base64.b64decode(delta)) + except ValueError: + pass + sample_rate_hz = stored_event.get("sample_rate_hz") + if isinstance(sample_rate_hz, int) and sample_rate_hz > 0: + self.output_sample_rate_hz = sample_rate_hz + + def count(self, event_type: str) -> int: + return sum(event.get("type") == event_type for event in self.events) + + def audio_bytes(self, response_id: str | None = None) -> bytes: + if response_id is not None: + return b"".join(self.response_audio.get(response_id, ())) + return b"".join( + chunk for response_id in self.response_ids for chunk in self.response_audio.get(response_id, ()) + ) + + def errors(self) -> list[dict[str, object]]: + return [event for event in self.events if event.get("type") == "error"] + + def first_received_at( + self, + *event_types: str, + after_s: float = 0.0, + ) -> float | None: + for event, received_at_s in zip(self.events, self.event_received_at_s, strict=True): + if received_at_s >= after_s and event.get("type") in event_types: + return received_at_s + return None + + def last_received_at(self, event_type: str) -> float | None: + for event, received_at_s in zip( + reversed(self.events), + reversed(self.event_received_at_s), + strict=True, + ): + if event.get("type") == event_type: + return received_at_s + return None + + def timing_summary( + self, + *, + after_s: float, + input_committed_at_s: float | None = None, + response_id: str | None = None, + ) -> dict[str, object]: + """Summarize engine token metrics and client-observed audio cadence.""" + stage0_metrics: dict[str, object] | None = None + response_created_at_s: float | None = None + audio_received_at_s: list[float] = [] + cumulative_audio_ms: list[float] = [] + for event, received_at_s in zip(self.events, self.event_received_at_s, strict=True): + if received_at_s < after_s: + continue + event_response_id = self.response_id(event) + if response_id is not None and event_response_id != response_id: + continue + if event.get("type") == "response.created" and response_created_at_s is None: + response_created_at_s = received_at_s + + stage_metrics = _event_stage_metrics(event) + stage0 = stage_metrics.get("0") if isinstance(stage_metrics, dict) else None + if isinstance(stage0, dict): + stage0_metrics = stage0 + + if event.get("type") != "response.audio.delta" or ( + response_id is not None and event_response_id != response_id + ): + continue + delta = event.get("delta") or event.get("audio") + if not isinstance(delta, str) or not delta: + continue + audio_received_at_s.append(received_at_s) + metadata = event.get("metadata") + duration_ms = metadata.get("audio_duration_ms") if isinstance(metadata, dict) else None + if isinstance(duration_ms, int | float) and math.isfinite(float(duration_ms)): + cumulative_audio_ms.append(max(0.0, float(duration_ms))) + + result: dict[str, object] = {} + if stage0_metrics is not None: + raw_itls = stage0_metrics.get("vllm_itls_ms") + itls = ( + [float(value) for value in raw_itls if isinstance(value, int | float)] + if isinstance(raw_itls, list) + else [] + ) + result["stage0_tokens"] = { + "source": "engine_stage_metrics", + "output_token_count": int(stage0_metrics.get("num_tokens_out") or 0), + "ttft_ms": float(stage0_metrics.get("vllm_ttft_ms") or 0.0), + "tpot_ms": float(stage0_metrics.get("vllm_tpot_ms") or 0.0), + "inter_token_interval_ms": _interval_summary(itls), + } + + if audio_received_at_s: + intervals_ms = [ + (current - previous) * 1000.0 for previous, current in zip(audio_received_at_s, audio_received_at_s[1:]) + ] + chunk_durations_ms: list[float] = [] + previous_duration_ms = 0.0 + for duration_ms in cumulative_audio_ms: + chunk_durations_ms.append( + duration_ms - previous_duration_ms if duration_ms >= previous_duration_ms else duration_ms + ) + previous_duration_ms = duration_ms + interval_summary = _interval_summary(intervals_ms) + result["audio_output"] = { + "source": "client_monotonic_receive", + "chunk_count": len(audio_received_at_s), + "response_created_to_first_audio_ms": ( + _rounded_ms((audio_received_at_s[0] - response_created_at_s) * 1000.0) + if response_created_at_s is not None + else None + ), + "commit_to_first_audio_ms": ( + _rounded_ms((audio_received_at_s[0] - input_committed_at_s) * 1000.0) + if input_committed_at_s is not None + else None + ), + "inter_chunk_interval_ms": interval_summary, + "chunk_duration_ms": _interval_summary(chunk_durations_ms), + "max_chunk_gap_ms": interval_summary["max"], + } + return result + + +class RealtimeDuplexClient: + """Small async client used by the user demo and reusable smoke probes.""" + + def __init__(self, url: str, *, max_size: int = 64 * 1024 * 1024) -> None: + self.url = url + self.max_size = max_size + self.events = RealtimeEventCollector() + self._ws: Any = None + self._reader_task: asyncio.Task[None] | None = None + + async def __aenter__(self) -> RealtimeDuplexClient: + self._ws = await websockets.connect(self.url, max_size=self.max_size) + self._reader_task = asyncio.create_task(self._read_events()) + return self + + async def __aexit__(self, exc_type, exc, traceback) -> None: + if self._ws is not None: + await self._ws.close() + if self._reader_task is not None: + self._reader_task.cancel() + try: + await self._reader_task + except asyncio.CancelledError: + pass + + async def _read_events(self) -> None: + try: + while True: + raw = await self._ws.recv() + if not isinstance(raw, str): + continue + event = json.loads(raw) + if isinstance(event, dict): + self.events.add(event) + except ConnectionClosed: + return + + async def send(self, event: dict[str, object]) -> None: + await self._ws.send(json.dumps(event)) + + async def configure( + self, + model: str, + *, + output_audio_format: str = "pcm16", + ref_audio: str | None = None, + session_id: str | None = None, + timeout_s: float = 20.0, + ) -> None: + session: dict[str, object] = { + "model": model, + "modalities": ["audio", "text"], + "input_audio_format": "pcm16", + "output_audio_format": output_audio_format, + "turn_detection": None, + "overlap_policy": "listen_only", + "playback_commit_policy": "ack_only", + "extra_body": { + "auto_response": True, + "minicpmo45_native_duplex": True, + "force_listen_count": 0, + }, + } + if ref_audio is not None: + session["ref_audio"] = ref_audio + if session_id: + session["session_id"] = session_id + await self.send({"type": "session.update", "session": session}) + await wait_for( + lambda: self.events.count("session.created") > 0, + timeout_s=timeout_s, + label="session.created", + ) + + async def stream_pcm16( + self, + pcm16: bytes, + *, + chunk_ms: int = 200, + realtime: bool = True, + ) -> None: + chunk_bytes = max( + PCM16_SAMPLE_RATE * PCM16_BYTES_PER_SAMPLE * chunk_ms // 1000, + PCM16_BYTES_PER_SAMPLE, + ) + audio_end_ms = 0 + for offset in range(0, len(pcm16), chunk_bytes): + chunk = pcm16[offset : offset + chunk_bytes] + duration_ms = len(chunk) * 1000 // (PCM16_SAMPLE_RATE * PCM16_BYTES_PER_SAMPLE) + audio_end_ms += duration_ms + await self.send( + { + "type": "input_audio_buffer.append", + "audio": base64.b64encode(chunk).decode("ascii"), + "input_audio_format": "pcm16", + "sample_rate_hz": PCM16_SAMPLE_RATE, + "duration_ms": duration_ms, + "audio_end_ms": audio_end_ms, + } + ) + if realtime: + await asyncio.sleep(duration_ms / 1000) + + async def commit(self) -> None: + await self.send({"type": "input_audio_buffer.commit", "final": True}) + + async def acknowledge_playback(self) -> None: + for response_id in self.events.response_ids: + pcm16 = self.events.audio_bytes(response_id) + if not pcm16: + continue + played_ms = len(pcm16) * 1000 // (self.events.output_sample_rate_hz * PCM16_BYTES_PER_SAMPLE) + await self.send( + { + "type": "playback.ack", + "response_id": response_id, + "item_id": f"item_{response_id}", + "played_ms": played_ms, + "committed_ms": played_ms, + } + ) + + async def close_session(self, *, timeout_s: float = 20.0) -> None: + await self.send({"type": "session.close"}) + await wait_for( + lambda: self.events.count("session.closed") > 0, + timeout_s=timeout_s, + label="session.closed", + ) diff --git a/vllm_omni/experimental/fullduplex/engine/__init__.py b/vllm_omni/experimental/fullduplex/engine/__init__.py new file mode 100644 index 000000000..c8e20437f --- /dev/null +++ b/vllm_omni/experimental/fullduplex/engine/__init__.py @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +"""Engine-side helpers for the experimental full-duplex runtime.""" diff --git a/vllm_omni/experimental/fullduplex/engine/contracts.py b/vllm_omni/experimental/fullduplex/engine/contracts.py new file mode 100644 index 000000000..ae449e82f --- /dev/null +++ b/vllm_omni/experimental/fullduplex/engine/contracts.py @@ -0,0 +1,276 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +"""Model-neutral contracts for the experimental duplex engine plugin. + +This module contains only immutable data transfer objects and narrow protocols. +Duplex control algorithms, session implementations, model policy, and Realtime +serving remain in sibling experimental modules. +""" + +from __future__ import annotations + +import base64 +from collections.abc import Iterable, Mapping +from dataclasses import dataclass, field +from enum import Enum +from types import MappingProxyType +from typing import Any, Protocol + +from vllm_omni.engine.messages import EngineQueueMessage +from vllm_omni.experimental.fullduplex.engine.messages import DuplexFence + + +class SessionMode(str, Enum): + TURN = "turn" + DUPLEX = "duplex" + + +class DuplexInputMode(str, Enum): + APPEND_TOKENS = "append_tokens" + APPEND_AUDIO_CHUNK = "append_audio_chunk" + REPLACE_LATEST_CHUNK = "replace_latest_chunk" + REENCODE_CONTEXT = "reencode_context" + ROLLBACK_TO_CHECKPOINT = "rollback_to_checkpoint" + TURN_COMMIT_ONLY = "turn_commit_only" + + +class DuplexOutputAction(str, Enum): + DIRECT_RESPONSE = "direct_response" + + +@dataclass +class DuplexRuntimeCapabilities: + input_modes: set[DuplexInputMode] = field(default_factory=lambda: {DuplexInputMode.TURN_COMMIT_ONLY}) + implementation_level: str = "serving_session_adapter" + + +@dataclass(frozen=True) +class DuplexAppendPlan: + prompt: dict[str, Any] + + +@dataclass(frozen=True) +class DuplexOutputDecision: + action: DuplexOutputAction + metadata: Mapping[str, Any] = field(default_factory=dict) + final_output_type: str = "text" + + def __post_init__(self) -> None: + object.__setattr__(self, "metadata", MappingProxyType(dict(self.metadata))) + + +class DuplexRuntimeExtension(Protocol): + """Pure model policy invoked by the experimental duplex control plane.""" + + def configure_sampling_params( + self, + *, + runtime_config: dict[str, Any], + defaults: tuple[object, ...], + ) -> tuple[object, ...]: ... + + def plan_append( + self, + *, + request_id: str, + fence: DuplexFence, + session_config: dict[str, Any], + runtime_config: dict[str, Any], + seq: int, + turn_seq: int, + mode: DuplexInputMode, + payload: object, + final: bool, + sampling_params: object, + ) -> DuplexAppendPlan: ... + + def decide_output( + self, + *, + stage_id: int, + final_stage_id: int, + segment_finished: bool, + segment_token_ids: tuple[int, ...], + segment_output_metadata: dict[str, Any], + output: object, + ) -> DuplexOutputDecision | None: ... + + +@dataclass(frozen=True) +class DuplexRequestIdentity: + session_id: str + fence: DuplexFence + + +@dataclass(frozen=True) +class DuplexStageRequestContext: + request_id: str + session_id: str + fence: DuplexFence + stage_id: int + final_stage_id: int + config_generation: int + sampling_params: tuple[object, ...] + session_config: Mapping[str, Any] = field(default_factory=dict) + runtime_config: Mapping[str, Any] = field(default_factory=dict) + + def __post_init__(self) -> None: + object.__setattr__(self, "sampling_params", tuple(self.sampling_params)) + object.__setattr__(self, "session_config", MappingProxyType(dict(self.session_config))) + object.__setattr__(self, "runtime_config", MappingProxyType(dict(self.runtime_config))) + + @property + def stage_sampling_params(self) -> object: + return self.sampling_params[self.stage_id] + + +@dataclass(frozen=True) +class DuplexStageSubmission: + context: DuplexStageRequestContext + prompt: Mapping[str, Any] + already_submitted: bool + + def __post_init__(self) -> None: + object.__setattr__(self, "prompt", MappingProxyType(dict(self.prompt))) + + +@dataclass(frozen=True) +class DuplexStageSubmissionResult: + request_id: str + stage_id: int + replica_id: int + + +@dataclass(frozen=True) +class DuplexOutputContext: + identity: DuplexRequestIdentity + final_stage_id: int + segment_finished: bool + segment_token_ids: tuple[int, ...] = () + segment_output_metadata: Mapping[str, Any] = field(default_factory=dict) + + def __post_init__(self) -> None: + object.__setattr__(self, "segment_token_ids", tuple(self.segment_token_ids)) + object.__setattr__( + self, + "segment_output_metadata", + MappingProxyType(dict(self.segment_output_metadata)), + ) + + +class DuplexStagePort(Protocol): + @property + def stage_count(self) -> int: ... + + def sampling_defaults(self) -> tuple[object, ...]: ... + + def ensure_request(self, context: DuplexStageRequestContext) -> None: ... + + async def submit(self, submission: DuplexStageSubmission) -> DuplexStageSubmissionResult: ... + + async def cleanup(self, request_ids: list[str], *, abort: bool = False) -> None: ... + + +class DuplexControlPlanePort(Protocol): + @property + def sessions(self) -> object: ... + + def accepts(self, message: object) -> bool: ... + + def dispatch(self, message: object) -> None: ... + + async def shutdown(self) -> None: ... + + def close_sessions_for_request_ids(self, request_ids: list[str]) -> dict[str, list[str]]: ... + + def finalize_closed_sessions(self, session_ids: Iterable[str]) -> None: ... + + def session_for_identity(self, identity: DuplexRequestIdentity | None) -> object | None: ... + + def decide_output( + self, + stage_id: int, + output: object, + context: DuplexOutputContext | None, + ) -> DuplexOutputDecision | None: ... + + +class CorrelatedRpcTransport(Protocol): + def execute( + self, + key: tuple[str, str], + message: EngineQueueMessage, + *, + timeout: float | None, + timeout_message: str, + block_on_submit: bool = False, + ) -> EngineQueueMessage: ... + + +def duplex_data_plane_request_info(result: dict[str, object]) -> tuple[str | None, int | None]: + stage_results = result.get("stage_results") + if not isinstance(stage_results, list): + return None, None + for item in stage_results: + if not isinstance(item, dict): + continue + inner = item.get("result") + if not isinstance(inner, dict) or inner.get("data_plane_append") is not True: + continue + request_id = inner.get("request_id") + if not isinstance(request_id, str) or not request_id: + continue + response_stage_id = inner.get("response_stage_id") + return request_id, response_stage_id if isinstance(response_stage_id, int) else None + return None, None + + +def duplex_resource_request_id(fence: DuplexFence, role: str) -> str: + if not role or any( + character not in "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_-" for character in role + ): + raise ValueError(f"invalid duplex resource role: {role!r}") + encoded_session_id = base64.urlsafe_b64encode(fence.session_id.encode("utf-8")).decode("ascii").rstrip("=") + return f"duplex-s.{encoded_session_id}.i.{fence.incarnation}.e.{fence.epoch}.r.{role}" + + +def duplex_resource_request_belongs_to_session(request_id: str, session_id: str) -> bool: + """Return whether a current-format resource request belongs to a session.""" + parts = request_id.split(".") + if len(parts) != 8 or parts[0] != "duplex-s" or parts[2] != "i" or parts[4] != "e" or parts[6] != "r": + return False + try: + int(parts[3]) + int(parts[5]) + except ValueError: + return False + role = parts[7] + if not role or any( + character not in "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_-" for character in role + ): + return False + encoded_session_id = base64.urlsafe_b64encode(session_id.encode("utf-8")).decode("ascii").rstrip("=") + return parts[1] == encoded_session_id + + +__all__ = [ + "CorrelatedRpcTransport", + "DuplexAppendPlan", + "DuplexControlPlanePort", + "DuplexInputMode", + "DuplexOutputAction", + "DuplexOutputContext", + "DuplexOutputDecision", + "DuplexRequestIdentity", + "DuplexRuntimeCapabilities", + "DuplexRuntimeExtension", + "DuplexStagePort", + "DuplexStageRequestContext", + "DuplexStageSubmission", + "DuplexStageSubmissionResult", + "SessionMode", + "duplex_data_plane_request_info", + "duplex_resource_request_belongs_to_session", + "duplex_resource_request_id", +] diff --git a/vllm_omni/experimental/fullduplex/engine/duplex_control_client.py b/vllm_omni/experimental/fullduplex/engine/duplex_control_client.py new file mode 100644 index 000000000..b37effe32 --- /dev/null +++ b/vllm_omni/experimental/fullduplex/engine/duplex_control_client.py @@ -0,0 +1,228 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from __future__ import annotations + +import uuid +from collections.abc import Callable + +from vllm_omni.experimental.fullduplex.engine.contracts import CorrelatedRpcTransport +from vllm_omni.experimental.fullduplex.engine.lease import DuplexLeaseActivity +from vllm_omni.experimental.fullduplex.engine.messages import ( + AppendDuplexInputMessage, + CloseDuplexSessionMessage, + DuplexControlResultMessage, + DuplexFence, + OpenDuplexSessionMessage, + ResumeDuplexSessionMessage, + SignalDuplexTurnMessage, + TouchDuplexSessionMessage, +) + +DuplexControlMessage = ( + OpenDuplexSessionMessage + | AppendDuplexInputMessage + | SignalDuplexTurnMessage + | CloseDuplexSessionMessage + | TouchDuplexSessionMessage + | ResumeDuplexSessionMessage +) + + +class DuplexControlRequestError(RuntimeError): + """Typed client-side failure returned by the duplex control plane.""" + + def __init__(self, result: dict[str, object]) -> None: + error = result.get("error") + error_data = error if isinstance(error, dict) else {} + self.result = result + self.code = str(error_data.get("code") or "internal_error") + self.retryable = bool(error_data.get("retryable", False)) + accepted_fence = result.get("accepted_fence") + self.accepted_fence = accepted_fence if isinstance(accepted_fence, DuplexFence) else None + lease_generation = result.get("lease_generation") + self.lease_generation = lease_generation if isinstance(lease_generation, int) else None + super().__init__(f"duplex {result.get('operation')} failed: {result}") + + +class DuplexControlClient: + """Builds duplex commands on top of an engine-owned RPC transport.""" + + def __init__( + self, + transport: CorrelatedRpcTransport, + *, + control_id_factory: Callable[[], str] | None = None, + ) -> None: + self._transport = transport + self._control_id_factory = control_id_factory or (lambda: uuid.uuid4().hex) + + def execute(self, message: DuplexControlMessage, *, timeout: float | None) -> dict[str, object]: + result_message = self._transport.execute( + ("duplex", message.control_id), + message, + timeout=timeout, + timeout_message=f"duplex control timed out after {timeout} seconds", + ) + if not isinstance(result_message, DuplexControlResultMessage): + raise RuntimeError(f"unexpected duplex control result type: {type(result_message).__name__}") + if result_message.fence != message.fence: + raise RuntimeError( + f"duplex control fence mismatch: expected {message.fence!r}, got {result_message.fence!r}" + ) + result = { + "fence": result_message.fence, + "operation": result_message.operation, + "session_id": result_message.session_id, + "ok": result_message.ok, + "stage_results": list(result_message.stage_results), + "unsupported_count": result_message.unsupported_count, + "error_count": result_message.error_count, + "accepted_fence": result_message.accepted_fence, + "lease_generation": result_message.lease_generation, + "error": ( + { + "code": result_message.error.code, + "message": result_message.error.message, + "retryable": result_message.error.retryable, + } + if result_message.error is not None + else None + ), + } + if not result_message.ok: + raise DuplexControlRequestError(result) + return result + + def open( + self, + session_id: str, + *, + session_mode: str, + capabilities: dict[str, object] | None, + session_config: dict[str, object] | None, + runtime_config: dict[str, object] | None, + fence: DuplexFence, + timeout: float | None, + ) -> dict[str, object]: + return self.execute( + OpenDuplexSessionMessage( + control_id=self._control_id_factory(), + fence=fence, + session_id=session_id, + session_mode=session_mode, + capabilities=dict(capabilities or {}), + session_config=dict(session_config or {}), + runtime_config=dict(runtime_config or {}), + ), + timeout=timeout, + ) + + def append( + self, + session_id: str, + *, + mode: str, + payload: object, + operation_id: str | None, + final: bool, + expected_epoch: int | None, + fence: DuplexFence, + timeout: float | None, + ) -> dict[str, object]: + if expected_epoch is not None and expected_epoch != fence.epoch: + raise ValueError("expected_epoch must match fence.epoch") + return self.execute( + AppendDuplexInputMessage( + control_id=self._control_id_factory(), + operation_id=operation_id, + fence=fence, + session_id=session_id, + expected_epoch=expected_epoch, + mode=mode, + payload=payload, + final=final, + ), + timeout=timeout, + ) + + def signal( + self, + session_id: str, + *, + event: str, + fence: DuplexFence, + next_fence: DuplexFence | None, + session_config: dict[str, object] | None, + runtime_config: dict[str, object] | None, + timeout: float | None, + ) -> dict[str, object]: + return self.execute( + SignalDuplexTurnMessage( + control_id=self._control_id_factory(), + fence=fence, + session_id=session_id, + event=event, + next_fence=next_fence, + session_config=dict(session_config) if session_config is not None else None, + runtime_config=dict(runtime_config) if runtime_config is not None else None, + ), + timeout=timeout, + ) + + def close( + self, + session_id: str, + *, + reason: str, + fence: DuplexFence, + timeout: float | None, + ) -> dict[str, object]: + return self.execute( + CloseDuplexSessionMessage( + control_id=self._control_id_factory(), + fence=fence, + session_id=session_id, + reason=reason, + ), + timeout=timeout, + ) + + def touch( + self, + session_id: str, + *, + fence: DuplexFence, + activity: DuplexLeaseActivity, + timeout: float | None, + ) -> dict[str, object]: + return self.execute( + TouchDuplexSessionMessage( + control_id=self._control_id_factory(), + fence=fence, + session_id=session_id, + activity=activity.value, + ), + timeout=timeout, + ) + + def resume( + self, + session_id: str, + *, + fence: DuplexFence, + expected_lease_generation: int, + timeout: float | None, + ) -> dict[str, object]: + return self.execute( + ResumeDuplexSessionMessage( + control_id=self._control_id_factory(), + fence=fence, + session_id=session_id, + expected_lease_generation=expected_lease_generation, + ), + timeout=timeout, + ) + + +__all__ = ["CorrelatedRpcTransport", "DuplexControlClient", "DuplexControlMessage"] diff --git a/vllm_omni/experimental/fullduplex/engine/duplex_control_plane.py b/vllm_omni/experimental/fullduplex/engine/duplex_control_plane.py new file mode 100644 index 000000000..a1d3dac00 --- /dev/null +++ b/vllm_omni/experimental/fullduplex/engine/duplex_control_plane.py @@ -0,0 +1,1063 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +"""Orchestrator-side control plane for experimental duplex sessions. + +``Orchestrator`` creates this component only when duplex control is enabled +and supplies a narrow :class:`DuplexStagePort` implementation. Queue messages +are routed through :meth:`DuplexControlPlane.handle`; stage outputs are passed +to :meth:`DuplexControlPlane.decide_output`; request cleanup is reported back +through the control plane so session leases and stage bindings stay coherent. + +The module owns duplex session/control algorithms. It deliberately does not +own stage pools, request queues, or OpenAI Realtime protocol state. +""" + +from __future__ import annotations + +import asyncio +from collections.abc import Callable, Iterable +from dataclasses import dataclass +from typing import Any, Protocol + +from vllm.logger import init_logger + +from vllm_omni.experimental.fullduplex.engine.contracts import ( + DuplexAppendPlan, + DuplexInputMode, + DuplexOutputContext, + DuplexOutputDecision, + DuplexRequestIdentity, + DuplexRuntimeCapabilities, + DuplexRuntimeExtension, + DuplexStagePort, + DuplexStageRequestContext, + DuplexStageSubmission, + DuplexStageSubmissionResult, + SessionMode, + duplex_resource_request_id, +) +from vllm_omni.experimental.fullduplex.engine.duplex_session import ( + DuplexAppendReservation, + DuplexFenceMismatchError, + DuplexSessionExpiry, + DuplexSessionRuntimeManager, + DuplexSessionRuntimeState, +) +from vllm_omni.experimental.fullduplex.engine.lease import DuplexLeaseActivity, DuplexLeaseConfig +from vllm_omni.experimental.fullduplex.engine.messages import ( + AppendDuplexInputMessage, + CloseDuplexSessionMessage, + DuplexControlError, + DuplexControlResultMessage, + DuplexFence, + DuplexSessionLifecycleMessage, + OpenDuplexSessionMessage, + ResumeDuplexSessionMessage, + SignalDuplexTurnMessage, + TouchDuplexSessionMessage, +) + +logger = init_logger(__name__) + + +@dataclass(frozen=True) +class _PendingControlCleanup: + kind: str + session_id: str + fence: DuplexFence + submitted_request_ids: tuple[str, ...] + reserved_request_ids: tuple[str, ...] = () + + +@dataclass(frozen=True) +class _PendingSubmissionCleanup: + session_id: str + request_ids: tuple[str, ...] + + +@dataclass(frozen=True) +class _PendingRequestCleanup: + session_id: str + fence: DuplexFence + lease_generation: int + request_ids: tuple[str, ...] + abort: bool + + +class DuplexResultSink(Protocol): + async def put(self, message: DuplexControlResultMessage) -> None: ... + + +class DuplexLifecycleSink(Protocol): + async def put(self, message: DuplexSessionLifecycleMessage) -> None: ... + + +class DuplexControlPlane: + _MESSAGE_TYPES = ( + OpenDuplexSessionMessage, + AppendDuplexInputMessage, + SignalDuplexTurnMessage, + CloseDuplexSessionMessage, + TouchDuplexSessionMessage, + ResumeDuplexSessionMessage, + ) + + def __init__( + self, + *, + extension: DuplexRuntimeExtension | None, + stage_port: DuplexStagePort, + result_sink: DuplexResultSink, + lifecycle_sink: DuplexLifecycleSink | None = None, + lease_config: DuplexLeaseConfig | None = None, + clock: Callable[[], float] | None = None, + max_sessions: int | None = None, + completed_append_limit: int = 256, + ) -> None: + self._extension = extension + self._stage_port = stage_port + self._result_sink = result_sink + self._lifecycle_sink = lifecycle_sink + self._lease_config = lease_config or DuplexLeaseConfig() + self._sessions = DuplexSessionRuntimeManager( + clock=clock, + max_sessions=max_sessions, + completed_append_limit=completed_append_limit, + ) + self._pending_expirations: dict[tuple[str, int], DuplexSessionExpiry] = {} + self._pending_control_cleanups: dict[tuple[str, str, int, int, int, int], _PendingControlCleanup] = {} + self._control_cleanup_tasks: dict[ + tuple[str, str, int, int, int, int], + asyncio.Task[None], + ] = {} + self._pending_submission_cleanups: dict[str, _PendingSubmissionCleanup] = {} + self._submission_cleanup_tasks: dict[str, asyncio.Task[None]] = {} + self._pending_request_cleanups: dict[tuple[str, int, int], _PendingRequestCleanup] = {} + self._request_cleanup_tasks: dict[tuple[str, int, int], asyncio.Task[None]] = {} + self._request_cleanups_in_progress: set[tuple[str, int, int]] = set() + self._session_control_tails: dict[str, asyncio.Task[None]] = {} + self._dispatched_control_tasks: set[asyncio.Task[None]] = set() + + @property + def sessions(self) -> DuplexSessionRuntimeManager: + return self._sessions + + @property + def pending_submission_cleanup_count(self) -> int: + return len(self._pending_submission_cleanups) + + def accepts(self, message: object) -> bool: + return isinstance(message, self._MESSAGE_TYPES) + + async def handle(self, message: object) -> None: + if isinstance(message, OpenDuplexSessionMessage): + await self.handle_open(message) + elif isinstance(message, AppendDuplexInputMessage): + await self.handle_append(message) + elif isinstance(message, SignalDuplexTurnMessage): + await self.handle_signal(message) + elif isinstance(message, CloseDuplexSessionMessage): + await self.handle_close(message) + elif isinstance(message, TouchDuplexSessionMessage): + await self.handle_touch(message) + elif isinstance(message, ResumeDuplexSessionMessage): + await self.handle_resume(message) + else: + raise TypeError(f"Unsupported duplex control message: {type(message).__name__}") + + def dispatch(self, message: object) -> None: + """Schedule one control command without blocking unrelated sessions.""" + if not self.accepts(message): + raise TypeError(f"Unsupported duplex control message: {type(message).__name__}") + session_id = message.session_id + predecessor = self._session_control_tails.get(session_id) + + async def run_ordered() -> None: + if predecessor is not None: + await predecessor + await self.handle(message) + + task = asyncio.create_task( + run_ordered(), + name=f"duplex-control-{session_id}-{message.control_id}", + ) + self._session_control_tails[session_id] = task + self._dispatched_control_tasks.add(task) + + def discard(completed: asyncio.Task[None]) -> None: + self._dispatched_control_tasks.discard(completed) + if self._session_control_tails.get(session_id) is completed: + self._session_control_tails.pop(session_id, None) + if not completed.cancelled(): + completed.exception() + + task.add_done_callback(discard) + + async def drain(self) -> None: + while self._dispatched_control_tasks: + await asyncio.gather(*tuple(self._dispatched_control_tasks)) + + async def shutdown(self) -> None: + tasks = tuple( + self._dispatched_control_tasks + | set(self._control_cleanup_tasks.values()) + | set(self._submission_cleanup_tasks.values()) + | set(self._request_cleanup_tasks.values()) + ) + for task in tasks: + task.cancel() + if tasks: + await asyncio.gather(*tasks, return_exceptions=True) + + @staticmethod + def coerce_capabilities(raw: dict[str, object]) -> DuplexRuntimeCapabilities: + input_modes: set[DuplexInputMode] = set() + values = raw.get("input_modes") + if values is not None and not isinstance(values, list): + raise TypeError("duplex input_modes capability must be a list") + if isinstance(values, list): + for value in values: + try: + input_modes.add(DuplexInputMode(value)) + except (TypeError, ValueError) as exc: + raise ValueError(f"unknown duplex input mode: {value!r}") from exc + return DuplexRuntimeCapabilities( + input_modes=input_modes or {DuplexInputMode.TURN_COMMIT_ONLY}, + implementation_level=str(raw.get("implementation_level") or "serving_session_adapter"), + ) + + def sampling_params_for_config(self, runtime_config: dict[str, Any]) -> list[object]: + defaults = self._stage_port.sampling_defaults() + if self._extension is None: + return list(defaults) + configured = self._extension.configure_sampling_params( + runtime_config=runtime_config, + defaults=defaults, + ) + if not isinstance(configured, tuple): + raise TypeError("duplex runtime extension must return sampling parameters as a tuple") + if len(configured) != len(defaults): + raise ValueError("duplex runtime extension must return one sampling parameter per stage") + return list(configured) + + @staticmethod + def stage_request_id(fence: DuplexFence, *, stage_id: int) -> str: + return duplex_resource_request_id(fence, f"stage{stage_id}") + + def ensure_stage_request( + self, + session: DuplexSessionRuntimeState, + *, + stage_id: int, + fence: DuplexFence | None = None, + ) -> DuplexStageRequestContext | None: + if stage_id >= self._stage_port.stage_count: + return None + effective_fence = fence or session.fence + request_id = self.stage_request_id(effective_fence, stage_id=stage_id) + session.reserve_stage_request(stage_id, request_id, fence=effective_fence) + context = DuplexStageRequestContext( + request_id=request_id, + session_id=session.session_id, + fence=effective_fence, + stage_id=stage_id, + final_stage_id=self._stage_port.stage_count - 1, + config_generation=session.config_generation, + sampling_params=tuple(self.sampling_params_for_config(session.runtime_config)), + session_config=session.session_config, + runtime_config=session.runtime_config, + ) + self._stage_port.ensure_request(context) + return context + + async def handle_open(self, message: OpenDuplexSessionMessage) -> None: + session: DuplexSessionRuntimeState | None = None + try: + session_mode = SessionMode(message.session_mode) + capabilities = self.coerce_capabilities(message.capabilities) + native_append_modes = capabilities.input_modes - {DuplexInputMode.TURN_COMMIT_ONLY} + if native_append_modes and self._extension is None: + raise RuntimeError("duplex_runtime_extension_not_configured") + session = self.sessions.open_session( + message.fence, + capabilities=capabilities, + session_config=message.session_config, + runtime_config=message.runtime_config, + lease_config=self._lease_config, + ) + request_context = self.ensure_stage_request(session, stage_id=0) if self._extension is not None else None + await self.put_result( + message.control_id, + fence=message.fence, + operation="open", + session_id=message.session_id, + stage_results=[ + { + "stage_id": -1, + "replica_id": -1, + "result": { + "supported": True, + "implementation_level": capabilities.implementation_level, + "data_plane_session": True, + "session_mode": session_mode.value, + "scheduler_request_context": request_context is not None, + "request_id": request_context.request_id if request_context is not None else None, + }, + } + ], + ) + except Exception as exc: + if self._control_error(exc).code == "resource_exhausted": + logger.info("open_duplex_session rejected: %s", exc) + else: + logger.exception("open_duplex_session failed: %s", exc) + if session is not None and self.sessions.get(message.session_id) is session: + reserved_request_ids = tuple(session.resource_request_ids()) + self.sessions.begin_close_session(session.fence, reason="open_rollback") + cleanup_key = self._cleanup_key("open_rollback", session.fence) + pending = _PendingControlCleanup( + kind="open_rollback", + session_id=session.session_id, + fence=session.fence, + reserved_request_ids=reserved_request_ids, + submitted_request_ids=(), + ) + self._pending_control_cleanups[cleanup_key] = pending + try: + await self._complete_control_cleanup(cleanup_key, pending) + except Exception as cleanup_exc: + logger.warning( + "duplex open rollback remains pending for session %s: %s", + session.session_id, + cleanup_exc, + ) + await self.put_result( + message.control_id, + fence=message.fence, + operation="open", + session_id=message.session_id, + stage_results=[], + error=exc, + ) + + async def handle_append(self, message: AppendDuplexInputMessage) -> None: + session: DuplexSessionRuntimeState | None = None + lease_operation_id = f"append:{message.control_id}" + operation_started = False + try: + session = self.sessions.require(message.session_id) + await self._complete_pending_submission_cleanup(message.session_id) + if message.expected_epoch is not None and message.expected_epoch != message.fence.epoch: + raise ValueError("expected_epoch must match fence.epoch") + mode = DuplexInputMode(message.mode) + if message.operation_id is not None: + completed = session.completed_append( + message.operation_id, + fence=message.fence, + mode=mode, + final=message.final, + ) + if completed is not None: + session.touch(message.fence, DuplexLeaseActivity.APPEND) + await self.put_result( + message.control_id, + fence=message.fence, + operation="append", + session_id=message.session_id, + stage_results=completed, + ) + return + session.begin_operation(message.fence, lease_operation_id) + operation_started = True + reservation = session.prepare_append(mode=mode, fence=message.fence) + stage_results = await self.append_via_data_plane( + message, + session=session, + reservation=reservation, + mode=mode, + ) + if message.operation_id is not None: + session.record_completed_append( + message.operation_id, + fence=message.fence, + mode=mode, + final=message.final, + stage_results=stage_results, + ) + await self.put_result( + message.control_id, + fence=message.fence, + operation="append", + session_id=message.session_id, + stage_results=stage_results, + ) + except Exception as exc: + logger.exception("append_duplex_input failed: %s", exc) + await self.put_result( + message.control_id, + fence=message.fence, + operation="append", + session_id=message.session_id, + stage_results=[], + error=exc, + ) + finally: + if session is not None and operation_started and lease_operation_id in session.lease.active_operations: + session.end_operation(session.fence, lease_operation_id) + + async def append_via_data_plane( + self, + message: AppendDuplexInputMessage, + *, + session: DuplexSessionRuntimeState, + reservation: DuplexAppendReservation, + mode: DuplexInputMode, + ) -> list[dict[str, object]]: + if self._extension is None and mode is DuplexInputMode.TURN_COMMIT_ONLY: + update = session.commit_append(reservation) + return [ + { + "stage_id": -1, + "replica_id": -1, + "result": { + "supported": True, + "data_plane_append": False, + "seq": update.seq, + "turn_id": update.turn_id, + "turn_seq": update.turn_seq, + "mode": mode.value, + }, + } + ] + if self._stage_port.stage_count == 0: + return [ + { + "stage_id": -1, + "replica_id": -1, + "result": {"supported": False, "error": "duplex_data_plane_has_no_stage"}, + } + ] + if self._extension is None: + raise RuntimeError("duplex_runtime_extension_not_configured") + + stage_id = 0 + request_id = self.stage_request_id(message.fence, stage_id=stage_id) + existing_binding = session.stage_bindings.get(stage_id) + already_submitted = existing_binding is not None and existing_binding.request_id == request_id + request_context = self.ensure_stage_request( + session, + stage_id=stage_id, + fence=message.fence, + ) + if request_context is None: + raise RuntimeError("duplex_data_plane_has_no_stage") + append_plan = self._extension.plan_append( + request_id=request_id, + fence=message.fence, + session_config=dict(request_context.session_config), + runtime_config=dict(request_context.runtime_config), + seq=reservation.update.seq, + turn_seq=reservation.update.turn_seq, + mode=mode, + payload=message.payload, + final=message.final, + sampling_params=request_context.stage_sampling_params, + ) + if not isinstance(append_plan, DuplexAppendPlan): + raise TypeError("duplex runtime extension plan_append() must return DuplexAppendPlan") + submission = DuplexStageSubmission( + context=request_context, + prompt=append_plan.prompt, + already_submitted=already_submitted, + ) + submission_result = await self._stage_port.submit(submission) + try: + if submission_result.request_id != request_id or submission_result.stage_id != stage_id: + raise RuntimeError("duplex stage adapter returned a mismatched submission result") + update = session.commit_append(reservation) + session.bind_stage_request(stage_id, request_id, fence=message.fence) + except BaseException: + self._pending_submission_cleanups[session.session_id] = _PendingSubmissionCleanup( + session_id=session.session_id, + request_ids=(request_id,), + ) + try: + await self._complete_pending_submission_cleanup(session.session_id) + except Exception as cleanup_exc: + logger.warning( + "duplex append compensation remains pending for session %s: %s", + session.session_id, + cleanup_exc, + ) + raise + return [ + { + "stage_id": stage_id, + "replica_id": submission_result.replica_id, + "result": { + "supported": True, + "implementation_level": session.capabilities.implementation_level, + "data_plane_append": True, + "request_id": request_id, + "response_stage_id": request_context.final_stage_id, + "seq": update.seq, + "turn_id": update.turn_id, + "response_seq": message.fence.response_seq, + "turn_seq": update.turn_seq, + "mode": mode.value, + "resumable": True, + }, + } + ] + + async def handle_signal(self, message: SignalDuplexTurnMessage) -> None: + try: + cancel_events = {"barge_in", "input.cancel", "response.cancel"} + if message.event not in {*cancel_events, "session.update"}: + raise ValueError(f"unsupported duplex runtime signal: {message.event}") + session = self.sessions.require(message.session_id) + await self._complete_pending_submission_cleanup(message.session_id) + effective_next_fence = message.next_fence + if message.event in cancel_events: + if effective_next_fence is None: + raise ValueError(f"{message.event} requires next_fence") + cleanup_key = self._cleanup_key("cancel", message.fence) + pending = self._pending_control_cleanups.get(cleanup_key) + if pending is None: + stale_request_ids = session.prepare_cancel_fence(message.fence, effective_next_fence) + pending = _PendingControlCleanup( + kind="cancel", + session_id=message.session_id, + fence=message.fence, + submitted_request_ids=tuple(stale_request_ids), + ) + self._pending_control_cleanups[cleanup_key] = pending + await self._complete_control_cleanup(cleanup_key, pending) + else: + session.accept_fence(message.fence) + if message.runtime_config is not None: + self.sampling_params_for_config(message.runtime_config) + session.replace_configs( + session_config=message.session_config, + runtime_config=message.runtime_config, + ) + session.touch(session.fence, DuplexLeaseActivity.SIGNAL) + await self.put_result( + message.control_id, + fence=message.fence, + operation="signal", + session_id=message.session_id, + stage_results=[ + { + "stage_id": -1, + "replica_id": -1, + "result": { + "supported": True, + "data_plane_signal": True, + "event": message.event, + "fence": message.fence, + "next_fence": effective_next_fence, + }, + } + ], + ) + except Exception as exc: + logger.exception("signal_duplex_turn failed: %s", exc) + await self.put_result( + message.control_id, + fence=message.fence, + operation="signal", + session_id=message.session_id, + stage_results=[], + error=exc, + ) + + async def handle_close(self, message: CloseDuplexSessionMessage) -> None: + try: + await self._complete_pending_submission_cleanup(message.session_id) + session = self.sessions.get(message.session_id) + if session is None: + await self.put_result( + message.control_id, + fence=message.fence, + operation="close", + session_id=message.session_id, + stage_results=[], + ) + return + cleanup_key = self._cleanup_key("close", message.fence) + pending = self._pending_control_cleanups.get(cleanup_key) + if pending is None: + submitted = tuple(session.resource_request_ids(submitted=True)) + reserved = tuple(session.resource_request_ids(submitted=False)) + self.sessions.begin_close_session(message.fence, reason=message.reason) + pending = _PendingControlCleanup( + kind="close", + session_id=message.session_id, + fence=message.fence, + submitted_request_ids=submitted, + reserved_request_ids=reserved, + ) + self._pending_control_cleanups[cleanup_key] = pending + await self._complete_control_cleanup(cleanup_key, pending) + await self.put_result( + message.control_id, + fence=message.fence, + operation="close", + session_id=message.session_id, + stage_results=[ + { + "stage_id": -1, + "replica_id": -1, + "result": { + "supported": True, + "data_plane_close": True, + "reason": message.reason, + }, + } + ], + ) + except Exception as exc: + logger.exception("close_duplex_session failed: %s", exc) + await self.put_result( + message.control_id, + fence=message.fence, + operation="close", + session_id=message.session_id, + stage_results=[], + error=exc, + ) + + async def handle_touch(self, message: TouchDuplexSessionMessage) -> None: + try: + session = self.sessions.require(message.session_id) + activity = DuplexLeaseActivity(message.activity) + if activity is DuplexLeaseActivity.DETACH: + session.detach(message.fence) + else: + session.touch(message.fence, activity) + await self.put_result( + message.control_id, + fence=message.fence, + operation="touch", + session_id=message.session_id, + stage_results=[ + { + "stage_id": -1, + "replica_id": -1, + "result": { + "supported": True, + "activity": activity.value, + "lease_generation": session.lease.generation, + }, + } + ], + ) + except Exception as exc: + logger.exception("touch_duplex_session failed: %s", exc) + await self.put_result( + message.control_id, + fence=message.fence, + operation="touch", + session_id=message.session_id, + stage_results=[], + error=exc, + ) + + async def handle_resume(self, message: ResumeDuplexSessionMessage) -> None: + try: + session = self.sessions.require(message.session_id) + generation = session.resume( + message.fence, + expected_lease_generation=message.expected_lease_generation, + ) + await self.put_result( + message.control_id, + fence=message.fence, + operation="resume", + session_id=message.session_id, + stage_results=[ + { + "stage_id": -1, + "replica_id": -1, + "result": { + "supported": True, + "lease_generation": generation, + }, + } + ], + ) + except Exception as exc: + logger.exception("resume_duplex_session failed: %s", exc) + await self.put_result( + message.control_id, + fence=message.fence, + operation="resume", + session_id=message.session_id, + stage_results=[], + error=exc, + ) + + async def reap_expired(self, now: float | None = None) -> int: + completed = 0 + for session_id in list(self._pending_submission_cleanups): + try: + await self._complete_pending_submission_cleanup(session_id) + except Exception as exc: + logger.warning( + "duplex append compensation remains pending for session %s: %s", + session_id, + exc, + ) + for key, pending in list(self._pending_control_cleanups.items()): + try: + await self._complete_control_cleanup(key, pending) + except Exception as exc: + logger.warning( + "duplex %s cleanup remains pending for session %s: %s", + pending.kind, + pending.session_id, + exc, + ) + for key, pending in list(self._pending_request_cleanups.items()): + if key in self._request_cleanups_in_progress: + continue + try: + await self._complete_request_cleanup(key, pending) + except Exception as exc: + logger.warning( + "duplex request cleanup remains pending for session %s: %s", + pending.session_id, + exc, + ) + continue + completed += 1 + for item in self.sessions.collect_expired( + now, + excluded_session_ids=set(self._pending_submission_cleanups), + ): + self._pending_expirations.setdefault( + (item.session_id, item.lease_generation), + item, + ) + for key, item in list(self._pending_expirations.items()): + try: + if item.submitted_request_ids: + await self._stage_port.cleanup(list(item.submitted_request_ids), abort=True) + if item.reserved_request_ids: + await self._stage_port.cleanup(list(item.reserved_request_ids)) + if self._lifecycle_sink is not None: + await self._lifecycle_sink.put( + DuplexSessionLifecycleMessage( + fence=item.fence, + session_id=item.session_id, + event="expired", + reason=item.reason, + lease_generation=item.lease_generation, + submitted_request_ids=list(item.submitted_request_ids), + reserved_request_ids=list(item.reserved_request_ids), + ) + ) + except Exception as exc: + logger.warning( + "duplex expiry cleanup remains pending for session %s: %s", + item.session_id, + exc, + ) + continue + session = self.sessions.get(item.session_id) + if session is not None and session.lease.generation == item.lease_generation: + self.sessions.finalize_close_session(session) + self._pending_expirations.pop(key, None) + completed += 1 + return completed + + async def _complete_pending_submission_cleanup(self, session_id: str) -> None: + pending = self._pending_submission_cleanups.get(session_id) + if pending is None: + return + task = self._submission_cleanup_tasks.get(session_id) + if task is not None and task.done(): + self._submission_cleanup_tasks.pop(session_id, None) + task = None + if task is None: + task = asyncio.create_task( + self._run_submission_cleanup(pending), + name=f"duplex-append-compensation-{session_id}", + ) + self._submission_cleanup_tasks[session_id] = task + + def discard(completed: asyncio.Task[None]) -> None: + if self._submission_cleanup_tasks.get(session_id) is completed: + self._submission_cleanup_tasks.pop(session_id, None) + + task.add_done_callback(discard) + await asyncio.shield(task) + + async def _run_submission_cleanup(self, pending: _PendingSubmissionCleanup) -> None: + await self._stage_port.cleanup(list(pending.request_ids), abort=True) + session = self.sessions.get(pending.session_id) + if session is not None: + session.release_request_ids(list(pending.request_ids)) + if self._pending_submission_cleanups.get(pending.session_id) is pending: + self._pending_submission_cleanups.pop(pending.session_id, None) + + async def _complete_request_cleanup( + self, + key: tuple[str, int, int], + pending: _PendingRequestCleanup, + ) -> None: + task = self._request_cleanup_tasks.get(key) + if task is not None and task.done(): + self._request_cleanup_tasks.pop(key, None) + task = None + if task is None: + task = asyncio.create_task( + self._run_request_cleanup(key, pending), + name=f"duplex-request-cleanup-{pending.session_id}", + ) + self._request_cleanup_tasks[key] = task + + def discard(completed: asyncio.Task[None]) -> None: + if self._request_cleanup_tasks.get(key) is completed: + self._request_cleanup_tasks.pop(key, None) + + task.add_done_callback(discard) + await asyncio.shield(task) + + async def _run_request_cleanup( + self, + key: tuple[str, int, int], + pending: _PendingRequestCleanup, + ) -> None: + await self._stage_port.cleanup(list(pending.request_ids), abort=pending.abort) + session = self.sessions.get(pending.session_id) + if ( + session is not None + and session.fence.incarnation == pending.fence.incarnation + and session.lease.generation == pending.lease_generation + ): + self.sessions.finalize_close_session(session) + if self._pending_request_cleanups.get(key) is pending: + self._pending_request_cleanups.pop(key, None) + self._request_cleanups_in_progress.discard(key) + + @staticmethod + def _cleanup_key(kind: str, fence: DuplexFence) -> tuple[str, str, int, int, int, int]: + return (kind, fence.session_id, fence.incarnation, fence.epoch, fence.turn_id, fence.response_seq) + + async def _complete_control_cleanup( + self, + key: tuple[str, str, int, int, int, int], + pending: _PendingControlCleanup, + ) -> None: + task = self._control_cleanup_tasks.get(key) + if task is not None and task.done(): + self._control_cleanup_tasks.pop(key, None) + task = None + if task is None: + task = asyncio.create_task( + self._run_control_cleanup(key, pending), + name=f"duplex-{pending.kind}-cleanup-{pending.session_id}", + ) + self._control_cleanup_tasks[key] = task + + def discard(completed: asyncio.Task[None]) -> None: + if self._control_cleanup_tasks.get(key) is completed: + self._control_cleanup_tasks.pop(key, None) + + task.add_done_callback(discard) + await asyncio.shield(task) + + async def _run_control_cleanup( + self, + key: tuple[str, str, int, int, int, int], + pending: _PendingControlCleanup, + ) -> None: + if pending.submitted_request_ids: + await self._stage_port.cleanup(list(pending.submitted_request_ids), abort=True) + if pending.reserved_request_ids: + await self._stage_port.cleanup(list(pending.reserved_request_ids)) + session = self.sessions.get(pending.session_id) + if session is not None: + if pending.kind == "cancel": + session.release_fence(pending.fence) + elif pending.kind in {"close", "open_rollback"}: + self.sessions.finalize_close_session(session) + self._pending_control_cleanups.pop(key, None) + + @classmethod + def _iter_result_dicts(cls, result: object): + if isinstance(result, dict): + yield result + elif isinstance(result, list | tuple): + for item in result: + yield from cls._iter_result_dicts(item) + + @classmethod + def _result_counts(cls, stage_results: list[dict[str, object]]) -> tuple[int, int]: + unsupported_count = 0 + error_count = 0 + for item in stage_results: + for result in cls._iter_result_dicts(item.get("result")): + if result.get("supported") is False: + unsupported_count += 1 + if result.get("error"): + error_count += 1 + return unsupported_count, error_count + + async def put_result( + self, + control_id: str, + *, + fence: DuplexFence, + operation: str, + session_id: str, + stage_results: list[dict[str, object]], + error: BaseException | str | None = None, + ) -> None: + control_error = self._control_error(error) if error is not None else None + if error is not None: + stage_results = [ + { + "stage_id": -1, + "replica_id": -1, + "result": {"supported": False, "error": control_error.message}, + } + ] + unsupported_count, error_count = self._result_counts(stage_results) + session = self.sessions.get(session_id) + await self._result_sink.put( + DuplexControlResultMessage( + control_id=control_id, + fence=fence, + operation=operation, + session_id=session_id, + ok=error_count == 0 and unsupported_count == 0, + stage_results=stage_results, + unsupported_count=unsupported_count, + error_count=error_count, + error=control_error, + accepted_fence=session.fence if session is not None else None, + lease_generation=session.lease.generation if session is not None else None, + ) + ) + + @staticmethod + def _control_error(error: BaseException | str) -> DuplexControlError: + message = str(error) + if isinstance(error, DuplexFenceMismatchError): + code = "stale_fence" + retryable = False + elif "unknown duplex input mode" in message: + code = "invalid_capability" + retryable = False + elif "duplex_session_capacity_exhausted" in message: + code = "resource_exhausted" + retryable = True + elif isinstance(error, KeyError): + code = "not_found" + retryable = False + elif isinstance(error, (TypeError, ValueError)): + code = "invalid_argument" + retryable = False + elif isinstance(error, TimeoutError): + code = "timeout" + retryable = True + else: + code = "failed_precondition" + retryable = False + return DuplexControlError(code=code, message=message, retryable=retryable) + + def decide_output( + self, + stage_id: int, + output: object, + context: DuplexOutputContext | None, + ) -> DuplexOutputDecision | None: + if context is None or self._extension is None: + return None + session = self._sessions.get(context.identity.session_id) + if session is None: + return None + try: + session.touch(context.identity.fence, DuplexLeaseActivity.MODEL_OUTPUT) + except RuntimeError: + return None + decision = self._extension.decide_output( + stage_id=stage_id, + final_stage_id=context.final_stage_id, + segment_finished=context.segment_finished, + segment_token_ids=context.segment_token_ids, + segment_output_metadata=dict(context.segment_output_metadata), + output=output, + ) + if decision is not None and not isinstance(decision, DuplexOutputDecision): + raise TypeError("duplex runtime extension decide_output() must return DuplexOutputDecision or None") + return decision + + def session_for_identity(self, identity: DuplexRequestIdentity | None) -> DuplexSessionRuntimeState | None: + if identity is None: + return None + return self._sessions.get(identity.session_id) + + def close_sessions_for_request_ids( + self, + request_ids: list[str], + *, + abort: bool = False, + cleanup_in_progress: bool = False, + ) -> dict[str, list[str]]: + closed = self._sessions.close_sessions_for_request_ids(request_ids) + for session_id, stale_request_ids in closed.items(): + session = self._sessions.get(session_id) + if session is None: + continue + key = (session_id, session.fence.incarnation, session.lease.generation) + existing = self._pending_request_cleanups.get(key) + merged_request_ids = tuple( + dict.fromkeys( + [ + *(existing.request_ids if existing is not None else ()), + *stale_request_ids, + ] + ) + ) + self._pending_request_cleanups[key] = _PendingRequestCleanup( + session_id=session_id, + fence=session.fence, + lease_generation=session.lease.generation, + request_ids=merged_request_ids, + abort=abort or (existing.abort if existing is not None else False), + ) + if cleanup_in_progress: + self._request_cleanups_in_progress.add(key) + return closed + + def defer_request_cleanups(self, session_ids: Iterable[str]) -> None: + session_id_set = set(session_ids) + active_keys = {key for key in self._request_cleanups_in_progress if key[0] in session_id_set} + self._request_cleanups_in_progress.difference_update(active_keys) + + def finalize_closed_sessions(self, session_ids: Iterable[str]) -> None: + session_id_set = set(session_ids) + for key in list(self._pending_request_cleanups): + if key[0] not in session_id_set: + continue + self._pending_request_cleanups.pop(key, None) + self._request_cleanups_in_progress.discard(key) + self._sessions.finalize_closed_sessions(session_ids) + + +__all__ = [ + "DuplexControlPlane", + "DuplexOutputContext", + "DuplexRequestIdentity", + "DuplexResultSink", + "DuplexLifecycleSink", + "DuplexStagePort", + "DuplexStageRequestContext", + "DuplexStageSubmission", + "DuplexStageSubmissionResult", +] diff --git a/vllm_omni/experimental/fullduplex/engine/duplex_runtime.py b/vllm_omni/experimental/fullduplex/engine/duplex_runtime.py new file mode 100644 index 000000000..acd977448 --- /dev/null +++ b/vllm_omni/experimental/fullduplex/engine/duplex_runtime.py @@ -0,0 +1,108 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +"""Experimental duplex runtime loading and compatibility exports.""" + +from __future__ import annotations + +from importlib import import_module + +from vllm_omni.experimental.fullduplex.engine.contracts import ( + DuplexAppendPlan, + DuplexInputMode, + DuplexOutputAction, + DuplexOutputDecision, + DuplexRuntimeCapabilities, + DuplexRuntimeExtension, + SessionMode, + duplex_data_plane_request_info, + duplex_resource_request_belongs_to_session, + duplex_resource_request_id, +) + + +def load_duplex_runtime_extension(path: str | None) -> DuplexRuntimeExtension | None: + if not path: + return None + module_name, separator, attribute_name = path.rpartition(".") + if not separator: + raise ValueError(f"Invalid duplex runtime extension path: {path!r}") + extension_type = getattr(import_module(module_name), attribute_name) + return validate_duplex_runtime_extension(extension_type()) + + +def validate_duplex_runtime_extension( + extension: object, + *, + sampling_defaults: tuple[object, ...] | None = None, +) -> DuplexRuntimeExtension: + required_methods = ( + "configure_sampling_params", + "plan_append", + "decide_output", + ) + missing = [name for name in required_methods if not callable(getattr(extension, name, None))] + if missing: + raise TypeError(f"Duplex runtime extension is missing callable method(s): {', '.join(missing)}") + typed_extension = extension # type: ignore[assignment] + if sampling_defaults is not None: + configured = typed_extension.configure_sampling_params( + runtime_config={}, + defaults=sampling_defaults, + ) + if not isinstance(configured, tuple): + raise TypeError("Duplex runtime extension must return sampling parameters as a tuple") + if len(configured) != len(sampling_defaults): + raise ValueError("Duplex runtime extension must return one sampling parameter per stage") + for stage_id, (value, default) in enumerate(zip(configured, sampling_defaults, strict=True)): + if default is not None and not isinstance(value, type(default)): + raise TypeError( + "Duplex runtime extension sampling parameter type mismatch " + f"for stage {stage_id}: expected {type(default).__name__}, got {type(value).__name__}" + ) + return typed_extension + + +from vllm_omni.experimental.fullduplex.engine.duplex_session import ( # noqa: E402, F401 + DuplexAppendReservation, + DuplexCompletedAppend, + DuplexFenceMismatchError, + DuplexInputAppend, + DuplexRequestResource, + DuplexSessionRuntimeManager, + DuplexSessionRuntimeState, + DuplexStageBinding, +) +from vllm_omni.experimental.fullduplex.engine.lease import ( # noqa: E402, F401 + DuplexLeaseActivity, + DuplexLeaseConfig, + DuplexLeaseState, + DuplexSessionExpiry, +) + +__all__ = [ + "DuplexAppendPlan", + "DuplexAppendReservation", + "DuplexCompletedAppend", + "DuplexFenceMismatchError", + "DuplexInputAppend", + "DuplexInputMode", + "DuplexLeaseActivity", + "DuplexLeaseConfig", + "DuplexLeaseState", + "DuplexOutputAction", + "DuplexOutputDecision", + "DuplexRequestResource", + "DuplexRuntimeCapabilities", + "DuplexRuntimeExtension", + "DuplexSessionExpiry", + "DuplexSessionRuntimeManager", + "DuplexSessionRuntimeState", + "DuplexStageBinding", + "SessionMode", + "duplex_data_plane_request_info", + "duplex_resource_request_belongs_to_session", + "duplex_resource_request_id", + "load_duplex_runtime_extension", + "validate_duplex_runtime_extension", +] diff --git a/vllm_omni/experimental/fullduplex/engine/duplex_session.py b/vllm_omni/experimental/fullduplex/engine/duplex_session.py new file mode 100644 index 000000000..244005f06 --- /dev/null +++ b/vllm_omni/experimental/fullduplex/engine/duplex_session.py @@ -0,0 +1,534 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from __future__ import annotations + +import time +from collections import OrderedDict +from collections.abc import Callable, Iterable +from dataclasses import dataclass, field +from typing import Any + +from vllm_omni.experimental.fullduplex.engine.contracts import ( + DuplexInputMode, + DuplexRuntimeCapabilities, +) +from vllm_omni.experimental.fullduplex.engine.lease import ( + DuplexLeaseActivity, + DuplexLeaseConfig, + DuplexLeaseState, + DuplexSessionExpiry, +) +from vllm_omni.experimental.fullduplex.engine.messages import DuplexFence + + +def _default_capabilities() -> DuplexRuntimeCapabilities: + return DuplexRuntimeCapabilities() + + +class DuplexFenceMismatchError(RuntimeError): + def __init__(self, expected: DuplexFence, actual: DuplexFence) -> None: + super().__init__(f"duplex fence mismatch: expected {expected!r}, got {actual!r}") + self.expected = expected + self.actual = actual + + +@dataclass +class DuplexStageBinding: + request_id: str + fence: DuplexFence + + +@dataclass +class DuplexRequestResource: + stage_id: int + request_id: str + fence: DuplexFence + submitted: bool = False + + +@dataclass +class DuplexInputAppend: + seq: int + turn_seq: int + turn_id: int + + +@dataclass(frozen=True) +class DuplexAppendReservation: + fence: DuplexFence + mode: DuplexInputMode + base_fence: DuplexFence + base_input_seq: int + base_input_turn_seq: int + base_append_turn_key: tuple[int, int, int] | None + update: DuplexInputAppend + + +@dataclass(frozen=True) +class DuplexCompletedAppend: + fence: DuplexFence + mode: DuplexInputMode + final: bool + stage_results: tuple[dict[str, object], ...] + + +@dataclass +class DuplexSessionRuntimeState: + """Engine resource handles associated with a core-owned identity fence.""" + + fence: DuplexFence + lease: DuplexLeaseState + _clock: Callable[[], float] = field(repr=False) + capabilities: DuplexRuntimeCapabilities = field(default_factory=_default_capabilities) + session_config: dict[str, Any] = field(default_factory=dict) + runtime_config: dict[str, Any] = field(default_factory=dict) + config_generation: int = 0 + stage_bindings: dict[int, DuplexStageBinding] = field(default_factory=dict) + request_resources: dict[tuple[int, str], DuplexRequestResource] = field(default_factory=dict) + input_seq: int = 0 + input_turn_seq: int = 0 + _append_turn_key: tuple[int, int, int] | None = None + completed_append_limit: int = 256 + completed_appends: OrderedDict[str, DuplexCompletedAppend] = field(default_factory=OrderedDict) + + def __post_init__(self) -> None: + if self.completed_append_limit <= 0: + raise ValueError("completed_append_limit must be positive") + + @property + def session_id(self) -> str: + return self.fence.session_id + + @property + def epoch(self) -> int: + return self.fence.epoch + + @property + def turn_id(self) -> int: + return self.fence.turn_id + + def _validate_fence(self, fence: DuplexFence) -> None: + if fence.session_id != self.session_id or fence.incarnation != self.fence.incarnation: + raise DuplexFenceMismatchError(self.fence, fence) + current = self.fence + if fence.epoch < current.epoch or ( + fence.epoch == current.epoch + and (fence.turn_id < current.turn_id or fence.response_seq < current.response_seq) + ): + raise DuplexFenceMismatchError(current, fence) + + def accept_fence(self, fence: DuplexFence) -> None: + self._validate_fence(fence) + if fence.epoch != self.fence.epoch: + self.input_seq = 0 + self.input_turn_seq = 0 + self._append_turn_key = None + self.completed_appends.clear() + self.fence = fence + + def touch(self, fence: DuplexFence, activity: DuplexLeaseActivity) -> None: + self._validate_fence(fence) + self.lease.touch(self._clock(), activity) + + def detach(self, fence: DuplexFence) -> None: + self._validate_fence(fence) + self.lease.detach(self._clock()) + + def resume(self, fence: DuplexFence, *, expected_lease_generation: int) -> int: + self._validate_fence(fence) + return self.lease.resume( + self._clock(), + expected_generation=expected_lease_generation, + ) + + def begin_operation(self, fence: DuplexFence, operation_id: str) -> None: + self._validate_fence(fence) + self.lease.begin_operation(self._clock(), operation_id) + + def end_operation(self, fence: DuplexFence, operation_id: str) -> None: + self._validate_fence(fence) + self.lease.end_operation(self._clock(), operation_id) + + def replace_session_config(self, session_config: dict[str, Any]) -> None: + self.session_config = dict(session_config) + self.config_generation += 1 + + def replace_runtime_config(self, runtime_config: dict[str, Any]) -> None: + self.runtime_config = dict(runtime_config) + self.config_generation += 1 + + def replace_configs( + self, + *, + session_config: dict[str, Any] | None = None, + runtime_config: dict[str, Any] | None = None, + ) -> None: + """Atomically publish one validated configuration generation.""" + if session_config is None and runtime_config is None: + return + next_session_config = self.session_config if session_config is None else dict(session_config) + next_runtime_config = self.runtime_config if runtime_config is None else dict(runtime_config) + self.session_config = next_session_config + self.runtime_config = next_runtime_config + self.config_generation += 1 + + def reserve_stage_request(self, stage_id: int, request_id: str, *, fence: DuplexFence) -> None: + self._validate_fence(fence) + resource_key = (stage_id, request_id) + existing = self.request_resources.get(resource_key) + if existing is not None: + if ( + existing.fence.session_id != fence.session_id + or existing.fence.incarnation != fence.incarnation + or existing.fence.epoch != fence.epoch + ): + raise ValueError(f"Duplex request resource already reserved with different identity: {request_id}") + return + self.request_resources[resource_key] = DuplexRequestResource( + stage_id=stage_id, + request_id=request_id, + fence=fence, + ) + + def bind_stage_request(self, stage_id: int, request_id: str, *, fence: DuplexFence) -> None: + self.reserve_stage_request(stage_id, request_id, fence=fence) + self.accept_fence(fence) + resource = self.request_resources[(stage_id, request_id)] + resource.fence = fence + resource.submitted = True + self.stage_bindings[stage_id] = DuplexStageBinding(request_id=request_id, fence=fence) + + def stage_request_ids(self, fence: DuplexFence | None = None) -> list[str]: + return [ + binding.request_id for binding in self.stage_bindings.values() if fence is None or binding.fence == fence + ] + + def resource_request_ids( + self, + fence: DuplexFence | None = None, + *, + submitted: bool | None = None, + ) -> list[str]: + return list( + dict.fromkeys( + resource.request_id + for resource in self.request_resources.values() + if (fence is None or resource.fence == fence) and (submitted is None or resource.submitted is submitted) + ) + ) + + def release_request_ids(self, request_ids: list[str]) -> None: + released = set(request_ids) + if not released: + return + self.stage_bindings = { + stage_id: binding for stage_id, binding in self.stage_bindings.items() if binding.request_id not in released + } + self.request_resources = { + resource_key: resource + for resource_key, resource in self.request_resources.items() + if resource.request_id not in released + } + + def prepare_append(self, *, mode: DuplexInputMode, fence: DuplexFence) -> DuplexAppendReservation: + if mode not in self.capabilities.input_modes: + raise ValueError(f"Duplex input mode {mode.value!r} is not supported by session {self.session_id}") + self._validate_fence(fence) + input_seq = 0 if fence.epoch != self.fence.epoch else self.input_seq + input_turn_seq = 0 if fence.epoch != self.fence.epoch else self.input_turn_seq + append_turn_key = None if fence.epoch != self.fence.epoch else self._append_turn_key + turn_key = (fence.epoch, fence.turn_id, fence.response_seq) + turn_seq = input_turn_seq + 1 if turn_key == append_turn_key else 1 + return DuplexAppendReservation( + fence=fence, + mode=mode, + base_fence=self.fence, + base_input_seq=self.input_seq, + base_input_turn_seq=self.input_turn_seq, + base_append_turn_key=self._append_turn_key, + update=DuplexInputAppend( + seq=input_seq + 1, + turn_seq=turn_seq, + turn_id=fence.turn_id, + ), + ) + + def commit_append(self, reservation: DuplexAppendReservation) -> DuplexInputAppend: + if ( + self.fence != reservation.base_fence + or self.input_seq != reservation.base_input_seq + or self.input_turn_seq != reservation.base_input_turn_seq + or self._append_turn_key != reservation.base_append_turn_key + ): + raise RuntimeError("duplex append reservation is stale") + self.accept_fence(reservation.fence) + self.input_seq = reservation.update.seq + self.input_turn_seq = reservation.update.turn_seq + self._append_turn_key = ( + reservation.fence.epoch, + reservation.fence.turn_id, + reservation.fence.response_seq, + ) + return reservation.update + + def completed_append( + self, + operation_id: str, + *, + fence: DuplexFence, + mode: DuplexInputMode, + final: bool, + ) -> list[dict[str, object]] | None: + self._validate_fence(fence) + completed = self.completed_appends.get(operation_id) + if completed is None: + return None + if completed.fence != fence or completed.mode != mode or completed.final != final: + raise ValueError(f"duplex append operation {operation_id!r} was reused with different metadata") + return [dict(result) for result in completed.stage_results] + + def record_completed_append( + self, + operation_id: str, + *, + fence: DuplexFence, + mode: DuplexInputMode, + final: bool, + stage_results: list[dict[str, object]], + ) -> None: + self.completed_appends[operation_id] = DuplexCompletedAppend( + fence=fence, + mode=mode, + final=final, + stage_results=tuple(dict(result) for result in stage_results), + ) + self.completed_appends.move_to_end(operation_id) + while len(self.completed_appends) > self.completed_append_limit: + self.completed_appends.popitem(last=False) + + def append_input(self, *, mode: DuplexInputMode, fence: DuplexFence) -> DuplexInputAppend: + return self.commit_append(self.prepare_append(mode=mode, fence=fence)) + + def release_fence(self, fence: DuplexFence) -> list[str]: + stale = self.resource_request_ids(fence) + self.stage_bindings = { + stage_id: binding for stage_id, binding in self.stage_bindings.items() if binding.fence != fence + } + self.request_resources = { + resource_key: resource + for resource_key, resource in self.request_resources.items() + if resource.fence != fence + } + return stale + + def cancel_fence(self, cancelled_fence: DuplexFence, next_fence: DuplexFence) -> list[str]: + stale = self.prepare_cancel_fence(cancelled_fence, next_fence) + self.release_fence(cancelled_fence) + return stale + + def prepare_cancel_fence(self, cancelled_fence: DuplexFence, next_fence: DuplexFence) -> list[str]: + """Advance the cancellation fence without dropping cleanup records.""" + if cancelled_fence.session_id != self.session_id or cancelled_fence.incarnation != self.fence.incarnation: + raise DuplexFenceMismatchError(self.fence, cancelled_fence) + if ( + next_fence.session_id != self.session_id + or next_fence.incarnation != self.fence.incarnation + or next_fence.epoch <= cancelled_fence.epoch + ): + raise DuplexFenceMismatchError(cancelled_fence, next_fence) + current_key = (self.fence.epoch, self.fence.turn_id, self.fence.response_seq) + cancelled_key = (cancelled_fence.epoch, cancelled_fence.turn_id, cancelled_fence.response_seq) + next_key = (next_fence.epoch, next_fence.turn_id, next_fence.response_seq) + if cancelled_key > current_key: + raise DuplexFenceMismatchError(self.fence, cancelled_fence) + if next_key > current_key: + self.accept_fence(next_fence) + return self.resource_request_ids(cancelled_fence) + + def begin_close(self, fence: DuplexFence, *, reason: str) -> bool: + """Make close irreversible while retaining resources for cleanup retry.""" + self.accept_fence(fence) + if self.lease.terminal_reason is not None: + return True + return self.lease.mark_terminal(reason) + + def finalize_close(self) -> list[str]: + return self.close() + + def close(self, fence: DuplexFence | None = None) -> list[str]: + if fence is not None: + self.accept_fence(fence) + stale = self.resource_request_ids() + self.stage_bindings.clear() + self.request_resources.clear() + return stale + + def terminate(self, fence: DuplexFence, *, reason: str) -> bool: + if not self.begin_close(fence, reason=reason): + return False + self.finalize_close() + return True + + +class DuplexSessionRuntimeManager: + def __init__( + self, + *, + clock: Callable[[], float] | None = None, + max_sessions: int | None = None, + completed_append_limit: int = 256, + ) -> None: + if max_sessions is not None and max_sessions <= 0: + raise ValueError("max_sessions must be positive or null") + if completed_append_limit <= 0: + raise ValueError("completed_append_limit must be positive") + self._sessions: dict[str, DuplexSessionRuntimeState] = {} + self._clock = clock or time.monotonic + self._max_sessions = max_sessions + self._completed_append_limit = completed_append_limit + + def open_session( + self, + fence: DuplexFence, + *, + capabilities: DuplexRuntimeCapabilities | None = None, + session_config: dict[str, Any] | None = None, + runtime_config: dict[str, Any] | None = None, + lease_config: DuplexLeaseConfig | None = None, + ) -> DuplexSessionRuntimeState: + if not isinstance(fence, DuplexFence): + raise TypeError("open_session requires DuplexFence") + if fence.session_id in self._sessions: + raise ValueError(f"Duplex session already exists: {fence.session_id}") + if self._max_sessions is not None and len(self._sessions) >= self._max_sessions: + raise RuntimeError(f"duplex_session_capacity_exhausted: limit={self._max_sessions}") + session = DuplexSessionRuntimeState( + fence=fence, + lease=DuplexLeaseState( + config=lease_config or DuplexLeaseConfig(), + generation=0, + last_activity=self._clock(), + ), + _clock=self._clock, + capabilities=capabilities or _default_capabilities(), + session_config=dict(session_config or {}), + runtime_config=dict(runtime_config or {}), + completed_append_limit=self._completed_append_limit, + ) + self._sessions[fence.session_id] = session + return session + + def get(self, session_id: str) -> DuplexSessionRuntimeState | None: + return self._sessions.get(session_id) + + def require(self, session_id: str) -> DuplexSessionRuntimeState: + session = self.get(session_id) + if session is None: + raise KeyError(f"Unknown duplex session: {session_id}") + return session + + def close_session( + self, + fence: DuplexFence, + *, + reason: str = "explicit_close", + ) -> DuplexSessionRuntimeState | None: + if not isinstance(fence, DuplexFence): + raise TypeError("close_session requires DuplexFence") + session = self._sessions.get(fence.session_id) + if session is not None: + if session.lease.terminal_reason is not None: + return None + if not session.terminate(fence, reason=reason): + return None + if self._sessions.get(fence.session_id) is session: + self._sessions.pop(fence.session_id) + return session + + def begin_close_session( + self, + fence: DuplexFence, + *, + reason: str = "explicit_close", + ) -> DuplexSessionRuntimeState | None: + if not isinstance(fence, DuplexFence): + raise TypeError("begin_close_session requires DuplexFence") + session = self._sessions.get(fence.session_id) + if session is not None: + session.begin_close(fence, reason=reason) + return session + + def finalize_close_session(self, session: DuplexSessionRuntimeState) -> None: + session.finalize_close() + if self._sessions.get(session.session_id) is session: + self._sessions.pop(session.session_id, None) + + def collect_expired( + self, + now: float | None = None, + *, + excluded_session_ids: set[str] | None = None, + ) -> list[DuplexSessionExpiry]: + effective_now = self._clock() if now is None else now + excluded = excluded_session_ids or set() + expired: list[DuplexSessionExpiry] = [] + for session_id, session in list(self._sessions.items()): + if session_id in excluded: + continue + if session.lease.disconnect_grace_expired(effective_now): + reason = "disconnect_grace_expired" + elif session.lease.idle_expired(effective_now): + reason = "idle_ttl_expired" + else: + continue + submitted = tuple(session.resource_request_ids(submitted=True)) + reserved = tuple(session.resource_request_ids(submitted=False)) + if not session.begin_close(session.fence, reason=reason): + continue + expired.append( + DuplexSessionExpiry( + session_id=session_id, + fence=session.fence, + lease_generation=session.lease.generation, + reason=reason, + submitted_request_ids=submitted, + reserved_request_ids=reserved, + ) + ) + return expired + + def close_sessions_for_request_ids(self, request_ids: list[str]) -> dict[str, list[str]]: + request_id_set = set(request_ids) + closed: dict[str, list[str]] = {} + for session_id, session in list(self._sessions.items()): + stale = session.resource_request_ids() + if request_id_set.isdisjoint(stale): + continue + if not session.begin_close(session.fence, reason="request_cleanup"): + continue + closed[session_id] = stale + return closed + + def finalize_closed_sessions(self, session_ids: Iterable[str]) -> None: + for session_id in session_ids: + session = self._sessions.get(session_id) + if session is None or session.lease.terminal_reason is None: + continue + self.finalize_close_session(session) + + +__all__ = [ + "DuplexAppendReservation", + "DuplexCompletedAppend", + "DuplexFenceMismatchError", + "DuplexInputAppend", + "DuplexLeaseActivity", + "DuplexLeaseConfig", + "DuplexLeaseState", + "DuplexRequestResource", + "DuplexSessionExpiry", + "DuplexSessionRuntimeManager", + "DuplexSessionRuntimeState", + "DuplexStageBinding", +] diff --git a/vllm_omni/experimental/fullduplex/engine/intermediate.py b/vllm_omni/experimental/fullduplex/engine/intermediate.py new file mode 100644 index 000000000..07c6ab668 --- /dev/null +++ b/vllm_omni/experimental/fullduplex/engine/intermediate.py @@ -0,0 +1,99 @@ +from __future__ import annotations + +from typing import TypedDict + + +class DuplexIntermediateBuffer(TypedDict, total=False): + """Structured keys carried through ``model_intermediate_buffer``. + + The buffer remains a dict for scheduler and msgspec compatibility, but + duplex-specific producers and consumers should use the helpers in this + module instead of scattering nested string keys across serving, runner, and + model code. + """ + + request_id: str + global_request_id: list[str] + prompt_token_ids: list[int] + llm_output_token_ids: list[int] + llm_output_text: list[str] + stream_output: bool + native_duplex: bool + ids: dict[str, object] + hidden_states: dict[str, object] + codes: dict[str, object] + meta: dict[str, object] + duplex: dict[str, object] + omni_payload: object + waveform: object + mel_spec: object + + +def build_duplex_intermediate_buffer( + *, + request_id: str, + prompt_token_ids: list[int] | None = None, + output_token_ids: list[int] | None = None, + output_text: str | None = None, + stream_output: bool = False, + native_duplex: bool = False, +) -> DuplexIntermediateBuffer: + buffer: DuplexIntermediateBuffer = { + "global_request_id": [str(request_id)], + "ids": {}, + } + if prompt_token_ids is not None: + prompt_ids = [int(token_id) for token_id in prompt_token_ids] + buffer["prompt_token_ids"] = prompt_ids + buffer["ids"]["prompt"] = prompt_ids + if output_token_ids is not None: + output_ids = [int(token_id) for token_id in output_token_ids] + buffer["llm_output_token_ids"] = output_ids + buffer["ids"]["output"] = output_ids + if output_text is not None: + buffer["llm_output_text"] = [output_text] + if stream_output: + buffer["stream_output"] = True + if native_duplex: + buffer["native_duplex"] = True + return buffer + + +def set_ref_audio(buffer: dict[str, object], waveform: object, sample_rate_hz: int) -> None: + buffer.setdefault("codes", {})["ref"] = waveform + buffer.setdefault("meta", {})["ref_audio_sr"] = int(sample_rate_hz) + + +def set_tts_handoff(buffer: dict[str, object], token_ids: object | None, hidden_states: object | None) -> None: + """Store the generic AR-stage token/hidden-state handoff consumed by TTS.""" + if token_ids is not None: + buffer.setdefault("ids", {})["tts"] = token_ids + if hidden_states is not None: + buffer.setdefault("hidden_states", {})["tts"] = hidden_states + + +def get_tts_handoff(info: dict[str, object]) -> tuple[object | None, object | None]: + """Read the generic AR-stage to TTS handoff, including legacy flat keys.""" + ids_info = info.get("ids") + hidden_info = info.get("hidden_states") + tts_token_ids = ids_info.get("tts") if isinstance(ids_info, dict) else None + tts_hidden_states = hidden_info.get("tts") if isinstance(hidden_info, dict) else None + if tts_token_ids is None: + tts_token_ids = info.get("tts_token_ids") + if tts_hidden_states is None: + tts_hidden_states = info.get("tts_hidden_states") + return tts_token_ids, tts_hidden_states + + +def get_stream_request_key(info: dict[str, object]) -> str: + key = info.get("global_request_id") or info.get("request_id") or info.get("_omni_req_id") + if isinstance(key, (list, tuple)): + key = key[0] if key else None + if isinstance(key, bytes): + key = key.decode("utf-8", errors="replace") + if key is None: + raise ValueError( + "Duplex streaming handoff requires a stable request id; " + "expected global_request_id, request_id, or _omni_req_id." + ) + return str(key) diff --git a/vllm_omni/experimental/fullduplex/engine/lease.py b/vllm_omni/experimental/fullduplex/engine/lease.py new file mode 100644 index 000000000..09430a7bb --- /dev/null +++ b/vllm_omni/experimental/fullduplex/engine/lease.py @@ -0,0 +1,129 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +"""Generic engine lease primitives shared by optional session runtimes.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from enum import Enum + +from vllm_omni.experimental.fullduplex.engine.messages import DuplexFence + + +class DuplexLeaseActivity(str, Enum): + APPEND = "append" + SIGNAL = "signal" + PLAYBACK_ACK = "playback_ack" + HEARTBEAT = "heartbeat" + ATTACH = "attach" + DETACH = "detach" + RESUME = "resume" + MODEL_OUTPUT = "model_output" + + +@dataclass(frozen=True) +class DuplexLeaseConfig: + idle_ttl_s: float | None = 300.0 + disconnect_grace_s: float = 30.0 + + def __post_init__(self) -> None: + if self.idle_ttl_s is not None and self.idle_ttl_s <= 0: + raise ValueError("idle_ttl_s must be positive or null") + if self.disconnect_grace_s <= 0: + raise ValueError("disconnect_grace_s must be positive") + + +@dataclass +class DuplexLeaseState: + config: DuplexLeaseConfig + generation: int + last_activity: float + detached_at: float | None = None + active_operations: set[str] = field(default_factory=set) + terminal_reason: str | None = None + + @property + def expires_at(self) -> float | None: + if self.config.idle_ttl_s is None: + return None + return self.last_activity + self.config.idle_ttl_s + + def _require_open(self) -> None: + if self.terminal_reason is not None: + raise RuntimeError(f"duplex lease is terminal: {self.terminal_reason}") + + def touch(self, now: float, activity: DuplexLeaseActivity) -> None: + self._require_open() + if not isinstance(activity, DuplexLeaseActivity): + raise TypeError("lease activity must be DuplexLeaseActivity") + self.last_activity = now + + def detach(self, now: float) -> None: + self.touch(now, DuplexLeaseActivity.DETACH) + self.detached_at = now + + def resume(self, now: float, *, expected_generation: int) -> int: + self._require_open() + if expected_generation != self.generation: + raise ValueError(f"duplex lease generation mismatch: expected {self.generation}, got {expected_generation}") + self.generation += 1 + self.detached_at = None + self.last_activity = now + return self.generation + + def begin_operation(self, now: float, operation_id: str) -> None: + self._require_open() + if not operation_id: + raise ValueError("duplex lease operation_id must not be empty") + if operation_id in self.active_operations: + raise ValueError(f"duplex lease operation already active: {operation_id}") + self.active_operations.add(operation_id) + self.last_activity = now + + def end_operation(self, now: float, operation_id: str) -> None: + self._require_open() + if operation_id not in self.active_operations: + raise KeyError(f"duplex lease operation is not active: {operation_id}") + self.active_operations.remove(operation_id) + self.last_activity = now + + def disconnect_grace_expired(self, now: float) -> bool: + return ( + self.terminal_reason is None + and self.detached_at is not None + and now >= self.detached_at + self.config.disconnect_grace_s + ) + + def idle_expired(self, now: float) -> bool: + expires_at = self.expires_at + return ( + self.terminal_reason is None and not self.active_operations and expires_at is not None and now >= expires_at + ) + + def mark_terminal(self, reason: str) -> bool: + if self.terminal_reason is not None: + return False + self.terminal_reason = reason + self.generation += 1 + self.detached_at = None + self.active_operations.clear() + return True + + +@dataclass(frozen=True) +class DuplexSessionExpiry: + session_id: str + fence: DuplexFence + lease_generation: int + reason: str + submitted_request_ids: tuple[str, ...] + reserved_request_ids: tuple[str, ...] + + +__all__ = [ + "DuplexLeaseActivity", + "DuplexLeaseConfig", + "DuplexLeaseState", + "DuplexSessionExpiry", +] diff --git a/vllm_omni/experimental/fullduplex/engine/messages.py b/vllm_omni/experimental/fullduplex/engine/messages.py new file mode 100644 index 000000000..94f154a8a --- /dev/null +++ b/vllm_omni/experimental/fullduplex/engine/messages.py @@ -0,0 +1,130 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +"""Queue contracts owned by the experimental duplex control plane.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Literal + +from vllm_omni.engine.messages import EngineQueueMessage + + +@dataclass(frozen=True, slots=True) +class DuplexFence: + """Identity fence carried by experimental duplex control messages.""" + + session_id: str + epoch: int = 0 + turn_id: int = 0 + response_seq: int = 0 + incarnation: int = 0 + + +class OpenDuplexSessionMessage(EngineQueueMessage, kw_only=True): + type: Literal["open_duplex_session"] = "open_duplex_session" + control_id: str + fence: DuplexFence + session_id: str + session_mode: str = "duplex" + capabilities: dict[str, object] + session_config: dict[str, object] | None = None + runtime_config: dict[str, object] | None = None + + +class AppendDuplexInputMessage(EngineQueueMessage, kw_only=True): + type: Literal["append_duplex_input"] = "append_duplex_input" + control_id: str + operation_id: str | None = None + fence: DuplexFence + session_id: str + expected_epoch: int | None = None + mode: str + payload: object + final: bool = False + + +class SignalDuplexTurnMessage(EngineQueueMessage, kw_only=True): + type: Literal["signal_duplex_turn"] = "signal_duplex_turn" + control_id: str + fence: DuplexFence + session_id: str + event: str + next_fence: DuplexFence | None = None + session_config: dict[str, object] | None = None + runtime_config: dict[str, object] | None = None + + +class CloseDuplexSessionMessage(EngineQueueMessage, kw_only=True): + type: Literal["close_duplex_session"] = "close_duplex_session" + control_id: str + fence: DuplexFence + session_id: str + reason: str = "client_close" + + +class TouchDuplexSessionMessage(EngineQueueMessage, kw_only=True): + type: Literal["touch_duplex_session"] = "touch_duplex_session" + control_id: str + fence: DuplexFence + session_id: str + activity: str + + +class ResumeDuplexSessionMessage(EngineQueueMessage, kw_only=True): + type: Literal["resume_duplex_session"] = "resume_duplex_session" + control_id: str + fence: DuplexFence + session_id: str + expected_lease_generation: int + + +class DuplexSessionLifecycleMessage(EngineQueueMessage, kw_only=True): + type: Literal["duplex_session_lifecycle"] = "duplex_session_lifecycle" + fence: DuplexFence + session_id: str + event: str + reason: str + lease_generation: int + submitted_request_ids: list[str] + reserved_request_ids: list[str] + + +class DuplexControlError(EngineQueueMessage, kw_only=True): + code: str + message: str + retryable: bool = False + + +class DuplexControlResultMessage(EngineQueueMessage, kw_only=True): + type: Literal["duplex_control_result"] = "duplex_control_result" + control_id: str + fence: DuplexFence + operation: str + session_id: str + ok: bool + stage_results: list[dict[str, object]] + unsupported_count: int = 0 + error_count: int = 0 + error: DuplexControlError | None = None + accepted_fence: DuplexFence | None = None + lease_generation: int | None = None + + @property + def rpc_correlation_key(self) -> tuple[str, str]: + return ("duplex", self.control_id) + + +__all__ = [ + "AppendDuplexInputMessage", + "CloseDuplexSessionMessage", + "DuplexControlError", + "DuplexControlResultMessage", + "DuplexFence", + "DuplexSessionLifecycleMessage", + "OpenDuplexSessionMessage", + "ResumeDuplexSessionMessage", + "SignalDuplexTurnMessage", + "TouchDuplexSessionMessage", +] diff --git a/vllm_omni/experimental/fullduplex/minicpmo45/__init__.py b/vllm_omni/experimental/fullduplex/minicpmo45/__init__.py new file mode 100644 index 000000000..6df010470 --- /dev/null +++ b/vllm_omni/experimental/fullduplex/minicpmo45/__init__.py @@ -0,0 +1,15 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from .adapter import MiniCPMO45ClientRuntimeConfigError, MiniCPMO45NativeDuplexServingAdapter +from .input import MiniCPMO45PcmAppendBuffer +from .policy import MiniCPMO45DuplexPolicy +from .stage0 import MiniCPMO45Stage0DuplexRuntime + +__all__ = [ + "MiniCPMO45DuplexPolicy", + "MiniCPMO45NativeDuplexServingAdapter", + "MiniCPMO45ClientRuntimeConfigError", + "MiniCPMO45PcmAppendBuffer", + "MiniCPMO45Stage0DuplexRuntime", +] diff --git a/vllm_omni/experimental/fullduplex/minicpmo45/adapter.py b/vllm_omni/experimental/fullduplex/minicpmo45/adapter.py new file mode 100644 index 000000000..10d75320d --- /dev/null +++ b/vllm_omni/experimental/fullduplex/minicpmo45/adapter.py @@ -0,0 +1,304 @@ +from __future__ import annotations + +import base64 +from copy import deepcopy +from typing import Any + +import numpy as np +from vllm.multimodal.media import MediaConnector + +from vllm_omni.experimental.fullduplex.minicpmo45.policy import MiniCPMO45DuplexPolicy +from vllm_omni.experimental.fullduplex.openai.protocol import DuplexSessionConfig +from vllm_omni.experimental.fullduplex.openai.runtime_adapter import ( + ServingRuntimeConfigError, +) + + +class MiniCPMO45ClientRuntimeConfigError(ServingRuntimeConfigError): + pass + + +class MiniCPMO45NativeDuplexServingAdapter: + """Serving-side MiniCPM-o 4.5 native duplex session preparation. + + The generic duplex WebSocket handler should not let client-supplied local + paths reach workers. This adapter follows the existing media connector + boundary: serving resolves client media URIs, then workers receive only + normalized PCM payloads. Server-owned model assets remain local paths here. + """ + + PRIVATE_RUNTIME_CONFIG_KEYS = frozenset( + { + "duplex_stage_sampling_params", + "duplex_stage_max_tokens", + "duplex_stage0_max_tokens", + "duplex_scheduler_token_id", + "duplex_first_append_context_tokens", + "ref_audio_data", + "ref_audio_format", + "ref_audio_sample_rate_hz", + } + ) + + @classmethod + def is_enabled(cls, config: DuplexSessionConfig) -> bool: + return config.extra_body.get("minicpmo45_native_duplex") is True + + @classmethod + def validate_client_config(cls, config: DuplexSessionConfig) -> None: + cls.validate_client_extra_body(config.extra_body) + + @classmethod + def validate_client_extra_body(cls, extra_body: object) -> None: + if not isinstance(extra_body, dict): + return + private_keys = sorted(cls.PRIVATE_RUNTIME_CONFIG_KEYS.intersection(extra_body)) + if private_keys: + raise MiniCPMO45ClientRuntimeConfigError( + "native duplex runtime configuration is server-owned: " + ", ".join(private_keys) + ) + + @classmethod + def runtime_config_for_update( + cls, + config: DuplexSessionConfig, + current: object, + ) -> dict[str, object]: + runtime_config = deepcopy(dict(current)) if isinstance(current, dict) else {} + runtime_config["instructions"] = config.instructions + stage_max_tokens = runtime_config.get("duplex_stage_max_tokens") + stage_max_tokens = deepcopy(stage_max_tokens) if isinstance(stage_max_tokens, dict) else {} + stage_max_tokens["0"] = ( + config.max_tokens if isinstance(config.max_tokens, int) and config.max_tokens > 0 else 20 + ) + stage_max_tokens.setdefault("1", 8192) + runtime_config["duplex_stage_max_tokens"] = stage_max_tokens + + stage_sampling = runtime_config.get("duplex_stage_sampling_params") + stage_sampling = deepcopy(stage_sampling) if isinstance(stage_sampling, dict) else {} + stage0 = stage_sampling.get("0") + stage0 = deepcopy(stage0) if isinstance(stage0, dict) else {} + stage0["temperature"] = config.temperature if config.temperature is not None else 0.7 + stage_sampling["0"] = stage0 + runtime_config["duplex_stage_sampling_params"] = stage_sampling + return runtime_config + + @classmethod + async def prepare_runtime_config(cls, config: DuplexSessionConfig, *, model_config: Any) -> dict[str, object]: + extra_body = dict(config.extra_body) + if any(key in extra_body for key in ("ref_audio_path", "tts_ref_audio_path")): + raise ValueError("ref_audio_path is not accepted by native duplex; use ref_audio URI instead") + cls.validate_client_config(config) + runtime_config: dict[str, object] = {"instructions": config.instructions} + cls._apply_default_scheduler_policy(runtime_config, config=config, model_config=model_config) + + ref_audio = config.ref_audio + if ref_audio is None and isinstance(extra_body.get("ref_audio"), str): + ref_audio = extra_body.pop("ref_audio") + if ref_audio is None and isinstance(extra_body.get("tts_ref_audio"), str): + ref_audio = extra_body.pop("tts_ref_audio") + + if ref_audio is None: + if any(str(modality).lower() == "audio" for modality in config.modalities): + raise MiniCPMO45ClientRuntimeConfigError( + "MiniCPM-o native duplex audio output requires ref_audio", + code="ref_audio_required", + ) + cls._apply_first_append_context_tokens( + runtime_config, + model_config=model_config, + instructions=config.instructions, + ref_sample_count=None, + ) + config.extra_body = extra_body + return runtime_config + else: + wav_np, sr = await cls.resolve_ref_audio(ref_audio, model_config=model_config) + + wav_np = cls.normalize_ref_audio(wav_np, int(sr), target_sr=16000) + # Trim to a whole number of pooled audio embeddings (100 ms frames) so + # the first-append scheduler reserve can count them exactly. + usable = (len(wav_np) // MiniCPMO45DuplexPolicy.SAMPLES_PER_AUDIO_TOKEN) * ( + MiniCPMO45DuplexPolicy.SAMPLES_PER_AUDIO_TOKEN + ) + wav_np = wav_np[:usable] + ref_audio_bytes = np.ascontiguousarray(wav_np, dtype=np.float32).tobytes() + runtime_config["ref_audio_data"] = base64.b64encode(ref_audio_bytes).decode("ascii") + runtime_config["ref_audio_format"] = "pcm_f32le" + runtime_config["ref_audio_sample_rate_hz"] = 16000 + cls._apply_first_append_context_tokens( + runtime_config, + model_config=model_config, + instructions=config.instructions, + ref_sample_count=len(wav_np), + ) + config.extra_body = extra_body + config.ref_audio = None + return runtime_config + + @classmethod + def _apply_default_scheduler_policy( + cls, + runtime_config: dict[str, object], + *, + config: DuplexSessionConfig, + model_config: Any, + ) -> None: + stage0_max_tokens = config.max_tokens if isinstance(config.max_tokens, int) and config.max_tokens > 0 else 20 + runtime_config["duplex_stage_max_tokens"] = {"0": stage0_max_tokens, "1": 8192} + stage0_params: dict[str, object] = { + "temperature": config.temperature if config.temperature is not None else 0.7, + "top_p": 0.8, + "top_k": 20, + "repetition_penalty": 1.05, + } + stop_token_ids = cls._native_stage0_stop_token_ids(model_config) + if stop_token_ids: + stage0_params["stop_token_ids"] = stop_token_ids + runtime_config["duplex_stage_sampling_params"] = {"0": stage0_params} + scheduler_token_id = cls._native_scheduler_token_id(model_config) + if scheduler_token_id is not None: + runtime_config["duplex_scheduler_token_id"] = scheduler_token_id + + @classmethod + def _apply_first_append_context_tokens( + cls, + runtime_config: dict[str, object], + *, + model_config: Any, + instructions: object, + ref_sample_count: int | None, + ) -> None: + """Precompute the exact session-context token count for the engine. + + The first data-plane append carries the system template and optional + reference-audio embeddings ahead of the first unit. The engine + reserves scheduler slots from this count; an inexact count turns into + pad embeddings inside the model KV (surplus) or truncated context + (deficit), so it is computed with the same template and pooling math + the worker uses. + """ + if "duplex_first_append_context_tokens" in runtime_config: + return + tokenizer = cls._load_native_tokenizer(model_config) + if tokenizer is None: + return + prefix, suffix = MiniCPMO45DuplexPolicy.session_context_texts( + instructions, + ref_sample_count is not None, + ) + try: + prefix_ids = tokenizer.encode(prefix, add_special_tokens=False) + suffix_ids = tokenizer.encode(suffix, add_special_tokens=False) + except Exception: + return + ref_tokens = MiniCPMO45DuplexPolicy.audio_token_count(ref_sample_count or 0) + runtime_config["duplex_first_append_context_tokens"] = len(prefix_ids) + ref_tokens + len(suffix_ids) + + @staticmethod + def _native_stage0_stop_token_ids(model_config: Any) -> list[int]: + tokenizer = MiniCPMO45NativeDuplexServingAdapter._load_native_tokenizer(model_config) + if tokenizer is None: + return [] + out: list[int] = [] + stop_token_fields = ( + "chunk_eos_token_id", + "chunk_tts_eos_token_id", + "listen_token_id", + "turn_eos_token_id", + ) + for field in stop_token_fields: + token = MiniCPMO45DuplexPolicy.SPECIAL_TOKEN_FIELDS[field] + token_id = MiniCPMO45NativeDuplexServingAdapter._convert_token_to_id(tokenizer, token) + if token_id is not None and token_id not in out: + out.append(token_id) + return out + + @staticmethod + def _native_scheduler_token_id(model_config: Any) -> int | None: + tokenizer = MiniCPMO45NativeDuplexServingAdapter._load_native_tokenizer(model_config) + if tokenizer is None: + return None + scheduler_tokens = ( + MiniCPMO45DuplexPolicy.SPECIAL_TOKEN_FIELDS["unit_token_id"], + MiniCPMO45DuplexPolicy.OPTIONAL_TOKEN_FIELDS["audio_placeholder_token_id"], + ) + for token in scheduler_tokens: + token_id = MiniCPMO45NativeDuplexServingAdapter._convert_token_to_id(tokenizer, token) + if token_id is not None: + return token_id + eos_id = getattr(tokenizer, "eos_token_id", None) + try: + return int(eos_id) + except (TypeError, ValueError): + return None + + @staticmethod + def _load_native_tokenizer(model_config: Any) -> Any | None: + model_path = getattr(model_config, "model", None) + if not isinstance(model_path, str) or not model_path: + return None + try: + from transformers import AutoTokenizer + + return AutoTokenizer.from_pretrained( + model_path, + trust_remote_code=True, + local_files_only=True, + ) + except Exception: + return None + + @staticmethod + def _convert_token_to_id(tokenizer: Any, token: str) -> int | None: + convert = getattr(tokenizer, "convert_tokens_to_ids", None) + value = None + if callable(convert): + value = convert(token) + if isinstance(value, list): + value = value[0] if len(value) == 1 else None + try: + token_id = int(value) + except (TypeError, ValueError): + token_id = -1 + unk_token_id = getattr(tokenizer, "unk_token_id", None) + if token_id >= 0 and token_id != unk_token_id: + return token_id + encode = getattr(tokenizer, "encode", None) + if callable(encode): + try: + ids = list(encode(token, add_special_tokens=False)) + except TypeError: + ids = list(encode(token)) + if len(ids) == 1: + try: + token_id = int(ids[0]) + except (TypeError, ValueError): + token_id = -1 + if token_id >= 0 and token_id != unk_token_id: + return token_id + return None + + @staticmethod + async def resolve_ref_audio(ref_audio: str, *, model_config: Any) -> tuple[np.ndarray, int]: + connector = MediaConnector( + allowed_local_media_path=getattr(model_config, "allowed_local_media_path", None), + allowed_media_domains=getattr(model_config, "allowed_media_domains", None), + ) + wav_np, sr = await connector.fetch_audio_async(ref_audio) + return np.asarray(wav_np, dtype=np.float32), int(sr) + + @staticmethod + def normalize_ref_audio(wav_np: np.ndarray, sample_rate: int, *, target_sr: int) -> np.ndarray: + wav_np = np.asarray(wav_np, dtype=np.float32) + if wav_np.ndim > 1: + wav_np = wav_np.mean(axis=-1) + wav_np = wav_np.reshape(-1) + if sample_rate <= 0 or sample_rate == target_sr or wav_np.size == 0: + return wav_np.astype(np.float32, copy=False) + import torch + import torchaudio + + audio = torch.from_numpy(wav_np).to(dtype=torch.float32).unsqueeze(0) + resampled = torchaudio.functional.resample(audio, int(sample_rate), int(target_sr)) + return resampled.squeeze(0).cpu().numpy().astype(np.float32, copy=False) diff --git a/vllm_omni/experimental/fullduplex/minicpmo45/compat.py b/vllm_omni/experimental/fullduplex/minicpmo45/compat.py new file mode 100644 index 000000000..9b44bed4c --- /dev/null +++ b/vllm_omni/experimental/fullduplex/minicpmo45/compat.py @@ -0,0 +1,43 @@ +from __future__ import annotations + +from typing import Any + + +def patch_minicpmo_remote_config(config: Any) -> None: + """Fill config fields expected by the vLLM MiniCPM implementation.""" + seen: set[int] = set() + + def patch_cfg(cfg: Any) -> None: + if cfg is None or id(cfg) in seen: + return + seen.add(id(cfg)) + if getattr(cfg, "base_model_tp_plan", None) is None: + setattr(cfg, "base_model_tp_plan", {}) + for value in getattr(cfg, "__dict__", {}).values(): + if value is cfg: + continue + if hasattr(value, "__dict__") and ( + value.__class__.__name__.endswith("Config") + or hasattr(value, "model_type") + or hasattr(value, "base_model_tp_plan") + ): + patch_cfg(value) + + patch_cfg(config) + tts_config = getattr(config, "tts_config", None) + if tts_config is None: + return + defaults = { + "top_p": 0.8, + "top_k": 100, + "temperature": 0.8, + "repetition_penalty": 1.05, + } + for name, value in defaults.items(): + if isinstance(tts_config, dict): + tts_config.setdefault(name, value) + elif not hasattr(tts_config, name): + setattr(tts_config, name, value) + + +__all__ = ["patch_minicpmo_remote_config"] diff --git a/vllm_omni/experimental/fullduplex/minicpmo45/data_plane.py b/vllm_omni/experimental/fullduplex/minicpmo45/data_plane.py new file mode 100644 index 000000000..d3b082ffc --- /dev/null +++ b/vllm_omni/experimental/fullduplex/minicpmo45/data_plane.py @@ -0,0 +1,838 @@ +from __future__ import annotations + +from collections.abc import Callable, Iterator, Mapping +from dataclasses import dataclass, field + +import numpy as np +from vllm.logger import init_logger + +from vllm_omni.experimental.fullduplex.engine.contracts import ( + duplex_resource_request_belongs_to_session, +) +from vllm_omni.experimental.fullduplex.output import get_duplex_output_decision + +logger = init_logger(__name__) + +EncodeAudio = Callable[[object, int, str, float | None], str | None] + + +@dataclass(frozen=True, slots=True) +class MiniCPMO45DataPlaneContext: + """Serving state needed to project one MiniCPM data-plane output.""" + + epoch: int = 0 + turn_id: int = 0 + active_response_turn_id: int | None = None + active_response_id: str | None = None + auto_responds: bool = False + response_format: str = "wav" + speed: float | None = None + modalities: tuple[str, ...] = () + + +@dataclass(slots=True) +class _TurnState: + sent_segment_text: str = "" + has_text: bool = False + tts_eos_done: bool = False + turn_eos_done: bool = False + + +@dataclass(slots=True) +class _RequestState: + audio_offset: int = 0 + uses_segment_text_metadata: bool = False + pending_audio_without_text: list[dict[str, object]] = field(default_factory=list) + terminal: bool = False + turns: dict[int | None, _TurnState] = field(default_factory=dict) + + def turn(self, turn_id: int | None) -> _TurnState: + return self.turns.setdefault(turn_id, _TurnState()) + + +class MiniCPMO45DataPlaneSession: + """MiniCPM output projector and request/turn cursor owner. + + The scheduler request owns cumulative Stage1 audio, while model turns own + transcript and terminal cursors. Keeping both levels in one object avoids + leaking MiniCPM-specific state into the generic Realtime handler. + """ + + def __init__(self, encode_audio: EncodeAudio) -> None: + self._encode_audio = encode_audio + self._requests: dict[str, _RequestState] = {} + + def begin_request(self, request_id: str) -> None: + state = self._requests.setdefault(request_id, _RequestState()) + state.terminal = False + # Preserve turn-scoped cursors for resumable requests. The legacy path + # only reset request-scoped terminal markers on append. + request_turn = state.turns.get(None) + if request_turn is not None: + request_turn.tts_eos_done = False + request_turn.turn_eos_done = False + + def is_terminal(self, request_id: str | None) -> bool: + if request_id is None: + return False + state = self._requests.get(request_id) + return state is not None and state.terminal + + def mark_terminal(self, request_id: str) -> None: + self._requests.setdefault(request_id, _RequestState()).terminal = True + + def close_stream(self, request_id: str) -> None: + """Release non-resumable request cursors after its drain finishes.""" + state = self._requests.get(request_id) + if state is None: + return + state.audio_offset = 0 + state.turns.pop(None, None) + + def close_request(self, request_id: str) -> None: + self._requests.pop(request_id, None) + + def close_session(self, session_id: str, *, active_request_id: str | None = None) -> None: + if active_request_id is not None: + self.close_request(active_request_id) + for request_id in list(self._requests): + if self.request_belongs_to_session(request_id, session_id): + self.close_request(request_id) + + def has_request(self, request_id: str) -> bool: + return request_id in self._requests + + def has_pending_audio(self, request_id: str) -> bool: + state = self._requests.get(request_id) + return bool(state and state.pending_audio_without_text) + + def audio_offset(self, request_id: str | None) -> int | None: + if request_id is None: + return None + state = self._requests.get(request_id) + return state.audio_offset if state is not None and state.audio_offset > 0 else None + + def project( + self, + result: object, + *, + context: MiniCPMO45DataPlaneContext | None = None, + ) -> Iterator[dict[str, object]]: + if not isinstance(result, dict): + return + outputs = result.get("data_plane_outputs") + if not isinstance(outputs, list): + return + for output in outputs: + yield from self.project_output(output, context=context) + + def project_output( + self, + output: object, + *, + context: MiniCPMO45DataPlaneContext | None = None, + ) -> Iterator[dict[str, object]]: + context = context or MiniCPMO45DataPlaneContext() + stage_metrics = _output_stage_metrics(output) + + def runtime_result(**values: object) -> dict[str, object]: + result = _runtime_result(**values) + if stage_metrics is not None: + result["stage_metrics"] = stage_metrics + return result + + request_id = getattr(output, "request_id", None) + if not isinstance(request_id, str) or not request_id: + request_id = None + request_state = self._requests.setdefault(request_id, _RequestState()) if request_id is not None else None + + outputs = getattr(output, "outputs", None) + completion = outputs[0] if isinstance(outputs, list) and outputs else None + text = getattr(completion, "text", "") if completion is not None else "" + direct_decision = get_duplex_output_decision(output) + direct_metadata = getattr(direct_decision, "metadata", None) + if isinstance(direct_metadata, Mapping): + mm_output = direct_metadata + else: + mm_output = getattr(output, "multimodal_output", None) + if not isinstance(mm_output, Mapping): + mm_output = getattr(completion, "multimodal_output", {}) if completion is not None else {} + if not mm_output: + inner_output = getattr(output, "request_output", None) + if inner_output is not None and inner_output is not output: + inner_mm_output = getattr(inner_output, "multimodal_output", None) + if isinstance(inner_mm_output, Mapping) and inner_mm_output: + mm_output = inner_mm_output + else: + inner_outputs = getattr(inner_output, "outputs", None) + inner_completion = inner_outputs[0] if isinstance(inner_outputs, list) and inner_outputs else None + inner_mm_output = ( + getattr(inner_completion, "multimodal_output", None) if inner_completion is not None else None + ) + if completion is None and inner_completion is not None: + completion = inner_completion + text = getattr(inner_completion, "text", "") or text + if isinstance(inner_mm_output, Mapping): + mm_output = inner_mm_output + mm_output = dict(mm_output) if isinstance(mm_output, Mapping) else {} + + output_turn_id = output_turn_id_from_metadata(mm_output) + output_epoch = output_epoch_from_metadata(mm_output) + expected_turn_id = context.active_response_turn_id + stale_turn = expected_turn_id is not None and output_turn_id is not None and output_turn_id < expected_turn_id + if expected_turn_id is None and output_turn_id is not None: + stale_turn = output_turn_id < context.turn_id + stale_epoch = output_epoch is not None and output_epoch != context.epoch + if context.auto_responds and (stale_turn or stale_epoch): + return + + mm_text = _llm_output_text(mm_output) + if mm_text: + text = mm_text + if request_state is not None: + request_state.uses_segment_text_metadata = True + elif request_state is not None and request_state.uses_segment_text_metadata: + text = "" + + finished = bool(getattr(output, "finished", False)) + tts_is_last_chunk = _bool_metadata(mm_output, ("tts_is_last_chunk",), default=False) + token_ids = _completion_token_ids(completion) + native_decision = _native_decision(completion, mm_output, token_ids=token_ids, finished=finished) + if native_decision == "listen": + listen_result = runtime_result( + stage_role="llm", + is_listen=True, + model_listen=True, + listen_source="model_listen", + data_plane_request_id=request_id, + end_of_turn=False, + ) + if output_turn_id is not None: + listen_result["model_turn_id"] = output_turn_id + yield listen_result + return + + raw_audio = next((mm_output[key] for key in ("audio", "model_outputs", "latent") if key in mm_output), None) + raw_audio_samples = _audio_num_samples(raw_audio) + offset_before = self.audio_offset(request_id) + audio_chunks = list( + self._encode_audio_chunks_with_duration( + mm_output, + request_id=request_id, + response_format=context.response_format, + speed=context.speed, + ) + ) + stage_turn_end = _bool_metadata(mm_output, ("turn_end", "end_of_turn"), default=False) + terminal_turn_state = request_state.turn(output_turn_id) if request_state is not None else None + stage_tts_eos = ( + context.auto_responds + and 151645 in token_ids + and not audio_chunks + and raw_audio_samples is not None + and (raw_audio_samples == 0 or raw_audio_samples == offset_before) + and (terminal_turn_state is None or not terminal_turn_state.tts_eos_done) + ) + tts_segment_end = bool(tts_is_last_chunk or stage_tts_eos) and ( + terminal_turn_state is None or not terminal_turn_state.tts_eos_done + ) + if tts_segment_end and terminal_turn_state is not None: + terminal_turn_state.tts_eos_done = True + stage_turn_end_new = bool(stage_turn_end) and ( + terminal_turn_state is None or not terminal_turn_state.turn_eos_done + ) + if stage_turn_end_new and terminal_turn_state is not None: + terminal_turn_state.turn_eos_done = True + unit_end_of_turn = stage_turn_end_new or (finished and not context.auto_responds) + + text_turn_id = output_turn_id if output_turn_id is not None else context.turn_id + text_turn_state = request_state.turn(text_turn_id) if request_state is not None else None + if audio_chunks: + delta_text = self.segment_text_delta(request_id, text, turn_id=text_turn_id) + last_idx = len(audio_chunks) - 1 + sample_rate_hz = _sample_rate_hz(mm_output) + audio_text_marks = _audio_text_marks(mm_output) + fallback_marks = _fallback_audio_text_marks(audio_chunks, delta_text) + audio_results: list[dict[str, object]] = [] + for idx, (audio, duration_ms) in enumerate(audio_chunks): + native_result = runtime_result( + stage_role="tts", + is_listen=False, + data_plane_request_id=request_id, + text=delta_text if idx == 0 else "", + audio_data=audio, + audio_format=context.response_format, + audio_duration_ms=duration_ms, + audio_text_mark=idx == last_idx, + sample_rate_hz=sample_rate_hz, + end_of_turn=unit_end_of_turn and idx == last_idx, + abort_data_plane_request=tts_segment_end and idx == last_idx, + ) + if output_turn_id is not None: + native_result["model_turn_id"] = output_turn_id + if audio_text_marks and idx == last_idx: + native_result["audio_text_marks"] = audio_text_marks + native_result["audio_text_marks_are_cumulative"] = True + elif idx < len(fallback_marks) and fallback_marks[idx]: + native_result["audio_text_marks"] = fallback_marks[idx] + native_result["audio_text_marks_are_cumulative"] = True + audio_results.append(native_result) + + if context.auto_responds: + if delta_text and text_turn_state is not None: + text_turn_state.has_text = True + future_model_turn = ( + context.active_response_turn_id is not None + and output_turn_id is not None + and output_turn_id > context.active_response_turn_id + ) + response_turn_bound = context.active_response_id is not None and ( + output_turn_id is None + or context.active_response_turn_id is None + or context.active_response_turn_id == output_turn_id + ) + turn_has_text = text_turn_state is not None and text_turn_state.has_text + if not future_model_turn and not response_turn_bound and not turn_has_text: + if request_state is not None: + if tts_segment_end: + request_state.pending_audio_without_text.clear() + else: + request_state.pending_audio_without_text.extend(audio_results) + if tts_segment_end: + terminal_result = dict(audio_results[-1]) + terminal_result.update( + audio_data="", + audio_duration_ms=0, + audio_text_mark=False, + end_of_turn=True, + ) + yield terminal_result + return + if request_state is not None and request_state.pending_audio_without_text: + pending = request_state.pending_audio_without_text + request_state.pending_audio_without_text = [] + yield from pending + yield from audio_results + return + + if context.auto_responds and request_state is not None and isinstance(text, str) and text: + pending_audio = request_state.pending_audio_without_text + request_state.pending_audio_without_text = [] + if pending_audio: + delta_text = self.segment_text_delta(request_id, text, turn_id=text_turn_id) + if delta_text: + pending_audio[0]["text"] = delta_text + if text_turn_state is not None: + text_turn_state.has_text = True + total_duration_ms = sum( + max(0, int(result.get("audio_duration_ms", 0) or 0)) for result in pending_audio + ) + if total_duration_ms > 0 and not pending_audio[-1].get("audio_text_marks"): + pending_audio[-1]["audio_text_marks"] = [ + {"text_chars": len(delta_text), "audio_end_ms": total_duration_ms} + ] + pending_audio[-1]["audio_text_marks_are_cumulative"] = True + if unit_end_of_turn: + pending_audio[-1]["end_of_turn"] = True + if tts_segment_end: + pending_audio[-1]["abort_data_plane_request"] = True + yield from pending_audio + return + request_state.pending_audio_without_text = pending_audio + + if tts_segment_end: + if unit_end_of_turn and request_state is not None and request_state.pending_audio_without_text: + request_state.pending_audio_without_text[-1]["end_of_turn"] = True + request_state.pending_audio_without_text[-1]["abort_data_plane_request"] = True + terminal_result = runtime_result( + stage_role="tts", + is_listen=False, + data_plane_request_id=request_id, + text="", + audio_data="", + audio_format=context.response_format, + audio_text_mark=False, + end_of_turn=unit_end_of_turn, + abort_data_plane_request=True, + ) + if output_turn_id is not None: + terminal_result["model_turn_id"] = output_turn_id + yield terminal_result + return + + if context.active_response_id is not None and unit_end_of_turn: + terminal_result = runtime_result( + stage_role="tts", + is_listen=False, + data_plane_request_id=request_id, + text="", + audio_data="", + audio_format=context.response_format, + audio_text_mark=False, + end_of_turn=True, + ) + if output_turn_id is not None: + terminal_result["model_turn_id"] = output_turn_id + yield terminal_result + return + + if request_id is not None and context.auto_responds and unit_end_of_turn and context.active_response_id is None: + # The legacy projector only removed the request-scoped pending + # buffer here when no explicit model-turn key was present. + if request_state is not None and output_turn_id is None: + request_state.pending_audio_without_text.clear() + terminal_result = runtime_result( + stage_role="tts", + is_listen=False, + data_plane_request_id=request_id, + text="", + audio_data="", + audio_format=context.response_format, + audio_text_mark=False, + end_of_turn=True, + abort_data_plane_request=True, + ) + if output_turn_id is not None: + terminal_result["model_turn_id"] = output_turn_id + yield terminal_result + return + + if ( + finished + and context.auto_responds + and context.active_response_id is not None + and request_id is not None + and not unit_end_of_turn + ): + listen_result = runtime_result( + stage_role="llm", + is_listen=True, + model_listen=False, + listen_source="auto_response_segment_complete", + reason="auto_response_segment_complete", + data_plane_request_id=request_id, + end_of_turn=False, + ) + if output_turn_id is not None: + listen_result["model_turn_id"] = output_turn_id + yield listen_result + return + + if not text: + if context.auto_responds: + return + if finished: + listen_result = runtime_result( + stage_role="llm", + is_listen=True, + model_listen=False, + listen_source="data_plane_finished_without_output", + reason="data_plane_finished_without_output", + data_plane_request_id=request_id, + end_of_turn=False, + ) + if output_turn_id is not None: + listen_result["model_turn_id"] = output_turn_id + yield listen_result + return + if context.auto_responds: + return + if "audio" in context.modalities: + yield runtime_result( + stage_role="tts", + error_code="runtime_data_plane_text_without_audio", + error="MiniCPM-o native duplex data-plane produced text without audio.", + data_plane_request_id=request_id, + ) + return + yield runtime_result( + stage_role="llm", + is_listen=False, + data_plane_request_id=request_id, + text=text if isinstance(text, str) else "", + audio_data="", + end_of_turn=unit_end_of_turn, + ) + + def segment_text_delta(self, request_id: str | None, text: object, *, turn_id: int | None = None) -> str: + if not isinstance(text, str) or not text: + return "" + if request_id is None: + return text + turn_state = self._requests.setdefault(request_id, _RequestState()).turn(turn_id) + sent_text = turn_state.sent_segment_text + if not sent_text: + delta_text = text + elif text == sent_text: + delta_text = "" + elif text.startswith(sent_text): + delta_text = text[len(sent_text) :] + else: + # The producer sends text for the current thinker segment, not a + # cumulative turn snapshot. Distinct adjacent segments need not + # have a prefix relationship and must both remain visible. + delta_text = text + turn_state.sent_segment_text = text + return delta_text + + def slice_cumulative_audio(self, request_id: str | None, audio_data: object) -> object: + if request_id is None: + return audio_data + num_samples = _audio_num_samples(audio_data) + if num_samples is None or num_samples <= 0: + return audio_data + state = self._requests.setdefault(request_id, _RequestState()) + prev_samples = state.audio_offset + if prev_samples <= 0: + state.audio_offset = num_samples + return audio_data + if num_samples == prev_samples: + return None + if num_samples < prev_samples: + state.audio_offset = num_samples + return audio_data + state.audio_offset = num_samples + try: + import torch + + if isinstance(audio_data, torch.Tensor): + return audio_data.reshape(-1)[prev_samples:].contiguous() + return np.asarray(audio_data, dtype=np.float32).reshape(-1)[prev_samples:] + except Exception: + logger.exception("Failed to slice cumulative duplex audio output") + return audio_data + + def _encode_audio_chunks_with_duration( + self, + mm_output: dict[str, object], + *, + request_id: str | None, + response_format: str, + speed: float | None, + ) -> Iterator[tuple[str, int]]: + sample_rate_hz = _sample_rate_hz(mm_output) + audio_data = next((mm_output[key] for key in ("audio", "model_outputs", "latent") if key in mm_output), None) + if isinstance(audio_data, list): + for value in audio_data: + encoded = self._encode_audio(value, sample_rate_hz, response_format, speed) + if encoded: + duration_ms = int((_audio_num_samples(value) or 0) * 1000 / max(1, sample_rate_hz)) + yield encoded, duration_ms + return + sliced = self.slice_cumulative_audio(request_id, audio_data) + encoded = self._encode_audio(sliced, sample_rate_hz, response_format, speed) + if encoded: + duration_ms = int((_audio_num_samples(sliced) or 0) * 1000 / max(1, sample_rate_hz)) + yield encoded, duration_ms + + @staticmethod + def request_belongs_to_session(request_id: str, session_id: str) -> bool: + return ( + duplex_resource_request_belongs_to_session(request_id, session_id) + or request_id.startswith(f"duplex-{session_id}-") + or request_id.startswith(f"chatcmpl-duplex-{session_id}-") + ) + + +def coerce_int(value: object) -> int | None: + if hasattr(value, "detach"): + try: + value = value.detach().cpu().reshape(-1) + if value.numel() == 0: + return None + value = value[0].item() + except Exception: + return None + elif hasattr(value, "reshape") and hasattr(value, "size"): + try: + value = value.reshape(-1) + if int(value.size) == 0: + return None + item = value[0] + value = item.item() if hasattr(item, "item") else item + except Exception: + return None + try: + return int(value) + except (TypeError, ValueError): + return None + + +def payload_turn_id(payload: object) -> int | None: + if not isinstance(payload, Mapping): + return None + return coerce_int(payload.get("duplex_turn_id")) + + +def output_turn_id_from_metadata(mm_output: dict[str, object]) -> int | None: + return _first_metadata_int(mm_output, "duplex_turn_id", "turn_id") + + +def output_epoch_from_metadata(mm_output: dict[str, object]) -> int | None: + return _first_metadata_int(mm_output, "duplex_epoch", "epoch") + + +def _first_metadata_int(mm_output: dict[str, object], *names: str) -> int | None: + candidates: list[object] = [] + meta = mm_output.get("meta") + for name in names: + candidates.extend((mm_output.get(name), mm_output.get(f"meta.{name}"))) + if isinstance(meta, dict): + candidates.append(meta.get(name)) + for value in candidates: + result = coerce_int(value) + if result is not None: + return result + return None + + +def _runtime_result(**values: object) -> dict[str, object]: + return { + "supported": True, + **values, + "uses_model_runner_scheduler": True, + "runner_kv_backed": True, + "runtime_impl": "scheduler_data_plane", + "owned_runtime": False, + } + + +def _output_stage_metrics(output: object) -> dict[str, dict[str, object]] | None: + metrics = getattr(output, "metrics", None) + if not isinstance(metrics, Mapping): + return None + stage_metrics = metrics.get("stage_metrics") + if not isinstance(stage_metrics, Mapping): + return None + snapshot = { + str(stage_id): dict(values) for stage_id, values in stage_metrics.items() if isinstance(values, Mapping) + } + return snapshot or None + + +def _native_decision( + completion: object, + mm_output: dict[str, object], + *, + token_ids: list[int], + finished: bool, +) -> str | None: + if not finished: + return None + if mm_output.get("duplex_native_decision") == "listen" or mm_output.get("model_listen") is True: + return "listen" + listen_id = _special_token_ids(mm_output).get("listen_token_id") + if listen_id is None: + return None + stop_reason = getattr(completion, "stop_reason", None) if completion is not None else None + if coerce_int(stop_reason) == listen_id: + return "listen" + return "listen" if token_ids and token_ids[-1] == listen_id else None + + +def _special_token_ids(mm_output: dict[str, object]) -> dict[str, int]: + sources: list[object] = [] + raw_special = mm_output.get("special_token_ids") + if isinstance(raw_special, dict): + sources.append(raw_special) + raw_meta = mm_output.get("meta") + if isinstance(raw_meta, dict): + sources.append(raw_meta) + sources.append( + { + key.removeprefix("meta."): value + for key, value in mm_output.items() + if isinstance(key, str) and key.startswith("meta.") + } + ) + out: dict[str, int] = {} + for source in sources: + if not isinstance(source, dict): + continue + for key, value in source.items(): + if not isinstance(key, str): + continue + token_id = coerce_int(value) + if token_id is not None and token_id >= 0: + out[key] = token_id + return out + + +def _completion_token_ids(completion: object) -> list[int]: + if completion is None: + return [] + for candidate in ( + getattr(completion, "token_ids", None), + getattr(completion, "cumulative_token_ids", None), + ): + token_ids = _coerce_int_list(candidate) + if token_ids: + return token_ids + return [] + + +def _coerce_int_list(value: object) -> list[int]: + if value is None: + return [] + if hasattr(value, "detach"): + try: + value = value.detach().cpu().reshape(-1).tolist() + except Exception: + return [] + if not isinstance(value, (list, tuple)): + return [] + return [item for raw in value if (item := coerce_int(raw)) is not None] + + +def _audio_num_samples(audio_data: object) -> int | None: + try: + import torch + + if isinstance(audio_data, torch.Tensor): + return int(audio_data.numel()) + return int(np.asarray(audio_data, dtype=np.float32).size) + except Exception: + return None + + +def _bool_metadata( + mm_output: dict[str, object], + names: tuple[str, ...], + *, + default: bool, +) -> bool: + def coerce(value: object) -> bool | None: + if value is None: + return None + if isinstance(value, bool): + return value + try: + import torch + + if isinstance(value, torch.Tensor): + return bool(value.reshape(-1)[-1].item()) if value.numel() else None + except Exception: + pass + if isinstance(value, np.ndarray): + return bool(value.reshape(-1)[-1].item()) if value.size else None + if isinstance(value, np.generic): + return bool(value.item()) + if isinstance(value, (list, tuple)): + return coerce(value[-1]) if value else None + if isinstance(value, (int, float)): + return bool(value) + return None + + meta = mm_output.get("meta") + for name in names: + for key in (name, f"meta.{name}"): + result = coerce(mm_output.get(key)) + if result is not None: + return result + if isinstance(meta, dict): + result = coerce(meta.get(name)) + if result is not None: + return result + return default + + +def _sample_rate_hz(mm_output: dict[str, object]) -> int: + sr_raw = mm_output.get("sr") + if sr_raw is None: + sr_raw = mm_output.get("sample_rate_hz", mm_output.get("sample_rate")) + meta = mm_output.get("meta") + if sr_raw is None and isinstance(meta, dict): + sr_raw = meta.get("sr") or meta.get("sample_rate_hz") or meta.get("sample_rate") + if sr_raw is None: + sr_raw = mm_output.get("meta.sr") or mm_output.get("meta.sample_rate_hz") or mm_output.get("meta.sample_rate") + if hasattr(sr_raw, "item"): + try: + return int(sr_raw.item()) + except Exception: + return 24000 + return int(sr_raw) if isinstance(sr_raw, (int, float)) else 24000 + + +def _audio_text_marks(mm_output: dict[str, object]) -> list[dict[str, object]]: + names = ("audio_text_marks", "text_audio_marks", "audio_text_alignment", "alignment_marks") + candidates = [mm_output.get(name) for name in names] + meta = mm_output.get("meta") + if isinstance(meta, dict): + candidates.extend(meta.get(name) for name in names) + candidates.extend( + value + for key, value in mm_output.items() + if isinstance(key, str) and key.startswith("meta.") and key.rsplit(".", 1)[-1] in names + ) + for raw in candidates: + if not isinstance(raw, list): + continue + marks: list[dict[str, object]] = [] + for item in raw: + if not isinstance(item, dict): + continue + text_chars = item.get("text_chars") + audio_end_ms = item.get("audio_end_ms", item.get("audio_ms")) + if isinstance(text_chars, (int, float)) and isinstance(audio_end_ms, (int, float)): + marks.append({"text_chars": max(0, int(text_chars)), "audio_end_ms": max(0, int(audio_end_ms))}) + if marks: + return marks + return [] + + +def _llm_output_text(mm_output: dict[str, object]) -> str: + candidates: list[object] = [ + mm_output.get("llm_output_text"), + mm_output.get("text"), + mm_output.get("llm_output_text_utf8"), + mm_output.get("meta.llm_output_text_utf8"), + ] + meta = mm_output.get("meta") + if isinstance(meta, dict): + candidates.extend((meta.get("llm_output_text"), meta.get("text"), meta.get("llm_output_text_utf8"))) + candidates.extend(mm_output.get(key) for key in ("meta.llm_output_text", "meta.text")) + for value in candidates: + if isinstance(value, str) and value: + return value + decoded = _decode_text_tensor(value) + if decoded: + return decoded + if isinstance(value, list): + text_chunks = [item for item in value if isinstance(item, str)] + if text_chunks: + return "".join(text_chunks) + return "" + + +def _decode_text_tensor(value: object) -> str: + if value is None: + return "" + try: + if isinstance(value, np.ndarray): + raw = value.astype(np.uint8, copy=False).reshape(-1).tobytes() + elif hasattr(value, "detach"): + raw = value.detach().cpu().numpy().astype(np.uint8, copy=False).reshape(-1).tobytes() + else: + return "" + return raw.decode("utf-8", errors="ignore") + except Exception: + return "" + + +def _fallback_audio_text_marks( + audio_chunks: list[tuple[str, int]], + delta_text: str, +) -> list[list[dict[str, int]] | None]: + if not delta_text: + return [] + total_duration_ms = sum(max(0, int(duration_ms)) for _, duration_ms in audio_chunks) + cumulative_duration_ms = 0 + marks: list[list[dict[str, int]] | None] = [] + for _, duration_ms in audio_chunks: + cumulative_duration_ms += max(0, int(duration_ms)) + if total_duration_ms <= 0: + marks.append(None) + continue + text_chars = int(len(delta_text) * min(1.0, cumulative_duration_ms / float(total_duration_ms))) + marks.append([{"text_chars": max(0, text_chars), "audio_end_ms": max(0, cumulative_duration_ms)}]) + return marks diff --git a/vllm_omni/experimental/fullduplex/minicpmo45/input.py b/vllm_omni/experimental/fullduplex/minicpmo45/input.py new file mode 100644 index 000000000..2a4eac3ea --- /dev/null +++ b/vllm_omni/experimental/fullduplex/minicpmo45/input.py @@ -0,0 +1,384 @@ +from __future__ import annotations + +import base64 +import binascii +from typing import Any + +import numpy as np + + +class MiniCPMO45PcmAppendReservation: + __slots__ = ( + "_active", + "_force_listen", + "_is_speech", + "_owner", + "_raw", + "_sample_rate_hz", + "_turn_had_speech", + "_video_frames", + "operation_id", + "payload", + ) + + def __init__( + self, + *, + owner: MiniCPMO45PcmAppendBuffer, + operation_id: str, + payload: dict[str, object] | None, + raw: bytes, + sample_rate_hz: int, + force_listen: bool, + is_speech: bool, + turn_had_speech: bool = False, + video_frames: list[str] | None = None, + ) -> None: + self._owner = owner + self.operation_id = operation_id + self.payload = payload + self._raw = raw + self._sample_rate_hz = sample_rate_hz + self._force_listen = force_listen + self._is_speech = is_speech + self._turn_had_speech = turn_had_speech + self._video_frames = list(video_frames or []) + self._active = True + + @property + def active(self) -> bool: + return self._active + + @property + def byte_count(self) -> int: + return len(self._raw) + + def commit(self) -> None: + self._owner._commit_reservation(self) + + def rollback(self) -> None: + self._owner._rollback_reservation(self) + + +def validate_native_ref_audio_config(session_config: dict[str, Any]) -> None: + extra_body = session_config.get("extra_body") + if not isinstance(extra_body, dict): + extra_body = {} + if any(key in session_config for key in ("ref_audio_path", "tts_ref_audio_path")) or any( + key in extra_body for key in ("ref_audio_path", "tts_ref_audio_path") + ): + raise ValueError("native duplex ref_audio_path is not accepted; resolve ref_audio in serving first") + + +def decode_native_ref_audio_from_config(session_config: dict[str, Any]) -> np.ndarray | None: + validate_native_ref_audio_config(session_config) + extra_body = session_config.get("extra_body") + if not isinstance(extra_body, dict): + extra_body = {} + audio_data = extra_body.get("ref_audio_data") + if audio_data is None: + return None + if not isinstance(audio_data, str): + raise TypeError("native duplex ref_audio_data must be base64 pcm_f32le") + fmt = extra_body.get("ref_audio_format") or "pcm_f32le" + if fmt != "pcm_f32le": + raise ValueError(f"unsupported native duplex ref_audio_format: {fmt!r}") + try: + raw = base64.b64decode(audio_data, validate=True) + except (ValueError, binascii.Error) as exc: + raise ValueError("invalid native duplex ref_audio_data") from exc + if len(raw) % 4: + raise ValueError("invalid native duplex ref_audio_data length") + return np.frombuffer(raw, dtype=" None: + self._buffer = bytearray() + self._sample_rate_hz: int | None = None + self._force_listen = False + self._is_speech = False + self._turn_had_speech = False + self._reservation_seq = 0 + self._reservations: list[MiniCPMO45PcmAppendReservation] = [] + # Omni duplex: queued camera frames (base64 JPEG), consumed FIFO at + # one frame per emitted model unit alongside the unit's audio. + self._frame_queue: list[str] = [] + + def clear(self) -> None: + for reservation in self._reservations: + reservation._active = False + self._reservations.clear() + self._buffer.clear() + self._frame_queue.clear() + self._sample_rate_hz = None + self._force_listen = False + self._is_speech = False + self._turn_had_speech = False + + def clear_force_listen(self) -> None: + self._force_listen = False + + def has_pending(self) -> bool: + return bool(self._buffer) + + def has_reserved(self) -> bool: + return bool(self._reservations) + + @property + def pending_byte_count(self) -> int: + return len(self._buffer) + + def _reserve_passthrough( + self, + payload: dict[str, object], + *, + operation_id: str, + ) -> MiniCPMO45PcmAppendReservation: + sample_rate_hz = payload.get("sample_rate_hz") + if isinstance(payload, dict) and "video_frames" in payload: + # Frames align to whole audio units; a passthrough payload has no + # unit framing for Stage0 to interleave against. + payload = {key: value for key, value in payload.items() if key != "video_frames"} + reservation = MiniCPMO45PcmAppendReservation( + owner=self, + operation_id=operation_id, + payload=payload, + raw=b"", + sample_rate_hz=sample_rate_hz if isinstance(sample_rate_hz, int) else 0, + force_listen=bool(payload.get("force_listen", False)), + is_speech=bool(payload.get("is_speech", False)), + turn_had_speech=bool(payload.get("is_speech", False)), + ) + self._reservations.append(reservation) + return reservation + + def prepare_append( + self, + payload: dict[str, object], + *, + operation_id: str, + chunk_period_ms: int, + flush: bool = False, + allow_emit: bool = True, + ) -> MiniCPMO45PcmAppendReservation | None: + fmt = payload.get("format") + sample_rate_hz = payload.get("sample_rate_hz") + audio = payload.get("audio") + if fmt != "pcm_f32le" or not isinstance(sample_rate_hz, int) or not isinstance(audio, str): + return self._reserve_passthrough(payload, operation_id=operation_id) + try: + raw = base64.b64decode(audio, validate=True) + except (binascii.Error, ValueError): + return self._reserve_passthrough(payload, operation_id=operation_id) + if len(raw) % 4 != 0: + return self._reserve_passthrough(payload, operation_id=operation_id) + + if self._sample_rate_hz is not None and self._sample_rate_hz != sample_rate_hz: + raise ValueError("MiniCPM-o native duplex audio append sample_rate_hz changed within a session") + self._sample_rate_hz = sample_rate_hz + self._buffer.extend(raw) + frames_in = payload.get("video_frames") + if isinstance(frames_in, list): + self._frame_queue.extend(frame for frame in frames_in if isinstance(frame, str) and frame) + self._turn_had_speech = self._turn_had_speech or bool(payload.get("is_speech", False)) + self._force_listen = self._force_listen or bool(payload.get("force_listen", False)) + self._is_speech = self._is_speech or bool(payload.get("is_speech", False)) + if not allow_emit: + return None + + min_samples = max(1, int(sample_rate_hz * max(1, int(chunk_period_ms)) / 1000)) + buffered_samples = len(self._buffer) // 4 + if not flush and buffered_samples < min_samples: + return None + + # Emit whole model chunks only: the engine reserves scheduler slots + # from the payload size and the worker consumes whole chunks per + # append, so partial chunks would turn into pad embeddings inside the + # model KV. On flush, the tail is zero-padded (silence) up to the + # chunk boundary instead. + if flush: + emit_samples = buffered_samples + remainder = emit_samples % min_samples + pad_samples = (min_samples - remainder) if remainder else 0 + else: + emit_samples = min_samples + pad_samples = 0 + emit_bytes = emit_samples * 4 + reserved_raw = bytes(self._buffer[:emit_bytes]) + emit_raw = reserved_raw + b"\x00" * (pad_samples * 4) + del self._buffer[:emit_bytes] + + out = dict(payload) + out.pop("force_speak", None) + out.pop("video_frames", None) + out["audio"] = base64.b64encode(emit_raw).decode("ascii") + out["sample_rate_hz"] = sample_rate_hz + # Omni duplex: attach at most one queued camera frame per emitted + # model unit (official cadence: one frame per 1 s chunk). The engine + # budgets 66 scheduler slots per attached frame from this payload. + # Official omni cadence is one frame per ~1 s chunk, and the first + # append consumes extra samples (1035 ms first window), so per-unit + # attachment could outrun the units Stage0 actually builds. Attach at + # most ONE frame per emitted payload; the rest stay queued. + attached_frames: list[str] = [] + if emit_samples + pad_samples >= min_samples and self._frame_queue: + attached_frames = [self._frame_queue.pop(0)] + out["video_frames"] = attached_frames + out["force_listen"] = self._force_listen + out["is_speech"] = self._is_speech + if not self._buffer: + self._force_listen = False + self._is_speech = False + reservation = MiniCPMO45PcmAppendReservation( + owner=self, + operation_id=operation_id, + payload=out, + raw=reserved_raw, + sample_rate_hz=sample_rate_hz, + force_listen=bool(out.get("force_listen", False)), + is_speech=bool(out.get("is_speech", False)), + turn_had_speech=self._turn_had_speech, + video_frames=attached_frames, + ) + self._reservations.append(reservation) + return reservation + + def prepare_commit( + self, + *, + operation_id: str, + chunk_period_ms: int, + ) -> MiniCPMO45PcmAppendReservation: + """Reserve the terminal payload without invalidating prior appends.""" + had_speech = self._turn_had_speech + reservation: MiniCPMO45PcmAppendReservation | None = None + if had_speech and self._buffer: + payload: dict[str, object] = { + "type": "audio", + "audio": "", + "format": "pcm_f32le", + "sample_rate_hz": self._sample_rate_hz or 16000, + "force_listen": self._force_listen, + "is_speech": self._is_speech, + } + reservation = self.prepare_append( + payload, + operation_id=operation_id, + chunk_period_ms=chunk_period_ms, + flush=True, + ) + assert reservation is not None + assert reservation.payload is not None + reservation.payload["final"] = True + if reservation is None: + reservation = MiniCPMO45PcmAppendReservation( + owner=self, + operation_id=operation_id, + payload=None, + raw=b"", + sample_rate_hz=self._sample_rate_hz or 0, + force_listen=self._force_listen, + is_speech=self._is_speech, + turn_had_speech=had_speech, + ) + self._reservations.append(reservation) + + # Open the next physical input generation without touching already + # reserved appends. A rollback restores this generation's metadata. + self._sample_rate_hz = None + self._force_listen = False + self._is_speech = False + self._turn_had_speech = False + return reservation + + def append( + self, + payload: dict[str, object], + *, + chunk_period_ms: int, + flush: bool = False, + allow_emit: bool = True, + ) -> dict[str, object] | None: + self._reservation_seq += 1 + reservation = self.prepare_append( + payload, + operation_id=f"immediate-{self._reservation_seq}", + chunk_period_ms=chunk_period_ms, + flush=flush, + allow_emit=allow_emit, + ) + if reservation is None: + return None + reservation.commit() + assert reservation.payload is not None + return reservation.payload + + def _commit_reservation(self, reservation: MiniCPMO45PcmAppendReservation) -> None: + if not reservation._active: + return + if not self._reservations or self._reservations[0] is not reservation: + raise RuntimeError("PCM append reservations must commit in wire order") + self._reservations.pop(0) + reservation._active = False + + def _rollback_reservation(self, reservation: MiniCPMO45PcmAppendReservation) -> None: + if not reservation._active: + return + try: + index = self._reservations.index(reservation) + except ValueError: + reservation._active = False + return + rolled_back = self._reservations[index:] + restored = b"".join(item._raw for item in rolled_back) + self._buffer[:0] = restored + restored_frames = [frame for item in rolled_back for frame in item._video_frames] + if restored_frames: + self._frame_queue[:0] = restored_frames + self._sample_rate_hz = self._sample_rate_hz or reservation._sample_rate_hz + self._force_listen = self._force_listen or any(item._force_listen for item in rolled_back) + self._is_speech = self._is_speech or any(item._is_speech for item in rolled_back) + self._turn_had_speech = self._turn_had_speech or any(item._turn_had_speech for item in rolled_back) + for item in rolled_back: + item._active = False + del self._reservations[index:] + + def flush(self, *, chunk_period_ms: int) -> dict[str, object] | None: + if not self._buffer: + return None + payload: dict[str, object] = { + "type": "audio", + "audio": "", + "format": "pcm_f32le", + "sample_rate_hz": self._sample_rate_hz or 16000, + "force_listen": self._force_listen, + "is_speech": self._is_speech, + } + return self.append(payload, chunk_period_ms=chunk_period_ms, flush=True) + + def commit(self, *, chunk_period_ms: int) -> dict[str, object] | None: + """Commit the real residual PCM and reset per-client-input state. + + Complete model units are emitted by :meth:`append` as soon as they are + available. A commit at that exact boundary therefore has no payload; + synthesizing another unit would add a model decision that does not + exist in the official continuous streaming loop. + """ + self._reservation_seq += 1 + reservation = self.prepare_commit( + operation_id=f"immediate-commit-{self._reservation_seq}", + chunk_period_ms=chunk_period_ms, + ) + reservation.commit() + return reservation.payload + + +__all__ = [ + "MiniCPMO45PcmAppendReservation", + "MiniCPMO45PcmAppendBuffer", + "decode_native_ref_audio_from_config", + "validate_native_ref_audio_config", +] diff --git a/vllm_omni/experimental/fullduplex/minicpmo45/policy.py b/vllm_omni/experimental/fullduplex/minicpmo45/policy.py new file mode 100644 index 000000000..c55b25360 --- /dev/null +++ b/vllm_omni/experimental/fullduplex/minicpmo45/policy.py @@ -0,0 +1,140 @@ +from __future__ import annotations + +from typing import Any + + +class MiniCPMO45DuplexPolicy: + """MiniCPM-o 4.5 native duplex model policy. + + Keep model-specific token names, input modes, and handoff type strings out + of the generic scheduler/orchestrator path. These names are part of the + MiniCPM-o 4.5 remote-code contract, not a general vLLM-Omni duplex schema. + """ + + # Audio framing contract shared by serving, orchestrator, and worker. + # MiniCPM-o consumes 1 s units at 16 kHz and pools audio to one embedding + # per 100 ms, so a unit contributes exactly 10 audio embeddings plus the + # open (and a closure for every unit after the first). + # Scheduler token budgets must match the worker-built embeddings exactly: + # surplus slots become pad embeddings inside the KV and measurably corrupt + # the model's listen/speak behavior. + SAMPLE_RATE_HZ = 16000 + CHUNK_SAMPLES = 16000 + SAMPLES_PER_AUDIO_TOKEN = 1600 + # Vision framing contract (omni duplex). Official streaming_prefill feeds + # each frame as + 64 resampler embeddings + inside the + # unit, ahead of the unit's audio embeddings (max_slice_nums=1 in + # streaming, so exactly one 64-token block per frame). + VISION_EMBEDS_PER_FRAME = 64 + VISION_TOKENS_PER_FRAME = VISION_EMBEDS_PER_FRAME + 2 # + embeds + + DEFAULT_MAX_NEW_SPEAK_TOKENS_PER_CHUNK = 20 + DEFAULT_MAX_SPEAK_CHARS_PER_CHUNK = 28 + DEFAULT_MIN_NEW_SPEAK_TOKENS_BEFORE_CHUNK_BOUNDARY = 8 + REPETITION_HISTORY_SIZE = 512 + + @classmethod + def audio_token_count(cls, sample_count: int) -> int: + """Audio embedding count for a clip of ``sample_count`` samples. + + Matches the whole-clip encoder math (hop 160 mel frames -> CNN stride 2 + -> avg-pool 5) for any multiple of ``SAMPLES_PER_AUDIO_TOKEN``; serving + normalizes clips to that boundary before this is used for budgets. + """ + return max(0, int(sample_count) // cls.SAMPLES_PER_AUDIO_TOKEN) + + @staticmethod + def session_context_texts(instructions: object, has_ref_audio: bool) -> tuple[str, str]: + """System-context prefix/suffix, matching MiniCPMODuplex.prepare().""" + system_prompt = ( + instructions if isinstance(instructions, str) and instructions else "Streaming Omni Conversation." + ) + prefix = f"<|im_start|>system\n{system_prompt}" + suffix = "<|im_end|>" + if has_ref_audio: + prefix += "\n<|audio_start|>" + suffix = "<|audio_end|>" + suffix + return prefix, suffix + + SPECIAL_TOKEN_FIELDS: dict[str, str] = { + "unit_token_id": "", + "unit_end_token_id": "", + "listen_token_id": "<|listen|>", + "speak_token_id": "<|speak|>", + "tts_bos_token_id": "<|tts_bos|>", + "tts_eos_token_id": "<|tts_eos|>", + "tts_pad_token_id": "<|tts_pad|>", + "chunk_eos_token_id": "<|chunk_eos|>", + "chunk_tts_eos_token_id": "<|chunk_tts_eos|>", + "turn_eos_token_id": "<|turn_eos|>", + } + OPTIONAL_TOKEN_FIELDS: dict[str, str] = { + "audio_placeholder_token_id": "<|audio|>", + "image_start_token_id": "", + "image_end_token_id": "", + } + + @classmethod + def token_ids_from_tokenizer(cls, tokenizer: Any) -> dict[str, int]: + convert = getattr(tokenizer, "convert_tokens_to_ids", None) + + def token_id(token: str) -> int: + if not callable(convert): + value = None + else: + value = convert(token) + if isinstance(value, list): + value = value[0] if len(value) == 1 else None + unk_token_id = getattr(tokenizer, "unk_token_id", None) + try: + candidate = int(value) + except (TypeError, ValueError): + candidate = -1 + if candidate >= 0 and candidate != unk_token_id: + return candidate + + encode = getattr(tokenizer, "encode", None) + if callable(encode): + try: + ids = list(encode(token, add_special_tokens=False)) + except TypeError: + ids = list(encode(token)) + if len(ids) == 1: + try: + candidate = int(ids[0]) + except (TypeError, ValueError): + candidate = -1 + if candidate >= 0 and candidate != unk_token_id: + return candidate + return -1 + + resolved = {field: token_id(token) for field, token in cls.SPECIAL_TOKEN_FIELDS.items()} + resolved.update({field: token_id(token) for field, token in cls.OPTIONAL_TOKEN_FIELDS.items()}) + if resolved.get("listen_token_id", -1) < 0: + eos_id = getattr(tokenizer, "eos_token_id", None) + try: + resolved["listen_token_id"] = int(eos_id) + except (TypeError, ValueError): + pass + return resolved + + @staticmethod + def native_forbidden_token_ids( + token_ids: dict[str, int], + *, + bad_token_ids: list[int] | tuple[int, ...] = (), + ) -> list[int]: + return [ + token_ids.get("tts_pad_token_id", -1), + *list(bad_token_ids), + token_ids.get("chunk_eos_token_id", -1), + ] + + @staticmethod + def native_special_token_ids( + token_ids: dict[str, int], + *, + tokenizer_special_ids: list[int] | tuple[int, ...] = (), + ) -> set[int]: + special = set(tokenizer_special_ids or []) + special.update(value for value in token_ids.values() if isinstance(value, int) and value >= 0) + return special diff --git a/vllm_omni/experimental/fullduplex/minicpmo45/runtime.py b/vllm_omni/experimental/fullduplex/minicpmo45/runtime.py new file mode 100644 index 000000000..b5fbd37a5 --- /dev/null +++ b/vllm_omni/experimental/fullduplex/minicpmo45/runtime.py @@ -0,0 +1,354 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from __future__ import annotations + +from base64 import b64decode +from binascii import Error as BinasciiError +from typing import Any + +from vllm.sampling_params import SamplingParams + +from vllm_omni.experimental.fullduplex.engine.duplex_runtime import ( + DuplexAppendPlan, + DuplexInputMode, + DuplexOutputAction, + DuplexOutputDecision, +) +from vllm_omni.experimental.fullduplex.engine.messages import DuplexFence + +_DUPLEX_CHUNK_SAMPLES = 16000 +_DUPLEX_SAMPLES_PER_AUDIO_TOKEN = 1600 +# + 64 resampler embeddings + per frame (max_slice_nums=1), +# matching MiniCPMO45DuplexPolicy.VISION_TOKENS_PER_FRAME. +_DUPLEX_VISION_TOKENS_PER_FRAME = 66 + + +def _duplex_frame_count(payload: object) -> int: + if not isinstance(payload, dict): + return 0 + frames = payload.get("video_frames") + if not isinstance(frames, list): + return 0 + return sum(1 for frame in frames if isinstance(frame, str) and frame) + + +def _duplex_pcm_sample_count(payload: object) -> int | None: + if not isinstance(payload, dict): + return None + audio = payload.get("audio") or payload.get("data") + if payload.get("format") != "pcm_f32le" or not isinstance(audio, str): + return None + try: + raw = b64decode(audio, validate=True) + except (BinasciiError, ValueError): + return None + return len(raw) // 4 + + +def duplex_payload_is_exact_chunks(payload: object) -> bool: + sample_count = _duplex_pcm_sample_count(payload) + return bool(sample_count) and sample_count % _DUPLEX_CHUNK_SAMPLES == 0 + + +def duplex_first_append_unit_count(payload: object) -> int | None: + sample_count = _duplex_pcm_sample_count(payload) + if not sample_count or sample_count % _DUPLEX_CHUNK_SAMPLES != 0: + return None + return max(1, sample_count // _DUPLEX_CHUNK_SAMPLES - 1) + + +def duplex_scheduler_token_budget(payload: object, *, default: int = 64) -> int: + vision_tokens = _duplex_frame_count(payload) * _DUPLEX_VISION_TOKENS_PER_FRAME + sample_count = _duplex_pcm_sample_count(payload) + if sample_count is None: + return max(1, int(default)) + vision_tokens + sample_count = max(1, sample_count) + if sample_count % _DUPLEX_CHUNK_SAMPLES == 0: + units = sample_count // _DUPLEX_CHUNK_SAMPLES + return units * (2 + _DUPLEX_CHUNK_SAMPLES // _DUPLEX_SAMPLES_PER_AUDIO_TOKEN) + vision_tokens + return max(16, min(768, sample_count // _DUPLEX_SAMPLES_PER_AUDIO_TOKEN + 8)) + vision_tokens + + +def duplex_first_append_context_reserve(runtime_config: object) -> int: + if not isinstance(runtime_config, dict): + return 48 + exact = runtime_config.get("duplex_first_append_context_tokens") + if isinstance(exact, int) and exact >= 0: + return exact + reserve = 48 + ref = runtime_config.get("ref_audio_data") + if isinstance(ref, str) and ref: + try: + raw = b64decode(ref, validate=True) + except (BinasciiError, ValueError): + raw = b"" + if raw: + reserve += max(0, (len(raw) // 4) // _DUPLEX_SAMPLES_PER_AUDIO_TOKEN + 8) + return reserve + + +def _duplex_force_listen_count(extra_body: object) -> int: + raw = extra_body.get("force_listen_count") if isinstance(extra_body, dict) else None + try: + return 0 if raw is None else max(0, int(raw)) + except (TypeError, ValueError): + return 0 + + +def build_duplex_data_plane_prompt( + *, + request_id: str, + fence: DuplexFence, + session_config: dict[str, Any], + runtime_config: dict[str, Any], + seq: int, + turn_seq: int, + mode: DuplexInputMode, + payload: object, + final: bool, +) -> dict[str, Any]: + token_budget = duplex_scheduler_token_budget(payload) + if seq <= 1: + context_reserve = duplex_first_append_context_reserve(runtime_config) + token_budget += context_reserve + first_units = duplex_first_append_unit_count(payload) + if first_units is not None: + token_budget = ( + context_reserve + first_units * 12 - 1 + _duplex_frame_count(payload) * _DUPLEX_VISION_TOKENS_PER_FRAME + ) + if seq > 1 and duplex_payload_is_exact_chunks(payload): + token_budget += 1 + if final and duplex_payload_is_exact_chunks(payload): + token_budget += 12 + extra_body = session_config.get("extra_body") + raw_token_id = runtime_config.get("duplex_scheduler_token_id") + try: + token_id = max(0, int(raw_token_id)) + except (TypeError, ValueError): + token_id = 0 + force_listen_count = _duplex_force_listen_count(extra_body) + if ( + force_listen_count > 0 + and turn_seq <= force_listen_count + and isinstance(payload, dict) + and payload.get("force_listen") is not True + ): + payload = {**payload, "force_listen": True} + return { + "prompt_token_ids": [token_id] * token_budget, + "model_intermediate_buffer": { + "request_id": request_id, + "global_request_id": [fence.session_id], + "duplex": { + "fence": fence, + "session_id": fence.session_id, + "incarnation": fence.incarnation, + "epoch": fence.epoch, + "seq": seq, + "turn_id": fence.turn_id, + "response_seq": fence.response_seq, + "turn_seq": turn_seq, + "mode": mode.value, + "payload": payload, + "final": final, + "data_plane": True, + "session_config": dict(session_config), + "runtime_config": dict(runtime_config), + "scheduler_token_budget": token_budget, + "scheduler_token_id": token_id, + }, + }, + } + + +def _coerce_int(value: object) -> int | None: + if hasattr(value, "detach"): + try: + value = value.detach().cpu().reshape(-1) + if value.numel() == 0: + return None + value = value[0].item() + except Exception: + return None + try: + return int(value) + except (TypeError, ValueError): + return None + + +def _coerce_int_list(value: object) -> list[int]: + if value is None: + return [] + if hasattr(value, "detach"): + try: + value = value.detach().cpu().reshape(-1).tolist() + except Exception: + return [] + if not isinstance(value, (list, tuple)): + return [] + return [token_id for item in value if (token_id := _coerce_int(item)) is not None] + + +def _first_completion(output: object) -> object | None: + outputs = getattr(output, "outputs", None) + return outputs[0] if isinstance(outputs, list) and outputs else None + + +def _multimodal_output(output: object, completion: object | None) -> dict[str, Any]: + metadata = getattr(output, "multimodal_output", None) + if isinstance(metadata, dict): + return metadata + metadata = getattr(completion, "multimodal_output", None) if completion is not None else None + return metadata if isinstance(metadata, dict) else {} + + +def _special_token_ids(metadata: dict[str, Any]) -> dict[str, int]: + sources: list[object] = [metadata.get("special_token_ids"), metadata.get("meta")] + sources.append( + { + key.removeprefix("meta."): value + for key, value in metadata.items() + if isinstance(key, str) and key.startswith("meta.") + } + ) + token_ids: dict[str, int] = {} + for source in sources: + if not isinstance(source, dict): + continue + for key, value in source.items(): + token_id = _coerce_int(value) + if isinstance(key, str) and token_id is not None and token_id >= 0: + token_ids[key] = token_id + return token_ids + + +def _completion_token_ids(completion: object | None) -> list[int]: + if completion is None: + return [] + for attribute in ("token_ids", "cumulative_token_ids"): + token_ids = _coerce_int_list(getattr(completion, attribute, None)) + if token_ids: + return token_ids + return [] + + +def _stage_config_value(runtime_config: dict[str, Any], key: str, stage_id: int) -> object | None: + raw = runtime_config.get(key) + if isinstance(raw, dict): + value = raw.get(stage_id) + return raw.get(str(stage_id)) if value is None else value + if isinstance(raw, (list, tuple)) and stage_id < len(raw): + return raw[stage_id] + return None + + +class MiniCPMO45DuplexRuntimeExtension: + def configure_sampling_params( + self, + *, + runtime_config: dict[str, Any], + defaults: tuple[object, ...], + ) -> tuple[object, ...]: + configured: list[object] = [] + for stage_id, default in enumerate(defaults): + max_tokens = _coerce_int(_stage_config_value(runtime_config, "duplex_stage_max_tokens", stage_id)) + raw_overrides = _stage_config_value(runtime_config, "duplex_stage_sampling_params", stage_id) + overrides = dict(raw_overrides) if isinstance(raw_overrides, dict) else {} + if not isinstance(default, SamplingParams) or (not overrides and (max_tokens is None or max_tokens <= 0)): + configured.append(default) + continue + params = default.clone() + if max_tokens is not None and max_tokens > 0: + params.max_tokens = max_tokens + for name, value in overrides.items(): + if not hasattr(params, name): + continue + setattr(params, name, value) + if name == "stop_token_ids": + all_stop_token_ids = getattr(params, "_all_stop_token_ids", None) + if isinstance(all_stop_token_ids, set): + all_stop_token_ids.update(int(token_id) for token_id in value) + configured.append(params) + return tuple(configured) + + def plan_append( + self, + *, + request_id: str, + fence: DuplexFence, + session_config: dict[str, Any], + runtime_config: dict[str, Any], + seq: int, + turn_seq: int, + mode: DuplexInputMode, + payload: object, + final: bool, + sampling_params: object, + ) -> DuplexAppendPlan: + del sampling_params + return DuplexAppendPlan( + prompt=build_duplex_data_plane_prompt( + request_id=request_id, + fence=fence, + session_config=session_config, + runtime_config=runtime_config, + seq=seq, + turn_seq=turn_seq, + mode=mode, + payload=payload, + final=final, + ) + ) + + def decide_output( + self, + *, + stage_id: int, + final_stage_id: int, + segment_finished: bool, + segment_token_ids: tuple[int, ...], + segment_output_metadata: dict[str, Any], + output: object, + ) -> DuplexOutputDecision | None: + if stage_id >= final_stage_id or not segment_finished: + return None + + completion = _first_completion(output) + output_metadata = _multimodal_output(output, completion) + special_token_ids = _special_token_ids(segment_output_metadata) + special_token_ids.update(_special_token_ids(output_metadata)) + listen_id = special_token_ids.get("listen_token_id") + if listen_id is None: + return None + + stop_reason = getattr(completion, "stop_reason", None) if completion is not None else None + token_ids = _completion_token_ids(completion) or list(segment_token_ids) + if _coerce_int(stop_reason) != listen_id and (not token_ids or token_ids[-1] != listen_id): + return None + + metadata = dict(output_metadata) + for key, value in special_token_ids.items(): + metadata.setdefault(f"meta.{key}", value) + metadata.update( + { + "duplex_direct_response": True, + "duplex_native_decision": "listen", + "model_listen": True, + "listen_source": "model_listen", + } + ) + return DuplexOutputDecision( + action=DuplexOutputAction.DIRECT_RESPONSE, + metadata=metadata, + ) + + +__all__ = [ + "MiniCPMO45DuplexRuntimeExtension", + "build_duplex_data_plane_prompt", + "duplex_first_append_context_reserve", + "duplex_first_append_unit_count", + "duplex_payload_is_exact_chunks", + "duplex_scheduler_token_budget", +] diff --git a/vllm_omni/experimental/fullduplex/minicpmo45/serving_adapter.py b/vllm_omni/experimental/fullduplex/minicpmo45/serving_adapter.py new file mode 100644 index 000000000..e811d73e5 --- /dev/null +++ b/vllm_omni/experimental/fullduplex/minicpmo45/serving_adapter.py @@ -0,0 +1,96 @@ +from __future__ import annotations + +from collections.abc import Callable, Mapping +from typing import Any + +from vllm_omni.experimental.fullduplex.minicpmo45.adapter import ( + MiniCPMO45NativeDuplexServingAdapter, +) +from vllm_omni.experimental.fullduplex.minicpmo45.data_plane import ( + MiniCPMO45DataPlaneContext, + MiniCPMO45DataPlaneSession, +) +from vllm_omni.experimental.fullduplex.minicpmo45.session import ( + MiniCPMO45ServingSessionState, +) +from vllm_omni.experimental.fullduplex.openai.protocol import DuplexCapabilities + +EncodeAudio = Callable[[object, int, str, float | None], str | None] + + +class MiniCPMO45ServingRuntimeAdapter: + """MiniCPM-owned serving state, input packing, and output projection.""" + + adapter_id = "minicpmo45" + clean_response_done_prefix = "" + interrupted_tts_prefix = "" + private_runtime_config_keys = MiniCPMO45NativeDuplexServingAdapter.PRIVATE_RUNTIME_CONFIG_KEYS + + def __init__(self, encode_audio: EncodeAudio) -> None: + self.session_states: dict[str, MiniCPMO45ServingSessionState] = {} + self.data_plane = MiniCPMO45DataPlaneSession(encode_audio) + + def create_session_state(self) -> MiniCPMO45ServingSessionState: + return MiniCPMO45ServingSessionState() + + def session_state(self, session_id: str) -> MiniCPMO45ServingSessionState: + state = self.session_states.get(session_id) + if state is None: + state = self.create_session_state() + self.session_states[session_id] = state + return state + + def remove_session_state(self, session_id: str) -> None: + self.session_states.pop(session_id, None) + + @staticmethod + def is_enabled(config: object) -> bool: + return MiniCPMO45NativeDuplexServingAdapter.is_enabled(config) # type: ignore[arg-type] + + @staticmethod + def capabilities(*, max_sessions: int) -> DuplexCapabilities: + return DuplexCapabilities.minicpmo45_native(max_sessions=max_sessions) + + @staticmethod + def validate_client_extra_body(extra_body: object) -> None: + MiniCPMO45NativeDuplexServingAdapter.validate_client_extra_body(extra_body) + + @staticmethod + async def prepare_runtime_config(config: object, *, model_config: Any) -> dict[str, object]: + return await MiniCPMO45NativeDuplexServingAdapter.prepare_runtime_config( + config, # type: ignore[arg-type] + model_config=model_config, + ) + + @staticmethod + def runtime_config_for_update( + config: object, + current: Mapping[str, object], + ) -> dict[str, object]: + return MiniCPMO45NativeDuplexServingAdapter.runtime_config_for_update( + config, # type: ignore[arg-type] + dict(current), + ) + + @staticmethod + def data_plane_context( + *, + epoch: int, + turn_id: int, + active_response_turn_id: int | None, + active_response_id: str | None, + auto_responds: bool, + response_format: str, + speed: float | None, + modalities: tuple[str, ...], + ) -> MiniCPMO45DataPlaneContext: + return MiniCPMO45DataPlaneContext( + epoch=epoch, + turn_id=turn_id, + active_response_turn_id=active_response_turn_id, + active_response_id=active_response_id, + auto_responds=auto_responds, + response_format=response_format, + speed=speed, + modalities=modalities, + ) diff --git a/vllm_omni/experimental/fullduplex/minicpmo45/session.py b/vllm_omni/experimental/fullduplex/minicpmo45/session.py new file mode 100644 index 000000000..763bcf8b3 --- /dev/null +++ b/vllm_omni/experimental/fullduplex/minicpmo45/session.py @@ -0,0 +1,56 @@ +from __future__ import annotations + +import asyncio +from collections.abc import Awaitable, Callable +from dataclasses import dataclass, field + +from vllm_omni.experimental.fullduplex.minicpmo45.input import ( + MiniCPMO45PcmAppendBuffer, +) + + +@dataclass(slots=True) +class MiniCPMO45ServingSessionState: + """Mutable serving state owned by one MiniCPM duplex session.""" + + audio_buffer: MiniCPMO45PcmAppendBuffer = field(default_factory=MiniCPMO45PcmAppendBuffer) + input_since_commit: bool = False + speech_since_commit: bool = False + committed_audio_payload: dict[str, object] | None = None + committed_audio_operation_id: str | None = None + committed_audio_reserved_bytes: int = 0 + deferred_response_create: bool = False + deferred_precreate_response: bool = False + data_plane_task: asyncio.Task[None] | None = None + data_plane_restart_requested: bool = False + continuation_owner_id: str | None = None + continuation_units: int = 0 + pending_silence_task: asyncio.Task[bool] | None = None + pending_silence_owner_id: str | None = None + silence_continuation_scheduler: Callable[..., Awaitable[bool]] | None = None + + def retain_committed_audio( + self, + payload: dict[str, object], + *, + operation_id: str | None, + reserved_bytes: int = 0, + ) -> None: + self.committed_audio_payload = payload + self.committed_audio_operation_id = operation_id + self.committed_audio_reserved_bytes += max(0, int(reserved_bytes)) + + def clear_committed_audio(self) -> int: + reserved_bytes = self.committed_audio_reserved_bytes + self.committed_audio_payload = None + self.committed_audio_operation_id = None + self.committed_audio_reserved_bytes = 0 + self.deferred_response_create = False + self.deferred_precreate_response = False + return reserved_bytes + + def clear_continuation(self) -> None: + self.continuation_owner_id = None + self.continuation_units = 0 + self.pending_silence_task = None + self.pending_silence_owner_id = None diff --git a/vllm_omni/experimental/fullduplex/minicpmo45/stage0.py b/vllm_omni/experimental/fullduplex/minicpmo45/stage0.py new file mode 100644 index 000000000..84aaf1cb8 --- /dev/null +++ b/vllm_omni/experimental/fullduplex/minicpmo45/stage0.py @@ -0,0 +1,795 @@ +from __future__ import annotations + +import base64 +import copy +import sys +import time +from contextlib import suppress +from dataclasses import dataclass, field +from typing import Any + +import numpy as np + +from vllm_omni.experimental.fullduplex.minicpmo45.policy import MiniCPMO45DuplexPolicy + +_MINICPMO45_SPECIAL_TOKEN_FIELDS = MiniCPMO45DuplexPolicy.SPECIAL_TOKEN_FIELDS +_MINICPMO45_OPTIONAL_TOKEN_FIELDS = MiniCPMO45DuplexPolicy.OPTIONAL_TOKEN_FIELDS + + +@dataclass +class _MiniCPMO45Stage0SessionState: + session_id: str + streaming_processor: Any | None = None + audio_buffer: np.ndarray = field(default_factory=lambda: np.array([], dtype=np.float32)) + audio_chunk_idx: int = 0 + context_embeds: list[Any] = field(default_factory=list) + context_token_ids: list[int] = field(default_factory=list) + current_turn_ended: bool = True + prepared_append_identity: tuple[int | None, int] | None = None + prepared_inputs_embeds: Any | None = None + prepared_input_token_ids: list[int] = field(default_factory=list) + prepared_result: dict[str, Any] = field(default_factory=dict) + audio_past_key_values: Any | None = None + pending_terminator_token: int | None = None + last_terminator_token: int | None = None + pending_speech_context: bool = False + pending_speech_append_identity: tuple[int | None, int] | None = None + pending_speech_response_open: bool = False + generated_text_tokens: list[int] = field(default_factory=list) + + +class MiniCPMO45Stage0DuplexRuntime: + """Build scheduler-owned MiniCPM-o 4.5 Stage0 duplex inputs.""" + + def __init__(self, stage_model: Any, *, model_path: str | None = None, device: str = "cuda") -> None: + self.stage_model = stage_model + self.model_path = model_path + self.device = device + self.sessions: dict[tuple[str, int], _MiniCPMO45Stage0SessionState] = {} + self.thinker = getattr(stage_model, "thinker", None) or getattr(stage_model, "model", None) or stage_model + self.processor = ( + getattr(stage_model, "processor", None) + or getattr(self.thinker, "processor", None) + or self._load_processor_from_path(model_path) + ) + self.tokenizer = ( + getattr(self.processor, "tokenizer", None) + if self.processor is not None + else getattr(stage_model, "tokenizer", None) + ) + self._init_token_ids() + + def _stage_runtime_ready(self) -> bool: + return self.processor is not None and self.tokenizer is not None and self.thinker is not None + + def _configure_streaming_processor( + self, + state: _MiniCPMO45Stage0SessionState | None = None, + ) -> Any | None: + processor = self.processor + if processor is None: + return None + if state is not None: + if state.streaming_processor is not None: + return state.streaming_processor + processor = copy.copy(processor) + shared_mel = getattr(self.processor, "_streaming_mel_processor", None) + if shared_mel is not None: + processor._streaming_mel_processor = copy.deepcopy(shared_mel) + state.streaming_processor = processor + if processor is None: + return + set_streaming_mode = getattr(processor, "set_streaming_mode", None) + if callable(set_streaming_mode): + set_streaming_mode( + mode="exact", + chunk_ms=int(self._stage_param("chunk_ms", 1000)), + first_chunk_ms=int(self._stage_param("first_chunk_ms", 1035)), + cnn_redundancy_ms=int(self._stage_param("cnn_redundancy_ms", 20)), + enable_sliding_window=True, + slide_trigger_seconds=30.0, + slide_stride_seconds=10.0, + ) + # Match official init_streaming_processor: reset the streaming mel-processor + # buffers at session init (modeling_minicpmo_unified.py:207). + reset_streaming = getattr(processor, "reset_streaming", None) + if callable(reset_streaming): + reset_streaming() + return processor + configure_streaming = getattr(processor, "configure_streaming", None) + if callable(configure_streaming): + configure_streaming( + chunk_ms=int(self._stage_param("chunk_ms", 1000)), + enable_sliding_window=True, + slide_trigger_seconds=30.0, + slide_stride_seconds=10.0, + ) + return processor + + def _prepare_session_context( + self, + state: _MiniCPMO45Stage0SessionState, + session_config: dict[str, Any], + *, + runtime_config: dict[str, Any] | None = None, + ) -> None: + if not self._stage_runtime_ready(): + return + self._require_special_token_ids() + ref_audio = self._decode_ref_audio_from_session_config(runtime_config or {}) + # Matches MiniCPMODuplex.prepare() in the released checkpoint's + # modeling_minicpmo.py: the <|audio_start|>/<|audio_end|> markers are + # only present when reference audio is embedded between them. The + # template is shared with the serving adapter so the first-append + # scheduler reserve can count these tokens exactly. + prefix, suffix = MiniCPMO45DuplexPolicy.session_context_texts( + session_config.get("instructions"), + ref_audio is not None, + ) + for token_id in self._encode_text(prefix): + state.context_embeds.append(self._embed_token(token_id)) + state.context_token_ids.append(token_id) + if ref_audio is not None: + ref_audio_embeds = self._stage_ref_audio_embeddings(ref_audio, state=state) + if ref_audio_embeds is not None: + ref_audio_embeds = self._as_2d_tensor(ref_audio_embeds) + state.context_embeds.append(ref_audio_embeds) + state.context_token_ids.extend([self.unit_token_id] * int(ref_audio_embeds.shape[0])) + for token_id in self._encode_text(suffix): + state.context_embeds.append(self._embed_token(token_id)) + state.context_token_ids.append(token_id) + + def _stage_prefill_embeddings_only( + self, + state: _MiniCPMO45Stage0SessionState, + audio_waveform: Any, + *, + video_frames: list[Any] | None = None, + epoch: int | None = None, + seq: int | None = None, + is_speech: bool = False, + final: bool = False, + ) -> dict[str, Any]: + """Build scheduler-owned Stage0 input embeddings for one audio append. + + Unlike the legacy worker-control path, this method never calls an eager + model forward. The normal vLLM runner consumes the returned embeddings + and owns attention metadata, block tables, KV cache, and sampling. + """ + start_time = time.time() + processor = self._configure_streaming_processor(state) + append_identity = (epoch, seq) if seq is not None else None + if ( + append_identity is not None + and state.prepared_append_identity == append_identity + and state.prepared_inputs_embeds is not None + ): + result = dict(state.prepared_result) + result["inputs_embeds"] = state.prepared_inputs_embeds + result["input_token_ids"] = list(state.prepared_input_token_ids) + return result + self._require_special_token_ids() + if audio_waveform is None or len(audio_waveform) == 0: + return self._stage_prefill_result(False, start_time, "empty audio") + # Omni duplex: encode this append's camera frames up front so the unit + # loop can interleave one block per unit, mirroring official + # streaming_prefill (feed , then image embeds, then audio). + frame_blocks: list[Any] = [] + if video_frames: + self._require_vision_token_ids() + frame_blocks = self._stage_vision_embeddings(video_frames) + if frame_blocks is None or len(frame_blocks) != len(video_frames): + return self._stage_prefill_result(False, start_time, "streaming vision embedding failed") + state.audio_buffer = np.concatenate([state.audio_buffer, np.asarray(audio_waveform, dtype=np.float32)]) + chunk_size = self._streaming_chunk_size(processor) + self._pad_first_audio_chunk_if_needed(state, processor) + if len(state.audio_buffer) < chunk_size: + return self._stage_prefill_result( + False, + start_time, + f"audio not enough: need {chunk_size} samples, only {len(state.audio_buffer)}", + ) + + embed_parts: list[Any] = [] + token_ids: list[int] = [] + if state.audio_chunk_idx == 0 and state.context_embeds: + embed_parts.extend(state.context_embeds) + token_ids.extend(state.context_token_ids) + + # Consume every complete chunk in the buffer so the appended span and + # the scheduler's slot reservation for this append agree exactly. A + # final append may zero-pad a real residual chunk, but it must not add + # a whole silence unit after all input was already consumed: official + # duplex generation runs once per microphone unit and client commit is + # not an additional model decision. + units_built = 0 + while True: + if len(state.audio_buffer) < chunk_size: + if not final or len(state.audio_buffer) == 0: + break + pad = np.zeros(chunk_size - len(state.audio_buffer), dtype=np.float32) + state.audio_buffer = np.concatenate([state.audio_buffer, pad]) + audio_chunk = state.audio_buffer[:chunk_size] + batch_feature = self._process_streaming_audio(audio_chunk, state.audio_chunk_idx, processor=processor) + for name, value in ( + ("chunk_idx", state.audio_chunk_idx), + ("use_extra_context", True), + ("prefix_extra_frames", 0 if state.audio_chunk_idx == 0 else 2), + ("suffix_extra_frames", 2), + ): + with suppress(Exception): + setattr(batch_feature, name, value) + audio_embeds = self._stage_audio_embeddings(batch_feature, state=state) + if audio_embeds is None: + if units_built == 0: + return self._stage_prefill_result(False, start_time, "streaming audio embedding returned empty") + break + if state.audio_chunk_idx > 0: + # Official duplex closes every unit (finalize_unit feeds the + # sampled terminator + ) before the next opens. + # The scheduler session update discards the previous segment's + # sampled terminator token, so it is re-injected here ahead of + # the closure; the model's listen/speak policy depends on + # seeing its own past decisions in context. + pending_terminator = state.pending_terminator_token + if pending_terminator is not None and units_built == 0: + state.pending_terminator_token = None + embed_parts.append(self._embed_token(pending_terminator)) + token_ids.append(int(pending_terminator)) + embed_parts.append(self._embed_token(self.unit_end_token_id)) + token_ids.append(self.unit_end_token_id) + embed_parts.append(self._embed_token(self.unit_token_id)) + token_ids.append(self.unit_token_id) + if frame_blocks: + # Official order inside a unit: + 64 resampler + # embeddings + ahead of the unit's audio embeddings + # (max_slice_nums=1 in streaming, one frame per unit). + vision_block = self._as_2d_tensor(frame_blocks.pop(0)) + embed_parts.append(self._embed_token(self.image_start_token_id)) + token_ids.append(int(self.image_start_token_id)) + embed_parts.append(vision_block) + token_ids.extend([self._vision_embedding_placeholder_token_id()] * int(vision_block.shape[0])) + embed_parts.append(self._embed_token(self.image_end_token_id)) + token_ids.append(int(self.image_end_token_id)) + embed_parts.append(audio_embeds) + token_ids.extend( + [self._audio_embedding_placeholder_token_id()] * int(self._as_2d_tensor(audio_embeds).shape[0]) + ) + state.audio_buffer = state.audio_buffer[ + self._consumed_audio_samples(state.audio_chunk_idx, chunk_size, processor=processor) : + ] + state.audio_chunk_idx += 1 + units_built += 1 + chunk_size = self._streaming_chunk_size(processor) + if frame_blocks: + # Each frame reserves one image block in the append plan. A frame + # without a matching audio unit would desynchronize that plan. + return self._stage_prefill_result( + False, + start_time, + f"{len(frame_blocks)} video frame(s) left without a matching audio unit", + ) + # Match official streaming_prefill: per chunk feed ONLY +audio. The assistant + # turn is opened once at session init; re-emitting the turn-open prefix per chunk + # re-opened the turn each chunk -> degenerate repetition. tts_bos/listen/turn_eos are + # model-generated and tracked via current_turn_ended (mirrors streaming_generate). + prompt_suffix_len = 0 + + import torch + + inputs_embeds = torch.cat([self._as_2d_tensor(embed) for embed in embed_parts], dim=0) + result = self._stage_prefill_result(True, start_time) + result.update( + { + "inputs_embeds": inputs_embeds, + "input_token_ids": token_ids, + "special_token_ids": self._special_token_ids(), + "num_input_tokens": int(inputs_embeds.shape[0]), + "prompt_suffix_len": prompt_suffix_len, + "uses_model_runner_scheduler": True, + "runner_kv_backed": True, + "runtime_impl": "scheduler_data_plane", + } + ) + if is_speech and (append_identity is None or state.pending_speech_append_identity != append_identity): + state.pending_speech_context = True + state.pending_speech_append_identity = append_identity + if append_identity is not None: + state.prepared_append_identity = append_identity + state.prepared_inputs_embeds = inputs_embeds + state.prepared_input_token_ids = list(token_ids) + state.prepared_result = {k: v for k, v in result.items() if k not in {"inputs_embeds", "input_token_ids"}} + return result + + @staticmethod + def _stage_prefill_result(success: bool, start_time: float, reason: str = "") -> dict[str, Any]: + return { + "success": success, + "prefill_success": success, + "is_buffering": not success, + "reason": reason, + "cost_all": time.time() - start_time, + "stage_runtime_ready": True, + } + + @staticmethod + def _as_2d_tensor(value: Any) -> Any: + if value.ndim == 1: + return value.unsqueeze(0) + if value.ndim == 3 and value.shape[0] == 1: + return value.squeeze(0) + return value + + def _embed_token(self, token_id: int) -> Any: + import torch + + token = torch.tensor([int(token_id)], dtype=torch.long, device=self._model_device()) + embedder = self._token_embedder() + embeds = embedder(token) + return self._as_2d_tensor(embeds) + + def _token_embedder(self) -> Any: + nested_embed = getattr(getattr(getattr(self.thinker, "llm", None), "model", None), "embed_tokens", None) + if callable(nested_embed): + return nested_embed + for target in (self.thinker, self.stage_model): + embedder = getattr(target, "get_input_embeddings", None) + if callable(embedder): + try: + embeddings = embedder() + if callable(embeddings): + return embeddings + except TypeError: + return embedder + raise AttributeError("MiniCPM-o stage0 model does not expose token embeddings") + + def _model_device(self) -> Any: + try: + return next(self.thinker.parameters()).device + except Exception: + pass + try: + return next(self.stage_model.parameters()).device + except Exception: + pass + return self.device + + def _streaming_chunk_size(self, processor: Any | None = None) -> int: + processor = processor or self.processor + get_chunk = getattr(processor, "get_streaming_chunk_size", None) + if callable(get_chunk): + return int(get_chunk()) + return 16000 + + def _sample_rate(self, processor: Any | None = None) -> int: + processor = processor or self.processor + return int( + self._stage_param( + "sample_rate", + getattr(getattr(processor, "_streaming_mel_processor", None), "sample_rate", 16000), + ) + ) + + def _first_chunk_samples(self, default_chunk_size: int, processor: Any | None = None) -> int: + processor = processor or self.processor + if getattr(processor, "_streaming_mel_processor", None) is None: + return default_chunk_size + return int(self._stage_param("first_chunk_ms", 1035) * self._sample_rate(processor) / 1000) + + def _pad_first_audio_chunk_if_needed( + self, + state: _MiniCPMO45Stage0SessionState, + processor: Any | None = None, + ) -> None: + if state.audio_chunk_idx != 0 or len(state.audio_buffer) == 0: + return + first_chunk_samples = self._first_chunk_samples( + self._streaming_chunk_size(processor), + processor, + ) + if len(state.audio_buffer) >= first_chunk_samples: + return + padding = np.zeros(first_chunk_samples - len(state.audio_buffer), dtype=np.float32) + state.audio_buffer = np.concatenate([padding, state.audio_buffer]) + + def _stage_param(self, name: str, default: Any) -> Any: + for target in (self.stage_model, self.thinker, getattr(self.thinker, "llm", None)): + value = getattr(target, name, None) + if value is not None: + return value + value = getattr(target, name.upper(), None) + if value is not None: + return value + return default + + def _consumed_audio_samples( + self, + chunk_idx: int, + default_chunk_size: int, + *, + processor: Any | None = None, + ) -> int: + processor = processor or self.processor + if chunk_idx != 0: + chunk_ms = int(self._stage_param("chunk_ms", 1000)) + return int(chunk_ms * self._sample_rate(processor) / 1000) + mel_processor = getattr(processor, "_streaming_mel_processor", None) + get_config = getattr(mel_processor, "get_config", None) + if callable(get_config): + cfg = get_config() + if isinstance(cfg, dict): + consumed_ms = int(cfg.get("effective_first_chunk_ms", self._stage_param("first_chunk_ms", 1035))) + return int(consumed_ms * self._sample_rate(processor) / 1000) + return default_chunk_size + + def _process_streaming_audio( + self, + audio_chunk: Any, + chunk_idx: int, + *, + processor: Any | None = None, + ) -> Any: + processor = processor or self.processor + process = getattr(processor, "process_audio_streaming", None) + if callable(process): + try: + return process(audio_chunk, reset=False, return_batch_feature=True) + except TypeError: + return process(audio_chunk, chunk_idx=chunk_idx) + return {"audio_features": audio_chunk, "audio_feature_lens": [[len(audio_chunk)]]} + + def _stage_audio_embeddings( + self, + batch_feature: Any, + *, + state: _MiniCPMO45Stage0SessionState | None = None, + ) -> Any | None: + if hasattr(batch_feature, "to"): + batch_feature = batch_feature.to(self.device) + self._ensure_dynamic_cache_compat() + has_audio_cache = state is not None and hasattr(self.thinker, "audio_past_key_values") + previous_audio_past_key_values = ( + getattr(self.thinker, "audio_past_key_values", None) if has_audio_cache else None + ) + if has_audio_cache: + self.thinker.audio_past_key_values = state.audio_past_key_values + try: + for target in (self.stage_model, self.thinker): + get_streaming = getattr(target, "get_audio_embedding_streaming", None) + if callable(get_streaming): + try: + result = self._cat_nested_tensors( + get_streaming( + batch_feature, + use_extra_context=True, + prefix_extra_frames=0 if int(getattr(batch_feature, "chunk_idx", 0)) == 0 else 2, + suffix_extra_frames=2, + ) + ) + if has_audio_cache: + state.audio_past_key_values = getattr(self.thinker, "audio_past_key_values", None) + return result + except TypeError: + result = self._cat_nested_tensors(get_streaming(batch_feature)) + if has_audio_cache: + state.audio_past_key_values = getattr(self.thinker, "audio_past_key_values", None) + return result + get_hidden = getattr(target, "get_audio_hidden_states", None) + if callable(get_hidden): + result = self._cat_nested_tensors(get_hidden(batch_feature)) + if has_audio_cache: + state.audio_past_key_values = getattr(self.thinker, "audio_past_key_values", None) + return result + return None + finally: + if has_audio_cache: + self.thinker.audio_past_key_values = previous_audio_past_key_values + + @staticmethod + def _decode_ref_audio_from_session_config(session_config: dict[str, Any]) -> Any | None: + from vllm_omni.experimental.fullduplex.minicpmo45.input import decode_native_ref_audio_from_config + + return decode_native_ref_audio_from_config({"extra_body": session_config}) + + def _stage_ref_audio_embeddings( + self, + ref_audio: Any, + *, + state: _MiniCPMO45Stage0SessionState | None = None, + ) -> Any | None: + process_audio = getattr(self.processor, "process_audio", None) + if callable(process_audio): + batch_feature = process_audio([ref_audio]) + if hasattr(batch_feature, "to"): + batch_feature = batch_feature.to(self.device) + self._ensure_dynamic_cache_compat() + for target in (self.stage_model, self.thinker): + get_audio_embedding = getattr(target, "get_audio_embedding", None) + if callable(get_audio_embedding): + try: + chunk_length = getattr(getattr(target, "config", None), "audio_chunk_length", None) + if chunk_length is not None: + return self._cat_nested_tensors( + get_audio_embedding(batch_feature, chunk_length=chunk_length) + ) + except TypeError: + pass + return self._cat_nested_tensors(get_audio_embedding(batch_feature)) + # The split vLLM stage0 wrapper ports official + # get_audio_embedding(chunk_length=...) as + # get_audio_hidden_states (chunk_length comes from config). + get_hidden = getattr(target, "get_audio_hidden_states", None) + if callable(get_hidden): + return self._cat_nested_tensors(get_hidden(batch_feature)) + # The split vLLM stage0 wrapper may only expose the streaming encoder + # path. Use it as a fallback so the server-resolved reference audio is + # still represented in the same system context location as official + # MiniCPM-o prepare(). + processor = state.streaming_processor if state is not None else self.processor + batch_feature = self._process_streaming_audio(ref_audio, 0, processor=processor) + for name, value in ( + ("chunk_idx", 0), + ("use_extra_context", True), + ("prefix_extra_frames", 0), + ("suffix_extra_frames", 2), + ): + with suppress(Exception): + setattr(batch_feature, name, value) + return self._stage_audio_embeddings(batch_feature, state=state) + + @staticmethod + def _ensure_dynamic_cache_compat() -> None: + try: + from transformers.cache_utils import DynamicCache + except Exception: + return + if hasattr(DynamicCache, "get_usable_length"): + return + + def get_usable_length(self, new_seq_length: int | None = None, layer_idx: int = 0) -> int: + get_seq_length = getattr(self, "get_seq_length", None) + if not callable(get_seq_length): + return 0 + try: + return int(get_seq_length(layer_idx)) + except TypeError: + return int(get_seq_length()) + + DynamicCache.get_usable_length = get_usable_length # type: ignore[attr-defined] + + @staticmethod + def _cat_nested_tensors(value: Any) -> Any | None: + import torch + + tensors = [] + + def collect(item: Any) -> None: + if item is None: + return + if hasattr(item, "detach"): + tensors.append(item) + return + if isinstance(item, dict): + for child in item.values(): + collect(child) + return + if isinstance(item, (list, tuple)): + for child in item: + collect(child) + + collect(value) + if not tensors: + return None + return torch.cat([tensor.reshape(-1, tensor.shape[-1]) for tensor in tensors], dim=0) + + def _encode_text(self, text: str) -> list[int]: + encode = getattr(self.tokenizer, "encode", None) + if callable(encode): + return list(encode(text, add_special_tokens=False)) + return [] + + def _special_token_ids(self) -> dict[str, int]: + return { + name: value + for name, value in { + "unit_token_id": self.unit_token_id, + "unit_end_token_id": self.unit_end_token_id, + "listen_token_id": self.listen_token_id, + "speak_token_id": self.speak_token_id, + "tts_bos_token_id": self.tts_bos_token_id, + "tts_eos_token_id": self.tts_eos_token_id, + "tts_pad_token_id": self.tts_pad_token_id, + "chunk_eos_token_id": self.chunk_eos_token_id, + "chunk_tts_eos_token_id": self.chunk_tts_eos_token_id, + "turn_eos_token_id": self.turn_eos_token_id, + }.items() + if isinstance(value, int) and value >= 0 + } + + @staticmethod + def _load_processor_from_path(model_path: str | None) -> Any | None: + if not model_path: + return None + if model_path not in sys.path: + sys.path.insert(0, model_path) + try: + from processing_minicpmo import MiniCPMOProcessor + + return MiniCPMOProcessor.from_pretrained(model_path, trust_remote_code=True) + except Exception: + return None + + def _init_token_ids(self) -> None: + if self.tokenizer is None: + for field_name in _MINICPMO45_SPECIAL_TOKEN_FIELDS: + setattr(self, field_name, -1) + for field_name in _MINICPMO45_OPTIONAL_TOKEN_FIELDS: + setattr(self, field_name, -1) + else: + for field_name, token in _MINICPMO45_SPECIAL_TOKEN_FIELDS.items(): + setattr(self, field_name, self._resolve_special_token_id(token)) + for field_name, token in _MINICPMO45_OPTIONAL_TOKEN_FIELDS.items(): + setattr(self, field_name, self._resolve_special_token_id(token)) + + def _resolve_special_token_id(self, token: str) -> int: + if self.tokenizer is None: + return -1 + + unk_token_id = getattr(self.tokenizer, "unk_token_id", None) + candidate = None + convert = getattr(self.tokenizer, "convert_tokens_to_ids", None) + if callable(convert): + value = convert(token) + if isinstance(value, list): + value = value[0] if len(value) == 1 else None + with suppress(TypeError, ValueError): + candidate = int(value) + if candidate is not None and candidate >= 0 and candidate != unk_token_id: + return candidate + + encode = getattr(self.tokenizer, "encode", None) + if callable(encode): + ids = list(encode(token, add_special_tokens=False)) + if len(ids) == 1: + value = int(ids[0]) + if value >= 0 and value != unk_token_id: + return value + return -1 + + def _require_special_token_ids(self) -> None: + missing = [ + token + for field_name, token in _MINICPMO45_SPECIAL_TOKEN_FIELDS.items() + if not isinstance(getattr(self, field_name, None), int) or getattr(self, field_name) < 0 + ] + if missing: + raise ValueError( + "MiniCPM-o 4.5 native duplex requires tokenizer-defined special " + f"tokens, missing or unknown: {', '.join(missing)}" + ) + + def _required_token_id(self, field_name: str) -> int: + token_id = getattr(self, field_name, None) + if not isinstance(token_id, int) or token_id < 0: + token = _MINICPMO45_SPECIAL_TOKEN_FIELDS.get(field_name, field_name) + raise ValueError(f"MiniCPM-o 4.5 missing required special token id for {token}") + return token_id + + def stage_padding_token_id(self) -> int: + return self._required_token_id("unit_end_token_id") + + def _audio_embedding_placeholder_token_id(self) -> int: + token_id = getattr(self, "audio_placeholder_token_id", -1) + if isinstance(token_id, int) and token_id >= 0: + return token_id + return self.stage_padding_token_id() + + @staticmethod + def _decode_audio_payload(payload: dict[str, Any]) -> Any: + audio = payload.get("audio") or payload.get("data") + if not isinstance(audio, str): + raise ValueError("audio append payload requires base64 audio") + fmt = payload.get("format") or "pcm_f32le" + if fmt != "pcm_f32le": + raise ValueError(f"MiniCPM-o stage0 expects pcm_f32le audio, got {fmt!r}") + return np.frombuffer(base64.b64decode(audio), dtype=np.float32) + + @staticmethod + def _decode_video_frames_payload(payload: dict[str, Any]) -> list[Any]: + """Decode omni-duplex camera frames (base64 JPEG/PNG) to PIL images.""" + frames = payload.get("video_frames") + if not isinstance(frames, list) or not frames: + return [] + from io import BytesIO + + from PIL import Image + + decoded: list[Any] = [] + for frame_b64 in frames: + if not isinstance(frame_b64, str) or not frame_b64: + continue + try: + raw = base64.b64decode(frame_b64, validate=True) + image = Image.open(BytesIO(raw)) + image.load() + except Exception as exc: # noqa: BLE001 - normalized below + raise ValueError("invalid omni duplex video frame payload") from exc + decoded.append(image.convert("RGB")) + return decoded + + def _vision_embedding_placeholder_token_id(self) -> int: + unk_token_id = getattr(self.tokenizer, "unk_token_id", None) + if isinstance(unk_token_id, int) and unk_token_id >= 0: + return unk_token_id + return self._required_optional_token_id("image_start_token_id") + + def _required_optional_token_id(self, field_name: str) -> int: + token_id = getattr(self, field_name, None) + if not isinstance(token_id, int) or token_id < 0: + token = _MINICPMO45_OPTIONAL_TOKEN_FIELDS.get(field_name, field_name) + raise ValueError(f"MiniCPM-o 4.5 missing required special token id for {token}") + return token_id + + def _require_vision_token_ids(self) -> None: + missing = [ + _MINICPMO45_OPTIONAL_TOKEN_FIELDS[field_name] + for field_name in ("image_start_token_id", "image_end_token_id") + if not isinstance(getattr(self, field_name, None), int) or getattr(self, field_name) < 0 + ] + if missing: + raise ValueError( + "MiniCPM-o 4.5 omni duplex requires tokenizer-defined image tokens, " + f"missing or unknown: {', '.join(missing)}" + ) + + def _stage_vision_embeddings(self, frames: list[Any]) -> list[Any] | None: + """Encode camera frames for omni duplex via the loaded vision tower. + + Semantics mirror ``MiniCPMODuplex.streaming_prefill`` (one 64-embedding + block per frame at ``max_slice_nums=1``), executed through the vLLM + wrapper's ``get_vision_hidden_states`` (vpm + resampler) since the + stage model does not expose the remote-code ``get_vision_embedding``. + """ + process_image = getattr(self.processor, "process_image", None) + if not callable(process_image): + return None + try: + processed = process_image(frames, max_slice_nums=1) + except Exception: # noqa: BLE001 - prefill fails with a reason + return None + targets = (self.stage_model, self.thinker, getattr(self.stage_model, "model", None)) + for target in targets: + if target is None: + continue + get_hidden = getattr(target, "get_vision_hidden_states", None) + vpm = getattr(target, "vpm", None) + if not callable(get_hidden) or vpm is None: + continue + try: + import torch + + vpm_param = next(vpm.parameters()) + device, dtype = vpm_param.device, vpm_param.dtype + pixel_nested = processed["pixel_values"] + tgt_nested = processed["tgt_sizes"] + flat_pixels: list[Any] = [] + flat_tgt: list[Any] = [] + for image_slices, image_tgt in zip(pixel_nested, tgt_nested): + for slice_pixels in image_slices: + flat_pixels.append(slice_pixels.to(device=device, dtype=dtype)) + tgt_tensor = image_tgt if hasattr(image_tgt, "reshape") else torch.tensor(image_tgt) + flat_tgt.append(tgt_tensor.reshape(-1, 2)) + tgt_sizes = torch.cat(flat_tgt, dim=0).to(device=device, dtype=torch.int64) + with torch.no_grad(): + hidden = get_hidden({"pixel_values": flat_pixels, "tgt_sizes": tgt_sizes}) + except Exception: # noqa: BLE001 - prefill fails with a reason + return None + expected = MiniCPMO45DuplexPolicy.VISION_EMBEDS_PER_FRAME + out: list[Any] = [] + for block in hidden: + block_2d = self._as_2d_tensor(block) + if int(block_2d.shape[0]) != expected: + return None + out.append(block_2d) + return out + return None diff --git a/vllm_omni/experimental/fullduplex/model_executor.py b/vllm_omni/experimental/fullduplex/model_executor.py new file mode 100644 index 000000000..eebe01876 --- /dev/null +++ b/vllm_omni/experimental/fullduplex/model_executor.py @@ -0,0 +1,106 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + + +@dataclass(frozen=True, slots=True) +class DuplexSamplingRow: + """Request-local context for the experimental duplex sampling hook.""" + + row_idx: int + request_id: str + session_id: str | None + incarnation: int + seq: int | None + payload: dict[str, object] | None + max_tokens: int | None + + +class DuplexSamplingHelper: + """Tracks duplex sampling rows outside the generic AR model runner.""" + + def __init__(self) -> None: + self.active_request_ids: set[str] = set() + self.hook_active = False + + @staticmethod + def _is_duplex_data_plane_info(info: object) -> bool: + duplex = info.get("duplex") if isinstance(info, dict) else None + return isinstance(duplex, dict) and duplex.get("data_plane") is True + + @staticmethod + def _request_intermediate_info(runner: object, req_id: str) -> dict[str, Any] | None: + model_intermediate_buffer = getattr(runner, "model_intermediate_buffer", {}) + info = model_intermediate_buffer.get(req_id) if isinstance(model_intermediate_buffer, dict) else None + if isinstance(info, dict): + return info + requests = getattr(runner, "requests", {}) + req_state = requests.get(req_id) if isinstance(requests, dict) else None + info = getattr(req_state, "additional_information_cpu", None) + return info if isinstance(info, dict) else None + + def refresh_active_request(self, runner: object, req_id: str) -> None: + if self._is_duplex_data_plane_info(self._request_intermediate_info(runner, req_id)): + self.active_request_ids.add(req_id) + else: + self.active_request_ids.discard(req_id) + + def update_states(self, runner: object, scheduler_output: object) -> None: + self.active_request_ids.difference_update(str(req_id) for req_id in scheduler_output.finished_req_ids) + for request in scheduler_output.scheduled_new_reqs: + self.refresh_active_request(runner, str(request.req_id)) + + def rows(self, runner: object) -> tuple[DuplexSamplingRow, ...]: + rows: list[DuplexSamplingRow] = [] + req_ids = [str(req_id) for req_id in getattr(runner.input_batch, "req_ids", [])] + requests = getattr(runner, "requests", {}) + for row_idx, req_id in enumerate(req_ids): + if req_id not in self.active_request_ids: + continue + info = self._request_intermediate_info(runner, req_id) + duplex = info.get("duplex") if isinstance(info, dict) else None + if not isinstance(duplex, dict) or duplex.get("data_plane") is not True: + continue + session_id = duplex.get("session_id") + if not isinstance(session_id, str) or not session_id: + session_id = None + try: + incarnation = int(duplex.get("incarnation", 0)) + except (TypeError, ValueError): + incarnation = 0 + try: + seq = int(duplex.get("seq")) + except (TypeError, ValueError): + seq = None + payload = duplex.get("payload") + if not isinstance(payload, dict): + payload = None + request = requests.get(req_id) if isinstance(requests, dict) else None + sampling_params = getattr(request, "sampling_params", None) + try: + max_tokens = int(getattr(sampling_params, "max_tokens", 0) or 0) + except (TypeError, ValueError): + max_tokens = 0 + rows.append( + DuplexSamplingRow( + row_idx=row_idx, + request_id=req_id, + session_id=session_id, + incarnation=incarnation, + seq=seq, + payload=payload, + max_tokens=max_tokens if max_tokens > 0 else None, + ) + ) + return tuple(rows) + + def clear(self) -> None: + self.active_request_ids.clear() + self.hook_active = False + + +__all__ = ["DuplexSamplingHelper", "DuplexSamplingRow"] diff --git a/vllm_omni/experimental/fullduplex/openai/__init__.py b/vllm_omni/experimental/fullduplex/openai/__init__.py new file mode 100644 index 000000000..1bb45f8d4 --- /dev/null +++ b/vllm_omni/experimental/fullduplex/openai/__init__.py @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from .websocket import DuplexWebSocketActor + +__all__ = ["DuplexWebSocketActor"] diff --git a/vllm_omni/experimental/fullduplex/openai/audio.py b/vllm_omni/experimental/fullduplex/openai/audio.py new file mode 100644 index 000000000..023bfdd9e --- /dev/null +++ b/vllm_omni/experimental/fullduplex/openai/audio.py @@ -0,0 +1,185 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from __future__ import annotations + +import base64 +import binascii +import io +import math +import wave + +import numpy as np + +try: + from audioop import alaw2lin, lin2alaw, lin2ulaw, ulaw2lin +except ImportError: # pragma: no cover - audioop is removed in newer Python. + alaw2lin = lin2alaw = lin2ulaw = ulaw2lin = None + + +MIN_INPUT_SAMPLE_RATE_HZ = 8_000 +MAX_INPUT_SAMPLE_RATE_HZ = 192_000 + + +def validate_input_sample_rate_hz(sample_rate_hz: int | float) -> int: + if isinstance(sample_rate_hz, bool) or (isinstance(sample_rate_hz, float) and not math.isfinite(sample_rate_hz)): + raise ValueError("sample_rate_hz must be a finite integer") + rate = int(sample_rate_hz) + if rate != sample_rate_hz: + raise ValueError("sample_rate_hz must be an integer") + if not MIN_INPUT_SAMPLE_RATE_HZ <= rate <= MAX_INPUT_SAMPLE_RATE_HZ: + raise ValueError(f"sample_rate_hz must be between {MIN_INPUT_SAMPLE_RATE_HZ} and {MAX_INPUT_SAMPLE_RATE_HZ}") + return rate + + +def resample_pcm16_mono(raw: bytes, *, source_rate_hz: int, target_rate_hz: int) -> bytes: + if source_rate_hz <= 0 or target_rate_hz <= 0 or source_rate_hz == target_rate_hz: + return raw + samples = np.frombuffer(raw, dtype=" bytes: + if ulaw2lin is not None: + return ulaw2lin(raw, 2) + data = np.frombuffer(raw, dtype=np.uint8) + value = np.bitwise_not(data).astype(np.int16) + sign = value & 0x80 + exponent = (value >> 4) & 0x07 + mantissa = value & 0x0F + sample = ((mantissa << 3) + 0x84) << exponent + return np.where(sign != 0, -(sample - 0x84), sample - 0x84).astype(" bytes: + if alaw2lin is not None: + return alaw2lin(raw, 2) + data = np.bitwise_xor(np.frombuffer(raw, dtype=np.uint8), 0x55).astype(np.int16) + sign = data & 0x80 + exponent = (data >> 4) & 0x07 + mantissa = data & 0x0F + sample = np.where(exponent == 0, (mantissa << 4) + 8, ((mantissa << 4) + 0x108) << (exponent - 1)) + return np.where(sign != 0, sample, -sample).astype(" bytes: + if lin2ulaw is not None: + return lin2ulaw(raw, 2) + pcm = np.clip(np.frombuffer(raw, dtype=" (0xFF << exp), exp + 1, exponent) + mantissa = (magnitude >> (exponent + 3)) & 0x0F + return (np.bitwise_not(sign | (exponent << 4) | mantissa) & 0xFF).astype(np.uint8).tobytes() + + +def encode_g711_alaw(raw: bytes) -> bytes: + if lin2alaw is not None: + return lin2alaw(raw, 2) + pcm = np.frombuffer(raw, dtype="= 0, 0x80, 0x00) + magnitude = np.abs(pcm) + exponent = np.zeros_like(magnitude) + for exp in range(1, 8): + exponent = np.where(magnitude >= (1 << (exp + 7)), exp, exponent) + mantissa = np.where( + exponent == 0, + (magnitude >> 4) & 0x0F, + (magnitude >> (exponent + 3)) & 0x0F, + ) + return ((sign | (exponent << 4) | mantissa) ^ 0x55).astype(np.uint8).tobytes() + + +def wav_payload_to_pcm16(raw: bytes) -> tuple[bytes | None, int | None]: + try: + with wave.open(io.BytesIO(raw), "rb") as wav_file: + channels = wav_file.getnchannels() + sample_width = wav_file.getsampwidth() + sample_rate_hz = wav_file.getframerate() + frames = wav_file.readframes(wav_file.getnframes()) + if sample_width != 2: + return None, sample_rate_hz + if channels <= 1: + return frames, sample_rate_hz + pcm = np.frombuffer(frames, dtype=" tuple[object, object, int | float | None]: + if not isinstance(audio, str) or not isinstance(fmt, str): + return audio, fmt, sample_rate_hz + normalized = fmt.lower() + if normalized not in {"pcm16", "pcm_s16le", "s16le", "g711_ulaw", "g711_alaw"}: + return audio, fmt, sample_rate_hz + if isinstance(sample_rate_hz, int | float): + sample_rate_hz = validate_input_sample_rate_hz(sample_rate_hz) + try: + raw = base64.b64decode(audio.strip(), validate=False) + except (binascii.Error, ValueError): + return audio, fmt, sample_rate_hz + if normalized == "g711_ulaw": + raw = decode_g711_ulaw(raw) + sample_rate_hz = sample_rate_hz if isinstance(sample_rate_hz, int | float) else 8_000 + elif normalized == "g711_alaw": + raw = decode_g711_alaw(raw) + sample_rate_hz = sample_rate_hz if isinstance(sample_rate_hz, int | float) else 8_000 + elif len(raw) % 2: + return audio, fmt, sample_rate_hz + if isinstance(sample_rate_hz, int | float) and int(sample_rate_hz) != target_sample_rate_hz: + raw = resample_pcm16_mono( + raw, + source_rate_hz=int(sample_rate_hz), + target_rate_hz=target_sample_rate_hz, + ) + sample_rate_hz = target_sample_rate_hz + pcm = np.frombuffer(raw, dtype=" tuple[str, str, int | None]: + target = target_fmt.lower() + if target not in {"g711_ulaw", "g711_alaw"}: + return audio, source_fmt, source_sample_rate_hz + try: + raw = base64.b64decode(audio, validate=False) + except (binascii.Error, ValueError): + return audio, source_fmt, source_sample_rate_hz + source = source_fmt.lower() + if source == "wav": + pcm_raw, wav_rate = wav_payload_to_pcm16(raw) + if pcm_raw is None: + return audio, source_fmt, source_sample_rate_hz + raw = pcm_raw + source_sample_rate_hz = source_sample_rate_hz or wav_rate + elif source not in {"pcm", "pcm16", "pcm_s16le", "s16le"} or len(raw) % 2: + return audio, source_fmt, source_sample_rate_hz + target_rate = target_sample_rate_hz or 8_000 + if source_sample_rate_hz is not None: + raw = resample_pcm16_mono(raw, source_rate_hz=source_sample_rate_hz, target_rate_hz=target_rate) + encoded = encode_g711_ulaw(raw) if target == "g711_ulaw" else encode_g711_alaw(raw) + return base64.b64encode(encoded).decode("ascii"), target, target_rate diff --git a/vllm_omni/experimental/fullduplex/openai/chat_fallback.py b/vllm_omni/experimental/fullduplex/openai/chat_fallback.py new file mode 100644 index 000000000..65d3b32fd --- /dev/null +++ b/vllm_omni/experimental/fullduplex/openai/chat_fallback.py @@ -0,0 +1,252 @@ +from __future__ import annotations + +import asyncio +import json +from collections.abc import AsyncGenerator +from typing import Any + +from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest +from vllm.entrypoints.openai.engine.protocol import ErrorResponse +from vllm.logger import init_logger + +from vllm_omni.experimental.fullduplex.openai.protocol import DuplexSession + +logger = init_logger(__name__) + + +class ChatFallbackProjectorMixin: + """Project generic chat completion output into duplex response events.""" + + async def _run_response(self, session: DuplexSession, send_json) -> None: + response_id = session.begin_response() + epoch = session.epoch + request_id = f"duplex-{session.session_id}-{epoch}-{session.input_commit_seq}" + session.bind_request(f"chatcmpl-{request_id}") + await send_json( + self._response_created_payload( + session, + response_id, + epoch=epoch, + request_id=session.active_request_id, + ) + ) + + try: + request = self._build_chat_request(session, request_id) + result = await self._chat_service.create_chat_completion(request, raw_request=None) + if isinstance(result, ErrorResponse): + await send_json({"type": "error", "error": result.message, "code": result.type or "chat_error"}) + session.end_response(commit_text=False) + return + if hasattr(result, "__aiter__"): + await self._drain_streaming_response(session, result, epoch, response_id, send_json) + else: + await self._emit_full_response(session, result, epoch, response_id, send_json) + if session.epoch == epoch: + should_commit = self._should_commit_response_to_history(session, response_id) + committed_message = session.end_response(commit_text=should_commit) + if should_commit: + session.register_history_item(f"item_{response_id}", committed_message) + await send_json( + { + "type": "response.done", + "session_id": session.session_id, + "response_id": response_id, + "epoch": epoch, + "committed": committed_message is not None, + "playback": session.playback.as_dict(), + } + ) + except asyncio.CancelledError: + raise + except Exception as exc: + logger.exception("Duplex response failed: %s", exc) + session.end_response(commit_text=False) + await send_json( + { + "type": "error", + "session_id": session.session_id, + "response_id": response_id, + "error": str(exc), + "code": "response_error", + } + ) + + def _build_chat_request(self, session: DuplexSession, request_id: str) -> ChatCompletionRequest: + response_config = session.response_config + messages: list[dict[str, object]] = [] + if response_config.instructions: + messages.append({"role": "system", "content": response_config.instructions}) + messages.extend(session.history) + + kwargs: dict[str, Any] = { + "model": response_config.model or self._chat_service.model_config.model, + "messages": messages, + "stream": True, + } + if response_config.temperature is not None: + kwargs["temperature"] = response_config.temperature + if response_config.max_tokens is not None: + kwargs["max_tokens"] = response_config.max_tokens + model_extra = dict(response_config.extra_body) + tools = model_extra.pop("realtime_response_tools", model_extra.pop("realtime_tools", None)) + tool_choice = model_extra.pop( + "realtime_response_tool_choice", + model_extra.pop("realtime_tool_choice", None), + ) + for protocol_key in ( + "realtime_response_conversation", + "realtime_response_metadata", + "realtime_response_prompt", + ): + model_extra.pop(protocol_key, None) + kwargs.update(model_extra) + if isinstance(tools, list): + kwargs["tools"] = tools + if isinstance(tool_choice, str | dict): + kwargs["tool_choice"] = tool_choice + + request = ChatCompletionRequest(**kwargs) + object.__setattr__(request, "modalities", response_config.modalities) + object.__setattr__(request, "request_id", request_id) + object.__setattr__( + request, + "chat_template_kwargs", + {"use_tts_template": response_config.use_tts_template}, + ) + return request + + async def _drain_streaming_response( + self, + session: DuplexSession, + result: AsyncGenerator[str, None], + epoch: int, + response_id: str, + send_json, + ) -> None: + async for raw_chunk in result: + if session.epoch != epoch: + return + for payload in self._parse_sse_payloads(raw_chunk): + if payload == "[DONE]": + continue + if isinstance(payload, dict): + await self._emit_chat_payload(session, payload, epoch, response_id, send_json) + + async def _emit_full_response( + self, + session: DuplexSession, + result: Any, + epoch: int, + response_id: str, + send_json, + ) -> None: + if hasattr(result, "model_dump"): + payload = result.model_dump(mode="json", exclude_unset=True) + else: + payload = {"response": str(result)} + await self._emit_chat_payload(session, payload, epoch, response_id, send_json) + + def _parse_sse_payloads(self, raw_chunk: str) -> list[dict[str, object] | str]: + payloads: list[dict[str, object] | str] = [] + for line in raw_chunk.splitlines(): + line = line.strip() + if not line.startswith("data:"): + continue + data = line[5:].strip() + if not data: + continue + if data == "[DONE]": + payloads.append(data) + continue + try: + parsed = json.loads(data) + except json.JSONDecodeError: + logger.debug("Skipping non-JSON duplex stream payload: %s", data) + continue + if isinstance(parsed, dict): + payloads.append(parsed) + return payloads + + async def _emit_chat_payload( + self, + session: DuplexSession, + payload: dict[str, object], + epoch: int, + response_id: str, + send_json, + ) -> None: + modality = payload.get("modality") + if modality not in {None, "text", "audio"}: + await send_json( + { + "type": "error", + "session_id": session.session_id, + "response_id": response_id, + "epoch": epoch, + "code": "unsupported_response_modality", + "error": f"Unsupported chat response modality: {modality}", + } + ) + return + choices = payload.get("choices") + if not isinstance(choices, list): + await send_json( + { + "type": "response.message", + "session_id": session.session_id, + "response_id": response_id, + "epoch": epoch, + "payload": payload, + } + ) + return + + for choice in choices: + if not isinstance(choice, dict): + continue + delta = choice.get("delta") + message = choice.get("message") + content = None + if isinstance(delta, dict): + content = delta.get("content") + elif isinstance(message, dict): + content = message.get("content") + + if isinstance(content, str) and content: + if modality == "audio": + session.mark_audio_sent() + await send_json( + { + "type": "response.output_audio.delta", + "session_id": session.session_id, + "response_id": response_id, + "epoch": epoch, + "audio": content, + "format": session.response_config.response_format, + } + ) + else: + session.append_assistant_text(content) + await send_json( + { + "type": "response.text.delta", + "session_id": session.session_id, + "response_id": response_id, + "epoch": epoch, + "delta": content, + } + ) + + finish_reason = choice.get("finish_reason") + if finish_reason is not None and modality != "audio": + await send_json( + { + "type": "response.output_item.done", + "session_id": session.session_id, + "response_id": response_id, + "epoch": epoch, + "finish_reason": finish_reason, + "modality": modality, + } + ) diff --git a/vllm_omni/experimental/fullduplex/openai/commit_policy.py b/vllm_omni/experimental/fullduplex/openai/commit_policy.py new file mode 100644 index 000000000..940509ec3 --- /dev/null +++ b/vllm_omni/experimental/fullduplex/openai/commit_policy.py @@ -0,0 +1,38 @@ +from __future__ import annotations + +from dataclasses import dataclass +from enum import Enum + + +class CommitAction(str, Enum): + """How one native audio commit participates in response scheduling.""" + + START_AUTO_RESPONSE = "start_auto_response" + DEFER_ACTIVE_RESPONSE = "defer_active_response" + COMMIT_ONLY = "commit_only" + + +@dataclass(frozen=True, slots=True) +class CommitSnapshot: + auto_responds: bool + speech_since_commit: bool + active_response_id: str | None + overlap_speech_ms: int + native_response_in_progress: bool + playback_active: bool + + +def decide_commit_action(snapshot: CommitSnapshot) -> CommitAction: + """Choose response scheduling from an immutable session snapshot.""" + if snapshot.auto_responds: + if snapshot.speech_since_commit: + return CommitAction.START_AUTO_RESPONSE + return CommitAction.COMMIT_ONLY + if snapshot.native_response_in_progress and snapshot.overlap_speech_ms > 0: + return CommitAction.DEFER_ACTIVE_RESPONSE + if snapshot.playback_active and snapshot.overlap_speech_ms > 0: + return CommitAction.DEFER_ACTIVE_RESPONSE + return CommitAction.COMMIT_ONLY + + +__all__ = ["CommitAction", "CommitSnapshot", "decide_commit_action"] diff --git a/vllm_omni/experimental/fullduplex/openai/protocol.py b/vllm_omni/experimental/fullduplex/openai/protocol.py new file mode 100644 index 000000000..0a87d55b4 --- /dev/null +++ b/vllm_omni/experimental/fullduplex/openai/protocol.py @@ -0,0 +1,1336 @@ +from __future__ import annotations + +import copy +from collections.abc import Mapping +from dataclasses import dataclass, field +from enum import Enum +from types import MappingProxyType +from uuid import uuid4 + + +class DuplexOverlapPolicy(str, Enum): + AUTO = "auto" + LISTEN_ONLY = "listen_only" + BARGE_IN_ON_SPEECH = "barge_in_on_speech" + + +class DuplexPlaybackCommitPolicy(str, Enum): + COMMIT_ALL_ON_DONE = "commit_all_on_done" + ACK_ONLY = "ack_only" + + +class DuplexSessionState(str, Enum): + OPEN = "open" + CLOSING = "closing" + CLOSED = "closed" + + +class DuplexTurnState(str, Enum): + IDLE = "idle" + USER_SPEAKING = "user_speaking" + USER_COMMITTED = "user_committed" + ASSISTANT_GENERATING = "assistant_generating" + ASSISTANT_PLAYING = "assistant_playing" + BARGE_IN = "barge_in" + + +class DuplexTurnEventType(str, Enum): + USER_STARTED = "user_started" + USER_COMMITTED = "user_committed" + ASSISTANT_STARTED = "assistant_started" + ASSISTANT_DONE = "assistant_done" + BARGE_IN = "barge_in" + PLAYBACK_ACK = "playback_ack" + TIMEOUT = "timeout" + CLOSE = "close" + + +@dataclass +class DuplexCapabilities: + """Runtime/model capabilities exposed by the duplex serving protocol. + + These are intentionally explicit so the serving layer does not assume all + duplex models support the same input append, rollback, or turn policy. + ``supports_core_kv_lease`` is reserved for scheduler-owned KV lifecycle; + model-owned decoder/TTS state must use ``supports_model_internal_state``. + ``supports_core_resumable_request`` means the scheduler can resume the + same request id across streaming updates, but it is not a KV lease by + itself. Realtime support means this endpoint can speak the native Realtime + event schema for the supported audio duplex paths while keeping model- or + scheduler-specific limits explicit in the capability payload. + """ + + supports_session_adapter: bool = True + supports_model_native_turn_policy: bool = False + supports_external_turn_signal: bool = True + supports_client_commit: bool = True + supports_barge_in: bool = True + supports_playback_ack: bool = True + supports_input_append: bool = False + supports_replace_latest_chunk: bool = True + supports_reencode_context: bool = True + supports_rollback_to_checkpoint: bool = False + supports_turn_commit_only: bool = True + supports_kv_lease: bool = False + supports_core_kv_lease: bool = False + supports_model_internal_state: bool = False + supports_stage_resumption: bool = False + supports_scheduler_native_append: bool = False + supports_core_resumable_request: bool = False + supports_stage_connector_handoff: bool = False + supports_independent_io_streams: bool = False + supports_realtime_endpoint: bool = False + supports_multi_session: bool = False + supports_multi_session_same_replica: bool = False + supports_session_lease: bool = False + supports_session_resume: bool = False + session_admission_mode: str = "serving_managed" + supports_audio_truncate: bool = False + requires_model_runner_kv: bool = False + requires_native_stage_role: bool = False + implementation_level: str = "serving_session_adapter" + adapter_patterns: list[str] = field(default_factory=lambda: ["chunk_group_append"]) + input_modes: list[str] = field(default_factory=lambda: ["turn_commit_only", "reencode_context"]) + signal_sources: list[str] = field(default_factory=lambda: ["client_event", "server_policy", "model_native"]) + stage_handoff_transport: str | None = None + chunk_period_ms: int | None = 1000 + target_barge_in_latency_ms: int | None = 1000 + + @classmethod + def minicpmo45_native(cls, *, max_sessions: int = 1) -> DuplexCapabilities: + supports_multi_session = max_sessions > 1 + return cls( + supports_model_native_turn_policy=True, + supports_barge_in=False, + supports_input_append=True, + supports_replace_latest_chunk=False, + supports_reencode_context=False, + supports_turn_commit_only=False, + supports_kv_lease=False, + supports_core_kv_lease=False, + supports_model_internal_state=True, + supports_stage_resumption=True, + supports_scheduler_native_append=False, + supports_core_resumable_request=True, + supports_stage_connector_handoff=True, + supports_independent_io_streams=True, + supports_realtime_endpoint=True, + supports_multi_session=supports_multi_session, + supports_multi_session_same_replica=supports_multi_session, + supports_session_lease=True, + supports_session_resume=True, + session_admission_mode="engine_managed", + supports_audio_truncate=True, + requires_model_runner_kv=True, + requires_native_stage_role=True, + implementation_level="model_native_duplex", + adapter_patterns=["scheduler_data_plane"], + input_modes=["append_audio_chunk"], + signal_sources=["model_native", "client_event", "server_policy"], + stage_handoff_transport="scheduler_data_plane", + chunk_period_ms=1000, + target_barge_in_latency_ms=None, + ) + + def as_dict(self) -> dict[str, object]: + return { + "supports_session_adapter": self.supports_session_adapter, + "supports_model_native_turn_policy": self.supports_model_native_turn_policy, + "supports_external_turn_signal": self.supports_external_turn_signal, + "supports_client_commit": self.supports_client_commit, + "supports_barge_in": self.supports_barge_in, + "supports_playback_ack": self.supports_playback_ack, + "supports_input_append": self.supports_input_append, + "supports_replace_latest_chunk": self.supports_replace_latest_chunk, + "supports_reencode_context": self.supports_reencode_context, + "supports_rollback_to_checkpoint": self.supports_rollback_to_checkpoint, + "supports_turn_commit_only": self.supports_turn_commit_only, + "supports_kv_lease": self.supports_kv_lease, + "supports_core_kv_lease": self.supports_core_kv_lease, + "supports_model_internal_state": self.supports_model_internal_state, + "supports_stage_resumption": self.supports_stage_resumption, + "supports_scheduler_native_append": self.supports_scheduler_native_append, + "supports_core_resumable_request": self.supports_core_resumable_request, + "supports_stage_connector_handoff": self.supports_stage_connector_handoff, + "supports_independent_io_streams": self.supports_independent_io_streams, + "supports_realtime_endpoint": self.supports_realtime_endpoint, + "supports_multi_session": self.supports_multi_session, + "supports_multi_session_same_replica": self.supports_multi_session_same_replica, + "supports_session_lease": self.supports_session_lease, + "supports_session_resume": self.supports_session_resume, + "session_admission_mode": self.session_admission_mode, + "supports_audio_truncate": self.supports_audio_truncate, + "requires_model_runner_kv": self.requires_model_runner_kv, + "requires_native_stage_role": self.requires_native_stage_role, + "implementation_level": self.implementation_level, + "adapter_patterns": self.adapter_patterns, + "input_modes": self.input_modes, + "signal_sources": self.signal_sources, + "stage_handoff_transport": self.stage_handoff_transport, + "chunk_period_ms": self.chunk_period_ms, + "target_barge_in_latency_ms": self.target_barge_in_latency_ms, + } + + +@dataclass +class DuplexPlaybackCursor: + generated_ms: int = 0 + sent_ms: int = 0 + played_ms: int = 0 + committed_ms: int = 0 + + def acknowledge(self, played_ms: int, committed_ms: int | None = None) -> None: + self.played_ms = max(self.played_ms, max(0, int(played_ms))) + if committed_ms is None: + committed_ms = self.played_ms + self.committed_ms = max(self.committed_ms, max(0, int(committed_ms))) + + def truncate_committed(self, committed_ms: int) -> None: + self.committed_ms = max(0, min(max(self.sent_ms, self.generated_ms), int(committed_ms))) + + def as_dict(self) -> dict[str, int]: + return { + "generated_ms": self.generated_ms, + "sent_ms": self.sent_ms, + "played_ms": self.played_ms, + "committed_ms": self.committed_ms, + } + + def snapshot(self) -> DuplexPlaybackView: + return DuplexPlaybackView(**self.as_dict()) + + +@dataclass(frozen=True, slots=True) +class DuplexPlaybackView: + generated_ms: int = 0 + sent_ms: int = 0 + played_ms: int = 0 + committed_ms: int = 0 + + def as_dict(self) -> dict[str, int]: + return { + "generated_ms": self.generated_ms, + "sent_ms": self.sent_ms, + "played_ms": self.played_ms, + "committed_ms": self.committed_ms, + } + + +@dataclass +class DuplexAudioChunk: + data: str + format: str = "wav" + sample_rate_hz: int | None = None + + +@dataclass +class DuplexSessionConfig: + model: str | None = None + modalities: list[str] = field(default_factory=lambda: ["text", "audio"]) + instructions: str | None = None + voice: str | None = None + ref_audio: str | None = None + response_format: str = "wav" + temperature: float | None = None + max_tokens: int | None = None + speed: float | None = None + use_tts_template: bool = True + idle_timeout_s: float = 300.0 + overlap_policy: str = DuplexOverlapPolicy.LISTEN_ONLY.value + overlap_short_ack_ms: int = 700 + overlap_barge_in_ms: int = 1200 + overlap_silence_rms: float = 0.003 + playback_commit_policy: str = DuplexPlaybackCommitPolicy.COMMIT_ALL_ON_DONE.value + extra_body: dict[str, object] = field(default_factory=dict) + + def as_dict(self) -> dict[str, object]: + return { + "model": self.model, + "modalities": list(self.modalities), + "instructions": self.instructions, + "voice": self.voice, + "ref_audio": self.ref_audio, + "response_format": self.response_format, + "temperature": self.temperature, + "max_tokens": self.max_tokens, + "speed": self.speed, + "use_tts_template": self.use_tts_template, + "idle_timeout_s": self.idle_timeout_s, + "overlap_policy": self.overlap_policy, + "overlap_short_ack_ms": self.overlap_short_ack_ms, + "overlap_barge_in_ms": self.overlap_barge_in_ms, + "overlap_silence_rms": self.overlap_silence_rms, + "playback_commit_policy": self.playback_commit_policy, + "extra_body": dict(self.extra_body), + } + + @classmethod + def from_event(cls, event: dict[str, object]) -> DuplexSessionConfig: + payload = event.get("session") + if isinstance(payload, dict): + source = payload + else: + source = event + + config = cls() + if isinstance(source.get("model"), str): + config.model = source["model"] + if isinstance(source.get("instructions"), str): + config.instructions = source["instructions"] + if isinstance(source.get("voice"), str): + config.voice = source["voice"] + if isinstance(source.get("ref_audio"), str): + config.ref_audio = source["ref_audio"] + if isinstance(source.get("response_format"), str): + config.response_format = source["response_format"] + if isinstance(source.get("use_tts_template"), bool): + config.use_tts_template = bool(source["use_tts_template"]) + if isinstance(source.get("temperature"), int | float): + config.temperature = float(source["temperature"]) + if isinstance(source.get("max_tokens"), int): + config.max_tokens = int(source["max_tokens"]) + if isinstance(source.get("speed"), int | float): + config.speed = float(source["speed"]) + if isinstance(source.get("idle_timeout_s"), int | float): + config.idle_timeout_s = float(source["idle_timeout_s"]) + if isinstance(source.get("overlap_policy"), str): + config.overlap_policy = cls._normalize_overlap_policy(source["overlap_policy"]) + if isinstance(source.get("overlap_short_ack_ms"), int | float): + config.overlap_short_ack_ms = max(0, int(source["overlap_short_ack_ms"])) + if isinstance(source.get("overlap_barge_in_ms"), int | float): + config.overlap_barge_in_ms = max(0, int(source["overlap_barge_in_ms"])) + if isinstance(source.get("overlap_silence_rms"), int | float): + config.overlap_silence_rms = max(0.0, float(source["overlap_silence_rms"])) + if isinstance(source.get("playback_commit_policy"), str): + config.playback_commit_policy = cls._normalize_playback_commit_policy(source["playback_commit_policy"]) + if isinstance(source.get("modalities"), list) and all(isinstance(x, str) for x in source["modalities"]): + config.modalities = list(source["modalities"]) + if isinstance(source.get("extra_body"), dict): + config.extra_body = dict(source["extra_body"]) + extra = config.extra_body + if isinstance(extra.get("overlap_policy"), str): + config.overlap_policy = cls._normalize_overlap_policy(extra["overlap_policy"]) + if isinstance(extra.get("overlap_short_ack_ms"), int | float): + config.overlap_short_ack_ms = max(0, int(extra["overlap_short_ack_ms"])) + if isinstance(extra.get("overlap_barge_in_ms"), int | float): + config.overlap_barge_in_ms = max(0, int(extra["overlap_barge_in_ms"])) + if isinstance(extra.get("overlap_silence_rms"), int | float): + config.overlap_silence_rms = max(0.0, float(extra["overlap_silence_rms"])) + if isinstance(extra.get("playback_commit_policy"), str): + config.playback_commit_policy = cls._normalize_playback_commit_policy(extra["playback_commit_policy"]) + return config + + @staticmethod + def _normalize_overlap_policy(value: str) -> str: + normalized = value.strip().lower() + if normalized in {policy.value for policy in DuplexOverlapPolicy}: + return normalized + return DuplexOverlapPolicy.LISTEN_ONLY.value + + @staticmethod + def _normalize_playback_commit_policy(value: str) -> str: + normalized = value.strip().lower() + if normalized in {policy.value for policy in DuplexPlaybackCommitPolicy}: + return normalized + return DuplexPlaybackCommitPolicy.COMMIT_ALL_ON_DONE.value + + +@dataclass(frozen=True) +class ResponseCreateOptions: + instructions: str | None = None + voice: str | None = None + response_format: str | None = None + temperature: float | None = None + max_tokens: int | None = None + speed: float | None = None + modalities: tuple[str, ...] | None = None + extra_body: Mapping[str, object] = field(default_factory=dict) + + def apply_to(self, config: DuplexSessionConfig) -> None: + for field_name in ("instructions", "voice", "response_format", "temperature", "max_tokens", "speed"): + value = getattr(self, field_name) + if value is not None: + setattr(config, field_name, value) + if self.modalities is not None: + config.modalities = list(self.modalities) + config.extra_body.update(self.extra_body) + + +@dataclass +class DuplexCommittedInput: + message: dict[str, object] + turn_id: int + epoch: int + input_commit_seq: int + + +@dataclass +class DuplexAssistantAudioTextMark: + text_chars: int + audio_end_ms: int + + +@dataclass +class InputBufferState: + commit_seq: int = 0 + pending_text: list[str] = field(default_factory=list) + pending_audio: list[DuplexAudioChunk] = field(default_factory=list) + overlap_speech_ms: int = 0 + reserved_input_bytes: int = 0 + pending_turns: int = 0 + + +@dataclass +class ResponseState: + active_request_id: str | None = None + active_response_id: str | None = None + active_response_turn_id: int | None = None + last_response_id: str | None = None + assistant_text_buffer: list[str] = field(default_factory=list) + assistant_audio_text_marks: list[DuplexAssistantAudioTextMark] = field(default_factory=list) + pending_options: ResponseCreateOptions | None = None + active_options: ResponseCreateOptions | None = None + active_config: DuplexSessionConfig | None = None + stage_metrics: dict[str, dict[str, object]] = field(default_factory=dict) + stage_metric_tpot_weighted_ms: dict[str, float] = field(default_factory=dict) + stage_metric_tpot_weight: dict[str, int] = field(default_factory=dict) + + +@dataclass +class PlaybackLedger: + current: DuplexPlaybackCursor = field(default_factory=DuplexPlaybackCursor) + by_response: dict[str, DuplexPlaybackCursor] = field(default_factory=dict) + + +@dataclass +class ConversationHistory: + messages: list[dict[str, object]] = field(default_factory=list) + item_ids: dict[str, dict[str, object]] = field(default_factory=dict) + item_audio_text_marks: dict[str, list[DuplexAssistantAudioTextMark]] = field(default_factory=dict) + pending_item_ids: dict[str, dict[str, object]] = field(default_factory=dict) + pending_item_audio_text_marks: dict[str, list[DuplexAssistantAudioTextMark]] = field(default_factory=dict) + pending_truncations_ms: dict[str, int] = field(default_factory=dict) + last_assistant_full_message: dict[str, object] | None = None + last_assistant_audio_text_marks: list[DuplexAssistantAudioTextMark] = field(default_factory=list) + + +@dataclass +class DuplexSession: + session_id: str + config: DuplexSessionConfig + capabilities: DuplexCapabilities = field(default_factory=DuplexCapabilities) + incarnation: int = 0 + state: DuplexSessionState = DuplexSessionState.OPEN + turn_state: DuplexTurnState = DuplexTurnState.IDLE + epoch: int = 0 + turn_id: int = 0 + _input: InputBufferState = field(default_factory=InputBufferState, repr=False) + _response: ResponseState = field(default_factory=ResponseState, repr=False) + _playback: PlaybackLedger = field(default_factory=PlaybackLedger, repr=False) + _conversation: ConversationHistory = field(default_factory=ConversationHistory, repr=False) + _runtime_config: dict[str, object] = field(default_factory=dict, repr=False) + + @property + def response_config(self) -> DuplexSessionConfig: + """Return the immutable-for-this-lifecycle response configuration. + + ``config`` remains the session defaults. A response takes a deep + snapshot when it begins so response.create overrides and concurrent + session updates cannot mutate each other's ownership domains. + """ + return self._response.active_config or self.config + + @property + def runtime_config(self) -> Mapping[str, object]: + return MappingProxyType(dict(self._runtime_config)) + + def replace_runtime_config(self, runtime_config: Mapping[str, object]) -> None: + self._runtime_config = dict(runtime_config) + + @property + def input_commit_seq(self) -> int: + return self._input.commit_seq + + @property + def pending_text(self) -> tuple[str, ...]: + return tuple(self._input.pending_text) + + @property + def pending_audio(self) -> tuple[DuplexAudioChunk, ...]: + return tuple(self._input.pending_audio) + + @property + def history(self) -> tuple[dict[str, object], ...]: + return tuple(dict(message) for message in self._conversation.messages) + + @property + def active_request_id(self) -> str | None: + return self._response.active_request_id + + @property + def active_response_id(self) -> str | None: + return self._response.active_response_id + + @property + def active_response_turn_id(self) -> int | None: + return self._response.active_response_turn_id + + @property + def last_response_id(self) -> str | None: + return self._response.last_response_id + + @property + def overlap_speech_ms(self) -> int: + return self._input.overlap_speech_ms + + @property + def assistant_text_buffer(self) -> tuple[str, ...]: + return tuple(self._response.assistant_text_buffer) + + @property + def assistant_audio_text_marks(self) -> tuple[DuplexAssistantAudioTextMark, ...]: + return tuple(self._response.assistant_audio_text_marks) + + @property + def last_assistant_full_message(self) -> dict[str, object] | None: + message = self._conversation.last_assistant_full_message + return dict(message) if message is not None else None + + @property + def last_assistant_audio_text_marks(self) -> tuple[DuplexAssistantAudioTextMark, ...]: + return tuple(self._conversation.last_assistant_audio_text_marks) + + @property + def playback(self) -> DuplexPlaybackView: + return self._playback.current.snapshot() + + @property + def response_playbacks(self) -> Mapping[str, DuplexPlaybackView]: + return MappingProxyType({key: value.snapshot() for key, value in self._playback.by_response.items()}) + + @property + def history_item_ids(self) -> Mapping[str, dict[str, object]]: + return MappingProxyType({key: dict(value) for key, value in self._conversation.item_ids.items()}) + + @property + def pending_history_item_ids(self) -> Mapping[str, dict[str, object]]: + return MappingProxyType({key: dict(value) for key, value in self._conversation.pending_item_ids.items()}) + + @property + def pending_history_truncations_ms(self) -> Mapping[str, int]: + return MappingProxyType(dict(self._conversation.pending_truncations_ms)) + + def replace_config(self, config: DuplexSessionConfig) -> None: + self.config = config + + def replace_capabilities(self, capabilities: DuplexCapabilities) -> None: + self.capabilities = capabilities + + def transition_turn(self, state: DuplexTurnState) -> None: + self.turn_state = state + + def transition_session(self, state: DuplexSessionState) -> None: + self.state = state + + def bind_request(self, request_id: str | None) -> None: + self._response.active_request_id = request_id + + def clear_request(self, expected_request_id: str | None = None) -> bool: + if expected_request_id is not None and self._response.active_request_id != expected_request_id: + return False + self._response.active_request_id = None + return True + + def bind_response_turn(self, turn_id: int | None) -> None: + self._response.active_response_turn_id = turn_id + + def active_response_accepts_model_turn(self, turn_id: int | None) -> bool: + if self._response.active_response_id is None: + return False + if turn_id is None: + return True + active_turn_id = self._response.active_response_turn_id + return active_turn_id is None or int(turn_id) == active_turn_id + + def append_history_message(self, message: dict[str, object]) -> None: + self._conversation.messages.append(message) + + def stage_pending_history_item(self, item_id: str, message: dict[str, object]) -> None: + self._conversation.pending_item_ids[item_id] = dict(message) + + def append_text(self, text: str) -> None: + if not text: + return + self._input.pending_text.append(text) + self.turn_state = DuplexTurnState.USER_SPEAKING + + @property + def pending_input_bytes(self) -> int: + return self._input.reserved_input_bytes + + @property + def pending_input_turns(self) -> int: + return self._input.pending_turns + + def reserve_input_bytes(self, size: int, *, limit: int) -> bool: + size = max(0, int(size)) + if self._input.reserved_input_bytes + size > int(limit): + return False + self._input.reserved_input_bytes += size + return True + + def release_input_bytes(self, size: int) -> None: + self._input.reserved_input_bytes = max(0, self._input.reserved_input_bytes - max(0, int(size))) + + def release_all_input_bytes(self) -> None: + self._input.reserved_input_bytes = 0 + + def reserve_pending_turn(self, *, limit: int) -> bool: + if self._input.pending_turns >= int(limit): + return False + self._input.pending_turns += 1 + return True + + def release_pending_turn(self) -> None: + self._input.pending_turns = max(0, self._input.pending_turns - 1) + + def clear_pending_turn_reservations(self) -> None: + self._input.pending_turns = 0 + + def append_audio(self, data: str, *, fmt: str = "wav", sample_rate_hz: int | None = None) -> None: + if not data: + return + self._input.pending_audio.append(DuplexAudioChunk(data=data, format=fmt, sample_rate_hz=sample_rate_hz)) + self.turn_state = DuplexTurnState.USER_SPEAKING + + def mark_user_input_activity(self) -> None: + self.turn_state = DuplexTurnState.USER_SPEAKING + + def mark_assistant_generating(self) -> None: + self.turn_state = DuplexTurnState.ASSISTANT_GENERATING + + def cancel_pending_input(self) -> dict[str, int]: + cancelled = { + "text_chunks": len(self._input.pending_text), + "audio_chunks": len(self._input.pending_audio), + } + self._input.pending_text.clear() + self._input.pending_audio.clear() + self._input.reserved_input_bytes = 0 + self._input.pending_turns = 0 + self.turn_state = DuplexTurnState.IDLE + return cancelled + + def commit_user_input(self) -> DuplexCommittedInput | None: + text = "".join(self._input.pending_text).strip() + audio_chunks = list(self._input.pending_audio) + if not text and not audio_chunks: + return None + + content: str | list[dict[str, object]] + if audio_chunks: + content_items: list[dict[str, object]] = [] + if text: + content_items.append({"type": "text", "text": text}) + for chunk in audio_chunks: + content_items.append( + { + "type": "audio_url", + "audio_url": { + "url": f"data:audio/{chunk.format};base64,{chunk.data}", + }, + } + ) + content = content_items + else: + content = text + + self._input.commit_seq += 1 + message = {"role": "user", "content": content} + self._conversation.messages.append(message) + self._input.pending_text.clear() + self._input.pending_audio.clear() + self._input.reserved_input_bytes = 0 + self.turn_state = DuplexTurnState.USER_COMMITTED + return DuplexCommittedInput( + message=message, + turn_id=self.turn_id, + epoch=self.epoch, + input_commit_seq=self._input.commit_seq, + ) + + def commit_native_audio_input( + self, + *, + transcript: str | None = None, + turn_id: int | None = None, + ) -> DuplexCommittedInput: + input_audio_part: dict[str, object] = { + "type": "audio_url", + "audio_url": {"url": "native-duplex:input-audio"}, + } + if transcript: + input_audio_part["transcript"] = transcript + self._input.commit_seq += 1 + message = {"role": "user", "content": [input_audio_part]} + if transcript: + message["transcript"] = transcript + self._conversation.messages.append(message) + self._input.pending_text.clear() + self._input.pending_audio.clear() + self.turn_state = DuplexTurnState.USER_COMMITTED + return DuplexCommittedInput( + message=message, + turn_id=self.turn_id if turn_id is None else int(turn_id), + epoch=self.epoch, + input_commit_seq=self._input.commit_seq, + ) + + def complete_model_turn(self, turn_id: int) -> None: + """Advance the model-owned output identity after its terminal signal.""" + completed_turn_id = int(turn_id) + if completed_turn_id >= self.turn_id: + self.turn_id = completed_turn_id + 1 + + def reserve_response_options(self, options: ResponseCreateOptions) -> None: + if self._response.active_response_id is not None: + raise RuntimeError("response options cannot be reserved while a response is active") + if self._response.pending_options is not None: + raise RuntimeError("response options are already reserved") + self._response.pending_options = options + + def discard_response_options(self) -> None: + self._response.pending_options = None + + def _activate_response_options(self) -> None: + options = self._response.pending_options + self._response.active_config = copy.deepcopy(self.config) + self._response.active_options = options + self._response.pending_options = None + if options is not None: + options.apply_to(self._response.active_config) + + def _restore_response_config(self) -> None: + self._response.active_config = None + self._response.active_options = None + self._response.pending_options = None + + def begin_response(self, *, turn_id: int | None = None) -> str: + self._activate_response_options() + response_id = f"resp-{self.session_id}-{self.epoch}-{uuid4().hex[:8]}" + self._response.active_response_id = response_id + self._response.active_response_turn_id = self.turn_id if turn_id is None else int(turn_id) + self._response.last_response_id = response_id + self._response.assistant_text_buffer.clear() + self._response.assistant_audio_text_marks.clear() + self._clear_response_metrics() + self._conversation.last_assistant_full_message = None + self._conversation.last_assistant_audio_text_marks.clear() + self._playback.current = DuplexPlaybackCursor() + self._playback.by_response[response_id] = self._playback.current + self.turn_state = DuplexTurnState.ASSISTANT_GENERATING + return response_id + + def _clear_response_metrics(self) -> None: + self._response.stage_metrics.clear() + self._response.stage_metric_tpot_weighted_ms.clear() + self._response.stage_metric_tpot_weight.clear() + + def accumulate_response_stage_metrics( + self, + stage_metrics: Mapping[object, object] | None, + ) -> dict[str, dict[str, object]]: + if self.active_response_id is None or not isinstance(stage_metrics, Mapping): + return copy.deepcopy(self._response.stage_metrics) + + additive_fields = ( + "num_tokens_in", + "num_tokens_out", + "stage_gen_time_ms", + "postprocess_time_ms", + "audio_generated_frames", + "audio_duration_s", + "image_pixels", + "output_unit_count", + ) + first_positive_fields = ( + "serving_time_to_first_output_ms", + "vllm_ttft_ms", + ) + interval_fields = ( + ("inter_output_latencies_ms", "inter_output_latency_ms"), + ("vllm_itls_ms", "vllm_itl_ms"), + ) + handled_fields = { + *additive_fields, + *first_positive_fields, + "vllm_tpot_ms", + *(name for pair in interval_fields for name in pair), + } + + for raw_stage_id, raw_values in stage_metrics.items(): + if not isinstance(raw_values, Mapping): + continue + stage_id = str(raw_stage_id) + current = self._response.stage_metrics.setdefault(stage_id, {}) + for name in additive_fields: + value = raw_values.get(name) + if isinstance(value, int | float) and not isinstance(value, bool): + current[name] = current.get(name, 0) + value + for name in first_positive_fields: + value = raw_values.get(name) + current_value = current.get(name) + if ( + isinstance(value, int | float) + and not isinstance(value, bool) + and value > 0 + and not (isinstance(current_value, int | float) and current_value > 0) + ): + current[name] = value + for list_name, mean_name in interval_fields: + values = raw_values.get(list_name) + if isinstance(values, list): + combined = list(current.get(list_name, [])) + combined.extend( + value for value in values if isinstance(value, int | float) and not isinstance(value, bool) + ) + current[list_name] = combined + current[mean_name] = sum(combined) / len(combined) if combined else 0.0 + + tpot_ms = raw_values.get("vllm_tpot_ms") + token_count = raw_values.get("num_tokens_out") + if isinstance(tpot_ms, int | float) and tpot_ms > 0: + weight = max(int(token_count) - 1, 1) if isinstance(token_count, int | float) else 1 + self._response.stage_metric_tpot_weighted_ms[stage_id] = ( + self._response.stage_metric_tpot_weighted_ms.get(stage_id, 0.0) + float(tpot_ms) * weight + ) + self._response.stage_metric_tpot_weight[stage_id] = ( + self._response.stage_metric_tpot_weight.get(stage_id, 0) + weight + ) + current["vllm_tpot_ms"] = self._response.stage_metric_tpot_weighted_ms[stage_id] / float( + self._response.stage_metric_tpot_weight[stage_id] + ) + + for name, value in raw_values.items(): + if name not in handled_fields: + current[str(name)] = copy.deepcopy(value) + + return copy.deepcopy(self._response.stage_metrics) + + def accumulate_overlap_speech(self, duration_ms: int) -> int: + self._input.overlap_speech_ms += max(0, int(duration_ms)) + return self._input.overlap_speech_ms + + def reset_overlap_speech(self) -> int: + previous = self._input.overlap_speech_ms + self._input.overlap_speech_ms = 0 + return previous + + def append_assistant_text(self, text: str) -> None: + if text: + self._response.assistant_text_buffer.append(text) + + def mark_audio_sent( + self, + duration_ms: int | None = None, + *, + text_chars: int | None = None, + audio_text_marks: list[dict[str, object]] | None = None, + ) -> None: + playback = self._playback.current + if duration_ms is not None: + playback.generated_ms = max(playback.generated_ms, duration_ms) + playback.sent_ms = max(playback.sent_ms, duration_ms) + if text_chars is not None and text_chars >= 0: + self._response.assistant_audio_text_marks.append( + DuplexAssistantAudioTextMark( + text_chars=int(text_chars), + audio_end_ms=max(0, int(duration_ms)), + ) + ) + if audio_text_marks: + for raw_mark in audio_text_marks: + if not isinstance(raw_mark, dict): + continue + raw_text_chars = raw_mark.get("text_chars") + raw_audio_end_ms = raw_mark.get("audio_end_ms", raw_mark.get("audio_ms")) + if not isinstance(raw_text_chars, int | float) or not isinstance(raw_audio_end_ms, int | float): + continue + self._response.assistant_audio_text_marks.append( + DuplexAssistantAudioTextMark( + text_chars=max(0, int(raw_text_chars)), + audio_end_ms=max(0, int(raw_audio_end_ms)), + ) + ) + self.turn_state = DuplexTurnState.ASSISTANT_PLAYING + + def _playback_cursor_for_response(self, response_id: str | None = None) -> DuplexPlaybackCursor: + if response_id is None: + return self._playback.current + playback = self._playback.by_response.get(response_id) + if playback is None: + # A restored or legacy session may not have response-scoped state. + # Keep its acknowledgement isolated from the active response. + playback = DuplexPlaybackCursor() + self._playback.by_response[response_id] = playback + return playback + + def playback_for_response(self, response_id: str | None = None) -> DuplexPlaybackView: + return self._playback_cursor_for_response(response_id).snapshot() + + def acknowledge_playback( + self, + played_ms: int, + committed_ms: int | None = None, + *, + response_id: str | None = None, + ) -> DuplexPlaybackView: + playback = self._playback_cursor_for_response(response_id) + playback.acknowledge(played_ms, committed_ms) + return playback.snapshot() + + def truncate_playback_commit( + self, + committed_ms: int, + *, + response_id: str | None = None, + ) -> DuplexPlaybackView: + playback = self._playback_cursor_for_response(response_id) + playback.truncate_committed(committed_ms) + return playback.snapshot() + + def release_response_playback(self, response_id: str | None) -> None: + if response_id is None or response_id == self.active_response_id: + return + self._playback.by_response.pop(response_id, None) + + def clear_playback_cursor(self) -> None: + self._playback.current = DuplexPlaybackCursor() + if self.active_response_id is not None: + self._playback.by_response[self.active_response_id] = self._playback.current + + def end_response( + self, + *, + commit_text: bool = True, + playback_commit_policy: str | None = None, + preserve_request: bool = False, + ) -> dict[str, object] | None: + assistant_text = "".join(self._response.assistant_text_buffer).strip() + message = None + if assistant_text: + self._conversation.last_assistant_full_message = {"role": "assistant", "content": assistant_text} + self._conversation.last_assistant_audio_text_marks = list(self._response.assistant_audio_text_marks) + if commit_text and assistant_text: + committed_text = self._playback_committed_text( + assistant_text, + playback_commit_policy=playback_commit_policy, + ) + else: + committed_text = "" + if commit_text and committed_text: + message = {"role": "assistant", "content": committed_text} + self._conversation.messages.append(message) + self._response.assistant_text_buffer.clear() + if not preserve_request: + self._response.active_request_id = None + self._response.active_response_id = None + self._response.active_response_turn_id = None + self._clear_response_metrics() + self.turn_state = DuplexTurnState.IDLE + self._restore_response_config() + return message + + def register_history_item(self, item_id: str | None, message: dict[str, object] | None) -> None: + if not item_id: + return + if message is None: + last_message = self._conversation.last_assistant_full_message + if last_message is None: + return + self._conversation.pending_item_ids[item_id] = dict(last_message) + if self._conversation.last_assistant_audio_text_marks: + self._conversation.pending_item_audio_text_marks[item_id] = list( + self._conversation.last_assistant_audio_text_marks + ) + pending_audio_ms = self._conversation.pending_truncations_ms.pop(item_id, None) + if pending_audio_ms is not None: + self.truncate_history_item( + item_id, + audio_end_ms=pending_audio_ms, + playback=self._playback_cursor_for_item_id(item_id), + ) + return + pending_audio_ms = self._conversation.pending_truncations_ms.pop(item_id, None) + if pending_audio_ms is not None: + self._truncate_message_to_audio_ms( + message, + audio_end_ms=pending_audio_ms, + marks=self._response.assistant_audio_text_marks or self._conversation.last_assistant_audio_text_marks, + playback=self._playback_cursor_for_item_id(item_id), + ) + if self._message_text_len(message) <= 0: + try: + self._conversation.messages.remove(message) + except ValueError: + pass + return + self._conversation.item_ids[item_id] = message + self._conversation.pending_item_ids.pop(item_id, None) + self._conversation.pending_item_audio_text_marks.pop(item_id, None) + if message.get("role") == "assistant": + marks = self._response.assistant_audio_text_marks or self._conversation.last_assistant_audio_text_marks + if marks: + self._conversation.item_audio_text_marks[item_id] = list(marks) + + def delete_history_item(self, item_id: str) -> bool: + message = self._conversation.item_ids.pop(item_id, None) + self._conversation.item_audio_text_marks.pop(item_id, None) + pending = self._conversation.pending_item_ids.pop(item_id, None) + self._conversation.pending_item_audio_text_marks.pop(item_id, None) + self._conversation.pending_truncations_ms.pop(item_id, None) + if message is None: + return pending is not None + try: + self._conversation.messages.remove(message) + except ValueError: + pass + return True + + def truncate_history_item( + self, + item_id: str, + *, + audio_end_ms: int, + playback: DuplexPlaybackCursor | DuplexPlaybackView | None = None, + ) -> bool: + playback = playback or self._playback_cursor_for_item_id(item_id) + message = self._conversation.item_ids.get(item_id) + if message is None: + pending = self._conversation.pending_item_ids.get(item_id) + if pending is None: + self._conversation.pending_truncations_ms[item_id] = max(0, int(audio_end_ms)) + return False + message = dict(pending) + changed = self._truncate_message_to_audio_ms( + message, + audio_end_ms=audio_end_ms, + marks=self._conversation.pending_item_audio_text_marks.get(item_id), + playback=playback, + ) + if not changed or self._message_text_len(message) <= 0: + if changed: + self._conversation.pending_item_ids.pop(item_id, None) + self._conversation.pending_item_audio_text_marks.pop(item_id, None) + self._conversation.pending_truncations_ms.pop(item_id, None) + return changed + self._conversation.messages.append(message) + self._conversation.item_ids[item_id] = message + if item_id in self._conversation.pending_item_audio_text_marks: + self._conversation.item_audio_text_marks[item_id] = list( + self._conversation.pending_item_audio_text_marks[item_id] + ) + self._conversation.pending_item_ids.pop(item_id, None) + self._conversation.pending_item_audio_text_marks.pop(item_id, None) + self._conversation.pending_truncations_ms.pop(item_id, None) + return True + changed = self._truncate_message_to_audio_ms( + message, + audio_end_ms=audio_end_ms, + marks=self._conversation.item_audio_text_marks.get(item_id), + playback=playback, + ) + if changed and self._message_text_len(message) <= 0: + self._conversation.item_ids.pop(item_id, None) + self._conversation.item_audio_text_marks.pop(item_id, None) + try: + self._conversation.messages.remove(message) + except ValueError: + pass + return changed + + def _truncate_message_to_audio_ms( + self, + message: dict[str, object], + *, + audio_end_ms: int, + marks: list[DuplexAssistantAudioTextMark] | None = None, + playback: DuplexPlaybackCursor | DuplexPlaybackView | None = None, + ) -> bool: + content = message.get("content") + if isinstance(content, str): + keep_chars = self._text_chars_for_audio_ms( + audio_end_ms, + len(content), + marks=marks, + playback=playback, + ) + message["content"] = content[:keep_chars].rstrip() + return True + if not isinstance(content, list): + return False + changed = False + for part in content: + if not isinstance(part, dict): + continue + part_type = part.get("type") + if part_type in {"output_audio", "audio", "audio_transcript"}: + transcript = part.get("transcript") + if isinstance(transcript, str): + keep_chars = self._text_chars_for_audio_ms( + audio_end_ms, + len(transcript), + marks=marks, + playback=playback, + ) + part["transcript"] = transcript[:keep_chars].rstrip() + changed = True + if part_type in {"output_text", "text"}: + text = part.get("text") + if isinstance(text, str): + keep_chars = self._text_chars_for_audio_ms( + audio_end_ms, + len(text), + marks=marks, + playback=playback, + ) + part["text"] = text[:keep_chars].rstrip() + changed = True + if not changed: + return False + return True + + @staticmethod + def _message_text_len(message: dict[str, object]) -> int: + content = message.get("content") + if isinstance(content, str): + return len(content) + if not isinstance(content, list): + return 0 + total = 0 + for part in content: + if not isinstance(part, dict): + continue + for key in ("text", "transcript"): + value = part.get(key) + if isinstance(value, str): + total += len(value) + return total + + def _playback_committed_text( + self, + assistant_text: str, + *, + playback_commit_policy: str | None = None, + ) -> str: + sent_ms = max(self.playback.sent_ms, self.playback.generated_ms) + committed_ms = self.playback.committed_ms + policy = playback_commit_policy or self.config.playback_commit_policy + if sent_ms <= 0 or committed_ms >= sent_ms: + return assistant_text + if committed_ms <= 0: + if policy == DuplexPlaybackCommitPolicy.COMMIT_ALL_ON_DONE.value: + return assistant_text + return "" + keep_chars = self._text_chars_for_audio_ms(committed_ms, len(assistant_text)) + if keep_chars <= 0: + return "" + return assistant_text[:keep_chars].rstrip() + + def _text_chars_for_audio_ms( + self, + audio_end_ms: int, + text_len: int, + *, + marks: list[DuplexAssistantAudioTextMark] | None = None, + playback: DuplexPlaybackCursor | DuplexPlaybackView | None = None, + ) -> int: + if text_len <= 0: + return 0 + audio_end_ms = max(0, int(audio_end_ms)) + marks = marks if marks is not None else self._response.assistant_audio_text_marks + playback = playback or self._playback.current + if not marks: + sent_ms = max(1, playback.sent_ms, playback.generated_ms) + return int(text_len * max(0.0, min(1.0, audio_end_ms / sent_ms))) + marks = sorted( + (mark for mark in marks if mark.audio_end_ms >= 0 and mark.text_chars >= 0), + key=lambda mark: mark.audio_end_ms, + ) + if not marks: + return 0 + if audio_end_ms <= 0: + return 0 + previous_ms = 0 + previous_chars = 0 + for mark in marks: + mark_ms = max(previous_ms, mark.audio_end_ms) + mark_chars = min(text_len, max(previous_chars, mark.text_chars)) + if audio_end_ms <= mark_ms: + if mark_ms <= previous_ms: + return mark_chars + ratio = (audio_end_ms - previous_ms) / max(1, mark_ms - previous_ms) + return int(previous_chars + (mark_chars - previous_chars) * max(0.0, min(1.0, ratio))) + previous_ms = mark_ms + previous_chars = mark_chars + final_ms = max(playback.sent_ms, playback.generated_ms, previous_ms) + if audio_end_ms >= final_ms: + return text_len + ratio = (audio_end_ms - previous_ms) / max(1, final_ms - previous_ms) + return int(previous_chars + (text_len - previous_chars) * max(0.0, min(1.0, ratio))) + + def _playback_cursor_for_item_id(self, item_id: str) -> DuplexPlaybackCursor | None: + if not item_id.startswith("item_"): + return None + return self._playback.by_response.get(item_id.removeprefix("item_")) + + def barge_in(self) -> int: + self.epoch += 1 + self._input.pending_text.clear() + self._input.pending_audio.clear() + self._response.assistant_text_buffer.clear() + self._response.assistant_audio_text_marks.clear() + self._response.active_request_id = None + self._response.active_response_id = None + self._response.active_response_turn_id = None + self._clear_response_metrics() + self._restore_response_config() + self.turn_state = DuplexTurnState.BARGE_IN + return self.epoch + + def mark_closing(self) -> None: + if self.state != DuplexSessionState.CLOSED: + self.state = DuplexSessionState.CLOSING + + def close(self) -> None: + self.state = DuplexSessionState.CLOSED + self.turn_state = DuplexTurnState.IDLE + self._response.active_response_turn_id = None + self._clear_response_metrics() + self._restore_response_config() + + def as_public_dict(self) -> dict[str, object]: + payload: dict[str, object] = { + "id": self.session_id, + "state": self.state.value, + "turn_state": self.turn_state.value, + "epoch": self.epoch, + "turn_id": self.turn_id, + "active_request_id": self.active_request_id, + "active_response_id": self.active_response_id, + "active_response_turn_id": self.active_response_turn_id, + "model": self.config.model, + "modalities": list(self.config.modalities), + "instructions": self.config.instructions, + "voice": self.config.voice, + "response_format": self.config.response_format, + "temperature": self.config.temperature, + "max_tokens": self.config.max_tokens, + "speed": self.config.speed, + "idle_timeout_s": self.config.idle_timeout_s, + "overlap_policy": self.config.overlap_policy, + "overlap_short_ack_ms": self.config.overlap_short_ack_ms, + "overlap_barge_in_ms": self.config.overlap_barge_in_ms, + "overlap_silence_rms": self.config.overlap_silence_rms, + "playback_commit_policy": self.config.playback_commit_policy, + "playback": self.playback.as_dict(), + "capabilities": self.capabilities.as_dict(), + } + if isinstance(self.config.extra_body.get("realtime_tools"), list): + payload["tools"] = self.config.extra_body["realtime_tools"] + if isinstance(self.config.extra_body.get("realtime_tool_choice"), str | dict): + payload["tool_choice"] = self.config.extra_body["realtime_tool_choice"] + if isinstance(self.config.extra_body.get("realtime_metadata"), dict): + payload["metadata"] = dict(self.config.extra_body["realtime_metadata"]) + if isinstance(self.config.extra_body.get("realtime_include"), list): + payload["include"] = list(self.config.extra_body["realtime_include"]) + if isinstance(self.config.extra_body.get("realtime_prompt"), dict): + payload["prompt"] = dict(self.config.extra_body["realtime_prompt"]) + if isinstance(self.config.extra_body.get("realtime_input_audio_transcription"), dict): + payload["input_audio_transcription"] = dict(self.config.extra_body["realtime_input_audio_transcription"]) + if isinstance(self.config.extra_body.get("realtime_input_audio_noise_reduction"), dict): + payload["input_audio_noise_reduction"] = dict( + self.config.extra_body["realtime_input_audio_noise_reduction"] + ) + if isinstance(self.config.extra_body.get("realtime_audio"), dict): + payload["audio"] = dict(self.config.extra_body["realtime_audio"]) + if isinstance(self.config.extra_body.get("realtime_tracing"), str | dict): + payload["tracing"] = self.config.extra_body["realtime_tracing"] + raw_realtime_session = self.config.extra_body.get("realtime_session_payload") + if isinstance(raw_realtime_session, dict): + for key, value in raw_realtime_session.items(): + if key not in payload and key != "extra_body": + payload[key] = value + return payload + + +class DuplexTurnController: + """Interaction controller that accepts signals from multiple sources.""" + + def signal( + self, + session: DuplexSession, + event_type: str, + payload: dict[str, object] | None = None, + ) -> dict[str, object]: + payload = payload or {} + if event_type == DuplexTurnEventType.USER_STARTED.value: + session.transition_turn(DuplexTurnState.USER_SPEAKING) + elif event_type == DuplexTurnEventType.USER_COMMITTED.value: + session.transition_turn(DuplexTurnState.USER_COMMITTED) + elif event_type == DuplexTurnEventType.ASSISTANT_STARTED.value: + session.transition_turn(DuplexTurnState.ASSISTANT_GENERATING) + elif event_type == DuplexTurnEventType.ASSISTANT_DONE.value: + session.transition_turn(DuplexTurnState.IDLE) + elif event_type == DuplexTurnEventType.PLAYBACK_ACK.value: + played_ms = int(payload.get("played_ms", 0) or 0) + committed_ms = payload.get("committed_ms") + session.acknowledge_playback( + played_ms, + int(committed_ms) if isinstance(committed_ms, int | float) else None, + ) + elif event_type == DuplexTurnEventType.BARGE_IN.value: + session.transition_turn(DuplexTurnState.BARGE_IN) + elif event_type in {DuplexTurnEventType.CLOSE.value, DuplexTurnEventType.TIMEOUT.value}: + session.transition_session(DuplexSessionState.CLOSING) + return { + "type": "turn.event", + "session_id": session.session_id, + "event": event_type, + "turn_state": session.turn_state.value, + "epoch": session.epoch, + } + + +class DuplexSessionRegistry: + def __init__(self, capabilities: DuplexCapabilities | None = None) -> None: + self._capabilities = capabilities or DuplexCapabilities() + self._sessions: dict[str, DuplexSession] = {} + self._next_incarnation_by_session_id: dict[str, int] = {} + + def create(self, config: DuplexSessionConfig | None = None, session_id: str | None = None) -> DuplexSession: + sid = session_id or f"duplex-{uuid4().hex}" + if sid in self._sessions: + raise ValueError(f"Duplex session already exists: {sid}") + incarnation = self._next_incarnation_by_session_id.get(sid, 0) + self._next_incarnation_by_session_id[sid] = incarnation + 1 + session = DuplexSession( + session_id=sid, + config=config or DuplexSessionConfig(), + capabilities=self._capabilities, + incarnation=incarnation, + ) + self._sessions[sid] = session + return session + + def active_count(self) -> int: + return len(self._sessions) + + def get(self, session_id: str) -> DuplexSession | None: + return self._sessions.get(session_id) + + def close(self, session_id: str) -> DuplexSession | None: + session = self._sessions.pop(session_id, None) + if session is not None: + session.close() + return session diff --git a/vllm_omni/experimental/fullduplex/openai/realtime_input.py b/vllm_omni/experimental/fullduplex/openai/realtime_input.py new file mode 100644 index 000000000..e3fe7f3ec --- /dev/null +++ b/vllm_omni/experimental/fullduplex/openai/realtime_input.py @@ -0,0 +1,1233 @@ +from __future__ import annotations + +import asyncio +import base64 +import binascii +from typing import Any +from uuid import uuid4 + +import numpy as np + +from vllm_omni.experimental.fullduplex.openai.audio import ( + convert_input_audio_with_rate, +) +from vllm_omni.experimental.fullduplex.openai.realtime_state import ( + REALTIME_INPUT_AUDIO_FORMATS, + REALTIME_OUTPUT_AUDIO_FORMATS, +) + + +class RealtimeInputTranslator: + """Translate and validate client Realtime events for the duplex core.""" + + async def discard_pending_input_audio( + self, + *, + audio_end_ms: int | None = None, + ) -> None: + """Drop Realtime input-buffer state that was consumed as overlap. + + The serving overlap policy may classify a short acknowledgement such + as "continue" as a side-channel signal instead of user turn input. The + Realtime protocol has already observed the append and may have emitted + ``speech_started``; reset that transient buffer so a later commit does + not accidentally attach the acknowledgement transcript/audio to the + next real user turn. + """ + if self._input_speech_started and self._active_input_item_id is not None: + await self._send_realtime_payload( + { + "type": "input_audio_buffer.speech_stopped", + "audio_end_ms": max(0, int(audio_end_ms or 0)), + "item_id": self._active_input_item_id, + } + ) + self._input_speech_started = False + self._active_input_item_id = None + self._input_audio_buffer_has_audio = False + self._input_audio_buffer_had_non_speech = False + self._input_audio_buffer_transcript_parts.clear() + + async def _send_realtime_input_ack(self, event: dict[str, object]) -> None: + if event.get("type") != "conversation.item.create": + return + item = event.get("item") + if not isinstance(item, dict): + return + item = self._normalize_conversation_item(item) + previous_item_id = event.get("previous_item_id") + if isinstance(previous_item_id, str): + item["_previous_item_id"] = previous_item_id + if item.get("role") != "user": + return + self._conversation_items[str(item["id"])] = item + for payload in self._conversation_item_added_events(item): + await self._send_realtime_payload(payload) + + async def _to_duplex_event(self, event: dict[str, object]) -> dict[str, object] | None: + event_type = event.get("type") + if event_type == "session.update": + session_payload = event.get("session") if isinstance(event.get("session"), dict) else event + format_error = self._validate_realtime_session_audio_formats(session_payload) + if format_error is not None: + await self._send_realtime_payload( + self._realtime_error_payload( + "unsupported_audio_format", + format_error, + event_id=event.get("event_id"), + ) + ) + return None + turn_detection_error = self._validate_realtime_turn_detection(session_payload) + if turn_detection_error is not None: + await self._send_realtime_payload( + self._realtime_error_payload( + "unsupported_turn_detection", + turn_detection_error, + event_id=event.get("event_id"), + param="turn_detection", + ) + ) + return None + self._apply_realtime_session_defaults(session_payload) + session_payload.update(self._realtime_overlap_fields(session_payload)) + if not self._opened: + self._opened = True + self._initial_session_update = True + return self._session_create_from_realtime(session_payload) + return { + "type": "turn.signal", + "event": "session.update", + "payload": session_payload, + } + if event_type == "conversation.item.create": + item = event.get("item") + format_error = self._validate_conversation_item_audio_formats(item) + if format_error is not None: + await self._send_realtime_payload( + self._realtime_error_payload( + "unsupported_audio_format", + format_error, + event_id=event.get("event_id"), + ) + ) + return None + return await self._conversation_item_to_duplex(event) + if event_type == "conversation.item.delete": + item_id = event.get("item_id") + if not isinstance(item_id, str) or not item_id: + await self._send_realtime_payload( + self._realtime_error_payload( + "missing_item_id", + "conversation.item.delete requires item_id", + event_id=event.get("event_id"), + ) + ) + return None + if item_id not in self._conversation_items: + await self._send_realtime_payload( + self._realtime_error_payload( + "item_not_found", + f"Conversation item not found: {item_id}", + event_id=event.get("event_id"), + ) + ) + return None + self._remove_conversation_item(item_id) + await self._pending_outbound.put( + { + "type": "turn.signal", + "event": "conversation.item.delete", + "payload": {"item_id": item_id}, + } + ) + return None + if event_type == "conversation.item.retrieve": + item_id = event.get("item_id") + if not isinstance(item_id, str) or not item_id: + await self._send_realtime_payload( + self._realtime_error_payload( + "missing_item_id", + "conversation.item.retrieve requires item_id", + event_id=event.get("event_id"), + ) + ) + return None + item = self._conversation_items.get(item_id) + if item is None: + await self._send_realtime_payload( + self._realtime_error_payload( + "item_not_found", + f"Conversation item not found: {item_id}", + event_id=event.get("event_id"), + ) + ) + return None + await self._send_realtime_payload({"type": "conversation.item.retrieved", "item": item}) + return None + if event_type == "conversation.item.truncate": + item_id = event.get("item_id") + audio_end_ms = event.get("audio_end_ms") + content_index = event.get("content_index", 0) + if not isinstance(item_id, str) or not item_id: + await self._send_realtime_payload( + self._realtime_error_payload( + "missing_item_id", + "conversation.item.truncate requires item_id", + event_id=event.get("event_id"), + ) + ) + return None + if not isinstance(audio_end_ms, int | float): + await self._send_realtime_payload( + self._realtime_error_payload( + "bad_event", + "conversation.item.truncate requires numeric audio_end_ms", + event_id=event.get("event_id"), + param="audio_end_ms", + ) + ) + return None + item = self._conversation_items.get(item_id) + if item is None: + await self._send_realtime_payload( + self._realtime_error_payload( + "item_not_found", + f"Conversation item not found: {item_id}", + event_id=event.get("event_id"), + ) + ) + return None + truncate_error = self._validate_realtime_item_truncate( + item, + content_index=int(content_index) if isinstance(content_index, int | float) else 0, + audio_end_ms=int(audio_end_ms), + ) + if truncate_error is not None: + await self._send_realtime_payload( + self._realtime_error_payload( + "bad_event", + truncate_error, + event_id=event.get("event_id"), + ) + ) + return None + self._item_truncation_cursors[item_id] = ( + int(content_index) if isinstance(content_index, int | float) else 0, + int(audio_end_ms), + ) + self._truncate_realtime_item_content( + item, + content_index=int(content_index) if isinstance(content_index, int | float) else 0, + audio_end_ms=int(audio_end_ms), + ) + await self._pending_outbound.put( + { + "type": "turn.signal", + "event": "conversation.item.truncate", + "payload": { + "item_id": item_id, + "content_index": content_index, + "audio_end_ms": audio_end_ms, + }, + } + ) + await self._pending_outbound.put( + { + "type": "playback.ack", + "item_id": item_id, + "committed_ms": int(audio_end_ms), + "played_ms": int(audio_end_ms), + "truncate": True, + } + ) + return None + if event_type == "input_audio_buffer.append": + audio = event.get("audio") or event.get("delta") + fmt, format_rate = self._parse_realtime_audio_format( + event.get("format") or event.get("input_audio_format") or self._input_audio_format + ) + sample_rate_hz = ( + event.get("sample_rate_hz") or event.get("sample_rate") or format_rate or self._input_sample_rate_hz + ) + if not self._is_supported_realtime_input_format(fmt): + await self._send_realtime_payload( + self._realtime_error_payload( + "unsupported_audio_format", + f"Unsupported input_audio_format: {fmt}", + event_id=event.get("event_id"), + ) + ) + return None + try: + audio, fmt, sample_rate_hz = convert_input_audio_with_rate( + audio, + fmt, + sample_rate_hz=sample_rate_hz if isinstance(sample_rate_hz, int | float) else None, + ) + except ValueError as exc: + await self._send_realtime_payload( + self._realtime_error_payload( + "bad_event", + str(exc), + event_id=event.get("event_id"), + param="sample_rate_hz", + ) + ) + return None + looks_like_speech = self._input_looks_like_speech(event, audio=audio, fmt=fmt) + self._input_audio_buffer_has_audio = self._input_audio_buffer_has_audio or ( + looks_like_speech and isinstance(audio, str) and bool(audio) + ) + self._input_audio_buffer_had_non_speech = self._input_audio_buffer_had_non_speech or ( + not looks_like_speech and isinstance(audio, str) and bool(audio) + ) + if looks_like_speech: + await self._emit_input_speech_started(event) + self._remember_input_transcript_hint(event) + payload = { + "type": "input_audio_buffer.append", + "audio": audio, + "format": fmt, + "sample_rate_hz": sample_rate_hz, + } + video_frames = event.get("video_frames") + if video_frames is not None: + frames_error = self._validate_realtime_video_frames(video_frames, event.get("max_slice_nums")) + if frames_error is not None: + await self._send_realtime_payload( + self._realtime_error_payload( + "invalid_video_frames", + frames_error, + event_id=event.get("event_id"), + param="video_frames", + ) + ) + return None + payload["video_frames"] = [frame for frame in video_frames if isinstance(frame, str) and frame] + self._copy_realtime_input_hints(event, payload) + if not looks_like_speech: + payload["is_speech"] = False + return payload + if event_type == "input_audio_buffer.commit": + if not self._input_audio_buffer_has_audio: + if self._input_audio_buffer_had_non_speech: + self._input_audio_buffer_had_non_speech = False + self._active_input_item_id = None + return { + "type": "input_audio_buffer.commit", + "final": event.get("final", True), + "response_create": False, + "is_speech": False, + } + await self._send_realtime_payload( + self._realtime_error_payload( + "input_audio_buffer_empty", + "input_audio_buffer.commit requires a non-empty input audio buffer", + event_id=event.get("event_id"), + ) + ) + return None + item_id = self._active_input_item_id or f"item_{uuid4().hex}" + await self._pending_commit_item_ids.put(item_id) + await self._emit_input_speech_stopped(event, item_id=item_id) + transcript = self._consume_input_transcript_hint() + self._active_input_item_id = None + self._input_audio_buffer_has_audio = False + self._input_audio_buffer_had_non_speech = False + payload = { + "type": "input_audio_buffer.commit", + "final": event.get("final", True), + "realtime_item_id": item_id, + "response_create": bool(event.get("response_create", False)), + } + if transcript: + payload["transcript"] = transcript + return payload + if event_type == "input_audio_buffer.clear": + self._input_speech_started = False + self._active_input_item_id = None + self._input_audio_buffer_has_audio = False + self._input_audio_buffer_had_non_speech = False + self._input_audio_buffer_transcript_parts.clear() + return {"type": "input_audio_buffer.clear", "reason": event_type} + if event_type == "output_audio_buffer.clear": + payload = {"type": "output_audio_buffer.clear", "reason": event_type} + response_id = event.get("response_id") + if not isinstance(response_id, str) or not response_id: + response_id = self._active_response_id or self._last_response_id + if self._response_is_done(response_id): + await self._send_realtime_payload( + { + "type": "output_audio_buffer.cleared", + "response_id": response_id, + } + ) + return None + if isinstance(response_id, str) and response_id: + payload["response_id"] = response_id + return payload + if event_type == "response.cancel": + payload = {"type": "response.cancel", "reason": event_type} + response_id = event.get("response_id") + if not isinstance(response_id, str) or not response_id: + response_id = self._active_response_id or self._last_response_id + if self._response_is_done(response_id): + await self._send_realtime_payload( + self._realtime_error_payload( + "response_not_active", + f"Response is already complete: {response_id}", + event_id=event.get("event_id"), + ) + ) + return None + if isinstance(response_id, str) and response_id: + payload["response_id"] = response_id + return payload + if event_type == "response.create": + response_payload = event.get("response") + if isinstance(response_payload, dict): + format_error = self._validate_realtime_response_audio_formats(response_payload) + if format_error is not None: + await self._send_realtime_payload( + self._realtime_error_payload( + "unsupported_audio_format", + format_error, + event_id=event.get("event_id"), + ) + ) + return None + return { + "type": "response.create", + "response": response_payload if isinstance(response_payload, dict) else {}, + } + if event_type in {"session.close", "close"}: + return {"type": "session.close"} + return event + + @staticmethod + def _normalize_conversation_item(item: dict[str, object]) -> dict[str, object]: + normalized = dict(item) + normalized.setdefault("id", f"item_{uuid4().hex}") + normalized.setdefault("object", "realtime.item") + normalized.setdefault("type", "message") + normalized.setdefault("status", "completed") + if "role" not in normalized and normalized.get("type") == "message": + normalized["role"] = "user" + if not isinstance(normalized.get("content"), list): + normalized["content"] = [] + return normalized + + @staticmethod + def _truncate_realtime_item_content( + item: dict[str, object], + *, + content_index: int, + audio_end_ms: int, + ) -> None: + content = item.get("content") + if not isinstance(content, list) or not content: + return + index = max(0, int(content_index)) + if index >= len(content): + return + part = content[index] + if not isinstance(part, dict): + return + transcript = part.get("transcript") + if not isinstance(transcript, str) or not transcript: + return + marks = part.get("audio_text_marks") + if isinstance(marks, list): + keep_chars = RealtimeInputTranslator._text_chars_for_audio_ms_from_marks( + audio_end_ms, + len(transcript), + marks, + final_ms=part.get("audio_duration_ms") or part.get("duration_ms") or part.get("audio_ms"), + ) + part["transcript"] = transcript[:keep_chars].rstrip() + return + duration_ms = part.get("audio_duration_ms") or part.get("duration_ms") or part.get("audio_ms") + if isinstance(duration_ms, int | float) and duration_ms > 0: + keep_chars = int(len(transcript) * max(0.0, min(1.0, int(audio_end_ms) / float(duration_ms)))) + part["transcript"] = transcript[:keep_chars].rstrip() + elif audio_end_ms <= 0: + part["transcript"] = "" + + @staticmethod + def _validate_realtime_item_truncate( + item: dict[str, object], + *, + content_index: int, + audio_end_ms: int, + ) -> str | None: + if item.get("type") != "message" or item.get("role") != "assistant": + return "conversation.item.truncate only supports assistant message items" + if audio_end_ms < 0: + return "conversation.item.truncate requires non-negative audio_end_ms" + content = item.get("content") + if not isinstance(content, list) or not content: + return None + index = max(0, int(content_index)) + if index >= len(content): + return f"conversation.item.truncate content_index out of range: {content_index}" + part = content[index] + if not isinstance(part, dict): + return "conversation.item.truncate target content part is invalid" + if part.get("type") not in {"audio", "output_audio"}: + return "conversation.item.truncate target content part must be audio" + duration_ms = part.get("audio_duration_ms") or part.get("duration_ms") or part.get("audio_ms") + if isinstance(duration_ms, int | float) and int(duration_ms) >= 0 and audio_end_ms > int(duration_ms): + return "conversation.item.truncate audio_end_ms exceeds item audio duration" + return None + + @staticmethod + def _text_chars_for_audio_ms_from_marks( + audio_end_ms: int, + text_len: int, + marks: list[object], + *, + final_ms: object | None = None, + ) -> int: + if text_len <= 0: + return 0 + clean_marks: list[tuple[int, int]] = [] + for mark in marks: + if not isinstance(mark, dict): + continue + raw_text_chars = mark.get("text_chars") + raw_audio_end_ms = mark.get("audio_end_ms", mark.get("audio_ms")) + if not isinstance(raw_text_chars, int | float) or not isinstance(raw_audio_end_ms, int | float): + continue + clean_marks.append((max(0, int(raw_audio_end_ms)), min(text_len, max(0, int(raw_text_chars))))) + if not clean_marks: + return 0 if audio_end_ms <= 0 else text_len + clean_marks.sort(key=lambda item: item[0]) + audio_end_ms = max(0, int(audio_end_ms)) + if audio_end_ms <= 0: + return 0 + previous_ms = 0 + previous_chars = 0 + for mark_ms, mark_chars in clean_marks: + mark_ms = max(previous_ms, mark_ms) + mark_chars = max(previous_chars, min(text_len, mark_chars)) + if audio_end_ms <= mark_ms: + if mark_ms <= previous_ms: + return mark_chars + ratio = (audio_end_ms - previous_ms) / max(1, mark_ms - previous_ms) + return int(previous_chars + (mark_chars - previous_chars) * max(0.0, min(1.0, ratio))) + previous_ms = mark_ms + previous_chars = mark_chars + if isinstance(final_ms, int | float) and int(final_ms) > previous_ms: + if audio_end_ms >= int(final_ms): + return text_len + ratio = (audio_end_ms - previous_ms) / max(1, int(final_ms) - previous_ms) + return int(previous_chars + (text_len - previous_chars) * max(0.0, min(1.0, ratio))) + return text_len if audio_end_ms >= previous_ms else previous_chars + + def _conversation_item_added_events(self, item: dict[str, object]) -> list[dict[str, object]]: + item_id = item.get("id") + explicit_previous_item_id = item.pop("_previous_item_id", None) + previous_item_id = ( + explicit_previous_item_id if isinstance(explicit_previous_item_id, str) else self._last_conversation_item_id + ) + if isinstance(item_id, str) and item_id: + self._last_conversation_item_id = item_id + return [ + { + "type": "conversation.item.added", + "previous_item_id": previous_item_id, + "item": item, + }, + { + "type": "conversation.item.created", + "previous_item_id": previous_item_id, + "item": item, + }, + ] + + def _conversation_item_done_event(self, item: dict[str, object]) -> dict[str, object]: + item_id = item.get("id") + previous_item_id = self._previous_item_id(item_id) if isinstance(item_id, str) else None + if isinstance(item_id, str) and item_id: + self._conversation_items[item_id] = item + self._last_conversation_item_id = item_id + return { + "type": "conversation.item.done", + "previous_item_id": previous_item_id, + "item": item, + } + + def _remove_conversation_item(self, item_id: str) -> bool: + removed = self._conversation_items.pop(item_id, None) is not None + if self._last_conversation_item_id == item_id: + self._last_conversation_item_id = next(reversed(self._conversation_items), None) + self._item_truncation_cursors.pop(item_id, None) + return removed + + def _response_output_item_added_events( + self, + *, + response_id: object, + item: dict[str, object], + ) -> list[dict[str, object]]: + return [ + { + "type": "response.output_item.added", + "response_id": response_id, + "output_index": 0, + "item": item, + }, + ] + + @staticmethod + def _response_content_part(*, transcript: str = "") -> dict[str, object]: + return { + "type": "audio", + "transcript": transcript, + } + + @staticmethod + def _response_text_content_part(*, text: str = "") -> dict[str, object]: + return { + "type": "text", + "text": text, + } + + @staticmethod + def _response_item_content_part( + *, + transcript: str = "", + audio_duration_ms: int | None = None, + audio_text_marks: list[dict[str, int]] | None = None, + ) -> dict[str, object]: + part: dict[str, object] = { + "type": "output_audio", + "transcript": transcript, + } + if audio_duration_ms is not None: + part["audio_duration_ms"] = int(audio_duration_ms) + if audio_text_marks: + part["audio_text_marks"] = [dict(mark) for mark in audio_text_marks] + return part + + @staticmethod + def _response_item_text_content_part(*, text: str = "") -> dict[str, object]: + return { + "type": "output_text", + "text": text, + } + + def _previous_item_id(self, item_id: str) -> str | None: + previous: str | None = None + for known_id in self._conversation_items: + if known_id == item_id: + return previous + previous = known_id + if self._last_conversation_item_id == item_id: + return previous + return self._last_conversation_item_id + + def _input_audio_buffer_committed_event( + self, + *, + item_id: str, + event: dict[str, Any], + ) -> dict[str, object]: + return { + "type": "input_audio_buffer.committed", + "previous_item_id": self._previous_item_id(item_id), + "item_id": item_id, + "event": event, + } + + def _session_create_from_realtime(self, session_payload: dict[str, object]) -> dict[str, object]: + self._apply_realtime_session_defaults(session_payload) + model = session_payload.get("model") + audio_config = session_payload.get("audio") + audio_input = audio_config.get("input") if isinstance(audio_config, dict) else None + audio_output = audio_config.get("output") if isinstance(audio_config, dict) else None + extra_body = ( + dict(session_payload.get("extra_body")) if isinstance(session_payload.get("extra_body"), dict) else {} + ) + extra_body["realtime_session_payload"] = self._json_safe_realtime_payload(session_payload) + if isinstance(session_payload.get("tools"), list): + extra_body["realtime_tools"] = session_payload["tools"] + if isinstance(session_payload.get("tool_choice"), str | dict): + extra_body["realtime_tool_choice"] = session_payload["tool_choice"] + if isinstance(session_payload.get("metadata"), dict): + extra_body["realtime_metadata"] = dict(session_payload["metadata"]) + if isinstance(session_payload.get("include"), list): + extra_body["realtime_include"] = list(session_payload["include"]) + if isinstance(session_payload.get("prompt"), dict): + extra_body["realtime_prompt"] = dict(session_payload["prompt"]) + input_audio_transcription = self._input_audio_transcription_config(session_payload) + if isinstance(input_audio_transcription, dict): + extra_body["realtime_input_audio_transcription"] = dict(input_audio_transcription) + if isinstance(session_payload.get("input_audio_noise_reduction"), dict): + extra_body["realtime_input_audio_noise_reduction"] = dict(session_payload["input_audio_noise_reduction"]) + if isinstance(audio_input, dict) and isinstance(audio_input.get("noise_reduction"), dict): + extra_body["realtime_input_audio_noise_reduction"] = dict(audio_input["noise_reduction"]) + if isinstance(session_payload.get("audio"), dict): + extra_body["realtime_audio"] = dict(session_payload["audio"]) + if isinstance(session_payload.get("tracing"), str | dict): + extra_body["realtime_tracing"] = session_payload["tracing"] + response_format = self._duplex_response_format(self._output_audio_format) + extra_body.setdefault("realtime_output_audio_format", self._output_audio_format) + overlap_fields = self._realtime_overlap_fields(session_payload) + voice = session_payload.get("voice") + if not isinstance(voice, str) and isinstance(audio_output, dict): + voice = audio_output.get("voice") + speed = session_payload.get("speed") + if not isinstance(speed, int | float) and isinstance(audio_output, dict): + speed = audio_output.get("speed") + return { + "type": "session.create", + "session_id": session_payload.get("session_id") or session_payload.get("id"), + "session": { + "model": model, + "modalities": ( + session_payload.get("modalities") or session_payload.get("output_modalities") or ["text", "audio"] + ), + "instructions": session_payload.get("instructions"), + "voice": voice, + "ref_audio": session_payload.get("ref_audio"), + "response_format": response_format, + "temperature": session_payload.get("temperature"), + "max_tokens": self.realtime_max_output_tokens( + session_payload.get("max_response_output_tokens") + or session_payload.get("max_output_tokens") + or session_payload.get("max_tokens") + ), + "speed": speed, + "idle_timeout_s": session_payload.get("idle_timeout_s") or 300.0, + **overlap_fields, + "extra_body": extra_body, + }, + } + + def _apply_realtime_session_defaults(self, session_payload: dict[str, object]) -> None: + input_format: object = session_payload.get("input_audio_format") + audio_config = session_payload.get("audio") + if input_format is None and isinstance(audio_config, dict): + audio_input = audio_config.get("input") + if isinstance(audio_input, dict): + input_format = audio_input.get("format") + input_format, input_rate = self._parse_realtime_audio_format(input_format) + if isinstance(input_format, str) and input_format.lower() in REALTIME_INPUT_AUDIO_FORMATS: + self._input_audio_format = input_format + output_format: object = session_payload.get("output_audio_format") or session_payload.get("response_format") + output_rate_raw: object | None = None + if output_format is None and isinstance(audio_config, dict): + audio_output = audio_config.get("output") + if isinstance(audio_output, dict): + output_format = audio_output.get("format") + output_rate_raw = audio_output.get("sample_rate_hz") or audio_output.get("sample_rate") + output_format, output_rate = self._parse_realtime_audio_format(output_format) + if output_rate is None and isinstance(output_rate_raw, int | float) and output_rate_raw > 0: + output_rate = int(output_rate_raw) + if isinstance(output_format, str) and output_format.lower() in REALTIME_OUTPUT_AUDIO_FORMATS: + self._output_audio_format = self._realtime_output_format(output_format) + sample_rate = session_payload.get("sample_rate_hz") or session_payload.get("sample_rate") + if sample_rate is None and isinstance(audio_config, dict): + audio_input = audio_config.get("input") + if isinstance(audio_input, dict): + sample_rate = audio_input.get("sample_rate_hz") or audio_input.get("sample_rate") + if sample_rate is None: + sample_rate = input_rate + if isinstance(sample_rate, int | float) and sample_rate > 0: + self._input_sample_rate_hz = int(sample_rate) + if isinstance(output_rate, int | float) and output_rate > 0: + self._output_sample_rate_hz = int(output_rate) + overlap_fields = self._realtime_overlap_fields(session_payload) + overlap_silence_rms = overlap_fields.get("overlap_silence_rms") + if isinstance(overlap_silence_rms, int | float): + self._overlap_silence_rms = max(0.0, float(overlap_silence_rms)) + + @classmethod + def _validate_realtime_session_audio_formats(cls, session_payload: dict[str, object]) -> str | None: + audio_config = session_payload.get("audio") + input_format: object = session_payload.get("input_audio_format") + if input_format is None and isinstance(audio_config, dict): + audio_input = audio_config.get("input") + if isinstance(audio_input, dict): + input_format = audio_input.get("format") + parsed_input, _ = cls._parse_realtime_audio_format(input_format) + if input_format is not None and not ( + isinstance(parsed_input, str) and parsed_input.lower() in REALTIME_INPUT_AUDIO_FORMATS + ): + return f"Unsupported input_audio_format: {input_format}" + + output_format: object = session_payload.get("output_audio_format") or session_payload.get("response_format") + if output_format is None and isinstance(audio_config, dict): + audio_output = audio_config.get("output") + if isinstance(audio_output, dict): + output_format = audio_output.get("format") + parsed_output, _ = cls._parse_realtime_audio_format(output_format) + if output_format is not None and not ( + isinstance(parsed_output, str) and parsed_output.lower() in REALTIME_OUTPUT_AUDIO_FORMATS + ): + return f"Unsupported output_audio_format: {output_format}" + return None + + @classmethod + def _validate_realtime_response_audio_formats(cls, response_payload: dict[str, object]) -> str | None: + output_format: object = response_payload.get("output_audio_format") or response_payload.get("response_format") + audio_config = response_payload.get("audio") + if output_format is None and isinstance(audio_config, dict): + audio_output = audio_config.get("output") + if isinstance(audio_output, dict): + output_format = audio_output.get("format") + parsed_output, _ = cls._parse_realtime_audio_format(output_format) + if output_format is not None and not ( + isinstance(parsed_output, str) and parsed_output.lower() in REALTIME_OUTPUT_AUDIO_FORMATS + ): + return f"Unsupported output_audio_format: {output_format}" + return None + + @classmethod + def _validate_conversation_item_audio_formats(cls, item: object) -> str | None: + if not isinstance(item, dict): + return None + content = item.get("content") + if not isinstance(content, list): + return None + for part in content: + if not isinstance(part, dict): + continue + if part.get("type") not in {"input_audio", "audio"}: + continue + raw_format = part.get("format") + parsed_format, _ = cls._parse_realtime_audio_format(raw_format) + if raw_format is not None and not ( + isinstance(parsed_format, str) and parsed_format.lower() in REALTIME_INPUT_AUDIO_FORMATS + ): + return f"Unsupported input_audio format in conversation.item.create: {raw_format}" + return None + + @staticmethod + def _json_safe_realtime_payload(payload: dict[str, object]) -> dict[str, object]: + clean: dict[str, object] = {} + for key, value in payload.items(): + if key == "extra_body": + continue + if isinstance(value, str | int | float | bool) or value is None: + clean[key] = value + elif isinstance(value, dict): + clean[key] = RealtimeInputTranslator._json_safe_realtime_payload(value) + elif isinstance(value, list): + clean[key] = [ + (RealtimeInputTranslator._json_safe_realtime_payload(item) if isinstance(item, dict) else item) + for item in value + if isinstance(item, str | int | float | bool | dict) or item is None + ] + return clean + + @staticmethod + def _parse_realtime_audio_format(raw_format: object) -> tuple[object, int | None]: + def normalize_format(fmt: str) -> str: + normalized = fmt.lower() + if normalized in {"audio/pcm", "pcm"}: + return "pcm16" + if normalized in {"audio/wav", "wav"}: + return "wav" + if normalized in {"audio/pcm16", "pcm16", "pcm_s16le", "s16le"}: + return "pcm16" + if normalized in {"audio/pcm_f32le", "pcm_f32le", "f32le"}: + return "pcm_f32le" + if normalized in {"audio/g711_ulaw", "g711_ulaw", "g711-ulaw", "ulaw", "mulaw"}: + return "g711_ulaw" + if normalized in {"audio/g711_alaw", "g711_alaw", "g711-alaw", "alaw"}: + return "g711_alaw" + return fmt + + if isinstance(raw_format, str): + return normalize_format(raw_format), None + if not isinstance(raw_format, dict): + return raw_format, None + rate = raw_format.get("rate") or raw_format.get("sample_rate_hz") or raw_format.get("sample_rate") + sample_rate_hz = int(rate) if isinstance(rate, int | float) and rate > 0 else None + fmt = raw_format.get("type") or raw_format.get("format") + if not isinstance(fmt, str): + return raw_format, sample_rate_hz + return normalize_format(fmt), sample_rate_hz + + @staticmethod + def _duplex_response_format(realtime_format: str) -> str: + normalized = realtime_format.lower() + if normalized in {"pcm16", "pcm_s16le", "s16le"}: + return "pcm" + if normalized in {"g711_ulaw", "g711_alaw"}: + return "pcm" + if normalized in {"wav", "pcm"}: + return normalized + return "wav" + + @staticmethod + def _realtime_output_format(duplex_format: object) -> str: + if isinstance(duplex_format, str) and duplex_format.lower() in {"g711_ulaw", "g711_alaw"}: + return duplex_format.lower() + if isinstance(duplex_format, str) and duplex_format.lower() == "pcm": + return "pcm16" + return str(duplex_format or "wav") + + @staticmethod + def _is_supported_realtime_input_format(fmt: object) -> bool: + return isinstance(fmt, str) and fmt.lower() in REALTIME_INPUT_AUDIO_FORMATS + + @staticmethod + def _input_explicitly_non_speech(event: dict[str, object]) -> bool: + for key in ("is_speech", "speech"): + value = event.get(key) + if isinstance(value, bool): + return not value + vad = event.get("vad") + if isinstance(vad, dict): + value = vad.get("is_speech") + if isinstance(value, bool): + return not value + probability = vad.get("speech_probability", vad.get("probability")) + if isinstance(probability, int | float): + return float(probability) < 0.5 + probability = event.get("speech_probability") + return isinstance(probability, int | float) and float(probability) < 0.5 + + def _input_looks_like_speech(self, event: dict[str, object], *, audio: object, fmt: object) -> bool: + if RealtimeInputTranslator._input_explicitly_non_speech(event): + return False + for key in ("is_speech", "speech"): + value = event.get(key) + if isinstance(value, bool): + return value + vad = event.get("vad") + if isinstance(vad, dict): + probability = vad.get("speech_probability", vad.get("probability")) + if isinstance(probability, int | float): + return float(probability) >= 0.5 + probability = event.get("speech_probability") + if isinstance(probability, int | float): + return float(probability) >= 0.5 + if fmt != "pcm_f32le" or not isinstance(audio, str): + return True + try: + raw = base64.b64decode(audio, validate=True) + except (binascii.Error, ValueError): + return True + if len(raw) < 4 or len(raw) % 4 != 0: + return True + samples = np.frombuffer(raw, dtype=np.float32) + if samples.size == 0: + return False + rms = float(np.sqrt(np.mean(np.square(samples.astype(np.float32))))) + threshold = event.get("overlap_silence_rms") + if not isinstance(threshold, int | float): + vad = event.get("vad") + if isinstance(vad, dict): + threshold = vad.get("silence_rms") + silence_rms = float(threshold) if isinstance(threshold, int | float) else self._overlap_silence_rms + return rms >= max(0.0, silence_rms) + + @staticmethod + def _copy_realtime_input_hints(source: dict[str, object], target: dict[str, object]) -> None: + for key in ( + "duration_ms", + "audio_duration_ms", + "audio_start_ms", + "audio_end_ms", + "is_speech", + "speech", + "speech_probability", + "vad", + "overlap_action", + "overlap", + "force_barge_in", + "force_listen", + "text", + "transcript", + ): + if key in source: + target[key] = source[key] + + @classmethod + def _realtime_overlap_fields(cls, session_payload: dict[str, object]) -> dict[str, object]: + fields: dict[str, object] = {} + if isinstance(session_payload.get("overlap_policy"), str): + fields["overlap_policy"] = session_payload["overlap_policy"] + for key in ("overlap_short_ack_ms", "overlap_barge_in_ms", "overlap_silence_rms"): + value = session_payload.get(key) + if isinstance(value, int | float): + fields[key] = value + + if isinstance(session_payload.get("playback_commit_policy"), str): + fields["playback_commit_policy"] = session_payload["playback_commit_policy"] + return fields + + @staticmethod + def _validate_realtime_video_frames(video_frames: object, max_slice_nums: object) -> str | None: + """Validate omni-duplex camera frames on input_audio_buffer.append. + + Wire contract matches the official MiniCPM-o-Demo omni client: one + base64 JPEG per ~1 s audio chunk. HD slicing (max_slice_nums > 1) is + not implemented by the duplex adapter yet and is rejected explicitly + rather than silently ignored. + """ + if max_slice_nums not in (None, 1): + return "max_slice_nums > 1 (HD slicing) is not implemented by the duplex Realtime adapter" + if not isinstance(video_frames, list): + return "video_frames must be a list of base64-encoded images" + frames = [frame for frame in video_frames if frame is not None] + if len(frames) > 2: + return "video_frames carries more than 2 frames for one append; send ~1 frame per 1 s chunk" + for frame in frames: + if not isinstance(frame, str) or not frame: + return "video_frames entries must be non-empty base64 strings" + if len(frame) > 4_000_000: + return "video_frames entry exceeds 4MB base64; reduce capture resolution or JPEG quality" + try: + header = base64.b64decode(frame[:64] + "=" * (-len(frame[:64]) % 4)) + except (binascii.Error, ValueError): + return "video_frames entries must be valid base64" + if not (header.startswith(b"\xff\xd8") or header.startswith(b"\x89PNG")): + return "video_frames entries must be JPEG or PNG images" + return None + + @staticmethod + def _validate_realtime_turn_detection(session_payload: dict[str, object]) -> str | None: + configured_values: list[tuple[str, object]] = [] + if "turn_detection" in session_payload: + configured_values.append(("turn_detection", session_payload["turn_detection"])) + audio_config = session_payload.get("audio") + if isinstance(audio_config, dict): + audio_input = audio_config.get("input") + if isinstance(audio_input, dict) and "turn_detection" in audio_input: + configured_values.append(("audio.input.turn_detection", audio_input["turn_detection"])) + for field_path, turn_detection in configured_values: + if turn_detection is None: + continue + turn_detection_type = turn_detection.get("type") if isinstance(turn_detection, dict) else turn_detection + return ( + f"{field_path}={turn_detection_type!r} is not implemented by the duplex Realtime adapter; " + "set turn_detection to null and commit input explicitly, or use the model-owned duplex policy" + ) + return None + + @staticmethod + def _input_audio_transcription_config(session_payload: dict[str, object]) -> dict[str, object] | None: + transcription = session_payload.get("input_audio_transcription") + if isinstance(transcription, dict): + return transcription + audio_config = session_payload.get("audio") + if not isinstance(audio_config, dict): + return None + audio_input = audio_config.get("input") + if not isinstance(audio_input, dict): + return None + transcription = audio_input.get("transcription") + return transcription if isinstance(transcription, dict) else None + + @staticmethod + def realtime_max_output_tokens(value: object) -> int | None: + """Normalize Realtime max output tokens. + + OpenAI Realtime clients commonly send ``"inf"`` for unbounded output. + The duplex core represents that as ``None`` because model-specific + defaults remain in force. + """ + if value is None: + return None + if isinstance(value, str) and value.strip().lower() in {"inf", "infinity", "unlimited"}: + return None + if isinstance(value, int) and value > 0: + return int(value) + return None + + async def _conversation_item_to_duplex(self, event: dict[str, object]) -> dict[str, object] | None: + item = event.get("item") + if not isinstance(item, dict): + return None + item = self._normalize_conversation_item(item) + previous_item_id = event.get("previous_item_id") + if isinstance(previous_item_id, str): + item["_previous_item_id"] = previous_item_id + item_id = str(item["id"]) + item_type = item.get("type") + role = item.get("role") + if item_type != "message" or role in {"assistant", "system"}: + return { + "type": "turn.signal", + "event": "conversation.item.create", + "payload": {"item": item}, + } + self._conversation_items[item_id] = item + content = item.get("content") + if not isinstance(content, list): + return None + text_chunks: list[str] = [] + audio_events: list[dict[str, object]] = [] + for part in content: + if not isinstance(part, dict): + continue + if part.get("type") in {"input_text", "text"} and isinstance(part.get("text"), str): + text_chunks.append(str(part["text"])) + if part.get("type") in {"input_audio", "audio"}: + audio = part.get("audio") or part.get("data") + fmt, format_rate = self._parse_realtime_audio_format(part.get("format") or self._input_audio_format) + sample_rate_hz = ( + part.get("sample_rate_hz") or part.get("sample_rate") or format_rate or self._input_sample_rate_hz + ) + if not self._is_supported_realtime_input_format(fmt): + continue + try: + audio, fmt, sample_rate_hz = convert_input_audio_with_rate( + audio, + fmt, + sample_rate_hz=sample_rate_hz if isinstance(sample_rate_hz, int | float) else None, + ) + except ValueError as exc: + await self._send_realtime_payload( + self._realtime_error_payload( + "bad_event", + str(exc), + event_id=event.get("event_id"), + param="sample_rate_hz", + ) + ) + return None + if not isinstance(audio, str) or not audio: + continue + speech_hints = dict(event) + speech_hints.update(part) + if not self._input_looks_like_speech(speech_hints, audio=audio, fmt=fmt): + continue + self._input_speech_started = True + payload = { + "type": "input_audio_buffer.append", + "audio": audio, + "format": fmt, + "sample_rate_hz": sample_rate_hz, + } + self._copy_realtime_input_hints(part, payload) + self._copy_realtime_input_hints(event, payload) + audio_events.append(payload) + if audio_events: + self._input_audio_buffer_has_audio = True + transcript = self._input_transcript_from_item(item) + for extra_event in audio_events[1:]: + self._pending_outbound.put_nowait(extra_event) + commit_payload: dict[str, object] = { + "type": "input_audio_buffer.commit", + "final": True, + "realtime_item_id": item_id, + "response_create": False, + } + if transcript: + commit_payload["transcript"] = transcript + self._pending_outbound.put_nowait(commit_payload) + return audio_events[0] + if not text_chunks: + return None + text_item = dict(item) + text_item["status"] = "completed" + return { + "type": "turn.signal", + "event": "conversation.item.create", + "payload": {"item": text_item}, + } + + def _remember_input_transcript_hint(self, event: dict[str, object]) -> None: + transcript = event.get("transcript") + if not isinstance(transcript, str): + transcript = event.get("text") if isinstance(event.get("text"), str) else None + if not isinstance(transcript, str): + hints = event.get("hints") + if isinstance(hints, dict): + transcript = hints.get("transcript") + if not isinstance(transcript, str): + transcript = hints.get("text") if isinstance(hints.get("text"), str) else None + if isinstance(transcript, str) and transcript: + if ( + self._input_audio_buffer_transcript_parts + and self._input_audio_buffer_transcript_parts[-1] == transcript + ): + return + self._input_audio_buffer_transcript_parts.append(transcript) + + def _consume_input_transcript_hint(self) -> str: + transcript = "".join(self._input_audio_buffer_transcript_parts).strip() + self._input_audio_buffer_transcript_parts.clear() + return transcript + + @staticmethod + def _input_transcript_from_item(item: dict[str, object]) -> str: + content = item.get("content") + if not isinstance(content, list): + return "" + parts: list[str] = [] + for part in content: + if not isinstance(part, dict): + continue + for key in ("transcript", "text"): + value = part.get(key) + if isinstance(value, str) and value: + if part.get("type") in {"input_audio", "audio", "audio_transcript", "transcript"}: + parts.append(value) + break + return "".join(parts).strip() + + @staticmethod + def _input_audio_transcription_completed_event( + item_id: str, + item: dict[str, object], + ) -> dict[str, object] | None: + content = item.get("content") + if not isinstance(content, list): + return None + transcript_parts: list[str] = [] + for index, part in enumerate(content): + if not isinstance(part, dict): + continue + if part.get("type") != "input_audio": + continue + transcript = part.get("transcript") + if isinstance(transcript, str) and transcript: + transcript_parts.append(transcript) + return { + "type": "conversation.item.input_audio_transcription.completed", + "item_id": item_id, + "content_index": index, + "transcript": "".join(transcript_parts), + } + return None + + def _pop_pending_commit_item_id(self) -> str: + try: + return self._pending_commit_item_ids.get_nowait() + except asyncio.QueueEmpty: + return f"item_{uuid4().hex}" + + async def _emit_input_speech_started(self, event: dict[str, object]) -> None: + if self._input_speech_started: + return + self._input_speech_started = True + if self._active_input_item_id is None: + self._active_input_item_id = f"item_{uuid4().hex}" + await self._send_realtime_payload( + { + "type": "input_audio_buffer.speech_started", + "audio_start_ms": int(event.get("audio_start_ms", 0) or 0), + "item_id": self._active_input_item_id, + } + ) + + async def _emit_input_speech_stopped(self, event: dict[str, object], *, item_id: str) -> None: + if not self._input_speech_started: + return + self._input_speech_started = False + audio_end_ms = event.get("audio_end_ms", event.get("audio_ms", 0)) + await self._send_realtime_payload( + { + "type": "input_audio_buffer.speech_stopped", + "audio_end_ms": int(audio_end_ms) if isinstance(audio_end_ms, int | float) else 0, + "item_id": item_id, + } + ) diff --git a/vllm_omni/experimental/fullduplex/openai/realtime_output.py b/vllm_omni/experimental/fullduplex/openai/realtime_output.py new file mode 100644 index 000000000..57c07c852 --- /dev/null +++ b/vllm_omni/experimental/fullduplex/openai/realtime_output.py @@ -0,0 +1,907 @@ +from __future__ import annotations + +from typing import Any +from uuid import uuid4 + +from vllm_omni.experimental.fullduplex.openai.audio import convert_output_audio +from vllm_omni.experimental.fullduplex.openai.realtime_state import ( + _RealtimeResponseState, +) + + +class RealtimeOutputProjector: + """Project internal duplex events onto the OpenAI Realtime schema.""" + + def _from_duplex_event(self, event: dict[str, Any]) -> list[dict[str, object]]: + event_type = event.get("type") + if event_type == "session.created": + session = self._realtime_session_payload(event.get("session")) + created: dict[str, object] = {"type": "session.created", "session": session} + for key in ( + "incarnation", + "attachment_generation", + "resume_token", + ): + if key in event: + created[key] = event[key] + payloads: list[dict[str, object]] = [created] + if self._initial_session_update: + payloads.append({"type": "session.updated", "session": session}) + self._initial_session_update = False + self._hold_realtime_output_until_session_created = False + if self._held_realtime_payloads: + payloads.extend(self._held_realtime_payloads) + self._held_realtime_payloads = [] + return payloads + if event_type == "session.updated": + session = self._realtime_session_payload(event.get("session")) + return [{"type": "session.updated", "session": session}] + if event_type in { + "session.resumed", + "session.heartbeat_ack", + "session.replaced", + "session.expired", + "session.resync_required", + }: + if event_type == "session.resumed": + self._hold_realtime_output_until_session_created = False + return [dict(event)] + if event_type == "response.created": + response_id = event.get("response_id") + if isinstance(response_id, str) and response_id: + self._active_response_id = response_id + self._last_response_id = response_id + item_id = self._response_item_id(response_id) + modalities = event.get("modalities") + has_audio_modality = not isinstance(modalities, list) or "audio" in modalities + item = { + "id": item_id, + "object": "realtime.item", + "type": "message", + "role": "assistant", + "status": "in_progress", + "content": [], + } + self._conversation_items[item_id] = item + payloads = [ + self._response_created_event(event), + *self._conversation_item_added_events(item), + *self._response_output_item_added_events(response_id=response_id, item=item), + ] + if has_audio_modality: + payloads.extend(self._ensure_response_audio_part_added(response_id)) + return payloads + if event_type == "response.listen": + return [ + { + "type": "response.listen", + "session_id": event.get("session_id"), + "epoch": event.get("epoch"), + "response": { + "object": "realtime.response", + "status": "listening", + "metadata": event, + }, + } + ] + if event_type == "response.speak": + response_id = event.get("response_id") + state = self._response_state(response_id) + if state is not None: + if state.speak_emitted: + return [] + state.speak_emitted = True + return [ + { + "type": "response.speak", + "response_id": response_id, + "item_id": self._response_item_id(response_id), + "output_index": 0, + "content_index": 0, + "metadata": self._response_speak_metadata(event), + } + ] + if event_type == "overlap.decision": + return [ + { + "type": "overlap.decision", + "session_id": event.get("session_id"), + "epoch": event.get("epoch"), + "policy": event.get("policy"), + "action": event.get("action"), + "reason": event.get("reason"), + "metadata": event, + } + ] + if event_type == "response.output_audio.delta": + response_id = event.get("response_id") + audio = event.get("audio", "") + payloads: list[dict[str, object]] = [] + if isinstance(audio, str) and audio: + payloads.extend(self._ensure_response_audio_part_added(response_id)) + payloads.extend(self._realtime_audio_delta_events(event, response_id, audio)) + self._refresh_in_progress_response_item(response_id) + text = event.get("text") + has_text = isinstance(text, str) and bool(text) + if has_text: + self._append_response_transcript(response_id, text) + self._refresh_in_progress_response_item(response_id) + # Keep the audio.delta + transcript.delta pair invariant even for + # text-less units (deduplicated continuations, turn-end flush): + # clients that treat the pair as unit-complete would otherwise + # wait on a transcript that never comes. + emit_transcript = has_text or (isinstance(audio, str) and bool(audio)) + if emit_transcript: + if not has_text: + text = "" + payloads.append( + { + "type": "response.audio_transcript.delta", + "response_id": response_id, + "item_id": self._response_item_id(response_id), + "output_index": 0, + "content_index": 0, + "delta": text, + } + ) + if event.get("end_of_turn") is True: + payloads.extend(self._realtime_audio_done_events(event, response_id)) + payloads.extend( + self._realtime_response_terminal_events( + event, + response_id, + status="completed", + status_details={ + "type": "completed", + "reason": event.get("finish_reason") or "stop", + }, + ) + ) + return payloads + if event_type == "response.text.delta": + response_id = event.get("response_id") + text = event.get("delta", "") + state = self._response_state(response_id) + if state is not None and isinstance(text, str) and text: + state.text_parts.append(text) + self._refresh_in_progress_response_item(response_id) + payloads = self._ensure_response_text_part_added(response_id) + payloads.append( + { + "type": "response.output_text.delta", + "response_id": response_id, + "item_id": self._response_item_id(response_id), + "output_index": 0, + "content_index": 1 if state is not None and state.audio_part_added else 0, + "delta": text, + } + ) + return payloads + if event_type == "response.done": + response_id = event.get("response_id") + status = event.get("status") if isinstance(event.get("status"), str) else "completed" + status_details = event.get("status_details") if isinstance(event.get("status_details"), dict) else None + return [ + *self._realtime_audio_done_events(event, response_id), + *self._realtime_response_terminal_events( + event, + response_id, + status=status, + status_details=status_details, + ), + ] + if event_type == "input.committed": + event_item_id = event.get("realtime_item_id") + item_id = ( + event_item_id + if isinstance(event_item_id, str) and event_item_id + else self._pop_pending_commit_item_id() + ) + item = self._conversation_items.get(item_id) + created_payload: list[dict[str, object]] = [] + if item is None: + message = event.get("message") + no_response = event.get("no_response") is True + is_speech = event.get("is_speech") + item = { + "id": item_id, + "object": "realtime.item", + "type": "message", + "role": "user", + "status": "completed", + "content": ( + [{"type": "input_audio", "transcript": "", "is_speech": False}] + if no_response and is_speech is False + else self._user_item_content_from_duplex_message(message) + ), + } + self._conversation_items[item_id] = item + created_payload.extend(self._conversation_item_added_events(item)) + item["status"] = "completed" + payloads = created_payload + [ + self._input_audio_buffer_committed_event(item_id=item_id, event=event), + ] + transcription_event = self._input_audio_transcription_completed_event(item_id, item) + if transcription_event is not None: + payloads.append(transcription_event) + payloads.append(self._conversation_item_done_event(item)) + return payloads + if event_type == "input.cancelled": + return [{"type": "input_audio_buffer.cleared"}] + if event_type == "input_audio_buffer.cleared": + return [{"type": "input_audio_buffer.cleared"}] + if event_type == "audio.cancelled": + response_id = event.get("response_id") + payloads: list[dict[str, object]] = [] + if event.get("reason") == "output_audio_buffer_clear": + if not isinstance(response_id, str) or not response_id: + response_id = self._active_response_id or self._last_response_id + payloads.append( + { + "type": "output_audio_buffer.cleared", + "response_id": response_id, + } + ) + if not isinstance(response_id, str) or not response_id: + return payloads + elif not isinstance(response_id, str) or not response_id: + response_id = self._active_response_id + if not isinstance(response_id, str) or not response_id: + return payloads + if self._response_is_done(response_id): + return payloads + committed_ms = event.get("committed_ms") + if isinstance(committed_ms, int | float): + item_id = self._response_item_id(response_id) + committed_audio_ms = max(0, int(committed_ms)) + self._item_truncation_cursors[item_id] = (0, committed_audio_ms) + item = self._conversation_items.get(item_id) + if item is not None: + self._truncate_realtime_item_content( + item, + content_index=0, + audio_end_ms=committed_audio_ms, + ) + payloads.append( + { + "type": "conversation.item.truncated", + "item_id": item_id, + "content_index": 0, + "audio_end_ms": committed_audio_ms, + "event": event, + } + ) + payloads.extend(self._realtime_audio_done_events(event, response_id)) + payloads.extend( + self._realtime_response_terminal_events( + event, + response_id, + status="cancelled", + status_details={ + "type": "cancelled", + "reason": event.get("reason") or "client_cancelled", + }, + ) + ) + if isinstance(response_id, str) and response_id == self._active_response_id: + self._active_response_id = None + return payloads + if event_type == "playback.acknowledged": + return [{"type": "playback.acknowledged", "event": event}] + if event_type == "conversation.item.created": + item = event.get("item") + if isinstance(item, dict) and isinstance(item.get("id"), str): + item_id = str(item["id"]) + already_known = item_id in self._conversation_items + self._conversation_items[item_id] = item + if already_known: + if item.get("status") == "completed": + return [self._conversation_item_done_event(item)] + return [] + payloads = self._conversation_item_added_events(item) + if item.get("status") == "completed": + payloads.append(self._conversation_item_done_event(item)) + return payloads + return [{"type": "conversation.item.created", "item": item, "event": event}] + if event_type == "conversation.item.deleted": + item_id = event.get("item_id") + if isinstance(item_id, str): + self._remove_conversation_item(item_id) + return [ + { + "type": "conversation.item.deleted", + "item_id": item_id, + "event": event, + } + ] + if event_type == "conversation.item.truncated": + item_id = event.get("item_id") + audio_end_ms = event.get("audio_end_ms") + content_index = event.get("content_index", 0) + if isinstance(item_id, str): + item = self._conversation_items.get(item_id) + if item is not None: + self._truncate_realtime_item_content( + item, + content_index=int(content_index) if isinstance(content_index, int | float) else 0, + audio_end_ms=int(audio_end_ms) if isinstance(audio_end_ms, int | float) else 0, + ) + return [ + { + "type": "conversation.item.truncated", + "item_id": item_id, + "content_index": content_index, + "audio_end_ms": audio_end_ms, + "event": event, + } + ] + if event_type == "response.output_item.done": + response_id = event.get("response_id") + return self._realtime_response_terminal_events( + event, + response_id, + status="completed", + status_details={ + "type": "completed", + "reason": event.get("finish_reason") or "stop", + }, + ) + if event_type == "error": + raw_error = event.get("error") + if isinstance(raw_error, dict): + return [event] + message = str(raw_error or event.get("message") or "Duplex runtime error") + code = str(event.get("code") or "duplex_error") + return [self._realtime_error_payload(code, message)] + if event_type == "session.closed": + return [{"type": "session.closed", "event": event}] + return [{"type": f"duplex.{event_type}", "event": event}] + + @staticmethod + def _user_item_content_from_duplex_message(message: object) -> list[dict[str, object]]: + if not isinstance(message, dict): + return [{"type": "input_audio", "transcript": ""}] + message_transcript = message.get("transcript") + content = message.get("content") + if isinstance(content, str): + return [{"type": "input_text", "text": content}] + if not isinstance(content, list): + return [ + { + "type": "input_audio", + "transcript": message_transcript if isinstance(message_transcript, str) else "", + } + ] + parts: list[dict[str, object]] = [] + for part in content: + if not isinstance(part, dict): + continue + part_type = part.get("type") + if part_type in {"text", "input_text"} and isinstance(part.get("text"), str): + parts.append({"type": "input_text", "text": part["text"]}) + continue + if part_type == "audio_url": + transcript = ( + part.get("transcript") + if isinstance(part.get("transcript"), str) + else message_transcript + if isinstance(message_transcript, str) + else "" + ) + parts.append({"type": "input_audio", "transcript": transcript}) + continue + if part_type in {"audio", "input_audio"}: + transcript = part.get("transcript") if isinstance(part.get("transcript"), str) else "" + parts.append({"type": "input_audio", "transcript": transcript}) + return parts or [{"type": "input_audio", "transcript": ""}] + + def _realtime_session_payload(self, session: object) -> object: + if not isinstance(session, dict): + return session + payload = dict(session) + payload.setdefault("object", "realtime.session") + payload.setdefault("type", "realtime") + payload.setdefault("id", payload.get("id") or self._default_session_id) + payload.setdefault("model", payload.get("model") or self._default_model) + payload.setdefault("input_audio_format", self._input_audio_format) + payload.setdefault("output_audio_format", self._output_audio_format) + payload.setdefault("modalities", payload.get("modalities") or ["text", "audio"]) + payload.setdefault("output_modalities", payload.get("output_modalities") or payload.get("modalities")) + payload.setdefault("object", "realtime.session") + payload.setdefault( + "audio", + { + "input": { + "format": self._realtime_audio_format_object( + self._input_audio_format, + sample_rate_hz=self._input_sample_rate_hz, + ), + "sample_rate_hz": self._input_sample_rate_hz, + }, + "output": { + "format": self._realtime_audio_format_object( + self._output_audio_format, + sample_rate_hz=self._output_sample_rate_hz, + ), + }, + }, + ) + payload.setdefault("turn_detection", payload.get("turn_detection")) + payload.setdefault("input_audio_transcription", payload.get("input_audio_transcription")) + payload.setdefault("tracing", payload.get("tracing")) + return payload + + @staticmethod + def _realtime_audio_format_object(fmt: object, *, sample_rate_hz: int | None = None) -> dict[str, object]: + if isinstance(fmt, str) and fmt.lower() in {"pcm16", "pcm_s16le", "s16le", "pcm"}: + payload: dict[str, object] = {"type": "audio/pcm"} + elif isinstance(fmt, str) and fmt.lower() == "pcm_f32le": + payload = {"type": "audio/pcm_f32le"} + elif isinstance(fmt, str) and fmt.lower() == "g711_ulaw": + payload = {"type": "audio/g711_ulaw"} + elif isinstance(fmt, str) and fmt.lower() == "g711_alaw": + payload = {"type": "audio/g711_alaw"} + else: + payload = {"type": "audio/wav"} + if sample_rate_hz is not None: + payload["rate"] = int(sample_rate_hz) + return payload + + def _response_state( + self, + response_id: object, + *, + event: dict[str, Any] | None = None, + create: bool = True, + ) -> _RealtimeResponseState | None: + if isinstance(response_id, str) and response_id: + key: str | int = response_id + item_id = f"item_{response_id}" + elif event is not None: + key = id(event) + item_id = f"item_{uuid4().hex}" + else: + return None + state = self._response_states.get(key) + if state is None and create: + state = _RealtimeResponseState(item_id=item_id) + self._response_states[key] = state + return state + + def _response_is_done(self, response_id: object) -> bool: + state = self._response_state(response_id, create=False) + return state is not None and state.done_emitted + + def _response_item_id(self, response_id: object) -> str: + state = self._response_state(response_id) + if state is not None: + return state.item_id + return f"item_{uuid4().hex}" + + def _response_done_output_item( + self, + response_id: object, + *, + status: str, + ) -> dict[str, object]: + item_id = self._response_item_id(response_id) + state = self._response_state(response_id) + transcript = state.transcript if state is not None else "" + text = state.text if state is not None else "" + audio_duration_ms = state.audio_duration_ms if state is not None else None + audio_text_marks = state.audio_text_marks if state is not None else None + content: list[dict[str, object]] = [] + if (state is not None and state.audio_part_added) or transcript or audio_duration_ms is not None: + content.append( + self._response_item_content_part( + transcript=transcript, + audio_duration_ms=audio_duration_ms, + audio_text_marks=audio_text_marks, + ) + ) + if text: + content.append(self._response_item_text_content_part(text=text)) + item = { + "id": item_id, + "object": "realtime.item", + "type": "message", + "role": "assistant", + "status": status, + "content": content, + } + self._apply_pending_item_truncation(item) + return item + + def _apply_pending_item_truncation(self, item: dict[str, object]) -> None: + item_id = item.get("id") + if not isinstance(item_id, str) or not item_id: + return + cursor = self._item_truncation_cursors.get(item_id) + if cursor is None: + return + content_index, audio_end_ms = cursor + self._truncate_realtime_item_content( + item, + content_index=content_index, + audio_end_ms=audio_end_ms, + ) + + def _refresh_in_progress_response_item(self, response_id: object) -> None: + if not isinstance(response_id, str) or not response_id: + return + item_id = self._response_item_id(response_id) + item = self._conversation_items.get(item_id) + if not isinstance(item, dict): + return + content = item.get("content") + if not isinstance(content, list): + content = [] + item["content"] = content + + state = self._response_state(response_id) + if state is None: + return + transcript = state.transcript + audio_duration_ms = state.audio_duration_ms + audio_text_marks = state.audio_text_marks + has_audio = state.audio_part_added or bool(transcript) or audio_duration_ms is not None + if has_audio: + audio_part = self._response_item_content_part( + transcript=transcript, + audio_duration_ms=audio_duration_ms, + audio_text_marks=audio_text_marks, + ) + if content and isinstance(content[0], dict) and content[0].get("type") in {"audio", "output_audio"}: + content[0] = audio_part + else: + content.insert(0, audio_part) + + text = state.text + if text: + text_index = ( + 1 + if content + and isinstance(content[0], dict) + and content[0].get("type") + in { + "audio", + "output_audio", + } + else 0 + ) + text_part = self._response_item_text_content_part(text=text) + if ( + len(content) > text_index + and isinstance(content[text_index], dict) + and content[text_index].get("type") in {"text", "output_text"} + ): + content[text_index] = text_part + else: + content.insert(text_index, text_part) + + self._apply_pending_item_truncation(item) + + def _append_response_transcript(self, response_id: object, text: str) -> None: + state = self._response_state(response_id) + if state is None or not text: + return + state.transcript_parts.append(text) + + def _ensure_response_text_part_added(self, response_id: object) -> list[dict[str, object]]: + state = self._response_state(response_id) + if state is None: + return [] + if state.text_part_added: + return [] + state.text_part_added = True + content_index = 1 if state.audio_part_added else 0 + return [ + { + "type": "response.content_part.added", + "response_id": response_id, + "item_id": self._response_item_id(response_id), + "output_index": 0, + "content_index": content_index, + "part": self._response_text_content_part(), + } + ] + + def _ensure_response_audio_part_added(self, response_id: object) -> list[dict[str, object]]: + state = self._response_state(response_id) + if state is None: + return [] + if state.audio_part_added: + return [] + state.audio_part_added = True + return [ + { + "type": "response.content_part.added", + "response_id": response_id, + "item_id": self._response_item_id(response_id), + "output_index": 0, + "content_index": 0, + "part": self._response_content_part(), + } + ] + + def _remember_response_audio_metadata(self, response_id: object, event: dict[str, Any]) -> None: + state = self._response_state(response_id) + if state is None: + return + duration = event.get("audio_duration_ms") + playback = event.get("playback") + if not isinstance(duration, int | float) and isinstance(playback, dict): + duration = playback.get("sent_ms") or playback.get("generated_ms") + if isinstance(duration, int | float): + state.audio_duration_ms = max(state.audio_duration_ms or 0, int(duration)) + marks = event.get("audio_text_marks") + if not isinstance(marks, list): + return + clean_marks: list[dict[str, int]] = [] + for mark in marks: + if not isinstance(mark, dict): + continue + text_chars = mark.get("text_chars") + audio_end_ms = mark.get("audio_end_ms", mark.get("audio_ms")) + if not isinstance(text_chars, int | float) or not isinstance(audio_end_ms, int | float): + continue + clean_marks.append( + { + "text_chars": max(0, int(text_chars)), + "audio_end_ms": max(0, int(audio_end_ms)), + } + ) + if clean_marks: + merged = list(state.audio_text_marks) + merged.extend(clean_marks) + deduped: dict[tuple[int, int], dict[str, int]] = {} + for mark in merged: + deduped[(int(mark["audio_end_ms"]), int(mark["text_chars"]))] = mark + state.audio_text_marks = sorted( + deduped.values(), + key=lambda mark: (mark["audio_end_ms"], mark["text_chars"]), + ) + + def _response_created_event(self, event: dict[str, Any]) -> dict[str, object]: + response_id = event.get("response_id") + metadata = event.get("metadata") + if not isinstance(metadata, dict): + metadata = {} + metadata = {**metadata, "duplex_event": event} + return { + "type": "response.created", + "response_id": response_id, + "response": { + "id": response_id, + "object": "realtime.response", + "status": "in_progress", + "status_details": None, + "output": [], + "modalities": event.get("modalities") or ["audio", "text"], + "metadata": metadata, + }, + } + + def _realtime_audio_delta_events( + self, + event: dict[str, Any], + response_id: object, + audio: str, + ) -> list[dict[str, object]]: + item_id = self._response_item_id(response_id) + fmt, format_rate = self._parse_realtime_audio_format(event.get("format", "wav")) + source_fmt = self._realtime_output_format(fmt) + source_sample_rate_hz = event.get("sample_rate_hz") or format_rate + target_sample_rate_hz = ( + self._output_sample_rate_hz if self._output_audio_format in {"g711_ulaw", "g711_alaw"} else None + ) + audio, fmt, converted_sample_rate_hz = convert_output_audio( + audio, + source_fmt=source_fmt, + target_fmt=self._output_audio_format, + source_sample_rate_hz=( + int(source_sample_rate_hz) if isinstance(source_sample_rate_hz, int | float) else None + ), + target_sample_rate_hz=target_sample_rate_hz, + ) + sample_rate_hz = ( + converted_sample_rate_hz + if fmt in {"g711_ulaw", "g711_alaw"} + else event.get("sample_rate_hz") or format_rate or self._output_sample_rate_hz + ) + metadata: dict[str, object] = {} + for key in ("session_id", "epoch", "model_speak", "end_of_turn", "playback", "vllm_omni"): + if key in event: + metadata[key] = event[key] + duration_ms = event.get("audio_duration_ms") + if isinstance(duration_ms, int | float): + metadata["audio_duration_ms"] = int(duration_ms) + marks = event.get("audio_text_marks") + if isinstance(marks, list): + metadata["audio_text_marks"] = marks + state = self._response_state(response_id) + if state is not None: + state.audio_delta_emitted = True + self._remember_response_audio_metadata(response_id, event) + payloads: list[dict[str, object]] = [] + if state is not None and not state.speak_emitted and metadata.get("model_speak") is True: + state.speak_emitted = True + payloads.insert( + 0, + { + "type": "response.speak", + "response_id": response_id, + "item_id": item_id, + "output_index": 0, + "content_index": 0, + "metadata": self._response_speak_metadata(event), + }, + ) + payloads.append( + { + "type": "response.audio.delta", + "response_id": response_id, + "item_id": item_id, + "output_index": 0, + "content_index": 0, + "delta": audio, + "format": fmt, + **({"sample_rate_hz": int(sample_rate_hz)} if isinstance(sample_rate_hz, int | float) else {}), + **({"metadata": metadata} if metadata else {}), + } + ) + return payloads + + @staticmethod + def _response_speak_metadata(event: dict[str, Any]) -> dict[str, object]: + return {key: event[key] for key in ("session_id", "epoch", "model_speak") if key in event} + + def _realtime_audio_done_events( + self, + event: dict[str, Any], + response_id: object, + ) -> list[dict[str, object]]: + item_id = self._response_item_id(response_id) + state = self._response_state(response_id, event=event) + transcript = state.transcript if state is not None else "" + payloads: list[dict[str, object]] = [] + if isinstance(response_id, str) and state is not None and not state.audio_delta_emitted and not transcript: + return payloads + if state is None or not state.audio_done_emitted: + if state is not None: + state.audio_done_emitted = True + payloads.append( + { + "type": "response.audio.done", + "response_id": response_id, + "item_id": item_id, + "output_index": 0, + "content_index": 0, + } + ) + if transcript: + payloads.append( + { + "type": "response.audio_transcript.done", + "response_id": response_id, + "item_id": item_id, + "output_index": 0, + "content_index": 0, + "transcript": transcript, + } + ) + return payloads + + def _realtime_response_terminal_events( + self, + event: dict[str, Any], + response_id: object, + *, + status: str = "completed", + status_details: dict[str, object] | None = None, + ) -> list[dict[str, object]]: + item_id = self._response_item_id(response_id) + state = self._response_state(response_id, event=event) + transcript = state.transcript if state is not None else "" + payloads: list[dict[str, object]] = [] + if state is not None and state.audio_part_added and not state.audio_part_done: + state.audio_part_done = True + payloads.append( + { + "type": "response.content_part.done", + "response_id": response_id, + "item_id": item_id, + "output_index": 0, + "content_index": 0, + "part": self._response_content_part(transcript=transcript), + } + ) + if state is not None and state.text_parts and not state.output_text_done: + state.output_text_done = True + content_index = 1 if state.audio_part_added else 0 + payloads.append( + { + "type": "response.output_text.done", + "response_id": response_id, + "item_id": item_id, + "output_index": 0, + "content_index": content_index, + "text": state.text, + } + ) + if state is not None and state.text_parts and not state.text_part_done: + state.text_part_done = True + content_index = 1 if state.audio_part_added else 0 + payloads.append( + { + "type": "response.content_part.done", + "response_id": response_id, + "item_id": item_id, + "output_index": 0, + "content_index": content_index, + "part": self._response_text_content_part(text=state.text), + } + ) + if state is None or not state.output_item_done: + if state is not None: + state.output_item_done = True + item = self._response_done_output_item(response_id, status=status) + payloads.append( + { + "type": "response.output_item.done", + "response_id": response_id, + "output_index": 0, + "item": item, + } + ) + if state is None or not state.conversation_item_done: + if state is not None: + state.conversation_item_done = True + payloads.append(self._conversation_item_done_event(item)) + done_event = self._realtime_response_done_event( + {**event, "response_id": response_id}, + state=state, + status=status, + status_details=status_details, + ) + if done_event is not None: + payloads.append(done_event) + payloads.append(self._rate_limits_updated_event()) + if isinstance(response_id, str) and response_id == self._active_response_id: + self._active_response_id = None + return payloads + + def _realtime_response_done_event( + self, + event: dict[str, Any], + *, + state: _RealtimeResponseState | None = None, + status: str = "completed", + status_details: dict[str, object] | None = None, + ) -> dict[str, object] | None: + response_id = event.get("response_id") + if state is None: + state = self._response_state(response_id, event=event) + if state is not None: + if state.done_emitted: + return None + state.done_emitted = True + return { + "type": "response.done", + "response_id": response_id, + "response": { + "id": response_id, + "object": "realtime.response", + "status": status, + "status_details": status_details, + "output": [self._response_done_output_item(response_id, status=status)], + "metadata": event, + }, + } + + @staticmethod + def _rate_limits_updated_event() -> dict[str, object]: + # vLLM-Omni does not yet expose a Realtime-specific quota budget. Emit + # the terminal event with an empty list so clients that sequence on + # rate_limits.updated do not need a private vLLM branch. + return {"type": "rate_limits.updated", "rate_limits": []} diff --git a/vllm_omni/experimental/fullduplex/openai/realtime_session.py b/vllm_omni/experimental/fullduplex/openai/realtime_session.py new file mode 100644 index 000000000..519c071cf --- /dev/null +++ b/vllm_omni/experimental/fullduplex/openai/realtime_session.py @@ -0,0 +1,139 @@ +from __future__ import annotations + +import json +from typing import Any +from uuid import uuid4 + +from fastapi import WebSocket + +from vllm_omni.experimental.fullduplex.openai import realtime_state as _state +from vllm_omni.experimental.fullduplex.openai.realtime_input import ( + RealtimeInputTranslator, +) +from vllm_omni.experimental.fullduplex.openai.realtime_output import ( + RealtimeOutputProjector, +) + +REALTIME_ERROR_TYPES_BY_CODE = _state.REALTIME_ERROR_TYPES_BY_CODE +REALTIME_INPUT_AUDIO_FORMATS = _state.REALTIME_INPUT_AUDIO_FORMATS +REALTIME_OUTPUT_AUDIO_FORMATS = _state.REALTIME_OUTPUT_AUDIO_FORMATS +RealtimeSessionState = _state.RealtimeSessionState +_RealtimeResponseState = _state._RealtimeResponseState + + +class NativeRealtimeSessionProtocol( + RealtimeInputTranslator, + RealtimeOutputProjector, + _state.RealtimeStateOwner, +): + """Compatibility facade for one native Realtime protocol session. + + Input translation, output projection, and mutable state have separate + owners. The facade retains the established import path and private helper + surface used by serving and protocol contract tests. + """ + + def __init__(self, query_params: Any) -> None: + self._state = RealtimeSessionState.from_query_params(query_params) + + def bind_sender(self, send_realtime_json) -> None: + self._send_realtime_json = send_realtime_json + + def _default_session_payload(self) -> dict[str, object]: + payload: dict[str, object] = { + "model": self._default_model, + "session_id": self._default_session_id, + } + if self._default_extra_body: + payload["extra_body"] = dict(self._default_extra_body) + return payload + + async def receive_internal_event_text(self, websocket: WebSocket) -> str: + if not self._pending_outbound.empty(): + return json.dumps(await self._pending_outbound.get()) + if not self._opened and not self._resume_only and not self._autostarted_default_session and self._default_model: + self._opened = True + self._autostarted_default_session = True + return json.dumps(self._session_create_from_realtime(self._default_session_payload())) + while True: + if not self._pending_outbound.empty(): + return json.dumps(await self._pending_outbound.get()) + raw = await websocket.receive_text() + try: + event = json.loads(raw) + except json.JSONDecodeError: + return raw + if not isinstance(event, dict): + return raw + if not self._opened and event.get("type") == "session.resume": + self._opened = True + translated = await self._to_duplex_event(event) + if translated is None: + if not self._pending_outbound.empty(): + return json.dumps(await self._pending_outbound.get()) + continue + return json.dumps(translated) + if not self._opened and self._resume_only: + translated = await self._to_duplex_event(event) + if translated is None: + if not self._pending_outbound.empty(): + return json.dumps(await self._pending_outbound.get()) + continue + return json.dumps(translated) + if not self._opened and event.get("type") != "session.update": + self._opened = True + await self._pending_outbound.put(self._session_create_from_realtime(self._default_session_payload())) + translated = await self._to_duplex_event(event) + if translated is not None: + await self._send_realtime_input_ack(event) + await self._pending_outbound.put(translated) + return json.dumps(await self._pending_outbound.get()) + translated = await self._to_duplex_event(event) + if translated is None: + if not self._pending_outbound.empty(): + return json.dumps(await self._pending_outbound.get()) + continue + await self._send_realtime_input_ack(event) + return json.dumps(translated) + + def encode_outbound_event(self, data: dict[str, Any]) -> list[dict[str, object]]: + payloads = self._from_duplex_event(data) + for payload in payloads: + self._attach_event_id(payload) + return payloads + + @staticmethod + def _attach_event_id(payload: dict[str, object]) -> None: + payload.setdefault("event_id", f"event_{uuid4().hex}") + + async def _send_realtime_payload(self, payload: dict[str, object]) -> None: + self._attach_event_id(payload) + if ( + self._hold_realtime_output_until_session_created + and payload.get("type") != "error" + and payload.get("type") != "session.created" + ): + self._held_realtime_payloads.append(payload) + return + if self._send_realtime_json is None: + raise RuntimeError("Native Realtime sender is not bound") + await self._send_realtime_json(payload) + + @staticmethod + def _realtime_error_payload( + code: str, + message: str, + *, + event_id: object | None = None, + param: object | None = None, + ) -> dict[str, object]: + error: dict[str, object] = { + "type": REALTIME_ERROR_TYPES_BY_CODE.get(code, "invalid_request_error"), + "code": code, + "message": message, + } + if isinstance(event_id, str) and event_id: + error["event_id"] = event_id + if isinstance(param, str) and param: + error["param"] = param + return {"type": "error", "error": error} diff --git a/vllm_omni/experimental/fullduplex/openai/realtime_state.py b/vllm_omni/experimental/fullduplex/openai/realtime_state.py new file mode 100644 index 000000000..ac3b2a8de --- /dev/null +++ b/vllm_omni/experimental/fullduplex/openai/realtime_state.py @@ -0,0 +1,197 @@ +from __future__ import annotations + +import asyncio +from dataclasses import dataclass, field +from typing import Any, Generic, TypeVar + +REALTIME_INPUT_AUDIO_FORMATS = { + "pcm16", + "pcm_s16le", + "s16le", + "pcm_f32le", + "g711_ulaw", + "g711_alaw", +} +REALTIME_OUTPUT_AUDIO_FORMATS = { + "pcm16", + "pcm_s16le", + "s16le", + "wav", + "pcm", + "g711_ulaw", + "g711_alaw", +} +REALTIME_ERROR_TYPES_BY_CODE = { + "bad_event": "invalid_request_error", + "bad_audio": "invalid_request_error", + "config_timeout": "invalid_request_error", + "invalid_json": "invalid_request_error", + "event_too_large": "invalid_request_error", + "unknown_event": "invalid_request_error", + "internal_error": "server_error", + "runtime_open_failed": "server_error", + "runtime_open_unsupported": "server_error", + "runtime_append_failed": "server_error", + "runtime_append_task_failed": "server_error", + "runtime_signal_failed": "server_error", + "runtime_close_failed": "server_error", + "runtime_abort_failed": "server_error", + "runtime_data_plane_stream_failed": "server_error", + "runtime_data_plane_text_without_audio": "server_error", + "response_error": "server_error", + "chat_error": "server_error", + "duplex_session_busy": "rate_limit_error", + "resource_exhausted": "rate_limit_error", + "response_already_active": "invalid_request_error", + "response_not_active": "invalid_request_error", + "response_create_without_input": "invalid_request_error", + "input_audio_buffer_empty": "invalid_request_error", + "missing_item_id": "invalid_request_error", + "item_not_found": "invalid_request_error", + "unsupported_audio_format": "invalid_request_error", + "unsupported_ref_audio_path": "invalid_request_error", + "ref_audio_required": "invalid_request_error", + "model_update_unsupported": "invalid_request_error", + "voice_update_after_audio_unsupported": "invalid_request_error", + "ref_audio_update_unsupported": "invalid_request_error", + "native_text_append_unsupported": "invalid_request_error", + "runtime_native_stage_role_required": "server_error", + "runtime_native_runner_kv_required": "server_error", +} + + +@dataclass(slots=True) +class _RealtimeResponseState: + item_id: str + transcript_parts: list[str] = field(default_factory=list) + text_parts: list[str] = field(default_factory=list) + audio_duration_ms: int | None = None + audio_text_marks: list[dict[str, int]] = field(default_factory=list) + audio_delta_emitted: bool = False + audio_done_emitted: bool = False + audio_part_added: bool = False + audio_part_done: bool = False + text_part_added: bool = False + text_part_done: bool = False + output_text_done: bool = False + output_item_done: bool = False + conversation_item_done: bool = False + speak_emitted: bool = False + done_emitted: bool = False + + @property + def transcript(self) -> str: + return "".join(self.transcript_parts) + + @property + def text(self) -> str: + return "".join(self.text_parts) + + +@dataclass(slots=True) +class RealtimeSessionState: + """Single mutable authority for one Realtime protocol session.""" + + _opened: bool = False + _autostarted_default_session: bool = False + _resume_only: bool = False + _pending_outbound: asyncio.Queue[dict[str, object]] = field(default_factory=asyncio.Queue) + _held_realtime_payloads: list[dict[str, object]] = field(default_factory=list) + _hold_realtime_output_until_session_created: bool = True + _default_model: object | None = None + _default_session_id: object | None = None + _default_extra_body: dict[str, object] = field(default_factory=dict) + _input_audio_format: str = "pcm16" + _input_sample_rate_hz: int = 16000 + _output_audio_format: str = "pcm16" + _overlap_silence_rms: float = 0.003 + _send_realtime_json: Any = None + _initial_session_update: bool = False + _input_speech_started: bool = False + _response_states: dict[str | int, _RealtimeResponseState] = field(default_factory=dict) + _item_truncation_cursors: dict[str, tuple[int, int]] = field(default_factory=dict) + _active_response_id: str | None = None + _last_response_id: str | None = None + _conversation_items: dict[str, dict[str, object]] = field(default_factory=dict) + _pending_commit_item_ids: asyncio.Queue[str] = field(default_factory=asyncio.Queue) + _last_conversation_item_id: str | None = None + _output_sample_rate_hz: int | None = None + _active_input_item_id: str | None = None + _input_audio_buffer_has_audio: bool = False + _input_audio_buffer_had_non_speech: bool = False + _input_audio_buffer_transcript_parts: list[str] = field(default_factory=list) + + @classmethod + def from_query_params(cls, query_params: Any) -> RealtimeSessionState: + if hasattr(query_params, "query_params"): + query_params = query_params.query_params + getter = query_params.get if hasattr(query_params, "get") else None + native_duplex = getter("minicpmo45_native_duplex") if getter is not None else None + resume_only = getter("resume") if getter is not None else None + autostart = getter("autostart") if getter is not None else None + default_extra_body = {} + if str(native_duplex).strip().lower() in {"1", "true", "yes", "on"}: + default_extra_body["minicpmo45_native_duplex"] = True + return cls( + _resume_only=( + str(resume_only).strip().lower() in {"1", "true", "yes", "on"} + or str(autostart).strip().lower() in {"0", "false", "no", "off"} + ), + _default_model=getter("model") if getter is not None else None, + _default_session_id=(getter("session_id") if getter is not None else None), + _default_extra_body=default_extra_body, + ) + + +_StateValue = TypeVar("_StateValue") + + +class _RealtimeStateField(Generic[_StateValue]): + """Explicit descriptor binding one protocol attribute to session state.""" + + def __set_name__(self, owner: type, name: str) -> None: + del owner + self._name = name + + def __get__(self, instance: Any, owner: type | None = None) -> _StateValue | _RealtimeStateField[_StateValue]: + del owner + if instance is None: + return self + return getattr(instance._state, self._name) + + def __set__(self, instance: Any, value: _StateValue) -> None: + setattr(instance._state, self._name, value) + + +class RealtimeStateOwner: + """Declare the mutable state surface shared by input/output projectors.""" + + _state: RealtimeSessionState + _opened: bool = _RealtimeStateField() + _autostarted_default_session: bool = _RealtimeStateField() + _resume_only: bool = _RealtimeStateField() + _pending_outbound: asyncio.Queue[dict[str, object]] = _RealtimeStateField() + _held_realtime_payloads: list[dict[str, object]] = _RealtimeStateField() + _hold_realtime_output_until_session_created: bool = _RealtimeStateField() + _default_model: object | None = _RealtimeStateField() + _default_session_id: object | None = _RealtimeStateField() + _default_extra_body: dict[str, object] = _RealtimeStateField() + _input_audio_format: str = _RealtimeStateField() + _input_sample_rate_hz: int = _RealtimeStateField() + _output_audio_format: str = _RealtimeStateField() + _overlap_silence_rms: float = _RealtimeStateField() + _send_realtime_json: Any = _RealtimeStateField() + _initial_session_update: bool = _RealtimeStateField() + _input_speech_started: bool = _RealtimeStateField() + _response_states: dict[str | int, _RealtimeResponseState] = _RealtimeStateField() + _item_truncation_cursors: dict[str, tuple[int, int]] = _RealtimeStateField() + _active_response_id: str | None = _RealtimeStateField() + _last_response_id: str | None = _RealtimeStateField() + _conversation_items: dict[str, dict[str, object]] = _RealtimeStateField() + _pending_commit_item_ids: asyncio.Queue[str] = _RealtimeStateField() + _last_conversation_item_id: str | None = _RealtimeStateField() + _output_sample_rate_hz: int | None = _RealtimeStateField() + _active_input_item_id: str | None = _RealtimeStateField() + _input_audio_buffer_has_audio: bool = _RealtimeStateField() + _input_audio_buffer_had_non_speech: bool = _RealtimeStateField() + _input_audio_buffer_transcript_parts: list[str] = _RealtimeStateField() diff --git a/vllm_omni/experimental/fullduplex/openai/runtime_adapter.py b/vllm_omni/experimental/fullduplex/openai/runtime_adapter.py new file mode 100644 index 000000000..f88cffd3b --- /dev/null +++ b/vllm_omni/experimental/fullduplex/openai/runtime_adapter.py @@ -0,0 +1,215 @@ +from __future__ import annotations + +import asyncio +from collections.abc import Awaitable, Callable, Iterable, Mapping, MutableMapping +from importlib import import_module +from typing import Any, Protocol + + +class ServingRuntimeConfigError(ValueError): + """A model serving plugin rejected client-visible runtime configuration.""" + + def __init__(self, message: str, *, code: str = "invalid_duplex_runtime_config") -> None: + super().__init__(message) + self.code = code + + +class PcmAppendReservation(Protocol): + operation_id: str + payload: dict[str, object] | None + + @property + def active(self) -> bool: ... + + @property + def byte_count(self) -> int: ... + + def commit(self) -> None: ... + + def rollback(self) -> None: ... + + +class PcmAppendBuffer(Protocol): + @property + def pending_byte_count(self) -> int: ... + + def clear(self) -> None: ... + + def clear_force_listen(self) -> None: ... + + def has_pending(self) -> bool: ... + + def has_reserved(self) -> bool: ... + + def prepare_append( + self, + payload: dict[str, object], + *, + operation_id: str, + chunk_period_ms: int, + allow_emit: bool, + ) -> PcmAppendReservation | None: ... + + def prepare_commit( + self, + *, + operation_id: str, + chunk_period_ms: int, + ) -> PcmAppendReservation: ... + + def flush(self, *, chunk_period_ms: int) -> dict[str, object] | None: ... + + +class ServingRuntimeSessionState(Protocol): + audio_buffer: PcmAppendBuffer + input_since_commit: bool + speech_since_commit: bool + committed_audio_payload: dict[str, object] | None + committed_audio_operation_id: str | None + committed_audio_reserved_bytes: int + deferred_response_create: bool + deferred_precreate_response: bool + data_plane_task: asyncio.Task[None] | None + data_plane_restart_requested: bool + continuation_owner_id: str | None + continuation_units: int + pending_silence_task: asyncio.Task[bool] | None + pending_silence_owner_id: str | None + silence_continuation_scheduler: Callable[..., Awaitable[bool]] | None + + def retain_committed_audio( + self, + payload: dict[str, object], + *, + operation_id: str | None, + reserved_bytes: int = 0, + ) -> None: ... + + def clear_committed_audio(self) -> int: ... + + def clear_continuation(self) -> None: ... + + +class RuntimeDataPlane(Protocol): + def begin_request(self, request_id: str) -> None: ... + + def is_terminal(self, request_id: str | None) -> bool: ... + + def mark_terminal(self, request_id: str) -> None: ... + + def close_stream(self, request_id: str) -> None: ... + + def close_session(self, session_id: str, *, active_request_id: str | None = None) -> None: ... + + def project(self, result: object, *, context: object | None = None) -> Iterable[dict[str, object]]: ... + + +class ServingRuntimeAdapter(Protocol): + adapter_id: str + session_states: MutableMapping[str, ServingRuntimeSessionState] + data_plane: RuntimeDataPlane + clean_response_done_prefix: str + interrupted_tts_prefix: str + private_runtime_config_keys: frozenset[str] + + def create_session_state(self) -> ServingRuntimeSessionState: ... + + def session_state(self, session_id: str) -> ServingRuntimeSessionState: ... + + def remove_session_state(self, session_id: str) -> None: ... + + def is_enabled(self, config: object) -> bool: ... + + def capabilities(self, *, max_sessions: int) -> object: ... + + def validate_client_extra_body(self, extra_body: object) -> None: ... + + async def prepare_runtime_config(self, config: object, *, model_config: Any) -> dict[str, object]: ... + + def runtime_config_for_update( + self, + config: object, + current: Mapping[str, object], + ) -> dict[str, object]: ... + + def data_plane_context( + self, + *, + epoch: int, + turn_id: int, + active_response_turn_id: int | None, + active_response_id: str | None, + auto_responds: bool, + response_format: str, + speed: float | None, + modalities: tuple[str, ...], + ) -> object: ... + + +def load_serving_runtime_adapter( + path: str, + encode_audio, +) -> ServingRuntimeAdapter: + module_name, separator, attribute_name = path.rpartition(".") + if not separator: + raise ValueError(f"Invalid duplex serving runtime adapter path: {path!r}") + adapter_type = getattr(import_module(module_name), attribute_name) + return validate_serving_runtime_adapter(adapter_type(encode_audio)) + + +def validate_serving_runtime_adapter(adapter: object) -> ServingRuntimeAdapter: + required_methods = ( + "create_session_state", + "session_state", + "remove_session_state", + "is_enabled", + "capabilities", + "validate_client_extra_body", + "prepare_runtime_config", + "runtime_config_for_update", + "data_plane_context", + ) + missing = [name for name in required_methods if not callable(getattr(adapter, name, None))] + if missing: + raise TypeError(f"Duplex serving runtime adapter is missing callable method(s): {', '.join(missing)}") + if not isinstance(getattr(adapter, "adapter_id", None), str) or not adapter.adapter_id: + raise TypeError("Duplex serving runtime adapter must declare adapter_id") + if not isinstance(getattr(adapter, "session_states", None), MutableMapping): + raise TypeError("Duplex serving runtime adapter must declare mutable session_states") + for prefix_name in ("clean_response_done_prefix", "interrupted_tts_prefix"): + if not isinstance(getattr(adapter, prefix_name, None), str): + raise TypeError(f"Duplex serving runtime adapter must declare {prefix_name}") + private_keys = getattr(adapter, "private_runtime_config_keys", None) + if not isinstance(private_keys, frozenset) or any(not isinstance(key, str) for key in private_keys): + raise TypeError("Duplex serving runtime adapter must declare private_runtime_config_keys as frozenset[str]") + data_plane = getattr(adapter, "data_plane", None) + if data_plane is None: + raise TypeError("Duplex serving runtime adapter must declare data_plane") + data_plane_methods = ( + "begin_request", + "is_terminal", + "mark_terminal", + "close_stream", + "close_session", + "project", + ) + missing_data_plane_methods = [name for name in data_plane_methods if not callable(getattr(data_plane, name, None))] + if missing_data_plane_methods: + raise TypeError( + "Duplex serving runtime adapter data_plane is missing callable method(s): " + + ", ".join(missing_data_plane_methods) + ) + return adapter # type: ignore[return-value] + + +def payload_turn_id(payload: object) -> int | None: + if not isinstance(payload, Mapping): + return None + return coerce_int(payload.get("duplex_turn_id", payload.get("model_turn_id"))) + + +def coerce_int(value: object) -> int | None: + try: + return int(value) if value is not None else None + except (TypeError, ValueError): + return None diff --git a/vllm_omni/experimental/fullduplex/openai/runtime_bridge.py b/vllm_omni/experimental/fullduplex/openai/runtime_bridge.py new file mode 100644 index 000000000..3d6674621 --- /dev/null +++ b/vllm_omni/experimental/fullduplex/openai/runtime_bridge.py @@ -0,0 +1,1298 @@ +from __future__ import annotations + +import asyncio +import base64 +import inspect +from collections.abc import Mapping + +import numpy as np +from vllm.logger import init_logger + +from vllm_omni.experimental.fullduplex.engine.duplex_control_client import DuplexControlRequestError +from vllm_omni.experimental.fullduplex.engine.duplex_runtime import duplex_data_plane_request_info +from vllm_omni.experimental.fullduplex.engine.messages import DuplexFence +from vllm_omni.experimental.fullduplex.openai.protocol import ( + DuplexSession, + DuplexSessionState, +) +from vllm_omni.experimental.fullduplex.openai.runtime_adapter import ( + coerce_int, + payload_turn_id, +) + +logger = init_logger(__name__) + + +class NativeRuntimeBridgeMixin: + """Bridge serving sessions to native runtime control and data-plane APIs.""" + + _NATIVE_RUNTIME_FENCE_CONTRACT = { + "open_duplex_session_async": ("fence",), + "append_duplex_input_async": ("fence",), + "signal_duplex_turn_async": ("fence", "next_fence"), + "close_duplex_session_async": ("fence",), + } + + async def _open_runtime_session(self, session: DuplexSession, send_json) -> dict[str, object] | bool: + contract_error = self._native_runtime_contract_error(session) + if contract_error is not None: + await send_json( + { + "type": "error", + "code": "runtime_contract_invalid", + "error": contract_error, + } + ) + return False + open_session = getattr(self._chat_service.engine_client, "open_duplex_session_async", None) + if not callable(open_session): + return True + try: + open_kwargs = { + "session_mode": "duplex", + "capabilities": session.capabilities.as_dict(), + "session_config": session.config.as_dict(), + "timeout": self._runtime_control_timeout_s(session), + } + if self._callable_accepts_keyword(open_session, "runtime_config"): + open_kwargs["runtime_config"] = dict(session.runtime_config) + if self._callable_accepts_keyword(open_session, "fence"): + open_kwargs["fence"] = DuplexFence( + session.session_id, + epoch=session.epoch, + turn_id=session.turn_id, + incarnation=session.incarnation, + ) + result = await open_session(session.session_id, **open_kwargs) + except Exception as exc: + if isinstance(exc, DuplexControlRequestError) and exc.code == "resource_exhausted": + logger.info("Duplex runtime session admission rejected: %s", exc) + else: + logger.exception("Failed to open duplex runtime session: %s", exc) + await self._send_runtime_error(send_json, "runtime_open_failed", exc, session=session) + return False + if ( + isinstance(result, dict) + and session.capabilities.implementation_level == "model_native_duplex" + and self._runtime_control_failed(result) + ): + await send_json( + { + "type": "error", + "error": "Native duplex runtime is not available for this session", + "code": ("runtime_open_unsupported" if result.get("unsupported_count") else "runtime_open_failed"), + "runtime_control": self._redact_runtime_control_result(result), + } + ) + return False + return result if isinstance(result, dict) else True + + async def _append_runtime_input( + self, + session: DuplexSession, + payload: object, + *, + operation_id: str | None = None, + final: bool, + send_json, + mode: str = "append_tokens", + expected_epoch: int | None = None, + ) -> tuple[bool, bool]: + if not session.capabilities.supports_input_append: + return True, False + append_input = getattr(self._chat_service.engine_client, "append_duplex_input_async", None) + if not callable(append_input): + return True, False + if expected_epoch is not None and session.epoch != expected_epoch: + return True, False + try: + append_kwargs = { + "mode": mode, + "payload": payload, + "final": final, + "timeout": self._runtime_control_timeout_s(session), + } + if operation_id is not None and self._callable_accepts_keyword(append_input, "operation_id"): + append_kwargs["operation_id"] = operation_id + if expected_epoch is not None and self._callable_accepts_keyword(append_input, "expected_epoch"): + append_kwargs["expected_epoch"] = expected_epoch + if self._callable_accepts_keyword(append_input, "fence"): + payload_turn = payload_turn_id(payload) + append_kwargs["fence"] = DuplexFence( + session.session_id, + epoch=session.epoch, + turn_id=( + payload_turn + if payload_turn is not None + else ( + session.active_response_turn_id + if session.active_response_turn_id is not None + else session.turn_id + ) + ), + incarnation=session.incarnation, + ) + if self._callable_accepts_keyword(append_input, "collect_outputs"): + append_kwargs["collect_outputs"] = False + result = await append_input(session.session_id, **append_kwargs) + except Exception as exc: + logger.exception("Failed to append duplex runtime input: %s", exc) + await self._send_runtime_error(send_json, "runtime_append_failed", exc, session=session) + return False, False + if isinstance(result, dict) and self._runtime_control_failed(result): + await self._send_runtime_control_error( + send_json, + "runtime_append_failed", + "Duplex runtime append failed", + result, + session=session, + ) + return False, False + if expected_epoch is not None and session.epoch != expected_epoch: + return True, False + await self._send_runtime_control_if_needed(send_json, result, session=session) + request_id, _ = self._data_plane_request_info(result) if isinstance(result, dict) else (None, None) + if request_id is not None: + self._serving_runtime_adapter.data_plane.begin_request(request_id) + close_reason, emitted_response = await self._send_native_duplex_events( + send_json, + result, + session=session, + expected_epoch=expected_epoch, + ) + # A resumable data-plane append remains active even when its persistent + # drain task already exists. Treating only a newly-created drain as + # activity clears active_request_id after later appends and prevents a + # terminal TTS segment from scheduling the next model decision. + emitted_response = emitted_response or request_id is not None + if close_reason is None and await self._start_native_data_plane_stream_task( + send_json, + result, + session=session, + expected_epoch=expected_epoch, + ): + emitted_response = True + if close_reason is not None: + if not await self._close_runtime_session(session, reason=close_reason, send_json=send_json): + return False, emitted_response + session.close() + await send_json( + { + "type": "session.closed", + "session_id": session.session_id, + "reason": close_reason, + } + ) + return False, emitted_response + return True, emitted_response + + @staticmethod + def _callable_accepts_keyword(fn, name: str) -> bool: + try: + params = inspect.signature(fn).parameters.values() + except (TypeError, ValueError): + return True + return any(param.kind == inspect.Parameter.VAR_KEYWORD or param.name == name for param in params) + + def _native_runtime_contract_error(self, session: DuplexSession) -> str | None: + if session.capabilities.implementation_level != "model_native_duplex": + return None + engine_client = self._chat_service.engine_client + invalid: list[str] = [] + for method_name, required_keywords in self._NATIVE_RUNTIME_FENCE_CONTRACT.items(): + method = getattr(engine_client, method_name, None) + if not callable(method): + invalid.append(f"{method_name} is missing") + continue + try: + parameters = tuple(inspect.signature(method).parameters.values()) + except (TypeError, ValueError): + invalid.append(f"{method_name} signature is not inspectable") + continue + accepts_kwargs = any(parameter.kind == inspect.Parameter.VAR_KEYWORD for parameter in parameters) + declared = {parameter.name for parameter in parameters} + missing = [name for name in required_keywords if not accepts_kwargs and name not in declared] + if missing: + invalid.append(f"{method_name} is missing keyword(s): {', '.join(missing)}") + if not invalid: + return None + return "Native duplex runtime requires the fenced control contract: " + "; ".join(invalid) + + async def _start_native_data_plane_stream_task( + self, + send_json, + result: object, + *, + session: DuplexSession, + expected_epoch: int | None = None, + ) -> bool: + request_id, _ = self._data_plane_request_info(result) + if request_id is None or self._data_plane_outputs_finished(result): + return False + session.bind_request(request_id) + + native = self._runtime_session_state(session) + old_task = native.data_plane_task + if old_task is not None and not old_task.done(): + if self._session_auto_responds(session): + # One persistent drain per session, like the official worker's + # single synchronous loop: the resumable data-plane request id + # is stable, and cancel/restart on every append orphans any + # decision that lands in the swap window. + native.data_plane_restart_requested = True + return False + old_task.cancel() + try: + await asyncio.wait_for(asyncio.gather(old_task, return_exceptions=True), timeout=0.25) + except asyncio.TimeoutError: + pass + + async def _run() -> None: + close_reason: str | None = None + try: + close_reason = await self._drain_native_data_plane_stream( + send_json, + result, + session=session, + expected_epoch=expected_epoch, + ) + if close_reason is not None: + if await self._close_runtime_session(session, reason=close_reason, send_json=send_json): + session.close() + await send_json( + { + "type": "session.closed", + "session_id": session.session_id, + "reason": close_reason, + } + ) + except asyncio.CancelledError: + raise + except Exception as exc: + logger.exception("Failed to drain duplex data-plane stream: %s", exc) + await self._send_runtime_error(send_json, "runtime_data_plane_stream_failed", exc, session=session) + if session.state != DuplexSessionState.CLOSED: + close_reason = "runtime_data_plane_stream_failed" + if await self._close_runtime_session(session, reason=close_reason, send_json=send_json): + session.close() + await send_json( + { + "type": "session.closed", + "session_id": session.session_id, + "reason": close_reason, + } + ) + finally: + if native.data_plane_task is task: + native.data_plane_task = None + restart_requested = ( + native.data_plane_restart_requested + and close_reason is None + and session.state != DuplexSessionState.CLOSED + and session.active_request_id == request_id + and (expected_epoch is None or session.epoch == expected_epoch) + ) + native.data_plane_restart_requested = False + if close_reason is None and not self._session_auto_responds(session): + # Auto-respond sessions keep one resumable stage-1 stream + # whose audio accumulates across speak units; the offset + # must survive drain-task turnover or every unit re-sends + # the reply audio from the start. + self._serving_runtime_adapter.data_plane.close_stream(request_id) + if restart_requested: + await self._start_native_data_plane_stream_task( + send_json, + result, + session=session, + expected_epoch=expected_epoch, + ) + elif close_reason is None: + await self._maybe_continue_native_response(send_json, session=session, expected_epoch=expected_epoch) + + task = asyncio.create_task(_run()) + native.data_plane_task = task + return True + + # One model unit (1 s at 16 kHz) of pcm_f32le silence, matching the + # official full-duplex behavior where the microphone keeps streaming + # silence while the assistant speaks; replies span multiple units. + _NATIVE_SILENCE_UNIT_PAYLOAD_AUDIO = base64.b64encode(bytes(16000 * 4)).decode("ascii") + _NATIVE_RESPONSE_MAX_CONTINUATION_UNITS = 8 + + def _native_silence_unit_payload(self) -> dict[str, object]: + return { + "type": "audio", + "audio": self._NATIVE_SILENCE_UNIT_PAYLOAD_AUDIO, + "format": "pcm_f32le", + "sample_rate_hz": 16000, + } + + def _native_response_continuations_remaining(self, session: DuplexSession, response_id: str) -> bool: + if self._session_auto_responds(session): + return True + native = self._runtime_session_state(session) + owner_id = f"response:{response_id}" + count = native.continuation_units if native.continuation_owner_id == owner_id else 0 + return count < self._NATIVE_RESPONSE_MAX_CONTINUATION_UNITS + + def _native_silence_continuation_is_stale( + self, + session: DuplexSession, + *, + request_id: str, + response_id: str | None, + response_owned: bool, + expected_epoch: int | None, + expected_incarnation: int, + expected_model_turn_id: int | None, + ) -> bool: + stale_common_owner = ( + session.state == DuplexSessionState.CLOSED + or session.incarnation != expected_incarnation + or session.active_request_id != request_id + or (expected_epoch is not None and session.epoch != expected_epoch) + ) + stale_response_owner = response_owned and session.active_response_id != response_id + stale_model_turn_owner = not response_owned and ( + session.active_response_id is not None or session.turn_id != expected_model_turn_id + ) + return stale_common_owner or stale_response_owner or stale_model_turn_owner + + async def _maybe_continue_native_response( + self, + send_json, + *, + session: DuplexSession, + expected_epoch: int | None, + expected_model_turn_id: int | None = None, + ) -> None: + """Give an active model turn another silence unit. + + A model turn can need another unit before it has emitted visible + text/audio and therefore before a Realtime response exists. Bind that + continuation to the model-turn identity; once a response exists, its + response/turn ownership remains the continuation fence. + """ + response_id = session.active_response_id + native = self._runtime_session_state(session) + if session.state == DuplexSessionState.CLOSED: + native.clear_continuation() + return + request_id = session.active_request_id + if request_id is None: + native.clear_continuation() + return + if expected_epoch is not None and session.epoch != expected_epoch: + return + auto_response = self._session_auto_responds(session) + response_owned = response_id is not None + if response_owned: + owner_id = f"response:{response_id}" + payload_turn_id = ( + session.active_response_turn_id if session.active_response_turn_id is not None else session.turn_id + ) + else: + if not auto_response or expected_model_turn_id is None or session.turn_id != expected_model_turn_id: + native.clear_continuation() + return + owner_id = f"model-turn:{expected_model_turn_id}" + payload_turn_id = expected_model_turn_id + count = native.continuation_units if native.continuation_owner_id == owner_id else 0 + if not auto_response and count >= self._NATIVE_RESPONSE_MAX_CONTINUATION_UNITS: + return + payload = self._native_silence_unit_payload() + payload["duplex_turn_id"] = payload_turn_id + + scheduler = native.silence_continuation_scheduler + if scheduler is None: + return + try: + scheduled = await scheduler( + payload, + request_id=request_id, + owner_id=owner_id, + response_id=response_id, + response_owned=response_owned, + expected_epoch=expected_epoch, + expected_incarnation=session.incarnation, + expected_model_turn_id=expected_model_turn_id, + send_json=send_json, + ) + except Exception as exc: + logger.exception("Failed to schedule duplex native response continuation: %s", exc) + scheduled = False + if scheduled: + native.continuation_owner_id = owner_id + native.continuation_units = count + 1 + + async def _cancel_native_data_plane_stream(self, session: DuplexSession) -> bool: + native = self._runtime_session_state(session) + task = native.data_plane_task + native.data_plane_task = None + native.data_plane_restart_requested = False + if task is None or task.done(): + return False + task.cancel() + try: + await asyncio.wait_for(asyncio.gather(task, return_exceptions=True), timeout=0.25) + except asyncio.TimeoutError: + # Keep barge-in/cancel responsive. The task still carries the old + # expected_epoch and late model output is filtered by the writer. + pass + return True + + async def _signal_runtime_session( + self, + session: DuplexSession, + event: str, + send_json=None, + *, + fence: DuplexFence | None = None, + next_fence: DuplexFence | None = None, + session_config: dict[str, object] | None = None, + runtime_config: dict[str, object] | None = None, + ) -> bool: + signal_turn = getattr(self._chat_service.engine_client, "signal_duplex_turn_async", None) + if not callable(signal_turn): + return True + try: + signal_kwargs = { + "event": event, + "timeout": self._runtime_control_timeout_s(session), + } + if self._callable_accepts_keyword(signal_turn, "fence"): + signal_kwargs["fence"] = fence or DuplexFence( + session.session_id, + epoch=session.epoch, + turn_id=session.turn_id, + incarnation=session.incarnation, + ) + if next_fence is not None and self._callable_accepts_keyword(signal_turn, "next_fence"): + signal_kwargs["next_fence"] = next_fence + if session_config is not None and self._callable_accepts_keyword(signal_turn, "session_config"): + signal_kwargs["session_config"] = session_config + if runtime_config is not None and self._callable_accepts_keyword(signal_turn, "runtime_config"): + signal_kwargs["runtime_config"] = runtime_config + result = await signal_turn(session.session_id, **signal_kwargs) + except Exception as exc: + logger.exception("Failed to signal duplex runtime session: %s", exc) + if send_json is not None: + await self._send_runtime_error(send_json, "runtime_signal_failed", exc, session=session) + return False + if isinstance(result, dict) and self._runtime_signal_failed(result): + logger.warning( + "Duplex runtime signal failed: session=%s event=%s result=%s", + session.session_id, + event, + self._redact_runtime_control_result(result), + ) + if send_json is not None: + await self._send_runtime_control_error( + send_json, + "runtime_signal_failed", + "Duplex runtime signal failed", + result, + session=session, + ) + return False + await self._send_runtime_control_if_needed(send_json, result, session=session) + return True + + async def _close_runtime_session(self, session: DuplexSession, *, reason: str, send_json=None) -> bool: + close_session = getattr(self._chat_service.engine_client, "close_duplex_session_async", None) + if not callable(close_session): + return True + try: + close_kwargs = { + "reason": reason, + "timeout": self._runtime_control_timeout_s(session), + } + if self._callable_accepts_keyword(close_session, "fence"): + close_kwargs["fence"] = DuplexFence( + session.session_id, + epoch=session.epoch, + turn_id=session.turn_id, + incarnation=session.incarnation, + ) + result = await close_session(session.session_id, **close_kwargs) + except Exception as exc: + logger.exception("Failed to close duplex runtime session: %s", exc) + if send_json is not None: + await self._send_runtime_error(send_json, "runtime_close_failed", exc, session=session) + return False + if isinstance(result, dict) and self._runtime_control_failed(result): + if send_json is not None: + await self._send_runtime_control_error( + send_json, + "runtime_close_failed", + "Duplex runtime close failed", + result, + session=session, + ) + return False + await self._send_runtime_control_if_needed(send_json, result, session=session) + return True + + @staticmethod + def _runtime_control_timeout_s(session: DuplexSession) -> float: + raw = session.config.extra_body.get("duplex_control_timeout_s") or session.config.extra_body.get( + "runtime_control_timeout_s" + ) + if isinstance(raw, int | float) and raw > 0: + return float(raw) + if session.capabilities.implementation_level == "model_native_duplex": + return 60.0 + return 10.0 + + async def _send_runtime_control_if_needed( + self, + send_json, + result: object, + *, + session: DuplexSession, + ) -> None: + if send_json is None: + return + if not isinstance(result, dict): + return + if not result.get("unsupported_count") and not result.get("error_count"): + return + await send_json( + { + "type": "runtime.control", + "session_id": session.session_id, + "epoch": session.epoch, + "result": self._redact_runtime_control_result(result), + } + ) + + @staticmethod + def _runtime_control_failed(result: dict[str, object]) -> bool: + if result.get("ok") is False: + return True + for key in ("unsupported_count", "error_count"): + value = result.get(key) + if isinstance(value, int | float) and value > 0: + return True + return False + + @classmethod + def _runtime_signal_failed(cls, result: dict[str, object]) -> bool: + error_count = result.get("error_count") + if isinstance(error_count, int | float) and error_count > 0: + return True + if result.get("ok") is not False: + return False + return not cls._runtime_signal_has_data_plane_ack(result) + + @classmethod + def _runtime_signal_has_data_plane_ack(cls, value: object) -> bool: + if isinstance(value, dict): + if value.get("data_plane_signal") is True and value.get("supported") is True: + return True + return any(cls._runtime_signal_has_data_plane_ack(child) for child in value.values()) + if isinstance(value, list | tuple): + return any(cls._runtime_signal_has_data_plane_ack(item) for item in value) + return False + + @classmethod + def _redact_runtime_control_result(cls, value: object) -> object: + if isinstance(value, dict): + redacted = { + key: cls._redact_runtime_control_result(child) + for key, child in value.items() + if key + not in { + "stage_handoff", + "tts_handoff", + "omni_payload", + "tts_hidden_states", + "tts_token_ids", + "traceback", + } + } + native_result = redacted.get("native_result") + if isinstance(native_result, dict): + if native_result.get("requires_stage_handoff") is True: + native_result.pop("requires_stage_handoff", None) + if native_result.get("requires_tts_stage") is True: + native_result.pop("requires_tts_stage", None) + return redacted + if isinstance(value, list | tuple): + return [cls._redact_runtime_control_result(item) for item in value] + return value + + async def _send_native_duplex_events( + self, + send_json, + result: object, + *, + session: DuplexSession, + expected_epoch: int | None = None, + ) -> tuple[str | None, bool]: + if send_json is None: + return None, False + if expected_epoch is not None and session.epoch != expected_epoch: + return None, False + close_reason: str | None = None + emitted_response = False + request_id, _ = self._data_plane_request_info(result) + if self._serving_runtime_adapter.data_plane.is_terminal(request_id): + return None, False + if request_id is not None and session.active_request_id is None: + session.bind_request(request_id) + context = self._runtime_data_plane_context(session) + for native_result in self._serving_runtime_adapter.data_plane.project(result, context=context): + close_reason_for_result, did_emit = await self._send_one_native_duplex_event( + send_json, + native_result, + session=session, + expected_epoch=expected_epoch, + ) + emitted_response = emitted_response or did_emit + close_reason = close_reason or close_reason_for_result + if expected_epoch is not None and session.epoch != expected_epoch: + return None, emitted_response + return close_reason, emitted_response + + async def _drain_native_data_plane_stream( + self, + send_json, + result: object, + *, + session: DuplexSession, + expected_epoch: int | None = None, + ) -> str | None: + request_id, response_stage_id = self._data_plane_request_info(result) + if request_id is None: + return None + if self._data_plane_outputs_finished(result): + return None + collect_outputs = getattr( + self._chat_service.engine_client, + "collect_duplex_data_plane_outputs_async", + None, + ) + if not callable(collect_outputs): + return None + + close_reason: str | None = None + empty_polls = 0 + while close_reason is None: + if self._serving_runtime_adapter.data_plane.is_terminal(request_id): + return None + if self._session_auto_responds(session): + active_request_id = session.active_request_id + if active_request_id is not None and active_request_id != request_id: + return None + if expected_epoch is not None and session.epoch != expected_epoch: + return None + if session.state == DuplexSessionState.CLOSED: + return None + outputs = await collect_outputs( + request_id, + response_stage_id=response_stage_id, + timeout=self._runtime_control_timeout_s(session), + ) + if expected_epoch is not None and session.epoch != expected_epoch: + return None + if not outputs: + # An empty poll means no output arrived within one control + # window, not that the stream is over. Exiting here orphans a + # decision that lands moments later: it would sit queued until + # the NEXT append starts a fresh drain task, adding one full + # chunk of latency to every model decision. + empty_polls += 1 + if empty_polls >= 3: + return None + continue + empty_polls = 0 + drain_result = { + "data_plane_outputs": outputs, + } + close_reason, emitted_response = await self._send_native_duplex_events( + send_json, + drain_result, + session=session, + expected_epoch=expected_epoch, + ) + if ( + self._data_plane_outputs_finished(drain_result) + and emitted_response + and not self._session_auto_responds(session) + ): + # Auto-respond sessions are resumable: every segment ends + # with finished=True but the stream continues with the next + # audio chunk. Exiting here would make each decision wait for + # the next append to start a fresh drain task (one full chunk + # of added latency); keep draining until epoch change, + # session close, or cancellation by a newer drain task. + return close_reason + # A batch without a client-visible event (e.g. the generic + # stage-0 final-output message that precedes the listen/speak + # decision in the same segment) must not terminate the drain; + # the real decision is still in flight. + return close_reason + + @staticmethod + def _data_plane_outputs_finished(result: object) -> bool: + if not isinstance(result, dict): + return False + outputs = result.get("data_plane_outputs") + if not isinstance(outputs, list) or not outputs: + return False + return bool(getattr(outputs[-1], "finished", False)) + + @staticmethod + def _data_plane_request_info(result: object) -> tuple[str | None, int | None]: + if not isinstance(result, dict): + return None, None + return duplex_data_plane_request_info(result) + + def _runtime_data_plane_context(self, session: DuplexSession) -> object: + response_config = session.response_config + return self._serving_runtime_adapter.data_plane_context( + epoch=session.epoch, + turn_id=session.turn_id, + active_response_turn_id=session.active_response_turn_id, + active_response_id=session.active_response_id, + auto_responds=self._session_auto_responds(session), + response_format=response_config.response_format, + speed=response_config.speed, + modalities=tuple(response_config.modalities), + ) + + async def _send_one_native_duplex_event( + self, + send_json, + native_result: dict[str, object], + *, + session: DuplexSession, + expected_epoch: int | None = None, + ) -> tuple[str | None, bool]: + close_reason: str | None = None + emitted_response = False + if expected_epoch is not None and session.epoch != expected_epoch: + return close_reason, emitted_response + data_plane_request_id = native_result.get("data_plane_request_id") + if isinstance(data_plane_request_id, str) and self._serving_runtime_adapter.data_plane.is_terminal( + data_plane_request_id + ): + return close_reason, emitted_response + active_request_matches = session.active_request_id == data_plane_request_id or ( + self._session_auto_responds(session) and session.active_request_id is None + ) + if isinstance(data_plane_request_id, str) and not active_request_matches: + return close_reason, emitted_response + if isinstance(native_result.get("error_code"), str): + response_id = session.active_response_id + await send_json( + { + "type": "error", + "code": native_result.get("error_code"), + "session_id": session.session_id, + "epoch": session.epoch, + "error": str(native_result.get("error") or "Duplex native data-plane error"), + } + ) + if response_id is not None: + session.end_response(commit_text=False) + await send_json( + { + "type": "response.done", + "session_id": session.session_id, + "response_id": response_id, + "epoch": session.epoch, + "committed": False, + "status": "failed", + "status_details": { + "type": "failed", + "reason": native_result.get("error_code"), + }, + "playback": session.playback.as_dict(), + } + ) + return close_reason, True + if native_result.get("requires_stage_handoff") is True or native_result.get("requires_tts_stage") is True: + # Stage0 announces a talker handoff before Stage1 has produced + # client-visible audio. A single client input stream may contain + # several model turns, including terminal-only turns. Reserve the + # protocol response only when Stage1 emits text/audio so empty + # model turns cannot create empty responses or steal later audio. + return close_reason, False + is_listen = native_result.get("is_listen") + model_turn_id = coerce_int(native_result.get("model_turn_id")) + if native_result.get("is_buffering") is True or native_result.get("prefill_success") is False: + if native_result.get("data_plane_request_id") == session.active_request_id: + session.clear_request() + payload = { + "type": "response.listen", + "session_id": session.session_id, + "epoch": session.epoch, + "reason": native_result.get("reason") or "buffering", + "model_listen": False, + "buffering": True, + } + self._attach_native_runtime_metadata(payload, native_result) + await send_json(payload) + return close_reason, emitted_response + if is_listen is True: + await self._end_active_response_before_future_model_turn( + send_json, + session=session, + model_turn_id=model_turn_id, + ) + if ( + session.active_response_id is not None + and model_turn_id is not None + and not session.active_response_accepts_model_turn(model_turn_id) + ): + return close_reason, emitted_response + non_terminal_auto_listen = ( + self._session_auto_responds(session) + and session.active_response_id is not None + and session.active_request_id is not None + and native_result.get("end_of_turn") is not True + ) + if non_terminal_auto_listen: + await self._maybe_continue_native_response( + send_json, + session=session, + expected_epoch=expected_epoch, + ) + return close_reason, emitted_response + auto_response = self._session_auto_responds(session) + if not auto_response and data_plane_request_id == session.active_request_id: + session.clear_request() + model_listen = native_result.get("model_listen") + if not isinstance(model_listen, bool): + model_listen = native_result.get("reason") in {None, "", "model_listen"} + response_id = session.active_response_id + if isinstance(data_plane_request_id, str) and not auto_response: + self._serving_runtime_adapter.data_plane.mark_terminal(data_plane_request_id) + emitted_response = True + payload = { + "type": "response.listen", + "session_id": session.session_id, + "epoch": session.epoch, + "reason": native_result.get("reason") or "model_listen", + "model_listen": model_listen, + } + self._attach_native_runtime_metadata(payload, native_result) + await send_json(payload) + if native_result.get("abort_data_plane_request") is True and isinstance(data_plane_request_id, str): + await self._abort_request_background( + session, + data_plane_request_id, + send_json, + notify=False, + ) + if response_id is not None: + if not self._session_auto_responds(session) and self._native_response_continuations_remaining( + session, response_id + ): + # The official model often listens for a silence beat or + # two before answering; keep the response open and give it + # the next silence unit as a decision point. + await self._maybe_continue_native_response( + send_json, + session=session, + expected_epoch=expected_epoch, + ) + return close_reason, emitted_response + session.end_response(commit_text=False, preserve_request=auto_response) + await send_json( + { + "type": "response.done", + "session_id": session.session_id, + "response_id": response_id, + "epoch": session.epoch, + "committed": False, + "playback": session.playback.as_dict(), + } + ) + return close_reason, emitted_response + + text = native_result.get("text") + audio = native_result.get("audio_data", native_result.get("audio")) + end_of_turn = bool(native_result.get("end_of_turn", False)) + has_text = isinstance(text, str) and bool(text) + has_audio = isinstance(audio, str) and bool(audio) + if not has_text and not has_audio and not end_of_turn: + tts_segment_ended = ( + native_result.get("stage_role") == "tts" and native_result.get("abort_data_plane_request") is True + ) + if ( + tts_segment_ended + and self._session_auto_responds(session) + and (model_turn_id is not None or session.active_response_id is not None) + ): + await self._maybe_continue_native_response( + send_json, + session=session, + expected_epoch=expected_epoch, + expected_model_turn_id=model_turn_id, + ) + return close_reason, emitted_response + if end_of_turn and not has_text and not has_audio and session.active_response_id is None: + if isinstance(data_plane_request_id, str): + if not self._session_auto_responds(session) and data_plane_request_id == session.active_request_id: + session.clear_request() + if not self._session_auto_responds(session): + self._serving_runtime_adapter.data_plane.mark_terminal(data_plane_request_id) + if model_turn_id is not None: + session.complete_model_turn(model_turn_id) + if self._session_auto_responds(session): + self._runtime_session_state(session).clear_continuation() + emitted_response = True + payload = { + "type": "response.listen", + "session_id": session.session_id, + "epoch": session.epoch, + "reason": "model_turn_completed_without_output", + "model_listen": True, + } + self._attach_native_runtime_metadata(payload, native_result) + await send_json(payload) + return close_reason, emitted_response + if session.active_response_id is None and model_turn_id is not None and model_turn_id < session.turn_id: + # A continuation append can already be in flight when the prior + # response reaches turn EOS. Its late audio still carries the + # completed model turn, so it must not reserve a second Realtime + # response for that turn. + return close_reason, emitted_response + await self._end_active_response_before_future_model_turn( + send_json, + session=session, + model_turn_id=model_turn_id, + ) + if ( + session.active_response_id is not None + and model_turn_id is not None + and not session.active_response_accepts_model_turn(model_turn_id) + ): + return close_reason, emitted_response + emitted_response = True + response_created = False + response_id = session.active_response_id + if response_id is None: + response_id = session.begin_response(turn_id=model_turn_id) + response_created = True + await send_json( + self._response_created_payload( + session, + response_id, + epoch=session.epoch, + ) + ) + response_stage_metrics = session.accumulate_response_stage_metrics( + native_result.get("stage_metrics") if isinstance(native_result.get("stage_metrics"), Mapping) else None + ) + if response_created: + speak_payload = { + "type": "response.speak", + "session_id": session.session_id, + "response_id": response_id, + "epoch": session.epoch, + "text": text if isinstance(text, str) else "", + "end_of_turn": end_of_turn, + "model_speak": True, + } + self._attach_native_runtime_metadata( + speak_payload, + native_result, + stage_metrics=response_stage_metrics, + ) + await send_json(speak_payload) + previous_sent_ms = session.playback.sent_ms + text_chars_before_append = len("".join(session.assistant_text_buffer)) + if isinstance(text, str): + session.append_assistant_text(text) + duration_ms = native_result.get("audio_duration_ms") + text_chars = len("".join(session.assistant_text_buffer)) + mark_duration_ms = None + mark_text_chars: int | None = text_chars + if native_result.get("audio_text_mark") is False: + mark_text_chars = None + if isinstance(duration_ms, int | float): + mark_duration_ms = int(duration_ms) + if native_result.get("audio_duration_is_cumulative") is not True: + mark_duration_ms += session.playback.sent_ms + audio_text_marks = native_result.get("audio_text_marks") + audio_text_marks = self._normalize_native_audio_text_marks( + audio_text_marks if isinstance(audio_text_marks, list) else None, + audio_offset_ms=( + 0 + if native_result.get("audio_text_marks_are_cumulative") is True + or native_result.get("audio_duration_is_cumulative") is True + else previous_sent_ms + ), + text_offset_chars=( + 0 if native_result.get("audio_text_marks_are_cumulative") is True else text_chars_before_append + ), + ) + session.mark_audio_sent( + mark_duration_ms, + text_chars=mark_text_chars if mark_duration_ms is not None else None, + audio_text_marks=audio_text_marks, + ) + payload = { + "type": "response.output_audio.delta", + "session_id": session.session_id, + "response_id": response_id, + "epoch": session.epoch, + "text": text if isinstance(text, str) else "", + "audio": audio if isinstance(audio, str) else "", + "format": ( + native_result.get("audio_format") + if isinstance(native_result.get("audio_format"), str) + else session.response_config.response_format + ), + "end_of_turn": end_of_turn, + "model_speak": True, + } + if mark_duration_ms is not None: + payload["audio_duration_ms"] = mark_duration_ms + if audio_text_marks: + payload["audio_text_marks"] = audio_text_marks + elif mark_duration_ms is not None and mark_text_chars is not None: + payload["audio_text_marks"] = [ + { + "text_chars": max(0, int(mark_text_chars)), + "audio_end_ms": max(0, int(mark_duration_ms)), + } + ] + payload["playback"] = session.playback.as_dict() + sample_rate_hz = native_result.get("sample_rate_hz") or native_result.get("audio_sample_rate_hz") + if isinstance(sample_rate_hz, int | float) and int(sample_rate_hz) > 0: + payload["sample_rate_hz"] = int(sample_rate_hz) + self._attach_native_runtime_metadata( + payload, + native_result, + stage_metrics=response_stage_metrics, + ) + await send_json(payload) + if ( + not end_of_turn + and native_result.get("stage_role") == "tts" + and native_result.get("abort_data_plane_request") is True + and self._session_auto_responds(session) + ): + await self._maybe_continue_native_response( + send_json, + session=session, + expected_epoch=expected_epoch, + ) + if end_of_turn: + data_plane_request_id = native_result.get("data_plane_request_id") + if isinstance(data_plane_request_id, str) and not self._session_auto_responds(session): + self._serving_runtime_adapter.data_plane.close_stream(data_plane_request_id) + if isinstance(data_plane_request_id, str): + if not self._session_auto_responds(session) and data_plane_request_id == session.active_request_id: + session.clear_request() + if not self._session_auto_responds(session): + self._serving_runtime_adapter.data_plane.mark_terminal(data_plane_request_id) + should_commit = self._should_commit_response_to_history(session, response_id) + committed_message = session.end_response( + commit_text=should_commit, + preserve_request=self._session_auto_responds(session), + ) + model_turn_id = coerce_int(native_result.get("model_turn_id")) + if model_turn_id is not None: + session.complete_model_turn(model_turn_id) + if should_commit: + session.register_history_item(f"item_{response_id}", committed_message) + await send_json( + { + "type": "response.done", + "session_id": session.session_id, + "response_id": response_id, + "epoch": session.epoch, + "committed": committed_message is not None, + "playback": session.playback.as_dict(), + } + ) + return close_reason, emitted_response + + async def _end_active_response_before_future_model_turn( + self, + send_json, + *, + session: DuplexSession, + model_turn_id: int | None, + ) -> None: + if not self._session_auto_responds(session): + return + response_id = session.active_response_id + active_turn_id = session.active_response_turn_id + if response_id is None or model_turn_id is None or active_turn_id is None: + return + if int(model_turn_id) <= int(active_turn_id): + return + session.complete_model_turn(int(model_turn_id) - 1) + should_commit = self._should_commit_response_to_history(session, response_id) + committed_message = session.end_response( + commit_text=should_commit, + preserve_request=True, + ) + if should_commit: + session.register_history_item(f"item_{response_id}", committed_message) + await send_json( + { + "type": "response.done", + "session_id": session.session_id, + "response_id": response_id, + "epoch": session.epoch, + "committed": committed_message is not None, + "playback": session.playback.as_dict(), + } + ) + + @staticmethod + def _normalize_native_audio_text_marks( + audio_text_marks: list[object] | None, + *, + audio_offset_ms: int, + text_offset_chars: int, + ) -> list[dict[str, int]] | None: + if not audio_text_marks: + return None + normalized: list[dict[str, int]] = [] + for raw_mark in audio_text_marks: + if not isinstance(raw_mark, dict): + continue + raw_text_chars = raw_mark.get("text_chars") + raw_audio_end_ms = raw_mark.get("audio_end_ms", raw_mark.get("audio_ms")) + if not isinstance(raw_text_chars, int | float) or not isinstance(raw_audio_end_ms, int | float): + continue + normalized.append( + { + "text_chars": max(0, int(raw_text_chars) + int(text_offset_chars)), + "audio_end_ms": max(0, int(raw_audio_end_ms) + int(audio_offset_ms)), + } + ) + return normalized or None + + def _cleanup_duplex_session_state(self, session: DuplexSession) -> None: + session_id = session.session_id + self._serving_runtime_adapter.remove_session_state(session_id) + self._serving_runtime_adapter.data_plane.close_session( + session_id, + active_request_id=session.active_request_id, + ) + + def _encode_native_data_plane_audio( + self, + audio_data: object, + sample_rate_hz: int, + response_format: str, + speed: float | None, + ) -> str | None: + if audio_data is None: + return None + try: + import torch + + from vllm_omni.entrypoints.openai.protocol.audio import CreateAudio + + if isinstance(audio_data, torch.Tensor): + audio_tensor = audio_data.detach().cpu().float().numpy() + else: + audio_tensor = np.asarray(audio_data, dtype=np.float32) + if audio_tensor.ndim > 1: + audio_tensor = audio_tensor.reshape(-1) + audio_response = self._chat_service.create_audio( + CreateAudio( + audio_tensor=audio_tensor, + sample_rate=sample_rate_hz, + response_format=response_format, + speed=float(speed) if isinstance(speed, int | float) and speed > 0 else 1.0, + stream_format="audio", + base64_encode=True, + ) + ) + return str(audio_response.audio_data) + except Exception: + logger.exception("Failed to encode duplex data-plane audio output") + return None + + @staticmethod + def _attach_native_runtime_metadata( + payload: dict[str, object], + native_result: dict[str, object], + *, + stage_metrics: Mapping[str, object] | None = None, + ) -> None: + metadata: dict[str, object] = {} + runtime_impl = native_result.get("runtime_impl") + if isinstance(runtime_impl, str) and runtime_impl: + metadata["runtime_impl"] = runtime_impl + owned_runtime = native_result.get("owned_runtime") + if isinstance(owned_runtime, bool): + metadata["owned_runtime"] = owned_runtime + model_turn_id = coerce_int(native_result.get("model_turn_id")) + if model_turn_id is not None: + metadata["model_turn_id"] = model_turn_id + for name in ( + "uses_model_runner_scheduler", + "runner_kv_backed", + ): + value = native_result.get(name) + if isinstance(value, bool): + metadata[name] = value + effective_stage_metrics = stage_metrics if stage_metrics is not None else native_result.get("stage_metrics") + if isinstance(effective_stage_metrics, Mapping): + metadata["stage_metrics"] = { + str(stage_id): dict(values) + for stage_id, values in effective_stage_metrics.items() + if isinstance(values, Mapping) + } + if metadata: + payload["vllm_omni"] = metadata + + async def _send_runtime_error( + self, + send_json, + code: str, + exc: Exception, + *, + session: DuplexSession | None = None, + ) -> None: + retryable = False + message = str(exc) + if isinstance(exc, DuplexControlRequestError): + code = exc.code + retryable = exc.retryable + error_data = exc.result.get("error") + if isinstance(error_data, dict) and isinstance(error_data.get("message"), str): + message = error_data["message"] + payload: dict[str, object] = { + "type": "error", + "code": code, + "error": message, + "retryable": retryable, + } + if session is not None: + payload["session_id"] = session.session_id + payload["epoch"] = session.epoch + await send_json(payload) + + async def _send_runtime_control_error( + self, + send_json, + code: str, + message: str, + result: dict[str, object], + *, + session: DuplexSession, + ) -> None: + await send_json( + { + "type": "error", + "code": code, + "error": message, + "session_id": session.session_id, + "epoch": session.epoch, + "runtime_control": self._redact_runtime_control_result(result), + } + ) diff --git a/vllm_omni/experimental/fullduplex/openai/serving.py b/vllm_omni/experimental/fullduplex/openai/serving.py new file mode 100644 index 000000000..3ce23ea8b --- /dev/null +++ b/vllm_omni/experimental/fullduplex/openai/serving.py @@ -0,0 +1,1764 @@ +from __future__ import annotations + +import asyncio +import base64 +import binascii +import inspect +import json +from collections.abc import Mapping +from contextlib import suppress +from dataclasses import dataclass +from typing import TYPE_CHECKING + +import numpy as np +from fastapi import WebSocket +from vllm.logger import init_logger + +from vllm_omni.config.stage_config import DuplexSessionRuntimeConfig +from vllm_omni.entrypoints.openai.duplex_capability import ( + should_enable_duplex_endpoint, +) +from vllm_omni.experimental.fullduplex.engine.duplex_runtime import duplex_resource_request_id +from vllm_omni.experimental.fullduplex.engine.messages import DuplexFence, DuplexSessionLifecycleMessage +from vllm_omni.experimental.fullduplex.openai.chat_fallback import ( + ChatFallbackProjectorMixin, +) +from vllm_omni.experimental.fullduplex.openai.protocol import ( + DuplexCapabilities, + DuplexCommittedInput, + DuplexOverlapPolicy, + DuplexPlaybackCommitPolicy, + DuplexSession, + DuplexSessionConfig, + DuplexSessionRegistry, + DuplexSessionState, + DuplexTurnController, + ResponseCreateOptions, +) +from vllm_omni.experimental.fullduplex.openai.realtime_session import ( + REALTIME_OUTPUT_AUDIO_FORMATS, + NativeRealtimeSessionProtocol, +) +from vllm_omni.experimental.fullduplex.openai.runtime_adapter import ( + ServingRuntimeAdapter, + ServingRuntimeConfigError, + ServingRuntimeSessionState, + load_serving_runtime_adapter, + validate_serving_runtime_adapter, +) +from vllm_omni.experimental.fullduplex.openai.runtime_bridge import ( + NativeRuntimeBridgeMixin, +) +from vllm_omni.experimental.fullduplex.openai.session_attachment import ( + DuplexJournalGapError, + DuplexSessionAttachmentRegistry, + InvalidResumeTokenError, +) +from vllm_omni.experimental.fullduplex.openai.session_runner import ( + DuplexSessionRunnerMixin, +) +from vllm_omni.experimental.fullduplex.openai.websocket import ( + DOMAIN_TERMINAL_EVENTS, + DuplexSessionTasks, + DuplexWebSocketActor, +) + +if TYPE_CHECKING: + from vllm_omni.entrypoints.openai.serving_chat import OmniOpenAIServingChat + +logger = init_logger(__name__) + +__all__ = ["OmniDuplexSessionHandler", "should_enable_duplex_endpoint"] + +_DEFAULT_CONFIG_TIMEOUT_S = 10.0 +_DEFAULT_IDLE_TIMEOUT_S = 300.0 + + +@dataclass(frozen=True) +class _DuplexSessionHandshake: + session: DuplexSession + resumed: bool = False + attachment_generation: int | None = None + + +class OmniDuplexSessionHandler( + DuplexSessionRunnerMixin, + NativeRuntimeBridgeMixin, + ChatFallbackProjectorMixin, +): + """WebSocket handler for RFC-style full-duplex session control. + + This owns the serving-side session actor, ordered inbound mailbox, + barge-in epoch, turn controller, and playback commit state. Generic sessions + can still fall back to chat requests, while MiniCPM-o 4.5 native sessions + route audio appends through scheduler data-plane stage requests. It + deliberately does not claim core persistent KV lease support. + """ + + def __init__( + self, + *, + chat_service: OmniOpenAIServingChat, + config_timeout_s: float = _DEFAULT_CONFIG_TIMEOUT_S, + idle_timeout_s: float = _DEFAULT_IDLE_TIMEOUT_S, + duplex_session_config: DuplexSessionRuntimeConfig | None = None, + serving_runtime_adapter: ServingRuntimeAdapter | None = None, + serving_runtime_adapter_path: str | None = None, + ) -> None: + self._chat_service = chat_service + self._config_timeout_s = config_timeout_s + self._idle_timeout_s = idle_timeout_s + self._duplex_session_config = duplex_session_config or DuplexSessionRuntimeConfig() + self._registry = DuplexSessionRegistry( + DuplexCapabilities( + supports_model_native_turn_policy=False, + supports_input_append=False, + supports_replace_latest_chunk=True, + supports_reencode_context=True, + supports_turn_commit_only=True, + supports_kv_lease=False, + ) + ) + self._turn_controller = DuplexTurnController() + adapter_path = serving_runtime_adapter_path or getattr( + chat_service, + "duplex_serving_adapter_path", + None, + ) + if serving_runtime_adapter is not None: + self._serving_runtime_adapter = validate_serving_runtime_adapter(serving_runtime_adapter) + elif isinstance(adapter_path, str) and adapter_path: + self._serving_runtime_adapter = load_serving_runtime_adapter( + adapter_path, + self._encode_native_data_plane_audio, + ) + else: + raise ValueError("A duplex serving runtime adapter must be explicitly configured") + self._session_tasks: dict[str, DuplexSessionTasks] = {} + self._realtime_protocols: dict[str, NativeRealtimeSessionProtocol] = {} + self._lease_generations: dict[str, int] = {} + self._resync_required_sessions: set[str] = set() + self._lifecycle_queue = getattr(self._chat_service.engine_client, "duplex_lifecycle_events", None) + self._lifecycle_task: asyncio.Task[None] | None = None + self._attachment_registry = DuplexSessionAttachmentRegistry( + replay_ttl_s=self._duplex_session_config.resume_replay_ttl_s, + replay_max_bytes_per_session=self._duplex_session_config.resume_replay_max_bytes_per_session, + disconnect_grace_s=self._duplex_session_config.disconnect_grace_s, + ) + + async def handle_realtime_session(self, websocket: WebSocket) -> None: + await self.handle_session( + websocket, + realtime_protocol=NativeRealtimeSessionProtocol(websocket.query_params), + ) + + def _ensure_lifecycle_listener(self) -> None: + if not isinstance(self._lifecycle_queue, asyncio.Queue): + return + if self._lifecycle_task is None or self._lifecycle_task.done(): + self._lifecycle_task = asyncio.create_task( + self._run_lifecycle_listener(), + name="duplex-serving-lifecycle", + ) + + async def _run_lifecycle_listener(self) -> None: + queue = self._lifecycle_queue + if not isinstance(queue, asyncio.Queue): + return + try: + while True: + message = await queue.get() + try: + if isinstance(message, DuplexSessionLifecycleMessage): + await self._apply_runtime_lifecycle(message) + finally: + queue.task_done() + if self._registry.active_count() == 0: + return + finally: + if self._lifecycle_task is asyncio.current_task(): + self._lifecycle_task = None + + async def _apply_runtime_lifecycle(self, message: DuplexSessionLifecycleMessage) -> None: + session = self._registry.get(message.session_id) + if session is None: + return + fence = message.fence + if ( + fence.session_id != session.session_id + or fence.incarnation != session.incarnation + or fence.epoch < session.epoch + ): + return + current_generation = self._lease_generations.get(session.session_id, 0) + if message.lease_generation < current_generation: + return + protocol = self._realtime_protocols.get(session.session_id) + expired_payload: dict[str, object] = { + "type": "session.expired", + "session_id": session.session_id, + "incarnation": session.incarnation, + "reason": message.reason, + } + if protocol is not None: + expired_payload = protocol.encode_outbound_event(expired_payload)[0] + with suppress(Exception): + await self._attachment_registry.send_event( + session.session_id, + expired_payload, + journal=False, + ) + + tasks = self._session_tasks.pop(session.session_id, None) + if tasks is not None: + await tasks.cancel_append_tasks() + active_response_task = tasks.active_response_task + if active_response_task is not None and not active_response_task.done(): + active_response_task.cancel() + with suppress(asyncio.CancelledError): + await active_response_task + native = self._serving_runtime_adapter.session_states.get(session.session_id) + if native is not None and native.data_plane_task is not None: + data_plane_task = native.data_plane_task + native.data_plane_task = None + data_plane_task.cancel() + with suppress(asyncio.CancelledError): + await data_plane_task + if session.active_response_id is not None: + session.end_response(commit_text=False) + self._cleanup_duplex_session_state(session) + self._registry.close(session.session_id) + self._realtime_protocols.pop(session.session_id, None) + self._lease_generations.pop(session.session_id, None) + self._resync_required_sessions.discard(session.session_id) + attachment = await self._attachment_registry.close(session.session_id) + if attachment is not None: + with suppress(Exception): + await attachment.close("session_expired") + + def _stop_lifecycle_listener_if_idle(self) -> None: + task = self._lifecycle_task + if self._registry.active_count() != 0 or task is None or task.done(): + return + if task is not asyncio.current_task(): + task.cancel() + self._lifecycle_task = None + + @staticmethod + def _native_audio_payload_size_bytes(payload: Mapping[str, object]) -> int: + audio = payload.get("audio") or payload.get("data") + if not isinstance(audio, str): + return 0 + try: + return len(base64.b64decode(audio, validate=True)) + except (ValueError, binascii.Error): + return 0 + + async def _apply_outbound_session_event( + self, + payload: dict[str, object], + *, + session: DuplexSession | None, + actor: DuplexWebSocketActor, + native: ServingRuntimeSessionState, + realtime_protocol: NativeRealtimeSessionProtocol | None, + ) -> tuple[bool, dict[str, object] | None]: + """Apply domain transitions before an event is queued for transport.""" + payload_type = payload.get("type") + is_terminal = payload_type in DOMAIN_TERMINAL_EVENTS + if is_terminal and session is not None: + payload_epoch = payload.get("epoch") + if isinstance(payload_epoch, int) and payload_epoch != session.epoch: + return False, None + if payload_type in {"response.done", "response.listen"} and ( + actor.closing or session.state == DuplexSessionState.CLOSED + ): + return False, None + elif actor._is_stale_model_output(payload): + return False, None + + if payload_type == "session.closed": + actor.close_reason = actor.close_reason or str(payload.get("reason") or "closed") + if session is not None: + session.mark_closing() + + if not is_terminal or session is None: + return True, None + + terminal_status = payload.get("status") + terminal_status_details = payload.get("status_details") + if terminal_status is None and isinstance(terminal_status_details, dict): + terminal_status = terminal_status_details.get("type") + response_terminal = payload_type == "response.done" or ( + payload_type == "response.listen" and session.active_response_id is not None + ) + can_promote_overlap = response_terminal and terminal_status not in {"cancelled", "failed"} + deferred_overlap_payload: dict[str, object] | None = None + continuous_input_crosses_terminal = ( + can_promote_overlap + and realtime_protocol is not None + and self._session_auto_responds(session) + and native.input_since_commit + and not native.deferred_response_create + ) + if continuous_input_crosses_terminal: + # Auto-response is a model-owned continuous stream. A response + # terminal advances the model turn without requiring the browser + # to close its Realtime input item. Keep any partial model unit so + # later PCM can complete it. Serving does not create or promote a + # new model generation here; Stage0 advances generation identity + # only after the model's terminal token. + session.reset_overlap_speech() + return True, None + realtime_input_still_open = ( + can_promote_overlap + and realtime_protocol is not None + and native.input_since_commit + and not native.deferred_response_create + ) + if realtime_input_still_open: + # response.done only closes the assistant response. It does not + # close the current Realtime input item, and it does not reserve + # or advance a model generation. + session.reset_overlap_speech() + return True, None + if can_promote_overlap and session.overlap_speech_ms > 0: + has_deferred_overlap = native.audio_buffer.has_pending() or native.committed_audio_payload is not None + should_promote_overlap = ( + session.state == DuplexSessionState.OPEN + and self._uses_native_input_append(session) + and has_deferred_overlap + and session.overlap_speech_ms > session.config.overlap_short_ack_ms + ) + if should_promote_overlap: + flushed_reserved_bytes = native.audio_buffer.pending_byte_count + deferred_overlap_payload = native.audio_buffer.flush( + chunk_period_ms=session.capabilities.chunk_period_ms or 1000 + ) + if native.committed_audio_payload is not None: + if deferred_overlap_payload is not None: + deferred_overlap_payload = self._merge_native_audio_payloads( + native.committed_audio_payload, + deferred_overlap_payload, + ) + else: + deferred_overlap_payload = native.committed_audio_payload + if self._session_auto_responds(session) and deferred_overlap_payload is not None: + deferred_overlap_payload = dict(deferred_overlap_payload) + deferred_overlap_payload["force_listen"] = False + if deferred_overlap_payload is not None: + native.retain_committed_audio( + deferred_overlap_payload, + operation_id=native.committed_audio_operation_id, + reserved_bytes=flushed_reserved_bytes, + ) + native.input_since_commit = deferred_overlap_payload is not None + if realtime_protocol is not None: + native.deferred_response_create = True + native.deferred_precreate_response = False + deferred_overlap_payload = None + else: + had_pending_overlap_audio = native.audio_buffer.has_pending() + native.audio_buffer.clear() + native.input_since_commit = False + native.speech_since_commit = False + if had_pending_overlap_audio and realtime_protocol is not None: + await realtime_protocol.discard_pending_input_audio(audio_end_ms=session.overlap_speech_ms) + if payload_type in {"audio.cancelled", "input.cancelled", "session.closed"}: + session.release_input_bytes(native.clear_committed_audio()) + + session.reset_overlap_speech() + if can_promote_overlap and native.deferred_response_create and native.committed_audio_payload is not None: + deferred_overlap_payload = native.committed_audio_payload + native.deferred_response_create = False + native.input_since_commit = False + native.speech_since_commit = False + return True, deferred_overlap_payload + + @staticmethod + def _advance_barge_in_epoch(session: DuplexSession) -> tuple[int, dict[str, int]]: + old_playback = session.playback.as_dict() + new_epoch = session.barge_in() + session.clear_playback_cursor() + return new_epoch, old_playback + + @staticmethod + def _commit_played_response_history( + session: DuplexSession, + response_id: str | None, + committed_ms: int, + ) -> None: + if not response_id or committed_ms < 0: + return + session.truncate_history_item( + f"item_{response_id}", + audio_end_ms=committed_ms, + playback=session.playback_for_response(response_id), + ) + + @staticmethod + def _should_commit_response_to_history(session: DuplexSession, response_id: str | None) -> bool: + if response_id is not None and response_id != session.active_response_id: + return True + mode = session.response_config.extra_body.get("realtime_response_conversation") + return not isinstance(mode, str) or mode.strip().lower() != "none" + + def _response_created_payload( + self, + session: DuplexSession, + response_id: str, + *, + epoch: int, + request_id: str | None = None, + ) -> dict[str, object]: + response_config = session.response_config + payload: dict[str, object] = { + "type": "response.created", + "session_id": session.session_id, + "response_id": response_id, + "epoch": epoch, + "modalities": list(response_config.modalities), + } + if request_id is not None: + payload["request_id"] = request_id + metadata = response_config.extra_body.get("realtime_response_metadata") + if not isinstance(metadata, dict): + metadata = response_config.extra_body.get("realtime_metadata") + if isinstance(metadata, dict): + payload["metadata"] = dict(metadata) + conversation = response_config.extra_body.get("realtime_response_conversation") + if isinstance(conversation, str): + payload["conversation"] = conversation + prompt = response_config.extra_body.get("realtime_response_prompt") + if isinstance(prompt, dict): + payload["prompt"] = dict(prompt) + return payload + + def _overlap_decision( + self, + session: DuplexSession, + event: dict[str, object], + payload: dict[str, object], + ) -> dict[str, object]: + """Classify input that arrives while assistant audio is active. + + This is a serving-side policy. The model still owns listen/speak + decisions for normal chunks; overlap policy only decides whether the + current assistant response should be interrupted before the new audio is + appended. + """ + duration_ms = self._input_audio_duration_ms(event, payload) + is_speech = self._input_looks_like_speech(event, payload, session=session) + if not session.capabilities.supports_barge_in and self._event_requests_barge_in(event): + return self._defer_unsupported_barge_in(session, duration_ms=duration_ms, is_speech=is_speech) + explicit = event.get("overlap_action") or event.get("overlap") + if isinstance(explicit, str): + normalized = explicit.strip().lower() + if normalized in {"barge_in", "interrupt", "cancel"}: + return { + "action": "barge_in", + "reason": "client_overlap_action", + "duration_ms": duration_ms, + "buffer_audio": True, + } + if normalized in {"listen", "continue", "continue_output", "ack"}: + session.reset_overlap_speech() + return { + "action": "listen", + "reason": "client_overlap_action", + "duration_ms": duration_ms, + "buffer_audio": ( + normalized == "listen" and is_speech and duration_ms > session.config.overlap_short_ack_ms + ), + "defer_runtime_append": True, + } + if normalized in {"drop", "ignore", "silence"}: + session.reset_overlap_speech() + return { + "action": "drop", + "reason": "client_overlap_action", + "duration_ms": duration_ms, + "buffer_audio": False, + } + + if bool(event.get("force_barge_in", False)): + return { + "action": "barge_in", + "reason": "client_force_barge_in", + "duration_ms": duration_ms, + "buffer_audio": True, + } + if self._session_auto_responds(session): + # Full-duplex input remains model-owned while output is active. Feed + # complete model units into the existing Stage0 stream immediately; + # playback only controls history ACKs, not model admission. + if is_speech: + session.accumulate_overlap_speech(duration_ms) + return { + "action": "listen", + "reason": "auto_response_continuous", + "duration_ms": duration_ms, + "overlap_speech_ms": session.overlap_speech_ms, + "buffer_audio": True, + "defer_runtime_append": False, + "force_listen": False, + "preserve_realtime_input": True, + } + if bool(event.get("force_listen", False)): + session.reset_overlap_speech() + return { + "action": "listen", + "reason": "client_force_listen", + "duration_ms": duration_ms, + "buffer_audio": is_speech, + "defer_runtime_append": True, + } + + policy = session.config.overlap_policy + if not is_speech: + if session.overlap_speech_ms <= 0: + session.reset_overlap_speech() + return { + "action": "drop", + "reason": "silence_or_noise", + "duration_ms": duration_ms, + "overlap_speech_ms": session.overlap_speech_ms, + "buffer_audio": False, + } + + if self._is_short_ack_transcript_hint(event, payload): + session.reset_overlap_speech() + return { + "action": "listen", + "reason": "short_ack_transcript", + "duration_ms": duration_ms, + "overlap_speech_ms": session.overlap_speech_ms, + "buffer_audio": False, + "defer_runtime_append": True, + } + + if policy == DuplexOverlapPolicy.LISTEN_ONLY.value: + session.accumulate_overlap_speech(duration_ms) + return { + "action": "listen", + "reason": "policy_listen_only", + "duration_ms": duration_ms, + "overlap_speech_ms": session.overlap_speech_ms, + "buffer_audio": True, + "defer_runtime_append": True, + } + + if policy == DuplexOverlapPolicy.BARGE_IN_ON_SPEECH.value and not session.capabilities.supports_barge_in: + return self._defer_unsupported_barge_in(session, duration_ms=duration_ms, is_speech=True) + + session.accumulate_overlap_speech(duration_ms) + if policy == DuplexOverlapPolicy.BARGE_IN_ON_SPEECH.value: + return { + "action": "barge_in", + "reason": "policy_barge_in_on_speech", + "duration_ms": duration_ms, + "overlap_speech_ms": session.overlap_speech_ms, + "buffer_audio": True, + } + + if ( + duration_ms <= session.config.overlap_short_ack_ms + and session.overlap_speech_ms <= session.config.overlap_short_ack_ms + ): + return { + "action": "listen", + "reason": "short_ack", + "duration_ms": duration_ms, + "overlap_speech_ms": session.overlap_speech_ms, + "buffer_audio": True, + "defer_runtime_append": True, + } + if session.overlap_speech_ms >= session.config.overlap_barge_in_ms: + if not session.capabilities.supports_barge_in: + return { + "action": "listen", + "reason": "barge_in_unsupported", + "duration_ms": duration_ms, + "overlap_speech_ms": session.overlap_speech_ms, + "buffer_audio": True, + "defer_runtime_append": True, + } + return { + "action": "barge_in", + "reason": "long_overlap_speech", + "duration_ms": duration_ms, + "overlap_speech_ms": session.overlap_speech_ms, + "buffer_audio": True, + } + return { + "action": "listen", + "reason": "accumulating_overlap_speech", + "duration_ms": duration_ms, + "overlap_speech_ms": session.overlap_speech_ms, + "buffer_audio": True, + "defer_runtime_append": True, + } + + @staticmethod + def _event_requests_barge_in(event: Mapping[str, object]) -> bool: + if event.get("force_barge_in") is True: + return True + explicit = event.get("overlap_action") or event.get("overlap") + return isinstance(explicit, str) and explicit.strip().lower() in { + "barge_in", + "interrupt", + "cancel", + } + + @staticmethod + def _defer_unsupported_barge_in( + session: DuplexSession, + *, + duration_ms: int, + is_speech: bool, + ) -> dict[str, object]: + if is_speech: + session.accumulate_overlap_speech(duration_ms) + return { + "action": "listen", + "reason": "barge_in_unsupported", + "duration_ms": duration_ms, + "overlap_speech_ms": session.overlap_speech_ms, + "buffer_audio": is_speech, + "defer_runtime_append": True, + } + + @staticmethod + def _barge_in_unsupported_error(session: DuplexSession) -> dict[str, object]: + return { + "type": "error", + "session_id": session.session_id, + "code": "barge_in_unsupported", + "error": "Barge-in is not supported by this duplex model", + } + + @staticmethod + def _is_short_ack_transcript_hint(event: dict[str, object], payload: dict[str, object]) -> bool: + raw_text = event.get("transcript") or event.get("text") or payload.get("transcript") or payload.get("text") + if not isinstance(raw_text, str): + return False + normalized = raw_text.strip().lower() + if not normalized: + return False + compact = "".join(ch for ch in normalized if ch.isalnum() or "\u4e00" <= ch <= "\u9fff") + if compact in { + "嗯", + "嗯嗯", + "对", + "对的", + "好", + "好的", + "继续", + "继续说", + "可以", + "是的", + "yes", + "yeah", + "yep", + "ok", + "okay", + "continue", + "goon", + "right", + }: + return True + return normalized in {"go on", "keep going", "please continue"} + + @staticmethod + def _input_audio_duration_ms(event: dict[str, object], payload: dict[str, object]) -> int: + for key in ("duration_ms", "audio_duration_ms"): + value = event.get(key) + if isinstance(value, int | float): + return max(0, int(value)) + fmt = payload.get("format") + sample_rate_hz = payload.get("sample_rate_hz") + audio = payload.get("audio") + if fmt == "pcm_f32le" and isinstance(sample_rate_hz, int) and sample_rate_hz > 0 and isinstance(audio, str): + try: + raw = base64.b64decode(audio, validate=True) + except (binascii.Error, ValueError): + return 0 + return int((len(raw) // 4) * 1000 / sample_rate_hz) + return 0 + + @staticmethod + def _merge_native_audio_payloads( + first: dict[str, object], + second: dict[str, object], + ) -> dict[str, object]: + if first.get("format") != "pcm_f32le" or second.get("format") != "pcm_f32le": + return second + first_rate = first.get("sample_rate_hz") + second_rate = second.get("sample_rate_hz") + if not isinstance(first_rate, int) or not isinstance(second_rate, int) or first_rate != second_rate: + return second + first_audio = first.get("audio") + second_audio = second.get("audio") + if not isinstance(first_audio, str) or not isinstance(second_audio, str): + return second + try: + first_raw = base64.b64decode(first_audio, validate=True) + second_raw = base64.b64decode(second_audio, validate=True) + except (binascii.Error, ValueError): + return second + merged = dict(second) + merged["audio"] = base64.b64encode(first_raw + second_raw).decode("ascii") + merged["sample_rate_hz"] = first_rate + merged_frames = [ + frame + for source in (first.get("video_frames"), second.get("video_frames")) + if isinstance(source, list) + for frame in source + if isinstance(frame, str) and frame + ] + if merged_frames: + merged["video_frames"] = merged_frames + else: + merged.pop("video_frames", None) + merged["force_listen"] = bool(first.get("force_listen", False)) or bool(second.get("force_listen", False)) + merged.pop("force_speak", None) + merged["is_speech"] = bool(first.get("is_speech", False)) or bool(second.get("is_speech", False)) + return merged + + @classmethod + def _should_force_listen_for_short_commit( + cls, + session: DuplexSession, + event: dict[str, object], + payload: dict[str, object], + ) -> bool: + """Keep very short committed Realtime chunks in listen mode. + + Realtime VAD can emit a commit for a short pause even when the user has + not actually yielded the turn. For MiniCPM-o native duplex, make that + policy explicit by steering the scheduler path to the model listen + token instead of letting a sub-second chunk start a response. + """ + if event.get("force_listen") is True or payload.get("force_listen") is True: + return True + if event.get("force_barge_in") is True: + return False + if event.get("response_create") is not True: + return False + duration_ms = cls._input_audio_duration_ms(event, payload) + return 0 < duration_ms <= session.config.overlap_short_ack_ms + + def _runtime_session_state(self, session: DuplexSession) -> ServingRuntimeSessionState: + return self._serving_runtime_adapter.session_state(session.session_id) + + # Temporary compatibility accessors for downstream tests and extensions. + def _minicpmo_session_state(self, session: DuplexSession) -> ServingRuntimeSessionState: + return self._runtime_session_state(session) + + @property + def _minicpmo_sessions(self): + return self._serving_runtime_adapter.session_states + + @property + def _minicpmo_data_plane(self): + return self._serving_runtime_adapter.data_plane + + def _should_force_listen_for_auto_response_overlap( + self, + session: DuplexSession, + event: dict[str, object], + payload: dict[str, object], + ) -> bool: + if not self._session_auto_responds(session): + return False + if event.get("force_barge_in") is True: + return False + return event.get("force_listen") is True or payload.get("force_listen") is True + + @staticmethod + def _assistant_playback_active(session: DuplexSession) -> bool: + return ( + session.config.playback_commit_policy == DuplexPlaybackCommitPolicy.ACK_ONLY.value + and session.playback.sent_ms > session.playback.committed_ms + ) + + @staticmethod + def _input_looks_like_speech( + event: dict[str, object], + payload: dict[str, object], + *, + session: DuplexSession, + ) -> bool: + for key in ("is_speech", "speech"): + value = event.get(key) + if isinstance(value, bool): + return value + vad = event.get("vad") + if isinstance(vad, dict): + value = vad.get("is_speech") + if isinstance(value, bool): + return value + probability = vad.get("speech_probability", vad.get("probability")) + if isinstance(probability, int | float): + return float(probability) >= 0.5 + probability = event.get("speech_probability") + if isinstance(probability, int | float): + return float(probability) >= 0.5 + + fmt = payload.get("format") + audio = payload.get("audio") + if fmt in {"pcm_f32le", "pcm16"} and isinstance(audio, str): + try: + raw = base64.b64decode(audio, validate=True) + except (binascii.Error, ValueError): + return True + if fmt == "pcm_f32le": + if len(raw) < 4 or len(raw) % 4 != 0: + return True + samples = np.frombuffer(raw, dtype=np.float32) + else: + if len(raw) < 2 or len(raw) % 2 != 0: + return True + samples = np.frombuffer(raw, dtype=np.int16).astype(np.float32) / 32768.0 + if samples.size == 0: + return False + rms = float(np.sqrt(np.mean(np.square(samples.astype(np.float32))))) + return rms >= session.config.overlap_silence_rms + return True + + async def _emit_overlap_decision( + self, + send_json, + session: DuplexSession, + decision: dict[str, object], + ) -> None: + await send_json( + { + "type": "overlap.decision", + "session_id": session.session_id, + "epoch": session.epoch, + "policy": session.config.overlap_policy, + **decision, + } + ) + + async def _open_session( + self, + websocket: WebSocket, + send_json, + *, + realtime_protocol: NativeRealtimeSessionProtocol | None = None, + attachment_send=None, + attachment_close=None, + ) -> _DuplexSessionHandshake | None: + raw = await self._receive_text( + websocket, + self._config_timeout_s, + realtime_protocol=realtime_protocol, + ) + if raw is None: + await send_json({"type": "error", "error": "Timeout waiting for session.create", "code": "config_timeout"}) + return None + try: + event = json.loads(raw) + except json.JSONDecodeError: + await send_json({"type": "error", "error": "Invalid JSON in session.create", "code": "invalid_json"}) + return None + if isinstance(event, dict) and event.get("type") == "session.resume": + if realtime_protocol is None: + await send_json( + { + "type": "error", + "error": "session.resume requires the Realtime protocol", + "code": "unsupported_session_resume", + } + ) + return None + return await self._resume_session_handshake( + event, + send_json=send_json, + realtime_protocol=realtime_protocol, + attachment_send=attachment_send, + attachment_close=attachment_close, + ) + if not isinstance(event, dict) or event.get("type") not in {"session.create", "open_session", "session.config"}: + await send_json( + { + "type": "error", + "error": f"Expected session.create, got: {event.get('type') if isinstance(event, dict) else None}", + "code": "bad_event", + } + ) + return None + + config = DuplexSessionConfig.from_event(event) + if config.idle_timeout_s == _DEFAULT_IDLE_TIMEOUT_S: + config.idle_timeout_s = self._idle_timeout_s + use_native_runtime = self._uses_serving_runtime_adapter(config) + runtime_config: dict[str, object] = {} + if use_native_runtime: + try: + runtime_config = await self._serving_runtime_adapter.prepare_runtime_config( + config, + model_config=getattr(self._chat_service, "model_config", None), + ) + except ServingRuntimeConfigError as exc: + await send_json({"type": "error", "error": str(exc), "code": exc.code}) + return None + except ValueError as exc: + await send_json({"type": "error", "error": str(exc), "code": "unsupported_ref_audio_path"}) + return None + session_id = event.get("session_id") if isinstance(event.get("session_id"), str) else None + session = self._registry.create(config=config, session_id=session_id) + if use_native_runtime: + session.replace_capabilities( + self._serving_runtime_adapter.capabilities( + max_sessions=self._duplex_session_config.max_sessions, + ) + ) + session.replace_runtime_config(runtime_config) + return _DuplexSessionHandshake(session=session) + + async def _resume_session_handshake( + self, + event: dict[str, object], + *, + send_json, + realtime_protocol: NativeRealtimeSessionProtocol, + attachment_send, + attachment_close, + ) -> _DuplexSessionHandshake | None: + session_id = event.get("session_id") + incarnation = event.get("incarnation") + resume_token = event.get("resume_token") + last_received = event.get("last_received_server_event_seq", 0) + if ( + not isinstance(session_id, str) + or not session_id + or not isinstance(incarnation, int) + or incarnation < 0 + or not isinstance(resume_token, str) + or not resume_token + or not isinstance(last_received, int) + or last_received < 0 + ): + await send_json( + { + "type": "error", + "error": ( + "session.resume requires session_id, incarnation, resume_token, " + "and a non-negative event sequence" + ), + "code": "invalid_session_resume", + } + ) + return None + session = self._registry.get(session_id) + if session is None or session.state != DuplexSessionState.OPEN: + await send_json( + { + "type": "error", + "error": f"Unknown or expired duplex session: {session_id}", + "code": "session_resume_expired", + } + ) + return None + if not session.capabilities.supports_session_resume: + await send_json( + { + "type": "error", + "error": f"Session does not support resume: {session_id}", + "code": "unsupported_session_resume", + } + ) + return None + try: + await self._attachment_registry.authenticate_resume( + session_id, + incarnation=incarnation, + resume_token=resume_token, + last_received_server_event_seq=last_received, + ) + except InvalidResumeTokenError: + await send_json( + { + "type": "error", + "error": "Invalid duplex session resume token", + "code": "invalid_resume_token", + } + ) + return None + except DuplexJournalGapError: + await send_json( + { + "type": "session.resync_required", + "session_id": session_id, + "reason": "journal_gap", + } + ) + return None + except (KeyError, ValueError) as exc: + await send_json( + { + "type": "error", + "error": str(exc), + "code": "session_resume_conflict", + } + ) + return None + + resume_runtime = getattr(self._chat_service.engine_client, "resume_duplex_session_async", None) + if not callable(resume_runtime): + await send_json( + { + "type": "error", + "error": "Duplex runtime does not expose session resume control", + "code": "runtime_resume_unsupported", + } + ) + return None + expected_generation = self._lease_generations.get(session_id, 0) + try: + runtime_result = await resume_runtime( + session_id, + fence=DuplexFence( + session.session_id, + epoch=session.epoch, + turn_id=session.turn_id, + incarnation=session.incarnation, + ), + expected_lease_generation=expected_generation, + ) + lease_generation = self._runtime_lease_generation(runtime_result) + if lease_generation is None: + raise RuntimeError("runtime resume result omitted lease_generation") + except Exception as exc: + await send_json( + { + "type": "error", + "error": str(exc), + "code": "runtime_resume_failed", + } + ) + return None + + if not callable(attachment_send) or not callable(attachment_close): + raise RuntimeError("session.resume requires transport attachment callbacks") + + def activation_payload_factory(token, generation: int) -> dict[str, object]: + internal = { + "type": "session.resumed", + "session_id": session_id, + "incarnation": incarnation, + "attachment_generation": generation, + "resume_token": token.plaintext, + } + return realtime_protocol.encode_outbound_event(internal)[0] + + try: + resumed = await self._attachment_registry.resume( + session_id, + incarnation=incarnation, + resume_token=resume_token, + last_received_server_event_seq=last_received, + send=attachment_send, + close=attachment_close, + activation_payload_factory=activation_payload_factory, + ) + except Exception as exc: + # The engine-side CAS already advanced even if the transport + # vanished before it received the rotated token. Keep that + # generation so the registry's one-shot recovery token can retry. + self._lease_generations[session_id] = lease_generation + await send_json( + { + "type": "error", + "error": str(exc), + "code": "session_resume_conflict", + } + ) + return None + self._lease_generations[session_id] = lease_generation + replaced = resumed.replaced_attachment + if replaced is not None: + replaced_payload = realtime_protocol.encode_outbound_event( + { + "type": "session.replaced", + "session_id": session_id, + "attachment_generation": replaced.generation, + } + )[0] + with suppress(Exception): + await replaced.send(replaced_payload) + with suppress(Exception): + await replaced.close("session_replaced") + return _DuplexSessionHandshake( + session=session, + resumed=True, + attachment_generation=resumed.attachment_generation, + ) + + @classmethod + def _runtime_lease_generation(cls, result: object) -> int | None: + if isinstance(result, dict): + generation = result.get("lease_generation") + if isinstance(generation, int): + return generation + for key in ("stage_results", "result"): + generation = cls._runtime_lease_generation(result.get(key)) + if generation is not None: + return generation + return None + if isinstance(result, list | tuple): + for item in result: + generation = cls._runtime_lease_generation(item) + if generation is not None: + return generation + return None + + def _uses_serving_runtime_adapter(self, config: DuplexSessionConfig) -> bool: + return self._serving_runtime_adapter.is_enabled(config) + + def _runtime_session_update_error( + self, + session: DuplexSession, + payload: dict[str, object], + ) -> dict[str, object] | None: + if not self._uses_native_input_append(session): + return None + try: + self._serving_runtime_adapter.validate_client_extra_body(payload.get("extra_body")) + except ServingRuntimeConfigError as exc: + return { + "type": "error", + "session_id": session.session_id, + "code": exc.code, + "error": str(exc), + } + return None + + def _runtime_config_for_session_update( + self, + session: DuplexSession, + candidate_config: DuplexSessionConfig, + ) -> dict[str, object]: + if not self._uses_native_input_append(session): + return dict(session.runtime_config) + return self._serving_runtime_adapter.runtime_config_for_update( + candidate_config, + dict(session.runtime_config), + ) + + def _runtime_session_candidate_update_error( + self, + session: DuplexSession, + candidate_config: DuplexSessionConfig, + ) -> dict[str, object] | None: + if not self._uses_native_input_append(session): + return None + if not self._config_requests_audio_output(candidate_config): + return None + if "ref_audio_data" in session.runtime_config: + return None + return { + "type": "error", + "session_id": session.session_id, + "code": "ref_audio_required", + "error": "MiniCPM-o native duplex audio output requires ref_audio", + } + + @staticmethod + def _uses_native_input_append(session: DuplexSession) -> bool: + return ( + session.capabilities.implementation_level == "model_native_duplex" + and session.capabilities.supports_input_append + ) + + @staticmethod + def _config_requests_audio_output(config: DuplexSessionConfig) -> bool: + return any(str(modality).lower() == "audio" for modality in config.modalities) + + @staticmethod + def _native_stage0_request_id(session: DuplexSession, epoch: int) -> str: + return duplex_resource_request_id( + DuplexFence( + session.session_id, + epoch=epoch, + incarnation=session.incarnation, + ), + "stage0", + ) + + @staticmethod + def _session_auto_responds(session: DuplexSession) -> bool: + """Full-duplex / model-driven mode. + + When set, the server runs per-chunk speak-generation continuously (like + the official MiniCPM-o ``duplex_generate`` loop) instead of waiting for an + explicit ``response.create``: each ~chunk_period of appended audio is + emitted and fed to the stage0 stream so the model itself decides to speak + or listen. Signaled by the client via ``extra_body.auto_response`` (or + ``extra_body.full_duplex``). + """ + extra = getattr(session.config, "extra_body", None) + if not isinstance(extra, dict): + return False + return extra.get("auto_response") is True or extra.get("full_duplex") is True + + async def _receive_text( + self, + websocket: WebSocket, + timeout_s: float, + *, + realtime_protocol: NativeRealtimeSessionProtocol | None = None, + ) -> str | None: + try: + if realtime_protocol is not None: + return await asyncio.wait_for( + realtime_protocol.receive_internal_event_text(websocket), + timeout=max(0.1, timeout_s), + ) + return await asyncio.wait_for(websocket.receive_text(), timeout=max(0.1, timeout_s)) + except asyncio.TimeoutError: + return None + + @staticmethod + def _input_committed_payload( + session: DuplexSession, + committed: DuplexCommittedInput, + *, + realtime_item_id: object | None = None, + ) -> dict[str, object]: + payload: dict[str, object] = { + "type": "input.committed", + "session_id": session.session_id, + "turn_id": committed.turn_id, + "epoch": committed.epoch, + "history_len": len(session.history), + "message": committed.message, + } + if isinstance(realtime_item_id, str) and realtime_item_id: + payload["realtime_item_id"] = realtime_item_id + return payload + + @staticmethod + def _commit_native_audio_input( + session: DuplexSession, + *, + realtime_item_id: object | None = None, + transcript: object | None = None, + turn_id: int | None = None, + ) -> DuplexCommittedInput: + clean_transcript = transcript.strip() if isinstance(transcript, str) else None + committed = session.commit_native_audio_input( + transcript=clean_transcript or None, + turn_id=turn_id, + ) + if isinstance(realtime_item_id, str) and realtime_item_id: + session.register_history_item(realtime_item_id, committed.message) + return committed + + @staticmethod + def _native_audio_committed_payload( + session: DuplexSession, + *, + committed: DuplexCommittedInput | None = None, + realtime_item_id: object | None = None, + transcript: object | None = None, + ) -> dict[str, object]: + message = committed.message if committed is not None else None + if not isinstance(message, dict): + input_audio_part: dict[str, object] = { + "type": "audio_url", + "audio_url": {"url": "native-duplex:input-audio"}, + } + if isinstance(transcript, str) and transcript: + input_audio_part["transcript"] = transcript + message = { + "role": "user", + "content": [input_audio_part], + } + payload: dict[str, object] = { + "type": "input.committed", + "session_id": session.session_id, + "turn_id": committed.turn_id if committed is not None else session.turn_id, + "epoch": committed.epoch if committed is not None else session.epoch, + "history_len": len(session.history), + "native_audio": True, + "message": message, + } + if isinstance(transcript, str) and transcript: + payload["transcript"] = transcript + if isinstance(realtime_item_id, str) and realtime_item_id: + payload["realtime_item_id"] = realtime_item_id + return payload + + @staticmethod + def _apply_session_update(session: DuplexSession, payload: dict[str, object]) -> dict[str, object] | None: + model = payload.get("model") + audio_config = payload.get("audio") + audio_input = audio_config.get("input") if isinstance(audio_config, dict) else None + audio_output = audio_config.get("output") if isinstance(audio_config, dict) else None + voice = payload.get("voice") + if not isinstance(voice, str) and isinstance(audio_output, dict): + voice = audio_output.get("voice") + if isinstance(model, str) and session.config.model is not None and model != session.config.model: + return { + "type": "error", + "session_id": session.session_id, + "code": "model_update_unsupported", + "error": "session.update cannot change model for an open realtime duplex session", + } + if isinstance(model, str) and session.config.model is None: + session.config.model = model + if isinstance(voice, str) and (session.playback.generated_ms > 0 or session.playback.sent_ms > 0): + return { + "type": "error", + "session_id": session.session_id, + "code": "voice_update_after_audio_unsupported", + "error": "session.update cannot change voice after audio output has started", + } + if isinstance(payload.get("ref_audio"), str): + return { + "type": "error", + "session_id": session.session_id, + "code": "ref_audio_update_unsupported", + "error": "session.update cannot change ref_audio after the native duplex runtime is open", + } + if isinstance(payload.get("instructions"), str): + session.config.instructions = str(payload["instructions"]) + elif "instructions" in payload and payload.get("instructions") is None: + session.config.instructions = None + if isinstance(voice, str): + session.config.voice = str(voice) + elif "voice" in payload and payload.get("voice") is None: + session.config.voice = None + response_format = payload.get("output_audio_format") or payload.get("response_format") + if response_format is None and isinstance(audio_config, dict): + if isinstance(audio_output, dict): + response_format = audio_output.get("format") + response_format, _ = NativeRealtimeSessionProtocol._parse_realtime_audio_format(response_format) + if isinstance(response_format, str) and response_format.lower() in REALTIME_OUTPUT_AUDIO_FORMATS: + session.config.response_format = NativeRealtimeSessionProtocol._duplex_response_format(response_format) + if isinstance(payload.get("temperature"), int | float): + session.config.temperature = float(payload["temperature"]) + speed = payload.get("speed") + if not isinstance(speed, int | float) and isinstance(audio_output, dict): + speed = audio_output.get("speed") + if isinstance(speed, int | float): + session.config.speed = float(speed) + max_tokens = ( + payload.get("max_response_output_tokens") + if "max_response_output_tokens" in payload + else payload.get("max_output_tokens") + if "max_output_tokens" in payload + else payload.get("max_tokens") + ) + if "max_response_output_tokens" in payload or "max_output_tokens" in payload or "max_tokens" in payload: + session.config.max_tokens = NativeRealtimeSessionProtocol.realtime_max_output_tokens(max_tokens) + if isinstance(payload.get("overlap_policy"), str): + session.config.overlap_policy = DuplexSessionConfig._normalize_overlap_policy( + str(payload["overlap_policy"]) + ) + if isinstance(payload.get("overlap_short_ack_ms"), int | float): + session.config.overlap_short_ack_ms = max(0, int(payload["overlap_short_ack_ms"])) + if isinstance(payload.get("overlap_barge_in_ms"), int | float): + session.config.overlap_barge_in_ms = max(0, int(payload["overlap_barge_in_ms"])) + if isinstance(payload.get("overlap_silence_rms"), int | float): + session.config.overlap_silence_rms = max(0.0, float(payload["overlap_silence_rms"])) + if isinstance(payload.get("playback_commit_policy"), str): + session.config.playback_commit_policy = DuplexSessionConfig._normalize_playback_commit_policy( + str(payload["playback_commit_policy"]) + ) + modalities = payload.get("modalities") or payload.get("output_modalities") + if isinstance(modalities, list) and all(isinstance(item, str) for item in modalities): + session.config.modalities = list(modalities) + if isinstance(payload.get("extra_body"), dict): + session.config.extra_body.update(payload["extra_body"]) + extra = payload["extra_body"] + if isinstance(extra.get("overlap_policy"), str): + session.config.overlap_policy = DuplexSessionConfig._normalize_overlap_policy( + str(extra["overlap_policy"]) + ) + if isinstance(extra.get("playback_commit_policy"), str): + session.config.playback_commit_policy = DuplexSessionConfig._normalize_playback_commit_policy( + str(extra["playback_commit_policy"]) + ) + if isinstance(payload.get("tools"), list): + session.config.extra_body["realtime_tools"] = payload["tools"] + elif "tools" in payload and payload.get("tools") is None: + session.config.extra_body.pop("realtime_tools", None) + if isinstance(payload.get("tool_choice"), str | dict): + session.config.extra_body["realtime_tool_choice"] = payload["tool_choice"] + elif "tool_choice" in payload and payload.get("tool_choice") is None: + session.config.extra_body.pop("realtime_tool_choice", None) + if isinstance(payload.get("metadata"), dict): + session.config.extra_body["realtime_metadata"] = dict(payload["metadata"]) + elif "metadata" in payload and payload.get("metadata") is None: + session.config.extra_body.pop("realtime_metadata", None) + if isinstance(payload.get("include"), list): + session.config.extra_body["realtime_include"] = list(payload["include"]) + elif "include" in payload and payload.get("include") is None: + session.config.extra_body.pop("realtime_include", None) + if isinstance(payload.get("prompt"), dict): + session.config.extra_body["realtime_prompt"] = dict(payload["prompt"]) + elif "prompt" in payload and payload.get("prompt") is None: + session.config.extra_body.pop("realtime_prompt", None) + input_audio_transcription = NativeRealtimeSessionProtocol._input_audio_transcription_config(payload) + if isinstance(input_audio_transcription, dict): + session.config.extra_body["realtime_input_audio_transcription"] = dict(input_audio_transcription) + elif "input_audio_transcription" in payload and payload.get("input_audio_transcription") is None: + session.config.extra_body.pop("realtime_input_audio_transcription", None) + if isinstance(payload.get("input_audio_noise_reduction"), dict): + session.config.extra_body["realtime_input_audio_noise_reduction"] = dict( + payload["input_audio_noise_reduction"] + ) + elif "input_audio_noise_reduction" in payload and payload.get("input_audio_noise_reduction") is None: + session.config.extra_body.pop("realtime_input_audio_noise_reduction", None) + if isinstance(audio_input, dict) and isinstance(audio_input.get("noise_reduction"), dict): + session.config.extra_body["realtime_input_audio_noise_reduction"] = dict(audio_input["noise_reduction"]) + elif isinstance(audio_input, dict) and audio_input.get("noise_reduction") is None: + session.config.extra_body.pop("realtime_input_audio_noise_reduction", None) + if isinstance(payload.get("audio"), dict): + session.config.extra_body["realtime_audio"] = dict(payload["audio"]) + elif "audio" in payload and payload.get("audio") is None: + session.config.extra_body.pop("realtime_audio", None) + if isinstance(payload.get("tracing"), str | dict): + session.config.extra_body["realtime_tracing"] = payload["tracing"] + elif "tracing" in payload and payload.get("tracing") is None: + session.config.extra_body.pop("realtime_tracing", None) + session.config.extra_body["realtime_session_payload"] = ( + NativeRealtimeSessionProtocol._json_safe_realtime_payload(payload) + ) + return None + + def _apply_response_create_options( + self, + session: DuplexSession, + payload: dict[str, object], + ) -> str | None: + """Reserve options that apply only to the next response lifecycle.""" + audio_config = payload.get("audio") + audio_output = audio_config.get("output") if isinstance(audio_config, dict) else None + if session.capabilities.implementation_level == "model_native_duplex": + nested_voice = audio_output.get("voice") if isinstance(audio_output, dict) else None + unsupported = ( + payload.get("instructions") is not None + or payload.get("voice") is not None + or nested_voice is not None + or payload.get("temperature") is not None + or any( + payload.get(field_name) is not None + for field_name in ("max_response_output_tokens", "max_output_tokens", "max_tokens") + ) + or payload.get("tools") is not None + or payload.get("tool_choice") is not None + ) + if unsupported: + return "unsupported_native_response_options" + + instructions = str(payload["instructions"]) if isinstance(payload.get("instructions"), str) else None + voice = payload.get("voice") + if not isinstance(voice, str) and isinstance(audio_output, dict): + voice = audio_output.get("voice") + voice = str(voice) if isinstance(voice, str) else None + response_format = payload.get("output_audio_format") or payload.get("response_format") + if response_format is None and isinstance(audio_config, dict): + if isinstance(audio_output, dict): + response_format = audio_output.get("format") + response_format, _ = NativeRealtimeSessionProtocol._parse_realtime_audio_format(response_format) + if isinstance(response_format, str) and response_format.lower() in REALTIME_OUTPUT_AUDIO_FORMATS: + response_format = NativeRealtimeSessionProtocol._duplex_response_format(response_format) + else: + response_format = None + temperature = float(payload["temperature"]) if isinstance(payload.get("temperature"), int | float) else None + speed = payload.get("speed") + if not isinstance(speed, int | float) and isinstance(audio_output, dict): + speed = audio_output.get("speed") + speed = float(speed) if isinstance(speed, int | float) else None + max_tokens = ( + payload.get("max_response_output_tokens") + if "max_response_output_tokens" in payload + else payload.get("max_output_tokens") + if "max_output_tokens" in payload + else payload.get("max_tokens") + ) + if "max_response_output_tokens" in payload or "max_output_tokens" in payload or "max_tokens" in payload: + max_tokens = NativeRealtimeSessionProtocol.realtime_max_output_tokens(max_tokens) + else: + max_tokens = None + modalities = payload.get("modalities") or payload.get("output_modalities") + if isinstance(modalities, list) and all(isinstance(item, str) for item in modalities): + modalities = tuple(modalities) + else: + modalities = None + response_extra: dict[str, object] = {} + conversation = payload.get("conversation") + if isinstance(conversation, str): + response_extra["realtime_response_conversation"] = conversation + metadata = payload.get("metadata") + if isinstance(metadata, dict): + response_extra["realtime_response_metadata"] = dict(metadata) + prompt = payload.get("prompt") + if isinstance(prompt, dict): + response_extra["realtime_response_prompt"] = dict(prompt) + if isinstance(payload.get("tools"), list): + response_extra["realtime_response_tools"] = payload["tools"] + if isinstance(payload.get("tool_choice"), str | dict): + response_extra["realtime_response_tool_choice"] = payload["tool_choice"] + extra_body = payload.get("extra_body") + if isinstance(extra_body, dict): + if session.capabilities.implementation_level == "model_native_duplex": + response_extra.update( + (key, value) + for key, value in extra_body.items() + if key not in self._serving_runtime_adapter.private_runtime_config_keys + ) + else: + response_extra.update(extra_body) + try: + session.reserve_response_options( + ResponseCreateOptions( + instructions=instructions, + voice=voice, + response_format=response_format, + temperature=temperature, + max_tokens=max_tokens, + speed=speed, + modalities=modalities, + extra_body=response_extra, + ) + ) + except RuntimeError: + return "response_already_active" + return None + + @staticmethod + def _realtime_item_to_history_message(item: object) -> dict[str, object] | None: + if not isinstance(item, dict): + return None + role = item.get("role") + if role not in {"system", "user", "assistant"}: + return None + content = item.get("content") + if isinstance(content, str): + text = content.strip() + return {"role": role, "content": text} if text else None + if not isinstance(content, list): + return None + text_chunks: list[str] = [] + audio_chunks: list[dict[str, object]] = [] + for part in content: + if not isinstance(part, dict): + continue + part_type = part.get("type") + if part_type in {"input_text", "text", "output_text"} and isinstance(part.get("text"), str): + text_chunks.append(str(part["text"])) + elif part_type in {"input_audio", "audio"}: + audio = part.get("audio") or part.get("data") + fmt = part.get("format") if isinstance(part.get("format"), str) else "wav" + if isinstance(audio, str) and audio: + audio_chunks.append( + { + "type": "audio_url", + "audio_url": { + "url": f"data:audio/{fmt};base64,{audio}", + }, + } + ) + elif part_type in {"audio_transcript", "transcript"} and isinstance(part.get("text"), str): + text_chunks.append(str(part["text"])) + text = "".join(text_chunks).strip() + if audio_chunks: + content_items: list[dict[str, object]] = [] + if text: + content_items.append({"type": "text", "text": text}) + content_items.extend(audio_chunks) + return {"role": role, "content": content_items} + if text: + return {"role": role, "content": text} + return None + + async def _handle_playback_ack(self, session: DuplexSession, event: dict[str, object], send_json) -> None: + played_ms = event.get("played_ms", event.get("audio_ms", 0)) + committed_ms = event.get("committed_ms") + if not isinstance(played_ms, int | float): + await send_json({"type": "error", "error": "playback.ack requires played_ms", "code": "bad_event"}) + return + committed_cursor = int(committed_ms) if isinstance(committed_ms, int | float) else int(played_ms) + item_id = event.get("item_id") + response_id = event.get("response_id") + response_id = response_id if isinstance(response_id, str) and response_id else None + if not isinstance(item_id, str) or not item_id: + item_id = f"item_{response_id}" if response_id is not None else None + elif response_id is None and item_id.startswith("item_"): + response_id = item_id.removeprefix("item_") + if response_id is None and item_id is None and len(session.pending_history_item_ids) == 1: + item_id = next(iter(session.pending_history_item_ids)) + if item_id.startswith("item_"): + response_id = item_id.removeprefix("item_") + if response_id is None and item_id is None and session.active_response_id is not None: + response_id = session.active_response_id + item_id = f"item_{response_id}" + if event.get("truncate") is True: + playback = session.acknowledge_playback( + int(played_ms), + committed_cursor, + response_id=response_id, + ) + playback = session.truncate_playback_commit( + committed_cursor, + response_id=response_id, + ) + else: + playback = session.acknowledge_playback( + int(played_ms), + committed_cursor, + response_id=response_id, + ) + committed_history = False + if isinstance(item_id, str) and item_id: + committed_history = session.truncate_history_item( + item_id, + audio_end_ms=committed_cursor, + playback=playback, + ) + elif session.pending_history_item_ids: + # A plain playback ack has no OpenAI item id. Commit the only + # uncommitted assistant candidate if the session has an unambiguous + # pending response; otherwise wait for conversation.item.truncate. + pending_ids = list(session.pending_history_item_ids) + if len(pending_ids) == 1: + item_id = pending_ids[0] + committed_history = session.truncate_history_item( + item_id, + audio_end_ms=committed_cursor, + playback=playback, + ) + elif session.active_response_id is not None: + item_id = f"item_{session.active_response_id}" + committed_history = session.truncate_history_item( + item_id, + audio_end_ms=committed_cursor, + playback=playback, + ) + elif session.last_assistant_full_message is not None: + if item_id is None and session.history_item_ids: + assistant_item_ids = [ + known_item_id + for known_item_id, message in session.history_item_ids.items() + if message.get("role") == "assistant" + ] + if len(assistant_item_ids) == 1: + item_id = assistant_item_ids[0] + if isinstance(item_id, str) and item_id: + committed_history = session.truncate_history_item( + item_id, + audio_end_ms=committed_cursor, + playback=playback, + ) + await send_json( + { + "type": "playback.acknowledged", + "session_id": session.session_id, + "epoch": session.epoch, + "item_id": item_id, + "played_ms": int(played_ms), + "committed_ms": committed_cursor, + "truncate": event.get("truncate") is True, + "playback": playback.as_dict(), + "history_committed": committed_history, + } + ) + if committed_history and committed_cursor >= max(playback.sent_ms, playback.generated_ms): + session.release_response_playback(response_id) + + async def _cancel_active_response( + self, + session: DuplexSession, + active_task: asyncio.Task[None] | None, + send_json, + *, + reason: str, + notify: bool = True, + ) -> bool: + has_running_task = active_task is not None and not active_task.done() + if not has_running_task and session.active_request_id is None and session.active_response_id is None: + return False + + old_epoch = session.epoch + old_request_id = session.active_request_id + old_response_id = session.active_response_id + committed_ms = session.playback.committed_ms + committed_message = session.end_response( + commit_text=self._should_commit_response_to_history(session, old_response_id), + playback_commit_policy=DuplexPlaybackCommitPolicy.ACK_ONLY.value, + ) + if old_response_id is not None: + item_id = f"item_{old_response_id}" + if committed_message is not None: + session.register_history_item(item_id, committed_message) + elif committed_ms > 0: + session.truncate_history_item(item_id, audio_end_ms=committed_ms) + new_epoch, old_playback = self._advance_barge_in_epoch(session) + if old_request_id is not None: + await self._abort_request_background( + session, + old_request_id, + send_json, + notify=notify, + ) + if has_running_task and active_task is not None: + active_task.cancel() + try: + await asyncio.wait_for(asyncio.gather(active_task, return_exceptions=True), timeout=0.25) + except asyncio.TimeoutError: + pass + if notify: + await send_json( + { + "type": "audio.cancelled", + "session_id": session.session_id, + "response_id": old_response_id, + "reason": reason, + "cancelled_epoch": old_epoch, + "epoch": new_epoch, + "committed_ms": committed_ms, + "playback": old_playback, + } + ) + return True + + async def _abort_request_background( + self, + session: DuplexSession, + request_id: str, + send_json, + *, + notify: bool, + ) -> None: + try: + abort_internal = getattr(self._chat_service.engine_client, "_abort_internal_requests", None) + if callable(abort_internal): + result = abort_internal([request_id]) + else: + result = self._chat_service.engine_client.abort([request_id]) + if inspect.isawaitable(result): + await result + except Exception as exc: + logger.exception("Failed to abort duplex request %s: %s", request_id, exc) + if notify and session.state != DuplexSessionState.CLOSED: + await self._send_runtime_error(send_json, "runtime_abort_failed", exc, session=session) + + async def _cancel_pending_input(self, session: DuplexSession, send_json, *, reason: str) -> None: + cancelled = session.cancel_pending_input() + self._advance_barge_in_epoch(session) + await send_json( + { + "type": "input.cancelled", + "session_id": session.session_id, + "reason": reason, + "epoch": session.epoch, + "cancelled": cancelled, + } + ) diff --git a/vllm_omni/experimental/fullduplex/openai/session_attachment.py b/vllm_omni/experimental/fullduplex/openai/session_attachment.py new file mode 100644 index 000000000..b0faa65d2 --- /dev/null +++ b/vllm_omni/experimental/fullduplex/openai/session_attachment.py @@ -0,0 +1,486 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from __future__ import annotations + +import asyncio +import hashlib +import hmac +import json +import secrets +import time +from collections import deque +from collections.abc import Awaitable, Callable, Mapping +from dataclasses import dataclass, field +from types import MappingProxyType + + +class InvalidResumeTokenError(RuntimeError): + pass + + +class DuplexJournalGapError(RuntimeError): + pass + + +class DuplexJournalOverflowError(RuntimeError): + pass + + +@dataclass(frozen=True) +class ResumeToken: + plaintext: str = field(repr=False) + + @classmethod + def generate(cls) -> ResumeToken: + return cls(secrets.token_urlsafe(32)) + + +@dataclass +class DuplexResumeCredential: + token_digest: bytes + + @classmethod + def from_token(cls, token: ResumeToken) -> DuplexResumeCredential: + return cls(token_digest=cls._digest(token.plaintext)) + + @staticmethod + def _digest(plaintext: str) -> bytes: + return hashlib.sha256(plaintext.encode("utf-8")).digest() + + def verify(self, plaintext: str) -> bool: + return hmac.compare_digest(self.token_digest, self._digest(plaintext)) + + def rotate(self) -> ResumeToken: + token = ResumeToken.generate() + self.token_digest = self._digest(token.plaintext) + return token + + +@dataclass(frozen=True) +class DuplexTransportAttachment: + generation: int + send: Callable[[dict[str, object]], Awaitable[None]] = field(repr=False) + close: Callable[[str], Awaitable[None]] = field(repr=False) + + +@dataclass(frozen=True) +class JournalEntry: + sequence: int + created_monotonic: float + encoded_bytes: int + payload: Mapping[str, object] = field(repr=False) + + def __post_init__(self) -> None: + object.__setattr__(self, "payload", MappingProxyType(dict(self.payload))) + + +class DuplexEventJournal: + def __init__( + self, + *, + max_bytes: int, + ttl_s: float, + clock: Callable[[], float] | None = None, + ) -> None: + if max_bytes <= 0: + raise ValueError("journal max_bytes must be positive") + if ttl_s <= 0: + raise ValueError("journal ttl_s must be positive") + self._max_bytes = max_bytes + self._ttl_s = ttl_s + self._clock = clock or time.monotonic + self._entries: deque[JournalEntry] = deque() + self._next_sequence = 1 + self._dropped_through = 0 + self._retained_bytes = 0 + self._overflowed = False + + @property + def retained_bytes(self) -> int: + return self._retained_bytes + + @property + def overflowed(self) -> bool: + return self._overflowed + + @property + def last_sequence(self) -> int: + return self._next_sequence - 1 + + def record(self, payload: Mapping[str, object]) -> JournalEntry: + if self._overflowed: + raise DuplexJournalOverflowError("duplex event journal already exceeded its byte limit") + self.prune() + sequence = self._next_sequence + sequenced_payload = dict(payload) + sequenced_payload["server_event_seq"] = sequence + encoded_bytes = len( + json.dumps( + sequenced_payload, + separators=(",", ":"), + ensure_ascii=False, + ).encode("utf-8") + ) + if self._retained_bytes + encoded_bytes > self._max_bytes: + self._overflowed = True + raise DuplexJournalOverflowError( + f"duplex event journal byte limit exceeded: {self._retained_bytes + encoded_bytes} > {self._max_bytes}" + ) + entry = JournalEntry( + sequence=sequence, + created_monotonic=self._clock(), + encoded_bytes=encoded_bytes, + payload=sequenced_payload, + ) + self._entries.append(entry) + self._retained_bytes += encoded_bytes + self._next_sequence += 1 + return entry + + def acknowledge(self, sequence: int) -> int: + if sequence < 0: + raise ValueError("acknowledged sequence must not be negative") + if sequence > self.last_sequence: + raise ValueError(f"acknowledged sequence {sequence} is newer than journal head {self.last_sequence}") + removed = 0 + while self._entries and self._entries[0].sequence <= sequence: + entry = self._entries.popleft() + self._retained_bytes -= entry.encoded_bytes + removed += 1 + self._dropped_through = max(self._dropped_through, sequence) + return removed + + def prune(self, now: float | None = None) -> int: + effective_now = self._clock() if now is None else now + cutoff = effective_now - self._ttl_s + removed = 0 + while self._entries and self._entries[0].created_monotonic <= cutoff: + entry = self._entries.popleft() + self._retained_bytes -= entry.encoded_bytes + self._dropped_through = max(self._dropped_through, entry.sequence) + removed += 1 + return removed + + def replay_after(self, sequence: int) -> tuple[JournalEntry, ...]: + if sequence < 0: + raise ValueError("replay sequence must not be negative") + self.prune() + if self._overflowed: + raise DuplexJournalGapError("duplex event journal overflowed; replay is incomplete") + if sequence < self._dropped_through: + raise DuplexJournalGapError( + f"requested sequence {sequence} is older than retained journal boundary {self._dropped_through}" + ) + if sequence > self.last_sequence: + raise ValueError(f"requested sequence {sequence} is newer than journal head {self.last_sequence}") + return tuple(entry for entry in self._entries if entry.sequence > sequence) + + +@dataclass(frozen=True) +class DuplexSessionAttachmentCreated: + session_id: str + incarnation: int + attachment_generation: int + resume_token: ResumeToken = field(repr=False) + + +@dataclass(frozen=True) +class DuplexSessionResumeResult: + session_id: str + incarnation: int + attachment_generation: int + resume_token: ResumeToken = field(repr=False) + replay_entries: tuple[JournalEntry, ...] = () + replaced_attachment: DuplexTransportAttachment | None = None + + +@dataclass +class _DuplexSessionAttachmentState: + session_id: str + incarnation: int + credential: DuplexResumeCredential + journal: DuplexEventJournal + attachment: DuplexTransportAttachment | None + attachment_generation: int + outbound_lock: asyncio.Lock = field(default_factory=asyncio.Lock, repr=False) + recovery_token_digest: bytes | None = field(default=None, repr=False) + grace_task: asyncio.Task[None] | None = field(default=None, repr=False) + + +class DuplexSessionAttachmentRegistry: + def __init__( + self, + *, + replay_ttl_s: float, + replay_max_bytes_per_session: int, + disconnect_grace_s: float = 30.0, + clock: Callable[[], float] | None = None, + ) -> None: + if replay_ttl_s <= 0: + raise ValueError("replay_ttl_s must be positive") + if replay_max_bytes_per_session <= 0: + raise ValueError("replay_max_bytes_per_session must be positive") + if disconnect_grace_s <= 0: + raise ValueError("disconnect_grace_s must be positive") + self._replay_ttl_s = replay_ttl_s + self._replay_max_bytes_per_session = replay_max_bytes_per_session + self._disconnect_grace_s = disconnect_grace_s + self._clock = clock or time.monotonic + self._sessions: dict[str, _DuplexSessionAttachmentState] = {} + self._lock = asyncio.Lock() + + def __repr__(self) -> str: + return f"{type(self).__name__}(session_ids={sorted(self._sessions)})" + + async def create( + self, + session_id: str, + *, + incarnation: int, + send: Callable[[dict[str, object]], Awaitable[None]], + close: Callable[[str], Awaitable[None]], + ) -> DuplexSessionAttachmentCreated: + async with self._lock: + if session_id in self._sessions: + raise ValueError(f"duplex attachment session already exists: {session_id}") + token = ResumeToken.generate() + generation = 1 + self._sessions[session_id] = _DuplexSessionAttachmentState( + session_id=session_id, + incarnation=incarnation, + credential=DuplexResumeCredential.from_token(token), + journal=DuplexEventJournal( + max_bytes=self._replay_max_bytes_per_session, + ttl_s=self._replay_ttl_s, + clock=self._clock, + ), + attachment=DuplexTransportAttachment( + generation=generation, + send=send, + close=close, + ), + attachment_generation=generation, + ) + return DuplexSessionAttachmentCreated( + session_id=session_id, + incarnation=incarnation, + attachment_generation=generation, + resume_token=token, + ) + + async def send_event( + self, + session_id: str, + payload: Mapping[str, object], + *, + journal: bool = True, + ) -> JournalEntry | None: + """Sequence and dispatch one event to the current attachment. + + The per-session lock keeps wire order equal to journal order without + serializing unrelated sessions. A detached session still records + replayable events, but has no transport side effect. + """ + async with self._lock: + state = self._require(session_id) + async with state.outbound_lock: + async with self._lock: + if self._sessions.get(session_id) is not state: + raise KeyError(f"unknown duplex attachment session: {session_id}") + entry = state.journal.record(payload) if journal else None + attachment = state.attachment + wire_payload = dict(entry.payload) if entry is not None else dict(payload) + if attachment is not None: + await attachment.send(wire_payload) + return entry + + async def acknowledge(self, session_id: str, sequence: int) -> int: + async with self._lock: + return self._require(session_id).journal.acknowledge(sequence) + + async def detach( + self, + session_id: str, + *, + attachment_generation: int, + on_grace_expired: Callable[[], Awaitable[None]] | None = None, + ) -> bool: + async with self._lock: + state = self._sessions.get(session_id) + if state is None or state.attachment_generation != attachment_generation: + return False + state.attachment = None + if state.grace_task is not None: + state.grace_task.cancel() + state.grace_task = ( + asyncio.create_task( + self._run_disconnect_grace( + state, + attachment_generation=attachment_generation, + callback=on_grace_expired, + ) + ) + if on_grace_expired is not None + else None + ) + return True + + async def is_current_attachment(self, session_id: str, attachment_generation: int) -> bool: + async with self._lock: + state = self._sessions.get(session_id) + return ( + state is not None + and state.attachment is not None + and state.attachment_generation == attachment_generation + ) + + async def authenticate_resume( + self, + session_id: str, + *, + incarnation: int, + resume_token: str, + last_received_server_event_seq: int, + ) -> None: + """Validate transport credentials before any engine resume control.""" + async with self._lock: + state = self._require(session_id) + self._validate_resume_identity( + state, + incarnation=incarnation, + resume_token=resume_token, + ) + state.journal.replay_after(last_received_server_event_seq) + + async def resume( + self, + session_id: str, + *, + incarnation: int, + resume_token: str, + last_received_server_event_seq: int, + send: Callable[[dict[str, object]], Awaitable[None]], + close: Callable[[str], Awaitable[None]], + activation_payload_factory: Callable[[ResumeToken, int], Mapping[str, object]] | None = None, + ) -> DuplexSessionResumeResult: + async with self._lock: + state = self._require(session_id) + async with state.outbound_lock: + async with self._lock: + if self._sessions.get(session_id) is not state: + raise KeyError(f"unknown duplex attachment session: {session_id}") + used_recovery = self._validate_resume_identity( + state, + incarnation=incarnation, + resume_token=resume_token, + ) + replay_entries = state.journal.replay_after(last_received_server_event_seq) + accepted_token_digest = ( + state.recovery_token_digest if used_recovery else bytes(state.credential.token_digest) + ) + if state.grace_task is not None: + state.grace_task.cancel() + state.grace_task = None + state.recovery_token_digest = None + rotated_token = state.credential.rotate() + replaced = state.attachment + state.attachment_generation += 1 + attachment_generation = state.attachment_generation + state.attachment = DuplexTransportAttachment( + generation=attachment_generation, + send=send, + close=close, + ) + if activation_payload_factory is not None: + try: + await send(dict(activation_payload_factory(rotated_token, attachment_generation))) + for entry in replay_entries: + await send(dict(entry.payload)) + except Exception: + async with self._lock: + if ( + self._sessions.get(session_id) is state + and state.attachment_generation == attachment_generation + ): + state.attachment = None + state.recovery_token_digest = accepted_token_digest + raise + return DuplexSessionResumeResult( + session_id=session_id, + incarnation=incarnation, + attachment_generation=attachment_generation, + resume_token=rotated_token, + replay_entries=replay_entries, + replaced_attachment=replaced, + ) + + async def close(self, session_id: str) -> DuplexTransportAttachment | None: + async with self._lock: + state = self._sessions.pop(session_id, None) + if state is not None and state.grace_task is not None: + state.grace_task.cancel() + return state.attachment if state is not None else None + + async def _run_disconnect_grace( + self, + state: _DuplexSessionAttachmentState, + *, + attachment_generation: int, + callback: Callable[[], Awaitable[None]] | None, + ) -> None: + try: + await asyncio.sleep(self._disconnect_grace_s) + except asyncio.CancelledError: + return + async with self._lock: + if ( + self._sessions.get(state.session_id) is not state + or state.attachment is not None + or state.attachment_generation != attachment_generation + ): + return + state.grace_task = None + if callback is not None: + await callback() + + def _require(self, session_id: str) -> _DuplexSessionAttachmentState: + state = self._sessions.get(session_id) + if state is None: + raise KeyError(f"unknown duplex attachment session: {session_id}") + return state + + @staticmethod + def _validate_resume_identity( + state: _DuplexSessionAttachmentState, + *, + incarnation: int, + resume_token: str, + ) -> bool: + if incarnation != state.incarnation: + raise ValueError(f"duplex attachment incarnation mismatch: expected {state.incarnation}, got {incarnation}") + if state.credential.verify(resume_token): + return False + recovery_digest = state.recovery_token_digest + if ( + state.attachment is None + and recovery_digest is not None + and hmac.compare_digest(recovery_digest, DuplexResumeCredential._digest(resume_token)) + ): + return True + raise InvalidResumeTokenError(f"invalid resume token for duplex session {state.session_id}") + + +__all__ = [ + "DuplexEventJournal", + "DuplexJournalGapError", + "DuplexJournalOverflowError", + "DuplexResumeCredential", + "DuplexSessionAttachmentCreated", + "DuplexSessionAttachmentRegistry", + "DuplexSessionResumeResult", + "DuplexTransportAttachment", + "InvalidResumeTokenError", + "JournalEntry", + "ResumeToken", +] diff --git a/vllm_omni/experimental/fullduplex/openai/session_runner.py b/vllm_omni/experimental/fullduplex/openai/session_runner.py new file mode 100644 index 000000000..7e2d1ecaf --- /dev/null +++ b/vllm_omni/experimental/fullduplex/openai/session_runner.py @@ -0,0 +1,1912 @@ +from __future__ import annotations + +import asyncio +import json +import uuid +from contextlib import suppress +from copy import deepcopy + +from fastapi import WebSocket, WebSocketDisconnect +from vllm.logger import init_logger + +from vllm_omni.experimental.fullduplex.engine.lease import DuplexLeaseActivity +from vllm_omni.experimental.fullduplex.engine.messages import DuplexFence +from vllm_omni.experimental.fullduplex.openai.audio import convert_input_audio_with_rate +from vllm_omni.experimental.fullduplex.openai.commit_policy import ( + CommitAction, + CommitSnapshot, + decide_commit_action, +) +from vllm_omni.experimental.fullduplex.openai.protocol import ( + DuplexPlaybackCommitPolicy, + DuplexSession, + DuplexSessionState, + DuplexTurnEventType, +) +from vllm_omni.experimental.fullduplex.openai.realtime_session import ( + NativeRealtimeSessionProtocol, +) +from vllm_omni.experimental.fullduplex.openai.runtime_adapter import ( + PcmAppendReservation, + ServingRuntimeSessionState, + payload_turn_id, +) +from vllm_omni.experimental.fullduplex.openai.session_attachment import ( + DuplexJournalOverflowError, +) +from vllm_omni.experimental.fullduplex.openai.websocket import ( + DuplexWebSocketActor, + is_input_event, + normalize_duplex_input_event, +) + +logger = init_logger(__name__) + +_MAX_EVENT_BYTES = 15 * 1024 * 1024 + + +class DuplexSessionRunnerMixin: + """Run one ordered WebSocket mailbox for a duplex session.""" + + async def handle_session( + self, + websocket: WebSocket, + *, + realtime_protocol: NativeRealtimeSessionProtocol | None = None, + ) -> None: + await websocket.accept() + session: DuplexSession | None = None + actor = DuplexWebSocketActor( + websocket, + current_epoch=lambda: session.epoch if session is not None else None, + session_closed=lambda: session is not None and session.state == DuplexSessionState.CLOSED, + outbound_protocol=realtime_protocol, + ) + runtime_opened = False + runtime_closed = False + attachment_ready = False + attachment_generation: int | None = None + resume_credential_delivered = False + transport_detached = False + pending_turn_reservations = 0 + + async def attachment_send(payload: dict[str, object]) -> None: + nonlocal resume_credential_delivered + try: + await websocket.send_json(payload) + except RuntimeError as exc: + message = str(exc) + if "after sending 'websocket.close'" in message or "response already completed" in message: + raise WebSocketDisconnect(code=1006) from exc + raise + if payload.get("type") == "session.created" and isinstance(payload.get("resume_token"), str): + resume_credential_delivered = True + + async def attachment_close(reason: str) -> None: + close = getattr(websocket, "close", None) + if callable(close): + await close(code=1000, reason=reason) + + async def send_attachment_payload(payload: dict[str, object], *, journal: bool) -> None: + assert session is not None + session_id = session.session_id + should_journal = journal and session_id not in self._resync_required_sessions + try: + await self._attachment_registry.send_event( + session_id, + payload, + journal=should_journal, + ) + except DuplexJournalOverflowError: + first_overflow = session_id not in self._resync_required_sessions + self._resync_required_sessions.add(session_id) + if first_overflow: + resync_payload: dict[str, object] = { + "type": "session.resync_required", + "session_id": session_id, + "reason": "journal_overflow", + } + if realtime_protocol is not None: + resync_payload = realtime_protocol.encode_outbound_event(resync_payload)[0] + await self._attachment_registry.send_event( + session_id, + resync_payload, + journal=False, + ) + await self._attachment_registry.send_event( + session_id, + payload, + journal=False, + ) + + if realtime_protocol is not None: + + async def send_realtime_raw(payload: dict[str, object]) -> None: + if attachment_ready and session is not None: + await send_attachment_payload( + payload, + journal=payload.get("type") not in {"session.created", "session.resumed"}, + ) + return + raw_payload = dict(payload) + raw_payload["_realtime_raw"] = True + await actor.send_json(raw_payload) + + realtime_protocol.bind_sender(send_realtime_raw) + native: ServingRuntimeSessionState = self._serving_runtime_adapter.create_session_state() + + def begin_close(reason: str) -> None: + actor.closing = True + actor.close_reason = reason + if session is not None: + session.mark_closing() + + event_emit_lock = asyncio.Lock() + + async def send_outbound(payload: dict[str, object]) -> None: + if not attachment_ready or session is None: + await actor.send_json(dict(payload)) + return + if realtime_protocol is None: + await send_attachment_payload( + payload, + journal=payload.get("type") not in {"session.created", "session.resumed"}, + ) + return + for projected in realtime_protocol.encode_outbound_event(payload): + await send_attachment_payload( + projected, + journal=projected.get("type") not in {"session.created", "session.resumed"}, + ) + + async def emit_event(payload: dict[str, object]) -> None: + deferred_precreate_response = False + async with event_emit_lock: + accepted, deferred_overlap_payload = await self._apply_outbound_session_event( + payload, + session=session, + actor=actor, + native=native, + realtime_protocol=realtime_protocol, + ) + if not accepted: + return + await send_outbound(payload) + if deferred_overlap_payload is not None: + deferred_precreate_response = native.deferred_precreate_response + native.deferred_precreate_response = False + if deferred_overlap_payload is not None and session is not None and not actor.closing: + await start_native_append( + deferred_overlap_payload, + final=True, + precreate_response=deferred_precreate_response, + operation_id=native.committed_audio_operation_id, + retained_committed_payload=( + deferred_overlap_payload if native.committed_audio_payload is deferred_overlap_payload else None + ), + ) + + writer_task = asyncio.create_task(actor.writer_loop(), name="duplex-session-writer") + reader_task: asyncio.Task[None] | None = None + + async def read_event_loop() -> None: + nonlocal pending_turn_reservations + assert session is not None + try: + while not actor.closing: + raw = await self._receive_text( + websocket, + session.config.idle_timeout_s, + realtime_protocol=realtime_protocol, + ) + if raw is None: + await actor.enqueue_event({"type": "__timeout__"}) + return + if len(raw.encode("utf-8")) > _MAX_EVENT_BYTES: + await emit_event( + {"type": "error", "error": "Duplex event too large", "code": "event_too_large"} + ) + continue + try: + event = json.loads(raw) + except json.JSONDecodeError: + await emit_event({"type": "error", "error": "Invalid JSON event", "code": "invalid_json"}) + continue + if not isinstance(event, dict): + await emit_event( + { + "type": "error", + "error": "Duplex event must be a JSON object", + "code": "bad_event", + } + ) + continue + event = normalize_duplex_input_event(event) + event_type = event.get("type") + if not isinstance(event_type, str): + await emit_event( + {"type": "error", "error": "Duplex event missing string type", "code": "bad_event"} + ) + continue + if event_type in {"input.commit", "input_audio_buffer.commit"}: + if not session.reserve_pending_turn( + limit=self._duplex_session_config.max_pending_turns_per_session + ): + await emit_event( + { + "type": "error", + "error": "Duplex session has too many pending input turns", + "code": "input_backpressure", + } + ) + continue + event["_duplex_pending_turn_reserved"] = True + pending_turn_reservations += 1 + if is_input_event(event_type) and native_response_in_progress(): + event["_duplex_overlap_candidate"] = True + await actor.enqueue_event(event) + except WebSocketDisconnect: + await actor.enqueue_event({"type": "__disconnect__"}) + + async def next_actor_event() -> dict[str, object]: + nonlocal pending_turn_reservations, transport_detached + event = await actor.next_event() + if event.pop("_duplex_pending_turn_reserved", False) and session is not None: + session.release_pending_turn() + pending_turn_reservations = max(0, pending_turn_reservations - 1) + if ( + attachment_ready + and attachment_generation is not None + and session is not None + and not await self._attachment_registry.is_current_attachment( + session.session_id, + attachment_generation, + ) + ): + transport_detached = True + return {"type": "__replaced_attachment__"} + return event + + def native_response_in_progress() -> bool: + if session is None: + return False + if session.active_response_id is not None: + return True + if ( + session.config.playback_commit_policy == DuplexPlaybackCommitPolicy.ACK_ONLY.value + and session.playback.sent_ms > session.playback.committed_ms + ): + return True + if actor.active_response_task is not None and not actor.active_response_task.done(): + return True + if actor.has_response_bound_append_tasks(): + return True + return False + + def clear_completed_pending_silence() -> None: + task = native.pending_silence_task + if task is not None and task.done(): + native.pending_silence_task = None + native.pending_silence_owner_id = None + + def mark_pending_silence_superseded() -> None: + task = native.pending_silence_task + if task is None: + return + if task.done(): + native.pending_silence_task = None + # Do not cancel the task or clear native_append_tail here. A + # silence append may already have reached the Engine, and local + # cancellation cannot retract that RPC. The sequencer preserves + # wire order; before_append will skip silence that has not started. + native.pending_silence_owner_id = None + + def real_native_input_waiting() -> bool: + clear_completed_pending_silence() + return ( + native.audio_buffer.has_pending() + or native.audio_buffer.has_reserved() + or actor.has_queued_input_events() + ) + + async def start_native_append( + payload: object, + *, + final: bool, + precreate_response: bool = False, + pcm_reservation: PcmAppendReservation | None = None, + operation_id: str | None = None, + retained_committed_payload: dict[str, object] | None = None, + silence_continuation: bool = False, + before_append=None, + ) -> asyncio.Task[bool] | None: + if session is None: + return + if not silence_continuation: + mark_pending_silence_superseded() + append_epoch = session.epoch + append_turn_id = payload_turn_id(payload) + if append_turn_id is None: + append_turn_id = session.turn_id + request_id = self._native_stage0_request_id(session, append_epoch) + if final or precreate_response: + session.bind_request(request_id) + if precreate_response: + session.bind_response_turn(append_turn_id) + if precreate_response and session.active_response_id is None: + response_id = session.begin_response(turn_id=append_turn_id) + await emit_event( + self._response_created_payload( + session, + response_id, + epoch=append_epoch, + ) + ) + precreated_response_id = session.active_response_id if precreate_response else None + + async def _run() -> bool: + nonlocal runtime_closed + try: + append_ok, emitted_response = await self._append_runtime_input( + session, + payload, + operation_id=(pcm_reservation.operation_id if pcm_reservation is not None else operation_id), + final=final, + send_json=emit_event, + mode="append_audio_chunk", + expected_epoch=append_epoch, + ) + if append_ok: + if pcm_reservation is not None: + pcm_reservation.commit() + session.release_input_bytes(pcm_reservation.byte_count) + if ( + retained_committed_payload is not None + and native.committed_audio_payload is retained_committed_payload + ): + session.release_input_bytes(native.clear_committed_audio()) + elif pcm_reservation is not None: + pcm_reservation.rollback() + if ( + not append_ok + and precreated_response_id is not None + and session.active_response_id == precreated_response_id + ): + session.end_response(commit_text=False) + await emit_event( + { + "type": "response.done", + "session_id": session.session_id, + "response_id": precreated_response_id, + "epoch": session.epoch, + "committed": False, + "status": "failed", + "status_details": { + "type": "failed", + "reason": "runtime_append_failed", + }, + "playback": session.playback.as_dict(), + } + ) + if not append_ok and session.state == DuplexSessionState.CLOSED: + runtime_closed = True + return False + if not emitted_response and session.epoch == append_epoch: + if session.active_request_id == self._native_stage0_request_id(session, append_epoch): + session.clear_request(request_id) + if final: + await emit_event( + self._turn_controller.signal(session, DuplexTurnEventType.USER_STARTED.value) + ) + return append_ok + except asyncio.CancelledError: + raise + except Exception as exc: + if pcm_reservation is not None: + pcm_reservation.rollback() + logger.exception("Native duplex append task failed: %s", exc) + await self._send_runtime_error(emit_event, "runtime_append_task_failed", exc, session=session) + if session.state != DuplexSessionState.CLOSED: + begin_close("runtime_append_task_failed") + if await self._close_runtime_session( + session, + reason="runtime_append_task_failed", + send_json=emit_event, + ): + runtime_closed = True + session.close() + await emit_event( + { + "type": "session.closed", + "session_id": session.session_id, + "reason": "runtime_append_task_failed", + } + ) + return False + + async def _run_in_wire_order(predecessor: asyncio.Task[bool] | None) -> bool: + if predecessor is not None: + try: + predecessor_ok = await predecessor + except asyncio.CancelledError: + current = asyncio.current_task() + if current is not None and current.cancelling(): + raise + predecessor_ok = False + except Exception: + predecessor_ok = False + if not predecessor_ok: + if pcm_reservation is not None: + pcm_reservation.rollback() + return False + if actor.closing or runtime_closed or session.state != DuplexSessionState.OPEN: + if pcm_reservation is not None: + pcm_reservation.rollback() + return False + if before_append is not None and not before_append(): + if pcm_reservation is not None: + pcm_reservation.rollback() + return True + if pcm_reservation is not None and not pcm_reservation.active: + return False + return await _run() + + predecessor = actor.native_append_tail + if predecessor is not None and predecessor.done(): + try: + predecessor_ok = predecessor.result() + except (asyncio.CancelledError, Exception): + predecessor_ok = False + if not predecessor_ok: + # Appends already queued behind a failed predecessor keep + # their captured dependency and stop. A later wire command + # is an explicit retry and starts a new chain. + predecessor = None + task = asyncio.create_task(_run_in_wire_order(predecessor)) + actor.native_append_tail = task + actor.track_append_task( + task, + epoch=append_epoch, + mode="append_audio_chunk", + final=final, + response_bound=final or precreate_response, + ) + if silence_continuation: + native.pending_silence_task = task + + def _clear_done_pending_silence(done: asyncio.Task[bool]) -> None: + if native.pending_silence_task is done: + native.pending_silence_task = None + native.pending_silence_owner_id = None + + task.add_done_callback(_clear_done_pending_silence) + # Let this wire-order effect start before the next mailbox event can + # cancel it. Later native appends still serialize on predecessor. + await asyncio.sleep(0) + return task + + async def schedule_native_silence_continuation( + payload: object, + *, + request_id: str, + owner_id: str, + response_id: str | None, + response_owned: bool, + expected_epoch: int | None, + expected_incarnation: int, + expected_model_turn_id: int | None, + send_json, + ) -> bool: + del send_json + if session is None: + return False + clear_completed_pending_silence() + pending_silence = native.pending_silence_task + if pending_silence is not None and not pending_silence.done(): + return False + if actor.native_append_tail is not None and not actor.native_append_tail.done(): + return False + if real_native_input_waiting(): + return False + + def _still_valid() -> bool: + return not real_native_input_waiting() and not self._native_silence_continuation_is_stale( + session, + request_id=request_id, + response_id=response_id, + response_owned=response_owned, + expected_epoch=expected_epoch, + expected_incarnation=expected_incarnation, + expected_model_turn_id=expected_model_turn_id, + ) + + native.pending_silence_owner_id = owner_id + task = await start_native_append( + payload, + final=False, + silence_continuation=True, + before_append=_still_valid, + ) + if task is None: + native.pending_silence_owner_id = None + return False + return True + + async def wait_for_native_append_tail() -> bool: + predecessor = actor.native_append_tail + if predecessor is None: + return True + try: + return await predecessor + except asyncio.CancelledError: + current = asyncio.current_task() + if current is not None and current.cancelling(): + raise + return False + except Exception: + # The append task already emitted its runtime error. Preserve + # wire order before the update path decides whether to continue. + return False + + async def start_runtime_append( + payload: object, + *, + final: bool, + mode: str = "append_tokens", + ) -> None: + """Schedule non-native runtime appends without blocking WS input. + + Model-native appends use ``start_native_append`` because they can + create data-plane response streams. Generic runtime appends still + need the same control-plane isolation so cancel/close can preempt a + slow engine append. + """ + if session is None: + return + append_epoch = session.epoch + + async def _run() -> None: + nonlocal runtime_closed + try: + append_ok, emitted_response = await self._append_runtime_input( + session, + payload, + final=final, + send_json=emit_event, + mode=mode, + expected_epoch=append_epoch, + ) + if not append_ok: + if session.state == DuplexSessionState.CLOSED: + runtime_closed = True + return + if not emitted_response and session.epoch == append_epoch: + await emit_event(self._turn_controller.signal(session, DuplexTurnEventType.USER_STARTED.value)) + except asyncio.CancelledError: + raise + except Exception as exc: + logger.exception("Duplex runtime append task failed: %s", exc) + await self._send_runtime_error(emit_event, "runtime_append_task_failed", exc, session=session) + + task = asyncio.create_task(_run()) + actor.track_append_task( + task, + epoch=append_epoch, + mode=mode, + final=final, + response_bound=final, + ) + + try: + handshake = await self._open_session( + websocket, + emit_event, + realtime_protocol=realtime_protocol, + attachment_send=attachment_send, + attachment_close=attachment_close, + ) + if handshake is None: + return + session = handshake.session + if handshake.resumed: + native = self._serving_runtime_adapter.session_states[session.session_id] + actor.tasks = self._session_tasks[session.session_id] + persisted_protocol = self._realtime_protocols.get(session.session_id) + if persisted_protocol is None: + raise RuntimeError(f"Missing Realtime protocol state for resumed session {session.session_id}") + realtime_protocol = persisted_protocol + realtime_protocol.bind_sender(send_realtime_raw) + attachment_generation = handshake.attachment_generation + attachment_ready = True + resume_credential_delivered = True + runtime_opened = True + self._ensure_lifecycle_listener() + reader_task = asyncio.create_task(read_event_loop(), name="duplex-session-reader") + else: + self._serving_runtime_adapter.session_states[session.session_id] = native + self._session_tasks[session.session_id] = actor.tasks + if realtime_protocol is not None: + self._realtime_protocols[session.session_id] = realtime_protocol + native.silence_continuation_scheduler = schedule_native_silence_continuation + if realtime_protocol is not None: + session.config.playback_commit_policy = DuplexPlaybackCommitPolicy.ACK_ONLY.value + if not handshake.resumed: + open_result = await self._open_runtime_session(session, emit_event) + if open_result is False: + return + runtime_opened = True + created_attachment = await self._attachment_registry.create( + session.session_id, + incarnation=session.incarnation, + send=attachment_send, + close=attachment_close, + ) + attachment_generation = created_attachment.attachment_generation + attachment_ready = True + self._lease_generations[session.session_id] = 0 + self._ensure_lifecycle_listener() + created_payload: dict[str, object] = { + "type": "session.created", + "session": session.as_public_dict(), + } + if session.capabilities.supports_session_resume: + created_payload.update( + { + "incarnation": session.incarnation, + "attachment_generation": attachment_generation, + "resume_token": created_attachment.resume_token.plaintext, + } + ) + if isinstance(open_result, dict): + created_payload["runtime_control"] = self._redact_runtime_control_result(open_result) + await emit_event(created_payload) + resume_credential_delivered = session.capabilities.supports_session_resume + reader_task = asyncio.create_task(read_event_loop(), name="duplex-session-reader") + + while True: + event = await next_actor_event() + event_type = event.get("type") + + if event_type == "__replaced_attachment__": + return + + if event_type == "__timeout__": + begin_close("timeout") + native.audio_buffer.clear() + session.release_all_input_bytes() + native.input_since_commit = False + native.speech_since_commit = False + native.clear_committed_audio() + await actor.cancel_append_tasks() + await self._cancel_native_data_plane_stream(session) + await self._cancel_active_response( + session, + actor.active_response_task, + emit_event, + reason="timeout", + notify=True, + ) + actor.active_response_task = None + runtime_closed = await self._close_runtime_session( + session, + reason="timeout", + send_json=emit_event, + ) + if not runtime_closed: + return + session.close() + await emit_event( + { + "type": "session.closed", + "session_id": session.session_id, + "reason": "timeout", + } + ) + return + + if event_type == "__disconnect__": + transport_detached = True + return + + if not isinstance(event_type, str): + continue + + if event_type == "session.heartbeat": + touch_session = getattr(self._chat_service.engine_client, "touch_duplex_session_async", None) + if not callable(touch_session): + await emit_event( + { + "type": "error", + "error": "Duplex runtime does not expose heartbeat control", + "code": "runtime_touch_unsupported", + } + ) + continue + try: + await touch_session( + session.session_id, + fence=DuplexFence( + session.session_id, + epoch=session.epoch, + turn_id=session.turn_id, + incarnation=session.incarnation, + ), + activity=DuplexLeaseActivity.HEARTBEAT, + ) + except Exception as exc: + await emit_event( + { + "type": "error", + "error": str(exc), + "code": "runtime_touch_failed", + } + ) + continue + await emit_event( + { + "type": "session.heartbeat_ack", + "session_id": session.session_id, + } + ) + continue + + if event_type == "session.event_ack": + acknowledged = event.get("server_event_seq") + if not isinstance(acknowledged, int) or acknowledged < 0: + await emit_event( + { + "type": "error", + "error": "session.event_ack requires a non-negative server_event_seq", + "code": "invalid_event_ack", + } + ) + continue + try: + await self._attachment_registry.acknowledge(session.session_id, acknowledged) + except ValueError as exc: + await emit_event( + { + "type": "error", + "error": str(exc), + "code": "invalid_event_ack", + } + ) + continue + + if event_type == "turn.signal" and event.get("event") in { + "input.cancel", + "response.cancel", + "barge_in", + }: + payload = event.get("payload") + normalized_event = dict(payload) if isinstance(payload, dict) else {} + normalized_event.update(event) + normalized_event["type"] = event["event"] + event = normalized_event + event_type = str(event["type"]) + + if event_type == "session.close": + with suppress(asyncio.TimeoutError): + await asyncio.wait_for(actor.output_queue.join(), timeout=1.0) + begin_close("session_close") + if session.state == DuplexSessionState.CLOSED: + runtime_closed = True + return + native.audio_buffer.clear() + session.release_all_input_bytes() + native.input_since_commit = False + native.speech_since_commit = False + native.clear_committed_audio() + await actor.cancel_append_tasks() + await self._cancel_native_data_plane_stream(session) + await self._cancel_active_response( + session, + actor.active_response_task, + emit_event, + reason="session_close", + ) + runtime_closed = await self._close_runtime_session( + session, + reason="session_close", + send_json=emit_event, + ) + actor.active_response_task = None + if not runtime_closed: + return + await emit_event({"type": "session.closed", "session_id": session.session_id}) + session.close() + return + + if event_type == "input_audio_buffer.clear": + native.audio_buffer.clear() + session.release_all_input_bytes() + native.input_since_commit = False + native.speech_since_commit = False + native.clear_committed_audio() + cancelled = session.cancel_pending_input() + await emit_event( + { + "type": "input_audio_buffer.cleared", + "session_id": session.session_id, + "epoch": session.epoch, + "drained_input_events": 0, + "cancelled": cancelled, + } + ) + continue + + if event_type == "barge_in" and not session.capabilities.supports_barge_in: + await emit_event(self._barge_in_unsupported_error(session)) + continue + + if event_type in {"input.cancel", "response.cancel", "barge_in", "output_audio_buffer.clear"}: + cancel_reason = ( + "output_audio_buffer_clear" if event_type == "output_audio_buffer.clear" else "barge_in" + ) + cancelled_fence = DuplexFence( + session.session_id, + epoch=session.epoch, + turn_id=session.turn_id, + incarnation=session.incarnation, + ) + if event_type == "response.cancel": + requested_response_id = event.get("response_id") + has_active_response_work = native_response_in_progress() + if ( + isinstance(requested_response_id, str) + and session.active_response_id is not None + and requested_response_id != session.active_response_id + ): + await emit_event( + { + "type": "error", + "session_id": session.session_id, + "code": "response_not_active", + "error": f"Response is not active: {requested_response_id}", + } + ) + continue + if not has_active_response_work: + if realtime_protocol is not None and isinstance(requested_response_id, str): + continue + await emit_event( + { + "type": "error", + "session_id": session.session_id, + "code": "response_not_active", + "error": "response.cancel requires an active response", + } + ) + continue + had_native_unbuffered_append = ( + self._uses_native_input_append(session) + and native.input_since_commit + and not native.audio_buffer.has_pending() + ) + playback_was_active = self._assistant_playback_active(session) + if event_type in {"input.cancel", "barge_in"}: + native.audio_buffer.clear() + session.release_all_input_bytes() + native.input_since_commit = False + native.speech_since_commit = False + native.clear_committed_audio() + had_native_append = await actor.cancel_append_tasks( + response_bound_only=event_type in {"response.cancel", "output_audio_buffer.clear"}, + ) + had_native_stream = native.data_plane_task is not None + cancelled = await self._cancel_active_response( + session, + actor.active_response_task, + emit_event, + reason=cancel_reason, + ) + had_native_stream = await self._cancel_native_data_plane_stream(session) or had_native_stream + if not cancelled and (had_native_stream or had_native_append or had_native_unbuffered_append): + old_epoch = session.epoch + old_response_id = session.active_response_id + committed_ms = session.playback.committed_ms + self._commit_played_response_history(session, old_response_id, committed_ms) + new_epoch, old_playback = self._advance_barge_in_epoch(session) + await emit_event( + { + "type": "audio.cancelled", + "session_id": session.session_id, + "response_id": old_response_id, + "reason": cancel_reason, + "cancelled_epoch": old_epoch, + "epoch": new_epoch, + "committed_ms": committed_ms, + "playback": old_playback, + } + ) + cancelled = True + if not cancelled and playback_was_active: + old_epoch = session.epoch + committed_ms = session.playback.committed_ms + self._commit_played_response_history(session, session.last_response_id, committed_ms) + new_epoch, old_playback = self._advance_barge_in_epoch(session) + await emit_event( + { + "type": "audio.cancelled", + "session_id": session.session_id, + "response_id": session.last_response_id, + "reason": cancel_reason, + "cancelled_epoch": old_epoch, + "epoch": new_epoch, + "committed_ms": committed_ms, + "playback": old_playback, + } + ) + cancelled = True + if not cancelled and event_type == "response.cancel": + old_epoch = session.epoch + old_response_id = session.active_response_id + committed_ms = session.playback.committed_ms + self._commit_played_response_history(session, old_response_id, committed_ms) + new_epoch, old_playback = self._advance_barge_in_epoch(session) + await emit_event( + { + "type": "audio.cancelled", + "session_id": session.session_id, + "response_id": old_response_id, + "reason": cancel_reason, + "cancelled_epoch": old_epoch, + "epoch": new_epoch, + "committed_ms": committed_ms, + "playback": old_playback, + } + ) + cancelled = True + if not cancelled and event_type == "output_audio_buffer.clear": + old_playback = session.playback.as_dict() + committed_ms = session.playback.committed_ms + session.clear_playback_cursor() + await emit_event( + { + "type": "audio.cancelled", + "session_id": session.session_id, + "response_id": session.active_response_id, + "reason": cancel_reason, + "cancelled_epoch": session.epoch, + "epoch": session.epoch, + "committed_ms": committed_ms, + "playback": old_playback, + } + ) + actor.active_response_task = None + continue + if not cancelled: + await self._cancel_pending_input(session, emit_event, reason="barge_in") + if not await self._signal_runtime_session( + session, + "barge_in", + emit_event, + fence=cancelled_fence, + next_fence=( + DuplexFence( + session.session_id, + epoch=session.epoch, + turn_id=session.turn_id, + incarnation=session.incarnation, + ) + if session.epoch > cancelled_fence.epoch + else None + ), + ): + continue + actor.active_response_task = None + continue + + if event_type == "turn.signal": + turn_event = event.get("event") + if isinstance(turn_event, str): + if turn_event == "barge_in" and not session.capabilities.supports_barge_in: + await emit_event(self._barge_in_unsupported_error(session)) + continue + if turn_event == "session.update": + payload = event.get("payload") + if not isinstance(payload, dict): + await emit_event( + { + "type": "error", + "session_id": session.session_id, + "code": "bad_event", + "error": "session.update requires a session payload", + } + ) + continue + if not await wait_for_native_append_tail(): + continue + runtime_update_error = self._runtime_session_update_error(session, payload) + if runtime_update_error is not None: + await emit_event(runtime_update_error) + continue + previous_config = session.config + candidate_config = deepcopy(previous_config) + session.replace_config(candidate_config) + try: + update_error = self._apply_session_update(session, payload) + finally: + session.replace_config(previous_config) + if update_error is not None: + await emit_event(update_error) + continue + runtime_update_error = self._runtime_session_candidate_update_error( + session, + candidate_config, + ) + if runtime_update_error is not None: + await emit_event(runtime_update_error) + continue + candidate_runtime_config = self._runtime_config_for_session_update( + session, + candidate_config, + ) + if not await self._signal_runtime_session( + session, + turn_event, + emit_event, + session_config=candidate_config.as_dict(), + runtime_config=candidate_runtime_config, + ): + continue + session.replace_config(candidate_config) + session.replace_runtime_config(candidate_runtime_config) + await emit_event( + { + "type": "session.updated", + "session": session.as_public_dict(), + } + ) + continue + if turn_event == "conversation.item.create": + payload = event.get("payload") + item = payload.get("item") if isinstance(payload, dict) else None + message = self._realtime_item_to_history_message(item) + item_id = item.get("id") if isinstance(item, dict) else None + if message is not None: + session.append_history_message(message) + session.register_history_item(item_id if isinstance(item_id, str) else None, message) + await emit_event( + { + "type": "conversation.item.created", + "session_id": session.session_id, + "item": item, + "created": message is not None, + } + ) + continue + if turn_event == "conversation.item.delete": + payload = event.get("payload") + item_id = payload.get("item_id") if isinstance(payload, dict) else None + deleted = session.delete_history_item(item_id) if isinstance(item_id, str) else False + await emit_event( + { + "type": "conversation.item.deleted", + "session_id": session.session_id, + "item_id": item_id, + "deleted": deleted, + } + ) + continue + if turn_event == "conversation.item.truncate": + payload = event.get("payload") + item_id = payload.get("item_id") if isinstance(payload, dict) else None + audio_end_ms = payload.get("audio_end_ms") if isinstance(payload, dict) else None + truncated = ( + session.truncate_history_item( + item_id, + audio_end_ms=int(audio_end_ms) if isinstance(audio_end_ms, int | float) else 0, + ) + if isinstance(item_id, str) + else False + ) + await emit_event( + { + "type": "conversation.item.truncated", + "session_id": session.session_id, + "item_id": item_id, + "content_index": ( + payload.get("content_index", 0) if isinstance(payload, dict) else 0 + ), + "audio_end_ms": audio_end_ms, + "truncated": truncated, + } + ) + continue + await emit_event(self._turn_controller.signal(session, turn_event, event)) + else: + await emit_event({"type": "error", "error": "turn.signal requires event", "code": "bad_event"}) + continue + + if event_type == "playback.ack": + await self._handle_playback_ack(session, event, emit_event) + continue + + if event_type == "input.text.append": + session.mark_user_input_activity() + text = event.get("text") + if not isinstance(text, str): + await emit_event( + { + "type": "error", + "error": "input.text.append requires text", + "code": "bad_event", + } + ) + continue + if self._uses_native_input_append(session): + await emit_event( + { + "type": "error", + "error": "The selected native duplex runtime accepts audio append only", + "code": "native_text_append_unsupported", + } + ) + continue + else: + session.append_text(text) + if session.capabilities.supports_input_append: + await start_runtime_append(text, final=False, mode="append_tokens") + continue + + if event_type == "input_audio_buffer.append": + session.mark_user_input_activity() + audio = event.get("audio") or event.get("data") + if not isinstance(audio, str): + await emit_event( + { + "type": "error", + "error": "input.audio.append requires audio", + "code": "bad_event", + } + ) + continue + if not session.capabilities.supports_barge_in and self._event_requests_barge_in(event): + await emit_event(self._barge_in_unsupported_error(session)) + event = dict(event) + event.pop("force_barge_in", None) + for key in ("overlap_action", "overlap"): + value = event.get(key) + if isinstance(value, str) and value.strip().lower() in { + "barge_in", + "interrupt", + "cancel", + }: + event.pop(key, None) + fmt = event.get("format") if isinstance(event.get("format"), str) else "pcm16" + default_sample_rate_hz = 16000 + sr_raw = event.get("sample_rate_hz") or event.get("sample_rate") + sample_rate_hz = sr_raw if isinstance(sr_raw, int | float) else default_sample_rate_hz + try: + audio, fmt, sample_rate_hz = convert_input_audio_with_rate( + audio, + fmt, + sample_rate_hz=sample_rate_hz, + ) + except ValueError as exc: + await emit_event({"type": "error", "error": str(exc), "code": "bad_event"}) + continue + if isinstance(fmt, str) and fmt.lower() in {"pcm16", "pcm_s16le", "s16le"}: + await emit_event( + { + "type": "error", + "error": "input_audio_buffer.append pcm16 audio could not be decoded", + "code": "bad_audio", + } + ) + continue + force_listen = bool(event.get("force_listen", False)) + payload = { + "type": "audio", + "audio": audio, + "format": fmt, + "sample_rate_hz": sample_rate_hz, + "force_listen": force_listen, + } + video_frames = event.get("video_frames") + if isinstance(video_frames, list): + frames = [frame for frame in video_frames if isinstance(frame, str) and frame] + if frames: + payload["video_frames"] = frames + # Speech/silence tag for the Stage0 turn-ended latch. + payload["is_speech"] = self._input_looks_like_speech(event, payload, session=session) + defer_native_append = False + buffer_overlap_audio = True + if self._uses_native_input_append(session): + mark_pending_silence_superseded() + overlap_active = not self._session_auto_responds(session) and native_response_in_progress() + if overlap_active: + decision = self._overlap_decision(session, event, payload) + await self._emit_overlap_decision(emit_event, session, decision) + action = decision.get("action") + if action == "drop": + if realtime_protocol is not None: + await realtime_protocol.discard_pending_input_audio( + audio_end_ms=self._input_audio_duration_ms(event, payload) + ) + continue + if action == "listen": + buffer_overlap_audio = bool(decision.get("buffer_audio", True)) + defer_native_append = bool(decision.get("defer_runtime_append", True)) + if ( + not buffer_overlap_audio + and realtime_protocol is not None + and decision.get("preserve_realtime_input") is not True + ): + await realtime_protocol.discard_pending_input_audio( + audio_end_ms=self._input_audio_duration_ms(event, payload) + ) + if decision.get("force_listen", True) is True: + payload["force_listen"] = True + else: + event["force_barge_in"] = True + cancelled_fence = DuplexFence( + session.session_id, + epoch=session.epoch, + turn_id=session.turn_id, + incarnation=session.incarnation, + ) + playback_was_active = self._assistant_playback_active(session) + buffer_overlap_audio = True + defer_native_append = False + native.audio_buffer.clear_force_listen() + session.reset_overlap_speech() + native.input_since_commit = False + native.speech_since_commit = False + await actor.cancel_append_tasks() + had_native_stream = native.data_plane_task is not None + cancelled = await self._cancel_active_response( + session, + actor.active_response_task, + emit_event, + reason="barge_in", + ) + had_native_stream = ( + await self._cancel_native_data_plane_stream(session) or had_native_stream + ) + if not cancelled and had_native_stream: + old_epoch = session.epoch + old_response_id = session.active_response_id + committed_ms = session.playback.committed_ms + self._commit_played_response_history(session, old_response_id, committed_ms) + new_epoch, old_playback = self._advance_barge_in_epoch(session) + await emit_event( + { + "type": "audio.cancelled", + "session_id": session.session_id, + "response_id": old_response_id, + "reason": "barge_in", + "cancelled_epoch": old_epoch, + "epoch": new_epoch, + "committed_ms": committed_ms, + "playback": old_playback, + } + ) + cancelled = True + if not cancelled and playback_was_active: + old_epoch = session.epoch + committed_ms = session.playback.committed_ms + self._commit_played_response_history( + session, session.last_response_id, committed_ms + ) + new_epoch, old_playback = self._advance_barge_in_epoch(session) + await emit_event( + { + "type": "audio.cancelled", + "session_id": session.session_id, + "response_id": session.last_response_id, + "reason": "barge_in", + "cancelled_epoch": old_epoch, + "epoch": new_epoch, + "committed_ms": committed_ms, + "playback": old_playback, + } + ) + cancelled = True + if session.epoch > cancelled_fence.epoch: + if not await self._signal_runtime_session( + session, + "barge_in", + emit_event, + fence=cancelled_fence, + next_fence=DuplexFence( + session.session_id, + epoch=session.epoch, + turn_id=session.turn_id, + incarnation=session.incarnation, + ), + ): + continue + actor.active_response_task = None + elif not self._session_auto_responds(session) and not self._input_looks_like_speech( + event, payload, session=session + ): + # Turn-mode only: skip silent chunks so they don't open a + # response. In auto-respond (full-duplex) mode the model owns + # the speak/listen decision and MUST receive silence units -- + # the official model typically starts speaking during the + # silence after a question. + await emit_event( + { + "type": "response.listen", + "session_id": session.session_id, + "epoch": session.epoch, + "reason": "silence_or_noise", + } + ) + continue + if self._should_force_listen_for_auto_response_overlap(session, event, payload): + # Auto-response keeps a long-lived native Stage0 stream. + # While assistant audio is still active, silence from the + # browser should advance the model in listen mode rather + # than letting the same stream start another assistant + # segment. Speech/barge-in chunks remain model-owned. + payload["force_listen"] = True + if not buffer_overlap_audio: + continue + session.mark_user_input_activity() + native.input_since_commit = True + native.speech_since_commit = native.speech_since_commit or self._input_looks_like_speech( + event, payload, session=session + ) + try: + raw_audio_bytes = self._native_audio_payload_size_bytes(payload) + if not session.reserve_input_bytes( + raw_audio_bytes, + limit=self._duplex_session_config.max_pending_input_bytes_per_session, + ): + await emit_event( + { + "type": "error", + "error": "Duplex session pending input exceeds server limit", + "code": "input_backpressure", + } + ) + continue + allow_emit = not defer_native_append and ( + realtime_protocol is None + or event_type != "input_audio_buffer.append" + # Full-duplex: emit each ~chunk_period of audio so the model + # runs per-chunk generation (speak/listen) without an explicit + # response.create, matching the official duplex_generate loop. + or self._session_auto_responds(session) + ) + pcm_reservation = native.audio_buffer.prepare_append( + payload, + operation_id=uuid.uuid4().hex, + chunk_period_ms=session.capabilities.chunk_period_ms or 1000, + allow_emit=allow_emit, + ) + except ValueError as exc: + session.release_input_bytes(raw_audio_bytes) + await emit_event({"type": "error", "error": str(exc), "code": "bad_event"}) + continue + if pcm_reservation is None: + continue + if pcm_reservation.byte_count == 0: + session.release_input_bytes(raw_audio_bytes) + payload = pcm_reservation.payload + else: + session.append_audio(audio, fmt=fmt, sample_rate_hz=sample_rate_hz) + if self._uses_native_input_append(session): + await start_native_append( + payload, + final=False, + pcm_reservation=pcm_reservation, + ) + continue + if session.capabilities.supports_input_append: + await start_runtime_append(payload, final=False, mode="append_audio_chunk") + continue + + if event_type in {"input.commit", "input_audio_buffer.commit", "response.create"}: + realtime_item_id = event.get("realtime_item_id") + realtime_validated_audio_commit = ( + event_type == "input_audio_buffer.commit" + and isinstance(realtime_item_id, str) + and bool(realtime_item_id) + ) + if ( + self._uses_native_input_append(session) + and event_type in {"input.commit", "input_audio_buffer.commit"} + and not await wait_for_native_append_tail() + ): + continue + if event_type == "input_audio_buffer.commit" and event.get("is_speech") is False: + native.input_since_commit = False + native.speech_since_commit = False + native.audio_buffer.clear() + session.release_all_input_bytes() + native.clear_committed_audio() + await emit_event( + { + "type": "input.committed", + "session_id": session.session_id, + "turn_id": session.turn_id, + "epoch": session.epoch, + "empty": True, + "is_speech": False, + "no_response": True, + } + ) + await emit_event( + { + "type": "response.listen", + "session_id": session.session_id, + "epoch": session.epoch, + "reason": "silence_or_noise", + } + ) + continue + should_create_response = ( + event_type == "response.create" + or bool(event.get("response_create", event_type == "input.commit")) + or (event_type == "input_audio_buffer.commit" and self._session_auto_responds(session)) + ) + precreate_response_requested = event_type == "response.create" or bool( + event.get("response_create", event_type == "input.commit") + ) + if event_type == "response.create": + response_payload = event.get("response") + if isinstance(response_payload, dict): + response_options_error = self._apply_response_create_options(session, response_payload) + if response_options_error is not None: + error_message = ( + "The selected native duplex runtime does not support generation " + "overrides for instructions, voice, temperature, max tokens, tools, or " + "tool_choice." + if response_options_error == "unsupported_native_response_options" + else "response.create cannot reserve options while another response is active." + ) + await emit_event( + { + "type": "error", + "session_id": session.session_id, + "epoch": session.epoch, + "code": response_options_error, + "error": error_message, + } + ) + continue + if self._uses_native_input_append(session) and event_type == "input_audio_buffer.commit": + has_pending_native_audio = ( + native.input_since_commit + or native.audio_buffer.has_pending() + or native.committed_audio_payload is not None + or realtime_validated_audio_commit + ) + if ( + not has_pending_native_audio + and not actor.native_append_tasks + and native.data_plane_task is None + ): + await emit_event( + { + "type": "error", + "session_id": session.session_id, + "epoch": session.epoch, + "code": "input_audio_buffer_empty", + "error": "input_audio_buffer.commit requires a non-empty input audio buffer.", + } + ) + continue + commit_action = decide_commit_action( + CommitSnapshot( + auto_responds=self._session_auto_responds(session), + speech_since_commit=native.speech_since_commit, + active_response_id=session.active_response_id, + overlap_speech_ms=session.overlap_speech_ms, + native_response_in_progress=native_response_in_progress(), + playback_active=self._assistant_playback_active(session), + ) + ) + if commit_action is CommitAction.DEFER_ACTIVE_RESPONSE: + if session.overlap_speech_ms <= session.config.overlap_short_ack_ms: + native.audio_buffer.clear() + session.release_all_input_bytes() + native.input_since_commit = False + native.speech_since_commit = False + native.clear_committed_audio() + if realtime_protocol is not None: + await realtime_protocol.discard_pending_input_audio( + audio_end_ms=session.overlap_speech_ms + ) + await emit_event( + { + "type": "input.committed", + "session_id": session.session_id, + "turn_id": session.turn_id, + "epoch": session.epoch, + "empty": True, + "is_speech": False, + "overlap_ack": True, + "no_response": True, + } + ) + session.reset_overlap_speech() + session.discard_response_options() + continue + + commit_reservation = native.audio_buffer.prepare_commit( + operation_id=uuid.uuid4().hex, + chunk_period_ms=session.capabilities.chunk_period_ms or 1000, + ) + deferred_payload = commit_reservation.payload + if deferred_payload is None: + commit_reservation.commit() + else: + if native.committed_audio_payload is not None: + deferred_payload = self._merge_native_audio_payloads( + native.committed_audio_payload, + deferred_payload, + ) + native.retain_committed_audio( + deferred_payload, + operation_id=commit_reservation.operation_id, + reserved_bytes=commit_reservation.byte_count, + ) + commit_reservation.commit() + native.deferred_response_create = should_create_response + native.deferred_precreate_response = precreate_response_requested + native.input_since_commit = False + native.speech_since_commit = False + committed = self._commit_native_audio_input( + session, + realtime_item_id=event.get("realtime_item_id"), + transcript=event.get("transcript"), + ) + committed_payload = self._native_audio_committed_payload( + session, + committed=committed, + realtime_item_id=event.get("realtime_item_id"), + transcript=event.get("transcript"), + ) + committed_payload["overlap_deferred"] = True + committed_payload["response_create_deferred"] = should_create_response + await emit_event(committed_payload) + continue + if commit_action is CommitAction.START_AUTO_RESPONSE: + commit_reservation = native.audio_buffer.prepare_commit( + operation_id=uuid.uuid4().hex, + chunk_period_ms=session.capabilities.chunk_period_ms or 1000, + ) + committed_input = commit_reservation.payload + final_payload = committed_input + if native.committed_audio_payload is not None: + if final_payload is not None: + final_payload = self._merge_native_audio_payloads( + native.committed_audio_payload, + final_payload, + ) + else: + final_payload = native.committed_audio_payload + commit_reservation.commit() + if final_payload is not None: + native.retain_committed_audio( + final_payload, + operation_id=commit_reservation.operation_id, + reserved_bytes=commit_reservation.byte_count, + ) + native.deferred_response_create = False + native.input_since_commit = False + native.speech_since_commit = False + data_plane_turn_id = session.turn_id + committed = self._commit_native_audio_input( + session, + realtime_item_id=event.get("realtime_item_id"), + transcript=event.get("transcript"), + turn_id=data_plane_turn_id, + ) + await emit_event( + self._native_audio_committed_payload( + session, + committed=committed, + realtime_item_id=event.get("realtime_item_id"), + transcript=event.get("transcript"), + ) + ) + if final_payload is not None: + await start_native_append( + { + **final_payload, + "duplex_turn_id": data_plane_turn_id, + }, + final=True, + precreate_response=False, + operation_id=commit_reservation.operation_id, + retained_committed_payload=final_payload, + ) + continue + if self._uses_native_input_append(session) and event_type == "response.create": + if ( + native_response_in_progress() + or actor.native_append_tasks + or native.data_plane_task is not None + ): + if session.active_response_id is None and ( + session.active_request_id is not None + or actor.native_append_tasks + or native.data_plane_task is not None + ): + continue + await emit_event( + { + "type": "error", + "session_id": session.session_id, + "epoch": session.epoch, + "code": "response_already_active", + "error": "response.create cannot start while another response is active.", + } + ) + session.discard_response_options() + continue + if native.committed_audio_payload is not None: + committed_payload = native.committed_audio_payload + operation_id = native.committed_audio_operation_id + if operation_id is None: + operation_id = uuid.uuid4().hex + native.committed_audio_operation_id = operation_id + await start_native_append( + committed_payload, + final=True, + precreate_response=True, + operation_id=operation_id, + retained_committed_payload=committed_payload, + ) + continue + await emit_event( + { + "type": "error", + "session_id": session.session_id, + "epoch": session.epoch, + "code": "response_create_without_input", + "error": "Native duplex response.create requires committed audio input.", + } + ) + session.discard_response_options() + continue + if self._uses_native_input_append(session) and not native_response_in_progress(): + commit_reservation = ( + native.audio_buffer.prepare_commit( + operation_id=uuid.uuid4().hex, + chunk_period_ms=session.capabilities.chunk_period_ms or 1000, + ) + if event_type in {"input.commit", "input_audio_buffer.commit"} + else None + ) + flushed_buffer_reserved_bytes = ( + native.audio_buffer.pending_byte_count if commit_reservation is None else 0 + ) + flushed = ( + commit_reservation.payload + if commit_reservation is not None + else native.audio_buffer.flush(chunk_period_ms=session.capabilities.chunk_period_ms or 1000) + ) + if native.committed_audio_payload is not None: + if flushed is not None: + flushed = self._merge_native_audio_payloads( + native.committed_audio_payload, + flushed, + ) + else: + flushed = native.committed_audio_payload + if commit_reservation is not None: + commit_reservation.commit() + if flushed is not None: + if self._should_force_listen_for_short_commit(session, event, flushed): + flushed = dict(flushed) + flushed["force_listen"] = True + native.input_since_commit = False + committed = self._commit_native_audio_input( + session, + realtime_item_id=event.get("realtime_item_id"), + transcript=event.get("transcript"), + ) + await emit_event( + self._native_audio_committed_payload( + session, + committed=committed, + realtime_item_id=event.get("realtime_item_id"), + transcript=event.get("transcript"), + ) + ) + if should_create_response: + native.retain_committed_audio( + flushed, + operation_id=( + commit_reservation.operation_id + if commit_reservation is not None + else uuid.uuid4().hex + ), + reserved_bytes=( + commit_reservation.byte_count + if commit_reservation is not None + else flushed_buffer_reserved_bytes + ), + ) + await start_native_append( + flushed, + final=True, + precreate_response=True, + operation_id=native.committed_audio_operation_id, + retained_committed_payload=flushed, + ) + else: + native.retain_committed_audio( + flushed, + operation_id=( + commit_reservation.operation_id + if commit_reservation is not None + else uuid.uuid4().hex + ), + reserved_bytes=( + commit_reservation.byte_count + if commit_reservation is not None + else flushed_buffer_reserved_bytes + ), + ) + native.deferred_response_create = False + continue + native_had_uncommitted_audio = self._uses_native_input_append(session) and ( + native.input_since_commit + or native.audio_buffer.has_pending() + or native.committed_audio_payload is not None + or realtime_validated_audio_commit + ) + committed = session.commit_user_input() + if self._uses_native_input_append(session) and event_type in { + "input_audio_buffer.commit", + "input.commit", + }: + native.input_since_commit = False + native.speech_since_commit = False + if committed is None and event_type != "response.create": + if self._uses_native_input_append(session): + committed = ( + self._commit_native_audio_input( + session, + realtime_item_id=event.get("realtime_item_id"), + transcript=event.get("transcript"), + ) + if native_had_uncommitted_audio + else None + ) + await emit_event( + self._native_audio_committed_payload( + session, + committed=committed, + realtime_item_id=event.get("realtime_item_id"), + transcript=event.get("transcript"), + ) + ) + else: + await emit_event( + { + "type": "input.committed", + "session_id": session.session_id, + "empty": True, + } + ) + continue + if committed is not None: + realtime_item_id = event.get("realtime_item_id") + if isinstance(realtime_item_id, str): + session.register_history_item(realtime_item_id, committed.message) + await emit_event( + self._input_committed_payload( + session, + committed, + realtime_item_id=realtime_item_id, + ) + ) + if not should_create_response: + continue + if actor.active_response_task is not None and not actor.active_response_task.done(): + await self._cancel_active_response( + session, + actor.active_response_task, + emit_event, + reason="new_response", + ) + actor.active_response_task = asyncio.create_task(self._run_response(session, emit_event)) + continue + + await emit_event( + { + "type": "error", + "error": f"Unknown duplex event: {event_type}", + "code": "unknown_event", + } + ) + + except WebSocketDisconnect: + logger.info("Duplex session disconnected") + transport_detached = True + except Exception as exc: + logger.exception("Duplex session failed: %s", exc) + with suppress(Exception): + await emit_event({"type": "error", "error": str(exc), "code": "internal_error"}) + finally: + if session is not None: + if reader_task is not None and not reader_task.done(): + reader_task.cancel() + with suppress(asyncio.CancelledError): + await reader_task + while pending_turn_reservations > 0: + session.release_pending_turn() + pending_turn_reservations -= 1 + resumable_detach = ( + transport_detached + and runtime_opened + and not runtime_closed + and session.state == DuplexSessionState.OPEN + and session.capabilities.supports_session_resume + and attachment_generation is not None + and resume_credential_delivered + ) + if resumable_detach: + + async def cancel_orphan_response_after_grace() -> None: + tasks = self._session_tasks.get(session.session_id) + current_session = self._registry.get(session.session_id) + if tasks is None or current_session is not session or session.state != DuplexSessionState.OPEN: + return + await tasks.cancel_append_tasks(response_bound_only=True) + await self._cancel_native_data_plane_stream(session) + await self._cancel_active_response( + session, + tasks.active_response_task, + emit_event, + reason="disconnect_grace_expired", + notify=False, + ) + tasks.active_response_task = None + + detached = await self._attachment_registry.detach( + session.session_id, + attachment_generation=attachment_generation, + on_grace_expired=cancel_orphan_response_after_grace, + ) + touch_session = getattr(self._chat_service.engine_client, "touch_duplex_session_async", None) + if detached and callable(touch_session): + with suppress(Exception): + await touch_session( + session.session_id, + fence=DuplexFence( + session.session_id, + epoch=session.epoch, + turn_id=session.turn_id, + incarnation=session.incarnation, + ), + activity=DuplexLeaseActivity.DETACH, + ) + else: + begin_close(actor.close_reason or "disconnect") + await actor.cancel_append_tasks() + await self._cancel_native_data_plane_stream(session) + await self._cancel_active_response( + session, + actor.active_response_task, + emit_event, + reason="disconnect", + notify=False, + ) + if runtime_opened and not runtime_closed and session.state != DuplexSessionState.CLOSED: + await self._close_runtime_session(session, reason="disconnect") + self._cleanup_duplex_session_state(session) + self._registry.close(session.session_id) + self._session_tasks.pop(session.session_id, None) + self._realtime_protocols.pop(session.session_id, None) + self._lease_generations.pop(session.session_id, None) + self._resync_required_sessions.discard(session.session_id) + with suppress(Exception): + await self._attachment_registry.close(session.session_id) + self._stop_lifecycle_listener_if_idle() + await actor.close_writer() + with suppress(Exception): + await asyncio.wait_for(actor.output_queue.join(), timeout=2.0) + if not writer_task.done(): + writer_task.cancel() + with suppress(asyncio.CancelledError): + await writer_task diff --git a/vllm_omni/experimental/fullduplex/openai/websocket.py b/vllm_omni/experimental/fullduplex/openai/websocket.py new file mode 100644 index 000000000..18cd11c8d --- /dev/null +++ b/vllm_omni/experimental/fullduplex/openai/websocket.py @@ -0,0 +1,265 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from __future__ import annotations + +import asyncio +from collections.abc import Callable +from dataclasses import dataclass, field +from typing import Any + +from fastapi import WebSocketDisconnect + +INPUT_EVENTS = frozenset( + { + "input.text.append", + "input_audio_buffer.append", + "input.commit", + "input_audio_buffer.commit", + "response.create", + } +) + +_INPUT_EVENT_ALIASES = { + "signal_turn": "turn.signal", + "close_session": "session.close", + "audio.playback_ack": "playback.ack", + "input_text.append": "input.text.append", + "push_text": "input.text.append", + "input.audio.append": "input_audio_buffer.append", + "push_chunk": "input_audio_buffer.append", +} +_WAV_AUDIO_ALIASES = frozenset({"input.audio.append", "push_chunk"}) + + +def normalize_duplex_input_event(event: dict[str, object]) -> dict[str, object]: + """Normalize compatibility aliases before an event enters the mailbox.""" + event_type = event.get("type") + if not isinstance(event_type, str): + return event + canonical_type = _INPUT_EVENT_ALIASES.get(event_type) + if canonical_type is None: + return event + normalized = dict(event) + normalized["type"] = canonical_type + if event_type in _WAV_AUDIO_ALIASES: + normalized.setdefault("format", "wav") + return normalized + + +MODEL_OUTPUT_EVENTS = frozenset( + { + "response.created", + "response.listen", + "response.speak", + "response.output_item.added", + "response.content_part.added", + "response.output_audio.delta", + "response.audio.delta", + "response.output_audio.done", + "response.audio.done", + "response.output_text.delta", + "response.output_text.done", + "response.text.delta", + "response.text.done", + "response.message", + "response.output_item.done", + "response.content_part.done", + "response.done", + "runtime.control", + } +) +DOMAIN_TERMINAL_EVENTS = frozenset( + { + "response.done", + "response.listen", + "audio.cancelled", + "input.cancelled", + "session.closed", + } +) + + +def is_input_event(event_type: object) -> bool: + return isinstance(event_type, str) and event_type in INPUT_EVENTS + + +@dataclass(frozen=True, slots=True) +class DuplexAppendTaskMeta: + epoch: int + mode: str + final: bool + response_bound: bool + + +@dataclass +class DuplexSessionTasks: + """Connection-independent task handles for one resumable session.""" + + native_append_tasks: dict[asyncio.Task[bool], DuplexAppendTaskMeta] = field(default_factory=dict) + native_append_tail: asyncio.Task[bool] | None = None + active_response_task: asyncio.Task[None] | None = None + + def track_append_task( + self, + task: asyncio.Task[bool], + *, + epoch: int, + mode: str, + final: bool, + response_bound: bool, + ) -> None: + self.native_append_tasks[task] = DuplexAppendTaskMeta(epoch, mode, final, response_bound) + task.add_done_callback(self.native_append_tasks.pop) + + def has_response_bound_append_tasks(self) -> bool: + return any(meta.response_bound for meta in self.native_append_tasks.values()) + + async def cancel_append_tasks(self, timeout_s: float = 0.25, *, response_bound_only: bool = False) -> bool: + tasks = [ + task for task, meta in self.native_append_tasks.items() if not response_bound_only or meta.response_bound + ] + if not tasks: + return False + for task in tasks: + task.cancel() + try: + await asyncio.wait_for(asyncio.gather(*tasks, return_exceptions=True), timeout=timeout_s) + except TimeoutError: + pass + return True + + +@dataclass +class DuplexWebSocketActor: + """Own ordered WebSocket I/O queues, but no domain identity.""" + + websocket: Any + current_epoch: Callable[[], int | None] | None = None + session_closed: Callable[[], bool] | None = None + output_queue: asyncio.Queue[dict[str, object] | None] = field(default_factory=asyncio.Queue) + mailbox: asyncio.Queue[dict[str, object]] = field(default_factory=asyncio.Queue) + outbound_protocol: Any | None = None + tasks: DuplexSessionTasks = field(default_factory=DuplexSessionTasks) + closing: bool = False + close_reason: str | None = None + stale_output_dropped: int = 0 + _queued_input_events: int = 0 + + @property + def native_append_tasks(self) -> dict[asyncio.Task[bool], DuplexAppendTaskMeta]: + return self.tasks.native_append_tasks + + @property + def active_response_task(self) -> asyncio.Task[None] | None: + return self.tasks.active_response_task + + @active_response_task.setter + def active_response_task(self, task: asyncio.Task[None] | None) -> None: + self.tasks.active_response_task = task + + @property + def native_append_tail(self) -> asyncio.Task[bool] | None: + return self.tasks.native_append_tail + + @native_append_tail.setter + def native_append_tail(self, task: asyncio.Task[bool] | None) -> None: + self.tasks.native_append_tail = task + + async def enqueue_event(self, event: dict[str, object]) -> None: + if is_input_event(event.get("type")): + self._queued_input_events += 1 + await self.mailbox.put(event) + + async def next_event(self) -> dict[str, object]: + event = await self.mailbox.get() + if is_input_event(event.get("type")): + self._queued_input_events = max(0, self._queued_input_events - 1) + self.mailbox.task_done() + return event + + def has_queued_input_events(self) -> bool: + return self._queued_input_events > 0 + + async def send_json(self, payload: dict[str, object]) -> None: + await self.output_queue.put(payload) + + async def close_writer(self) -> None: + await self.output_queue.put(None) + + async def writer_loop(self) -> None: + while True: + payload = await self.output_queue.get() + try: + if payload is None: + return + raw_realtime = payload.pop("_realtime_raw", False) is True + if not raw_realtime and self._is_stale_model_output(payload): + self.stale_output_dropped += 1 + continue + try: + if raw_realtime: + await self.websocket.send_json(payload) + elif self.outbound_protocol is not None: + for projected in self.outbound_protocol.encode_outbound_event(payload): + await self.websocket.send_json(projected) + else: + await self.websocket.send_json(payload) + except (WebSocketDisconnect, RuntimeError): + return + finally: + self.output_queue.task_done() + + def _is_stale_model_output(self, payload: dict[str, object]) -> bool: + event_type = payload.get("type") + if event_type in DOMAIN_TERMINAL_EVENTS: + return False + if event_type not in MODEL_OUTPUT_EVENTS: + return False + if self.closing and event_type not in {"response.listen", "runtime.control"}: + return True + expected_epoch = self.current_epoch() if self.current_epoch is not None else None + if ( + self.session_closed is not None + and self.session_closed() + and event_type + not in { + "response.listen", + "runtime.control", + } + ): + return True + epoch = payload.get("epoch") + return isinstance(epoch, int) and isinstance(expected_epoch, int) and epoch != expected_epoch + + def track_append_task( + self, + task: asyncio.Task[bool], + *, + epoch: int, + mode: str, + final: bool, + response_bound: bool, + ) -> None: + self.tasks.track_append_task( + task, + epoch=epoch, + mode=mode, + final=final, + response_bound=response_bound, + ) + + def has_response_bound_append_tasks(self) -> bool: + return self.tasks.has_response_bound_append_tasks() + + async def cancel_append_tasks(self, timeout_s: float = 0.25, *, response_bound_only: bool = False) -> bool: + return await self.tasks.cancel_append_tasks(timeout_s, response_bound_only=response_bound_only) + + +__all__ = [ + "DOMAIN_TERMINAL_EVENTS", + "DuplexAppendTaskMeta", + "DuplexSessionTasks", + "DuplexWebSocketActor", + "is_input_event", +] diff --git a/vllm_omni/experimental/fullduplex/output.py b/vllm_omni/experimental/fullduplex/output.py new file mode 100644 index 000000000..bfe2243f9 --- /dev/null +++ b/vllm_omni/experimental/fullduplex/output.py @@ -0,0 +1,36 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from __future__ import annotations + +from typing import Any + +from vllm_omni.outputs import OmniRequestOutput + +DUPLEX_OUTPUT_DECISION_KEY = "_vllm_omni.experimental.fullduplex.duplex_output_decision" + + +def attach_duplex_output_decision( + output: OmniRequestOutput, + decision: object, +) -> OmniRequestOutput: + custom_output = dict(output._custom_output) + custom_output[DUPLEX_OUTPUT_DECISION_KEY] = decision + output._custom_output = custom_output + return output + + +def get_duplex_output_decision(output: object) -> Any | None: + if not isinstance(output, OmniRequestOutput): + return None + custom_output = output._custom_output + if not isinstance(custom_output, dict): + return None + return custom_output.get(DUPLEX_OUTPUT_DECISION_KEY) + + +__all__ = [ + "DUPLEX_OUTPUT_DECISION_KEY", + "attach_duplex_output_decision", + "get_duplex_output_decision", +] diff --git a/vllm_omni/experimental/fullduplex/request_client.py b/vllm_omni/experimental/fullduplex/request_client.py new file mode 100644 index 000000000..d39a037a2 --- /dev/null +++ b/vllm_omni/experimental/fullduplex/request_client.py @@ -0,0 +1,383 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from __future__ import annotations + +import asyncio +import time +from collections.abc import Callable, MutableMapping +from typing import Protocol + +from vllm_omni.engine.messages import ErrorMessage, OutputMessage +from vllm_omni.entrypoints.client_request_state import ClientRequestState +from vllm_omni.experimental.fullduplex.engine.contracts import ( + duplex_data_plane_request_info, + duplex_resource_request_id, +) +from vllm_omni.experimental.fullduplex.engine.lease import DuplexLeaseActivity +from vllm_omni.experimental.fullduplex.engine.messages import DuplexFence +from vllm_omni.experimental.fullduplex.output import get_duplex_output_decision +from vllm_omni.metrics.stats import OrchestratorAggregator as OrchestratorMetrics +from vllm_omni.outputs import OmniRequestOutput + + +class DuplexEnginePort(Protocol): + """Engine control methods required by the experimental request client. + + The protocol keeps request preregistration and output collection testable + without coupling this component to ``AsyncOmniEngine``. + """ + + async def open_duplex_session_async(self, session_id: str, **kwargs) -> dict[str, object]: ... + + async def append_duplex_input_async(self, session_id: str, **kwargs) -> dict[str, object]: ... + + async def signal_duplex_turn_async(self, session_id: str, **kwargs) -> dict[str, object]: ... + + async def close_duplex_session_async(self, session_id: str, **kwargs) -> dict[str, object]: ... + + async def touch_duplex_session_async(self, session_id: str, **kwargs) -> dict[str, object]: ... + + async def resume_duplex_session_async(self, session_id: str, **kwargs) -> dict[str, object]: ... + + +class DuplexRequestOutputPort: + """Narrow output-processing surface owned by the AsyncOmni entrypoint.""" + + def __init__( + self, + *, + request_states: MutableMapping[str, ClientRequestState], + num_stages: int, + log_stats: bool, + start_output_handler: Callable[[], object], + process_single_result: Callable[..., OmniRequestOutput | None], + ) -> None: + self.request_states = request_states + self.num_stages = num_stages + self.log_stats = log_stats + self._start_output_handler = start_output_handler + self._process_single_result = process_single_result + + def start_output_handler(self) -> None: + self._start_output_handler() + + def process_single_result(self, *args, **kwargs) -> OmniRequestOutput | None: + return self._process_single_result(*args, **kwargs) + + +class DuplexRequestClient: + """Owns duplex request identity, preregistration and output collection.""" + + def __init__(self, engine: DuplexEnginePort, output_port: DuplexRequestOutputPort) -> None: + self.engine = engine + self.output_port = output_port + + @staticmethod + def request_info(result: dict[str, object]) -> tuple[str | None, int | None]: + return duplex_data_plane_request_info(result) + + async def open( + self, + session_id: str, + *, + session_mode: str, + capabilities: dict[str, object] | None, + session_config: dict[str, object] | None, + runtime_config: dict[str, object] | None, + fence: DuplexFence, + timeout: float | None, + ) -> dict[str, object]: + # Lifecycle expiry messages share the engine output channel. Start its + # sole dispatcher before open so an idle session can be reaped even if + # it never submits an append request. + self.output_port.start_output_handler() + kwargs: dict[str, object] = { + "session_mode": session_mode, + "capabilities": capabilities, + "session_config": session_config, + "fence": fence, + "timeout": timeout, + } + if runtime_config is not None: + kwargs["runtime_config"] = runtime_config + return await self.engine.open_duplex_session_async(session_id, **kwargs) + + async def append( + self, + session_id: str, + *, + mode: str, + payload: object, + operation_id: str | None, + final: bool, + expected_epoch: int | None, + fence: DuplexFence, + timeout: float | None, + collect_outputs: bool, + ) -> dict[str, object]: + kwargs: dict[str, object] = { + "mode": mode, + "payload": payload, + "final": final, + "expected_epoch": expected_epoch, + "timeout": timeout, + "fence": fence, + } + if operation_id is not None: + kwargs["operation_id"] = operation_id + + expected_request_id = duplex_resource_request_id(fence, "stage0") + request_state = self.output_port.request_states.get(expected_request_id) + created_request_state = request_state is None + if request_state is None: + request_state = ClientRequestState(expected_request_id) + self.output_port.request_states[expected_request_id] = request_state + if request_state.metrics is None: + wall_start_ts = time.time() + request_state.metrics = OrchestratorMetrics( + self.output_port.num_stages, + self.output_port.log_stats, + wall_start_ts, + max(0, self.output_port.num_stages - 1), + ) + request_state.request_arrival_ts = wall_start_ts + self.output_port.start_output_handler() + try: + result = await self.engine.append_duplex_input_async(session_id, **kwargs) + except BaseException: + if created_request_state and request_state.queue.empty(): + self.output_port.request_states.pop(expected_request_id, None) + raise + + request_id, response_stage_id = duplex_data_plane_request_info(result) + if request_id is None: + if created_request_state and request_state.queue.empty(): + self.output_port.request_states.pop(expected_request_id, None) + return result + if request_id != expected_request_id: + if created_request_state and request_state.queue.empty(): + self.output_port.request_states.pop(expected_request_id, None) + raise RuntimeError( + f"duplex data-plane request id mismatch: expected {expected_request_id!r}, got {request_id!r}" + ) + if not collect_outputs: + return result + outputs = await self.collect_outputs( + request_id, + request_state, + response_stage_id=response_stage_id, + timeout=timeout, + ) + if outputs: + result = dict(result) + result["data_plane_outputs"] = outputs + return result + + async def collect_registered_outputs( + self, + request_id: str, + *, + response_stage_id: int | None, + timeout: float | None, + ) -> list[OmniRequestOutput]: + self.output_port.start_output_handler() + request_state = self.output_port.request_states.get(request_id) + if request_state is None: + return [] + return await self.collect_outputs( + request_id, + request_state, + response_stage_id=response_stage_id, + timeout=timeout, + ) + + async def signal( + self, + session_id: str, + *, + event: str, + fence: DuplexFence, + next_fence: DuplexFence | None, + session_config: dict[str, object] | None, + runtime_config: dict[str, object] | None, + timeout: float | None, + ) -> dict[str, object]: + kwargs: dict[str, object] = {"event": event, "fence": fence, "timeout": timeout} + if next_fence is not None: + kwargs["next_fence"] = next_fence + if session_config is not None: + kwargs["session_config"] = session_config + if runtime_config is not None: + kwargs["runtime_config"] = runtime_config + result = await self.engine.signal_duplex_turn_async(session_id, **kwargs) + if event in {"barge_in", "input.cancel", "response.cancel"}: + self.output_port.request_states.pop(duplex_resource_request_id(fence, "stage0"), None) + return result + + async def close( + self, + session_id: str, + *, + reason: str, + fence: DuplexFence, + timeout: float | None, + ) -> dict[str, object]: + result = await self.engine.close_duplex_session_async( + session_id, + reason=reason, + fence=fence, + timeout=timeout, + ) + self.output_port.request_states.pop(duplex_resource_request_id(fence, "stage0"), None) + return result + + async def touch( + self, + session_id: str, + *, + fence: DuplexFence, + activity: DuplexLeaseActivity, + timeout: float | None, + ) -> dict[str, object]: + return await self.engine.touch_duplex_session_async( + session_id, + fence=fence, + activity=activity, + timeout=timeout, + ) + + async def resume( + self, + session_id: str, + *, + fence: DuplexFence, + expected_lease_generation: int, + timeout: float | None, + ) -> dict[str, object]: + return await self.engine.resume_duplex_session_async( + session_id, + fence=fence, + expected_lease_generation=expected_lease_generation, + timeout=timeout, + ) + + async def collect_outputs( + self, + request_id: str, + request_state: ClientRequestState, + *, + response_stage_id: int | None, + timeout: float | None, + ) -> list[OmniRequestOutput]: + deadline = None if timeout is None else time.monotonic() + timeout + wall_start_ts = request_state.request_arrival_ts or time.time() + final_stage_id = response_stage_id if response_stage_id is not None else max(0, self.output_port.num_stages - 1) + num_stages = max(self.output_port.num_stages, final_stage_id + 1) + metrics = getattr(request_state, "metrics", None) or OrchestratorMetrics( + num_stages, + self.output_port.log_stats, + wall_start_ts, + final_stage_id, + ) + stage_event_cursor = metrics.stage_event_cursor(request_id) + request_start_ts = {request_id: wall_start_ts} + outputs: list[OmniRequestOutput] = [] + while True: + remaining = None if deadline is None else max(0.0, deadline - time.monotonic()) + if remaining == 0.0: + break + try: + message = await asyncio.wait_for(request_state.queue.get(), timeout=remaining) + except asyncio.TimeoutError: + break + if isinstance(message, ErrorMessage): + raise RuntimeError(message.error) + if not isinstance(message, OutputMessage): + continue + engine_outputs = message.engine_outputs + is_response_stage = response_stage_id is None or message.stage_id >= response_stage_id + is_direct_response = self.is_direct_response(engine_outputs) + output_to_collect = None + if isinstance(engine_outputs, OmniRequestOutput): + output_to_collect = engine_outputs + if message.metrics is not None: + existing_metrics = dict(output_to_collect.metrics) + metrics.process_stage_metrics( + result={"metrics": message.metrics}, + stage_type="", + stage_id=message.stage_id, + req_id=request_id, + engine_outputs=engine_outputs, + finished=message.finished, + final_output_type=engine_outputs.final_output_type, + output_to_yield=output_to_collect, + event_cursor=stage_event_cursor, + ) + generated_metrics = dict(output_to_collect.metrics) + existing_stage_metrics = existing_metrics.get("stage_metrics") + generated_stage_metrics = generated_metrics.get("stage_metrics") + if isinstance(existing_stage_metrics, dict) and isinstance(generated_stage_metrics, dict): + generated_metrics["stage_metrics"] = { + **existing_stage_metrics, + **generated_stage_metrics, + } + output_to_collect.metrics = { + **existing_metrics, + **generated_metrics, + } + elif is_response_stage: + output_to_collect = self.output_port.process_single_result( + message, + message.stage_id, + metrics, + request_start_ts, + wall_start_ts, + final_stage_id, + stage_event_cursor, + ) + if output_to_collect is not None and (is_response_stage or is_direct_response): + if message.finished: + output_to_collect.finished = True + outputs.append(output_to_collect) + if message.finished or outputs: + break + return outputs + + @classmethod + def is_direct_response(cls, output: object) -> bool: + multimodal_output = cls.multimodal_output(output) + if not multimodal_output: + return False + return multimodal_output.get("duplex_direct_response") is True or multimodal_output.get( + "duplex_native_decision" + ) in {"listen", "speak"} + + @classmethod + def multimodal_output(cls, output: object) -> dict[str, object]: + if isinstance(output, OmniRequestOutput): + decision = get_duplex_output_decision(output) + if decision is not None: + return dict(decision.metadata) + private_output = getattr(output, "_multimodal_output", None) + multimodal_output = output.multimodal_output + if isinstance(multimodal_output, dict) and multimodal_output: + return multimodal_output + inner_output = getattr(output, "request_output", None) + if inner_output is not None and inner_output is not output: + inner_multimodal_output = cls.multimodal_output(inner_output) + if inner_multimodal_output: + return inner_multimodal_output + if isinstance(private_output, dict): + return private_output + return multimodal_output if isinstance(multimodal_output, dict) else {} + multimodal_output = getattr(output, "multimodal_output", None) + if isinstance(multimodal_output, dict): + return multimodal_output + outputs = getattr(output, "outputs", None) + completion = outputs[0] if isinstance(outputs, list) and outputs else None + multimodal_output = getattr(completion, "multimodal_output", None) if completion is not None else None + return multimodal_output if isinstance(multimodal_output, dict) else {} + + +__all__ = ["DuplexEnginePort", "DuplexRequestClient", "DuplexRequestOutputPort"] diff --git a/vllm_omni/inputs/data.py b/vllm_omni/inputs/data.py index f576b9724..1eb4dc202 100644 --- a/vllm_omni/inputs/data.py +++ b/vllm_omni/inputs/data.py @@ -21,15 +21,16 @@ from vllm.inputs.engine import TokensInput class OmniTextPrompt(TextPrompt): - """Text prompt with optional embeddings and additional information. + """Text prompt with optional embeddings and stage payloads. Extends TextPrompt to support prompt embeddings and additional information payloads for direct transfer between pipeline stages. Attributes: prompt_embeds: Optional tensor containing prompt embeddings - additional_information: Optional dictionary containing additional - information (tensors or lists) to pass along with the prompt + additional_information: Legacy optional dictionary containing + additional information to pass along with the prompt. + model_intermediate_buffer: Optional runner-owned stage payload. """ negative_prompt: NotRequired[str] @@ -39,18 +40,20 @@ class OmniTextPrompt(TextPrompt): prompt_embeds: NotRequired[torch.Tensor] negative_prompt_embeds: NotRequired[torch.Tensor] additional_information: NotRequired[dict[str, Any]] + model_intermediate_buffer: NotRequired[dict[str, Any]] class OmniTokensPrompt(TokensPrompt): - """Tokens prompt with optional embeddings and additional information. + """Tokens prompt with optional embeddings and stage payloads. Extends TokensPrompt to support prompt embeddings and additional information payloads for direct transfer between pipeline stages. Attributes: prompt_embeds: Optional tensor containing prompt embeddings - additional_information: Optional dictionary containing additional - information (tensors or lists) to pass along with the prompt + additional_information: Legacy optional dictionary containing + additional information to pass along with the prompt. + model_intermediate_buffer: Optional runner-owned stage payload. """ negative_prompt: NotRequired[str] @@ -61,10 +64,11 @@ class OmniTokensPrompt(TokensPrompt): # New: optional additional information dictionary # Values may be torch.Tensor or list additional_information: NotRequired[dict[str, Any]] + model_intermediate_buffer: NotRequired[dict[str, Any]] class OmniTokenInputs(TokensInput): - """Token inputs with optional embeddings and additional information. + """Token inputs with optional embeddings and stage payloads. Extends TokensInput to support prompt embeddings and additional information payloads for direct transfer between pipeline stages. @@ -72,8 +76,9 @@ class OmniTokenInputs(TokensInput): Attributes: prompt_embeds: Optional tensor containing prompt embeddings aligned with token IDs - additional_information: Optional dictionary containing additional - information (tensors or lists) to pass along with the inputs + additional_information: Legacy optional dictionary containing + additional information to pass along with the inputs. + model_intermediate_buffer: Optional runner-owned stage payload. """ # New: optional prompt embeddings aligned with token ids @@ -84,18 +89,20 @@ class OmniTokenInputs(TokensInput): # New: optional additional information dictionary # Values may be torch.Tensor or list additional_information: NotRequired[dict[str, Any]] + model_intermediate_buffer: NotRequired[dict[str, Any]] class OmniEmbedsPrompt(EmbedsPrompt): - """Embeddings prompt with optional additional information. + """Embeddings prompt with optional stage payloads. Extends EmbedsPrompt to support additional information payloads for direct transfer between pipeline stages. Attributes: prompt_embeds: Optional tensor containing prompt embeddings - additional_information: Optional dictionary containing additional - information (tensors or lists) to pass along with the prompt + additional_information: Legacy optional dictionary containing + additional information to pass along with the prompt. + model_intermediate_buffer: Optional runner-owned stage payload. """ # New: optional prompt embeddings aligned with token ids @@ -105,6 +112,7 @@ class OmniEmbedsPrompt(EmbedsPrompt): # New: optional additional information dictionary # Values may be torch.Tensor or list additional_information: NotRequired[dict[str, Any]] + model_intermediate_buffer: NotRequired[dict[str, Any]] class OmniCustomPrompt(TypedDict, total=False): @@ -143,6 +151,7 @@ def token_inputs_omni( cache_salt: str | None = None, prompt_embeds: torch.Tensor | None = None, additional_information: dict[str, Any] | None = None, + model_intermediate_buffer: dict[str, Any] | None = None, ) -> OmniTokenInputs: """Construct token inputs with optional embeddings and metadata. @@ -154,8 +163,9 @@ def token_inputs_omni( prompt: Optional prompt string cache_salt: Optional cache salt for prefix caching prompt_embeds: Optional tensor containing prompt embeddings - additional_information: Optional dictionary containing additional - information (tensors or lists) + additional_information: Legacy optional dictionary containing + additional information (tensors or lists). + model_intermediate_buffer: Optional runner-owned stage payload. Returns: OmniTokenInputs instance with the provided data @@ -170,6 +180,8 @@ def token_inputs_omni( inputs["prompt_embeds"] = prompt_embeds if additional_information is not None: inputs["additional_information"] = additional_information + if model_intermediate_buffer is not None: + inputs["model_intermediate_buffer"] = model_intermediate_buffer return inputs diff --git a/vllm_omni/inputs/preprocess.py b/vllm_omni/inputs/preprocess.py index 7282d7a52..3da1ef897 100644 --- a/vllm_omni/inputs/preprocess.py +++ b/vllm_omni/inputs/preprocess.py @@ -60,6 +60,9 @@ class OmniInputPreprocessor(InputPreprocessor): additional_information = parsed_content.get("additional_information") if additional_information is not None: inputs["additional_information"] = additional_information + model_intermediate_buffer = parsed_content.get("model_intermediate_buffer") + if model_intermediate_buffer is not None: + inputs["model_intermediate_buffer"] = model_intermediate_buffer elif "mm_processor_kwargs" in parsed_content: # Presence — not truthiness. An explicitly-set empty dict still # signals "route through the multimodal processor" (needed for @@ -81,6 +84,7 @@ class OmniInputPreprocessor(InputPreprocessor): prompt_token_ids, prompt_embeds=parsed_content.get("prompt_embeds"), additional_information=parsed_content.get("additional_information"), + model_intermediate_buffer=parsed_content.get("model_intermediate_buffer"), ) prompt_embeds = parsed_content.get("prompt_embeds") if prompt_embeds is not None: @@ -88,6 +92,9 @@ class OmniInputPreprocessor(InputPreprocessor): additional_information = parsed_content.get("additional_information") if additional_information is not None: inputs["additional_information"] = additional_information + model_intermediate_buffer = parsed_content.get("model_intermediate_buffer") + if model_intermediate_buffer is not None: + inputs["model_intermediate_buffer"] = model_intermediate_buffer if cache_salt := parsed_content.get("cache_salt"): inputs["cache_salt"] = cache_salt @@ -101,6 +108,7 @@ class OmniInputPreprocessor(InputPreprocessor): prompt_token_ids = self._truncate_inputs(parsed_content["prompt_token_ids"], tokenization_kwargs) prompt_embeds = parsed_content.get("prompt_embeds") additional_information = parsed_content.get("additional_information") + model_intermediate_buffer = parsed_content.get("model_intermediate_buffer") multi_modal_data = parsed_content.get("multi_modal_data") @@ -119,11 +127,14 @@ class OmniInputPreprocessor(InputPreprocessor): prompt_token_ids=prompt_token_ids, prompt_embeds=prompt_embeds, additional_information=additional_information, + model_intermediate_buffer=model_intermediate_buffer, ) if prompt_embeds is not None: inputs["prompt_embeds"] = prompt_embeds if additional_information is not None: inputs["additional_information"] = additional_information + if model_intermediate_buffer is not None: + inputs["model_intermediate_buffer"] = model_intermediate_buffer if prompt_text := parsed_content.get("prompt"): inputs["prompt"] = prompt_text if cache_salt := parsed_content.get("cache_salt"): @@ -147,6 +158,9 @@ class OmniInputPreprocessor(InputPreprocessor): additional_information = parsed_content.get("additional_information") if additional_information is not None: inputs["additional_information"] = additional_information # type: ignore[typeddict-unknown-key] + model_intermediate_buffer = parsed_content.get("model_intermediate_buffer") + if model_intermediate_buffer is not None: + inputs["model_intermediate_buffer"] = model_intermediate_buffer # type: ignore[typeddict-unknown-key] return inputs diff --git a/vllm_omni/metrics/stats.py b/vllm_omni/metrics/stats.py index a282d7d50..3d08adae0 100644 --- a/vllm_omni/metrics/stats.py +++ b/vllm_omni/metrics/stats.py @@ -381,6 +381,7 @@ class OrchestratorAggregator: finished: bool, final_output_type: str | None, output_to_yield: Any | None, + event_cursor: int = 0, ) -> None: """Process and record stage metrics. @@ -409,7 +410,7 @@ class OrchestratorAggregator: return rid_key = str(req_id) - stage_snapshot = self._build_stage_metrics_snapshot(rid_key) + stage_snapshot = self._build_stage_metrics_snapshot(rid_key, event_cursor=event_cursor) # 3. Not finished yet — expose incremental per-stage snapshot for streaming clients. if not finished: @@ -423,7 +424,11 @@ class OrchestratorAggregator: # token fields only for text stages (OpenAI-style completion token accounting). output_to_yield.metrics = {"stage_metrics": stage_snapshot} stage_event = next( - (evt for evt in reversed(self.stage_events.get(rid_key, [])) if evt.stage_id == stage_id), + ( + evt + for evt in reversed(self.stage_events.get(rid_key, [])[event_cursor:]) + if evt.stage_id == stage_id + ), None, ) if stage_event is not None and stage_event.final_output_type == "text": @@ -541,10 +546,18 @@ class OrchestratorAggregator: current[defs.TIME_PER_OUTPUT_UNIT_MS] = remaining_ms / float(output_count - 1) if output_count > 1 else 0.0 return current - def _build_stage_metrics_snapshot(self, req_id: str) -> dict[str, dict[str, Any]]: + def stage_event_cursor(self, req_id: str) -> int: + return len(self.stage_events.get(str(req_id), ())) + + def _build_stage_metrics_snapshot( + self, + req_id: str, + *, + event_cursor: int = 0, + ) -> dict[str, dict[str, Any]]: """Aggregate per-stage metrics for ``req_id`` (string key), for streaming/benchmark clients.""" snapshot: dict[str, dict[str, Any]] = {} - for evt in self.stage_events.get(req_id, []): + for evt in self.stage_events.get(req_id, [])[event_cursor:]: sid = int(evt.stage_id) if evt.stage_id is not None else -1 if sid < 0: continue diff --git a/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni.py b/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni.py index d9b48807a..ba51c9cfc 100644 --- a/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni.py +++ b/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni.py @@ -18,10 +18,13 @@ # limitations under the License. from collections.abc import Iterable +from contextlib import suppress from functools import cached_property +from typing import Any import torch import torch.nn as nn +import torch.nn.functional as F from vllm.config import VllmConfig from vllm.logger import init_logger from vllm.model_executor.models.interfaces import SupportsMRoPE, SupportsMultiModal, SupportsPP @@ -31,6 +34,9 @@ from vllm.sequence import IntermediateTensors from vllm.v1.outputs import SamplerOutput from vllm.v1.sample.metadata import SamplingMetadata +from vllm_omni.experimental.fullduplex.engine.intermediate import get_stream_request_key +from vllm_omni.experimental.fullduplex.minicpmo45.policy import MiniCPMO45DuplexPolicy +from vllm_omni.experimental.fullduplex.model_executor import DuplexSamplingRow from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni_llm import ( MiniCPMO45OmniLLMDummyInputsBuilder, MiniCPMO45OmniLLMMultiModalProcessor, @@ -52,11 +58,9 @@ logger = init_logger(__name__) class MiniCPMO45OmniForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP, SupportsMRoPE): """MiniCPM-o 4.5 Omni model for conditional generation. - Two-stage pipeline: - - thinker (model_stage="llm"): image / video / audio encoders + 3D - resampler + the omni LLM that emits text + hidden states. - - talker (model_stage="tts"): MiniCPMTTS + the in-process Token2Wav - vocoder that emits the final audio waveform directly. + This model has two pipeline stages: + - llm: multimodal thinker and text generation + - tts: talker generation followed by its built-in Token2Wav vocoder """ @classmethod @@ -80,6 +84,11 @@ class MiniCPMO45OmniForConditionalGeneration(nn.Module, SupportsMultiModal, Supp # Store configs self.config = config self.multimodal_config = multimodal_config + from vllm_omni.experimental.fullduplex.minicpmo45.compat import ( + patch_minicpmo_remote_config, + ) + + patch_minicpmo_remote_config(config) self.model_stage = vllm_config.model_config.model_stage @@ -89,8 +98,7 @@ class MiniCPMO45OmniForConditionalGeneration(nn.Module, SupportsMultiModal, Supp vllm_config=vllm_config, prefix=maybe_prefix(prefix, "thinker"), hf_config=config, - # Registry key — must match the entries declared in - # vllm_omni/model_executor/models/registry.py::_OMNI_MODELS. + # Use registry architecture key architectures=["MiniCPMO45OmniLLMForConditionalGeneration"], ) self.model = self.thinker @@ -98,21 +106,18 @@ class MiniCPMO45OmniForConditionalGeneration(nn.Module, SupportsMultiModal, Supp elif self.model_stage == "tts": self.thinker = None - # Initialize talker model — runs MiniCPMTTS + the in-process - # Token2Wav vocoder and emits the final audio waveform directly. + # Initialize talker model (LLM generation) self.talker = init_vllm_registered_model( vllm_config=vllm_config, prefix=maybe_prefix(prefix, "talker"), hf_config=config, - # Registry key — must match the entries declared in - # vllm_omni/model_executor/models/registry.py::_OMNI_MODELS. + # Use registry architecture key architectures=["MiniCPMO45OmniTTSForConditionalGeneration"], ) # Initialize multimodal components if needed if hasattr(self.talker, "init_multi_modal"): self.talker.init_multi_modal(config) self.model = self.talker - else: raise ValueError(f"Invalid model stage: {self.model_stage}. Must be one of: 'llm', 'tts'") @@ -124,6 +129,8 @@ class MiniCPMO45OmniForConditionalGeneration(nn.Module, SupportsMultiModal, Supp ) self._language_model_names = ["model"] + self.prefer_model_sampler = self.model_stage in {"llm", "tts"} + self.has_preprocess = self.model_stage == "llm" @cached_property def sampler(self): @@ -133,6 +140,90 @@ class MiniCPMO45OmniForConditionalGeneration(nn.Module, SupportsMultiModal, Supp return Sampler() + def prepare_duplex_sampling( + self, + logits: torch.Tensor, + sampling_metadata: SamplingMetadata, + rows: tuple[DuplexSamplingRow, ...], + ) -> None: + """Apply MiniCPM duplex policy before the standard model sampler.""" + del sampling_metadata + self._minicpmo45_active_duplex_rows = [row.row_idx for row in rows] + self._minicpmo45_duplex_row_sessions = { + row.row_idx: (row.session_id, row.incarnation) for row in rows if row.session_id is not None + } + request_sessions = getattr(self, "_minicpmo45_duplex_request_sessions", None) + if not isinstance(request_sessions, dict): + request_sessions = {} + self._minicpmo45_duplex_request_sessions = request_sessions + request_sessions.update( + {row.request_id: (row.session_id, row.incarnation) for row in rows if row.session_id is not None} + ) + self._minicpmo45_duplex_row_payloads = {row.row_idx: row.payload for row in rows if row.payload is not None} + self._minicpmo45_duplex_row_max_tokens = { + row.row_idx: row.max_tokens for row in rows if row.max_tokens is not None + } + if self.model_stage != "llm" or not rows or logits.ndim != 2: + return + + token_ids = self._minicpmo45_native_duplex_token_ids() + listen_id = int(token_ids.get("listen_token_id", -1)) + tts_bos_id = int(token_ids.get("tts_bos_token_id", -1)) + turn_eos_id = int(token_ids.get("turn_eos_token_id", -1)) + if listen_id < 0 or listen_id >= logits.shape[-1]: + return + + force_listen_segments = getattr( + self, + "_minicpmo45_force_listen_applied_segments", + None, + ) + if not isinstance(force_listen_segments, set): + force_listen_segments = set() + self._minicpmo45_force_listen_applied_segments = force_listen_segments + helper = getattr(self, "_minicpmo45_duplex_data_plane_helper", None) + helper_sessions = getattr(helper, "sessions", None) if helper is not None else None + for row in rows: + row_idx = row.row_idx + if row_idx < 0 or row_idx >= logits.shape[0]: + continue + payload = row.payload + if not isinstance(payload, dict): + continue + force_listen = payload.get("force_listen") is True + is_speech = payload.get("is_speech") + redirect_listen = False + segment_key = (row.request_id, row.seq if row.seq is not None else -1) + session_key = (row.session_id, row.incarnation) if row.session_id is not None else None + if turn_eos_id >= 0 and session_key is not None: + state = helper_sessions.get(session_key) if isinstance(helper_sessions, dict) else None + pending_speech_context = ( + bool(getattr(state, "pending_speech_context", False)) if state is not None else False + ) + if is_speech is True: + if state is not None: + if not getattr(state, "current_turn_ended", True): + redirect_listen = True + with suppress(Exception): + state.last_terminator_token = None + else: + turn_ended = bool(getattr(state, "current_turn_ended", True)) if state is not None else False + if turn_ended and not pending_speech_context: + force_listen = True + + if not force_listen and not redirect_listen: + continue + if force_listen and segment_key in force_listen_segments: + continue + if force_listen: + logits[row_idx, :] = float("-inf") + logits[row_idx, listen_id] = 0.0 + force_listen_segments.add(segment_key) + elif 0 <= tts_bos_id < logits.shape[-1]: + if logits[row_idx, listen_id] > logits[row_idx, tts_bos_id]: + logits[row_idx, tts_bos_id] = logits[row_idx, listen_id] + logits[row_idx, listen_id] = float("-inf") + # -------------------- Device utilities -------------------- @staticmethod def _module_device(module: nn.Module) -> torch.device: @@ -150,7 +241,7 @@ class MiniCPMO45OmniForConditionalGeneration(nn.Module, SupportsMultiModal, Supp thinker_device: str | torch.device | None = None, talker_device: str | torch.device | None = None, ) -> None: - """Optionally move thinker / talker to different devices. + """Optionally move the thinker and talker to different devices. Example: model.move_submodules_to_devices( @@ -168,7 +259,22 @@ class MiniCPMO45OmniForConditionalGeneration(nn.Module, SupportsMultiModal, Supp input_ids: torch.Tensor, multimodal_embeddings=None, ) -> torch.Tensor: - return self.model.get_input_embeddings(input_ids, multimodal_embeddings) + embed_fn = getattr(self.model, "get_input_embeddings", None) + if callable(embed_fn): + try: + return embed_fn(input_ids, multimodal_embeddings) + except TypeError: + embeddings = embed_fn() + if callable(embeddings): + return embeddings(input_ids) + except AttributeError: + pass + + embed_tokens = getattr(getattr(getattr(self.model, "llm", None), "model", None), "embed_tokens", None) + if callable(embed_tokens): + return embed_tokens(input_ids) + + raise AttributeError(f"{type(self.model).__name__} does not expose token embeddings") def embed_input_ids( self, @@ -181,6 +287,182 @@ class MiniCPMO45OmniForConditionalGeneration(nn.Module, SupportsMultiModal, Supp return self.get_input_embeddings(input_ids) return super().embed_input_ids(input_ids, multimodal_embeddings, is_multimodal=is_multimodal) + def preprocess( + self, + input_ids: torch.Tensor, + input_embeds: torch.Tensor | None = None, + **kwargs, + ) -> tuple[torch.Tensor, torch.Tensor, dict[str, object]]: + """Model-runner data-plane hook for MiniCPM-o 4.5 duplex audio. + + The scheduler owns the request, block table, attention metadata, KV, + and sampler. This hook only turns the current duplex audio append into + the prompt embeddings consumed by the normal runner forward. + """ + if self.model_stage != "llm": + embeds = input_embeds if input_embeds is not None else self.get_input_embeddings(input_ids) + return input_ids, embeds, {} + + duplex = kwargs.get("duplex") + if not isinstance(duplex, dict) or duplex.get("data_plane") is not True: + embeds = input_embeds if input_embeds is not None else self.get_input_embeddings(input_ids) + return input_ids, embeds, {} + + prompt_len_meta = kwargs.get("duplex_prompt_len") + token_offset_meta = kwargs.get("duplex_token_offset", 0) + if ( + isinstance(prompt_len_meta, int) + and isinstance(token_offset_meta, int) + and token_offset_meta >= prompt_len_meta + ): + # Decode step of the resumable duplex request: input_ids are the + # runner-sampled tokens and the normal embedding lookup is the + # correct input. Slicing the (prompt-only) duplex embeddings here + # would come up empty and pad-fill, feeding a embedding in + # place of every sampled token and corrupting generation. + embeds = input_embeds if input_embeds is not None else self.get_input_embeddings(input_ids) + return input_ids, embeds, {} + + helper = self._duplex_data_plane_helper() + session_id = str(duplex.get("session_id") or "") + try: + incarnation = int(duplex.get("incarnation", 0)) + except (TypeError, ValueError): + incarnation = 0 + payload = duplex.get("payload") + if not session_id or not isinstance(payload, dict): + embeds = input_embeds if input_embeds is not None else self.get_input_embeddings(input_ids) + return input_ids, embeds, {"duplex": {"prefill_success": False, "reason": "bad_duplex_payload"}} + + session_key = (session_id, incarnation) + state = helper.sessions.get(session_key) + if state is None: + from vllm_omni.experimental.fullduplex.minicpmo45.stage0 import ( + _MiniCPMO45Stage0SessionState, + ) + + state = _MiniCPMO45Stage0SessionState(session_id=session_id) + helper.sessions[session_key] = state + session_config = duplex.get("session_config") + session_config = dict(session_config) if isinstance(session_config, dict) else {} + runtime_config = duplex.get("runtime_config") + runtime_config = dict(runtime_config) if isinstance(runtime_config, dict) else {} + if hasattr(helper.thinker, "audio_past_key_values"): + helper.thinker.audio_past_key_values = None + helper._configure_streaming_processor(state) + helper._prepare_session_context(state, session_config, runtime_config=runtime_config) + + audio_waveform = helper._decode_audio_payload(payload) + try: + video_frames = helper._decode_video_frames_payload(payload) + except ValueError as exc: + embeds = input_embeds if input_embeds is not None else self.get_input_embeddings(input_ids) + return input_ids, embeds, {"duplex": {"prefill_success": False, "reason": str(exc)}} + seq = duplex.get("seq") + try: + seq = int(seq) if seq is not None else None + except (TypeError, ValueError): + seq = None + epoch = duplex.get("epoch") + try: + epoch = int(epoch) if epoch is not None else None + except (TypeError, ValueError): + epoch = None + result = helper._stage_prefill_embeddings_only( + state, + audio_waveform, + video_frames=video_frames, + epoch=epoch, + seq=seq, + is_speech=bool(payload.get("is_speech", False)), + final=bool(duplex.get("final")), + ) + update_result = dict(result) + update_result.pop("inputs_embeds", None) + if result.get("success") is not True: + embeds = input_embeds if input_embeds is not None else self.get_input_embeddings(input_ids) + return input_ids, embeds, {"duplex": update_result} + + target_dtype = ( + input_embeds.dtype if input_embeds is not None else self.get_input_embeddings(input_ids[:1]).dtype + ) + full_req_embeds = result["inputs_embeds"].to(device=input_ids.device, dtype=target_dtype) + full_input_token_ids = list(result.get("input_token_ids") or []) + prompt_len = kwargs.get("duplex_prompt_len") + try: + prompt_len = int(prompt_len) if prompt_len is not None else int(full_req_embeds.shape[0]) + except (TypeError, ValueError): + prompt_len = int(full_req_embeds.shape[0]) + pad_token_id = helper.stage_padding_token_id() + if prompt_len > int(full_req_embeds.shape[0]): + pad_len = prompt_len - int(full_req_embeds.shape[0]) + pad_ids = torch.full( + (pad_len,), + pad_token_id, + dtype=input_ids.dtype, + device=input_ids.device, + ) + pad_embeds = self.get_input_embeddings(pad_ids).to(dtype=full_req_embeds.dtype) + # The appended duplex tokens occupy the tail of the request prompt + # and the runner schedules the span [num_computed_tokens, prompt_len). + # Padding must therefore sit in front of the real chunk embeddings; + # otherwise the audio lands outside the scheduled span, is never + # forwarded, and generation runs on pad tokens only. Keeping the + # embeddings last also places the decode position directly after the + # final audio embedding, matching the official listen/speak decision + # point. + full_req_embeds = torch.cat([pad_embeds, full_req_embeds], dim=0) + full_input_token_ids = [pad_token_id] * pad_len + full_input_token_ids + elif prompt_len < int(full_req_embeds.shape[0]): + logger.warning( + "MiniCPM-o duplex append produced %d embeddings but the scheduler " + "reserved only %d prompt slots; the tail will be truncated. " + "Increase the duplex scheduler token budget.", + int(full_req_embeds.shape[0]), + prompt_len, + ) + + span_len = int(input_ids.shape[0]) + token_offset = kwargs.get("duplex_token_offset", 0) + try: + token_offset = max(0, int(token_offset)) + except (TypeError, ValueError): + token_offset = 0 + req_embeds = full_req_embeds[token_offset : token_offset + span_len] + if req_embeds.shape[0] < span_len: + pad_ids = torch.full( + (span_len - req_embeds.shape[0],), + pad_token_id, + dtype=input_ids.dtype, + device=input_ids.device, + ) + pad_embeds = self.get_input_embeddings(pad_ids).to(dtype=req_embeds.dtype) + req_embeds = torch.cat([req_embeds, pad_embeds], dim=0) + elif req_embeds.shape[0] > span_len: + req_embeds = req_embeds[:span_len] + + input_token_ids = full_input_token_ids[token_offset : token_offset + span_len] + if len(input_token_ids) < span_len: + input_token_ids.extend([pad_token_id] * (span_len - len(input_token_ids))) + if input_token_ids: + req_input_ids = torch.tensor(input_token_ids, dtype=input_ids.dtype, device=input_ids.device) + update_result["duplex_prompt_token_ids"] = full_input_token_ids + else: + req_input_ids = torch.full_like(input_ids, helper._required_token_id("unit_token_id")) + return req_input_ids, req_embeds, {"duplex": update_result} + + def _duplex_data_plane_helper(self): + helper = getattr(self, "_minicpmo45_duplex_data_plane_helper", None) + if helper is not None: + return helper + from vllm_omni.experimental.fullduplex.minicpmo45.stage0 import MiniCPMO45Stage0DuplexRuntime + + model_path = getattr(getattr(self.vllm_config, "model_config", None), "model", None) + device = str(self._module_device(self.thinker if self.thinker is not None else self)) + helper = MiniCPMO45Stage0DuplexRuntime(self, model_path=model_path, device=device) + self._minicpmo45_duplex_data_plane_helper = helper + return helper + def get_multimodal_embeddings(self, **kwargs): # Delegate to the active stage submodule when it implements MM encoding. mm_fn = getattr(self.model, "get_multimodal_embeddings", None) @@ -192,6 +474,174 @@ class MiniCPMO45OmniForConditionalGeneration(nn.Module, SupportsMultiModal, Supp """vLLM V1 encoder profiling calls this; the inherited Protocol stub returns None.""" return self.get_multimodal_embeddings(**kwargs) + def _run_tts_request( + self, + *, + input_ids: torch.Tensor | None, + positions: torch.Tensor | None, + inputs_embeds: torch.Tensor | None, + talker_info: dict[str, Any], + device: torch.device, + ) -> tuple[dict[str, Any] | None, bool, bool]: + meta_info = talker_info.get("meta") if isinstance(talker_info.get("meta"), dict) else {} + if talker_info.get("native_duplex") is True: + # ids/hidden_states may be accumulated for the talker KV stream, + # but displayed transcript belongs to the current thinker segment. + tts_text = meta_info.get("native_duplex_segment_text", "") + else: + tts_text = talker_info.get("llm_output_text", "") + if isinstance(tts_text, list): + tts_text = ( + tts_text[-1] + if talker_info.get("native_duplex") is True and tts_text + else (tts_text[0] if tts_text else "") + ) + if not isinstance(tts_text, str): + tts_text = "" + + with torch.inference_mode(): + talker_result = self.talker( + input_ids=input_ids, + positions=positions, + inputs_embeds=inputs_embeds, + additional_information=talker_info, + ) + chunk_flags = getattr(self.talker, "_ar_last_chunk_flags", [True]) + chunk_is_last = bool(chunk_flags[-1]) if chunk_flags else True + turn_end_flags = getattr(self.talker, "_ar_turn_end_flags", [False]) + turn_ended = bool(turn_end_flags[-1]) if turn_end_flags else False + if talker_info.get("native_duplex") is True: + emitted_text = getattr(self.talker, "_ar_last_emitted_text", "") + tts_text = emitted_text if isinstance(emitted_text, str) else "" + + if not (isinstance(talker_result, tuple) and len(talker_result) == 2): + return None, chunk_is_last, turn_ended + + mel_spec, waveform = talker_result + mm_out: dict[str, Any] = {} + duplex_info = talker_info.get("duplex") if isinstance(talker_info.get("duplex"), dict) else {} + if talker_info.get("native_duplex") is True: + turn_id = duplex_info.get("turn_id") + if isinstance(turn_id, int): + mm_out["meta.duplex_turn_id"] = torch.tensor([turn_id], dtype=torch.int32, device=device) + epoch = duplex_info.get("epoch") + if isinstance(epoch, int): + mm_out["meta.duplex_epoch"] = torch.tensor([epoch], dtype=torch.int32, device=device) + mm_out["meta.tts_is_last_chunk"] = torch.tensor( + [int(chunk_is_last)], + dtype=torch.int32, + device=device, + ) + if talker_info.get("native_duplex") is True: + mm_out["meta.turn_end"] = torch.tensor( + [int(turn_ended)], + dtype=torch.int32, + device=device, + ) + if tts_text or talker_info.get("native_duplex") is True: + mm_out["meta.llm_output_text_utf8"] = torch.tensor( + list(tts_text.encode("utf-8")), + dtype=torch.uint8, + device=device, + ) + mm_out["meta.audio_text_total_chars"] = torch.tensor( + [len(tts_text)], + dtype=torch.int32, + device=device, + ) + if mel_spec is not None: + mm_out["mel_spec"] = [mel_spec] + if waveform is not None: + mm_out["model_outputs"] = [waveform] + elif mel_spec is not None: + mm_out["model_outputs"] = [mel_spec] + return mm_out, chunk_is_last, turn_ended + + @staticmethod + def _slice_tts_request_rows(value: torch.Tensor | None, start: int, end: int) -> torch.Tensor | None: + return value[start:end] if isinstance(value, torch.Tensor) else value + + def _run_batched_tts_requests( + self, + *, + input_ids: torch.Tensor | None, + positions: torch.Tensor | None, + inputs_embeds: torch.Tensor | None, + runtime_info: list[object], + request_token_spans: object, + num_tokens: int, + hidden_dim: int, + device: torch.device, + ) -> OmniOutput: + if not isinstance(request_token_spans, list) or len(request_token_spans) != len(runtime_info): + raise RuntimeError( + "MiniCPM-o 4.5 batched TTS requires one request_token_spans entry " + f"per request; got {request_token_spans!r} for {len(runtime_info)} requests" + ) + + request_ids: list[str] = [] + request_outputs: list[dict[str, Any]] = [] + row_chunk_flags = [True] * num_tokens + row_turn_end_flags = [False] * num_tokens + previous_end = 0 + for index, raw_info in enumerate(runtime_info): + if not isinstance(raw_info, dict): + raise RuntimeError(f"MiniCPM-o 4.5 TTS request metadata at index {index} is not a dict") + span = request_token_spans[index] + if not ( + isinstance(span, (list, tuple)) and len(span) == 2 and all(isinstance(value, int) for value in span) + ): + raise RuntimeError(f"Invalid MiniCPM-o 4.5 TTS token span at index {index}: {span!r}") + start, end = span + if start != previous_end or start < 0 or end < start or end > num_tokens: + raise RuntimeError( + f"Invalid MiniCPM-o 4.5 TTS token span at index {index}: " + f"expected start {previous_end} and 0 <= start <= end <= {num_tokens}, got {span!r}" + ) + previous_end = end + if start == end: + continue + mm_out, chunk_is_last, turn_ended = self._run_tts_request( + input_ids=self._slice_tts_request_rows(input_ids, start, end), + positions=self._slice_tts_request_rows(positions, start, end), + inputs_embeds=self._slice_tts_request_rows(inputs_embeds, start, end), + talker_info=raw_info, + device=device, + ) + row_chunk_flags[start:end] = [chunk_is_last] * (end - start) + row_turn_end_flags[start:end] = [turn_ended] * (end - start) + if mm_out is not None: + request_ids.append(get_stream_request_key(raw_info)) + request_outputs.append(mm_out) + # vLLM pads the flattened token tensor to a CUDA Graph capture size. + # The spans describe only scheduled request rows, so an uncovered tail + # is graph padding rather than a missing request. Contiguity checks + # above still reject gaps between real requests. + + self.talker._ar_last_chunk_flags = row_chunk_flags + self.talker._ar_turn_end_flags = row_turn_end_flags + dummy_hidden = torch.zeros(num_tokens, hidden_dim, device=device) + if not request_outputs: + return OmniOutput(text_hidden_states=dummy_hidden, multimodal_outputs=None) + + sparse_output: dict[str, Any] = { + "meta.req_id": request_ids, + "meta.sparse_audio": ["1"], + } + output_keys = set().union(*(output.keys() for output in request_outputs)) + for key in output_keys: + values: list[Any] = [] + for output in request_outputs: + value = output.get(key) + if isinstance(value, list) and len(value) == 1: + value = value[0] + if value is None: + dtype = torch.uint8 if key == "meta.llm_output_text_utf8" else torch.float32 + value = torch.empty(0, dtype=dtype, device=device) + values.append(value) + sparse_output[key] = values + return OmniOutput(text_hidden_states=dummy_hidden, multimodal_outputs=sparse_output) + def forward( self, input_ids: torch.Tensor, @@ -208,10 +658,8 @@ class MiniCPMO45OmniForConditionalGeneration(nn.Module, SupportsMultiModal, Supp Forward pass for MiniCPM-o Omni model. Workflow: - 1) Thinker (model_stage="llm"): Image / video / audio encoders + - 3D resampler + omni LLM → text + hidden states. - 2) Talker (model_stage="tts"): MiniCPMTTS + the in-process - Token2Wav vocoder → audio waveform (final pipeline output). + 1) LLM: multimodal thinker → hidden states and text tokens + 2) TTS: talker + Token2Wav → speech waveform """ if self.model_stage == "llm": # Normalize to batched inputs if caller provides 1D/2D unbatched tensors @@ -275,13 +723,62 @@ class MiniCPMO45OmniForConditionalGeneration(nn.Module, SupportsMultiModal, Supp text_hidden_states = text_hidden_states.squeeze(0) # Return hidden states with latent in multimodal_outputs for stage_input_processors + multimodal_outputs = {"latent": text_hidden_states} + runtime_info = kwargs.get("runtime_additional_information") + if runtime_info and isinstance(runtime_info, list) and len(runtime_info) > 0: + duplex_rows = [] + for req_info in runtime_info: + duplex_info = req_info.get("duplex") if isinstance(req_info, dict) else None + duplex_rows.append(duplex_info if isinstance(duplex_info, dict) else {}) + + prompt_rows = [] + for duplex_info in duplex_rows: + prompt_token_ids = duplex_info.get("duplex_prompt_token_ids") + # This is a complete per-handoff snapshot, not a generated + # tensor delta. Keep it as row-local metadata so output + # accumulation replaces the previous value instead of + # attempting to concatenate variable-length prompts. + prompt_rows.append(list(prompt_token_ids) if isinstance(prompt_token_ids, list) else None) + if any(row is not None for row in prompt_rows): + multimodal_outputs["duplex_prompt_token_ids"] = prompt_rows + + special_keys = { + key + for duplex_info in duplex_rows + for key, value in ( + duplex_info.get("special_token_ids", {}).items() + if isinstance(duplex_info.get("special_token_ids"), dict) + else () + ) + if isinstance(key, str) and isinstance(value, int) and value >= 0 + } + if special_keys: + multimodal_outputs["meta"] = { + key: [ + torch.tensor( + [int(value)], + dtype=torch.long, + device=text_hidden_states.device, + ) + if isinstance(value, int) and value >= 0 + else None + for duplex_info in duplex_rows + for value in [ + ( + duplex_info.get("special_token_ids", {}).get(key) + if isinstance(duplex_info.get("special_token_ids"), dict) + else None + ) + ] + ] + for key in sorted(special_keys) + } return OmniOutput( text_hidden_states=text_hidden_states, - multimodal_outputs={"latent": text_hidden_states}, + multimodal_outputs=multimodal_outputs, ) - # Talker stage: runs MiniCPMTTS + the in-process Token2Wav vocoder and - # emits the final audio waveform directly. + # Talker stage: runs TTS generation and its built-in Token2Wav vocoder. if self.model_stage == "tts": if input_ids is not None: num_tokens = input_ids.shape[0] @@ -302,34 +799,33 @@ class MiniCPMO45OmniForConditionalGeneration(nn.Module, SupportsMultiModal, Supp return OmniOutput(text_hidden_states=dummy_hidden, multimodal_outputs=None) runtime_info = kwargs.get("runtime_additional_information") - talker_info = {} - if runtime_info and isinstance(runtime_info, list) and len(runtime_info) > 0: - talker_info = runtime_info[0] if isinstance(runtime_info[0], dict) else {} - - with torch.inference_mode(): - talker_result = self.talker( + request_token_spans = kwargs.get("request_token_spans") + if isinstance(runtime_info, list) and (len(runtime_info) > 1 or request_token_spans is not None): + return self._run_batched_tts_requests( input_ids=input_ids, positions=positions, inputs_embeds=inputs_embeds, - additional_information=talker_info, + runtime_info=runtime_info, + request_token_spans=request_token_spans, + num_tokens=num_tokens, + hidden_dim=hidden_dim, + device=device, ) - + talker_info = {} + if runtime_info and isinstance(runtime_info, list) and len(runtime_info) > 0: + talker_info = runtime_info[0] if isinstance(runtime_info[0], dict) else {} dummy_hidden = torch.zeros(num_tokens, hidden_dim, device=device) - - # Talker returns a (mel_spec, waveform_or_None) tuple. MiniCPM-o - # 4.5 emits only the waveform (mel_spec is always None); keep the - # 2-slot unpack so a future mel-emitting variant can plug in - # without changing the wrapper. - mm_out: dict = {} - if isinstance(talker_result, tuple) and len(talker_result) == 2: - _, waveform = talker_result - if waveform is not None: - mm_out["model_outputs"] = [waveform] - - return OmniOutput( - text_hidden_states=dummy_hidden, - multimodal_outputs=mm_out if mm_out else None, + mm_out, _, _ = self._run_tts_request( + input_ids=input_ids, + positions=positions, + inputs_embeds=inputs_embeds, + talker_info=talker_info, + device=device, ) + if mm_out is not None: + return OmniOutput(text_hidden_states=dummy_hidden, multimodal_outputs=mm_out) + + return OmniOutput(text_hidden_states=dummy_hidden, multimodal_outputs=None) raise ValueError(f"Unsupported model stage: {self.model_stage}") @@ -341,26 +837,431 @@ class MiniCPMO45OmniForConditionalGeneration(nn.Module, SupportsMultiModal, Supp # Use model for logits computation return self.model.compute_logits(hidden_states) + def on_requests_finished(self, finished_req_ids: set[str] | list[str]) -> None: + request_sessions = getattr(self, "_minicpmo45_duplex_request_sessions", None) + helper = getattr(self, "_minicpmo45_duplex_data_plane_helper", None) + sessions = getattr(helper, "sessions", None) if helper is not None else None + if isinstance(request_sessions, dict): + for request_id in finished_req_ids: + session_key = request_sessions.pop(request_id, None) + if session_key is not None and isinstance(sessions, dict): + sessions.pop(session_key, None) + forced_segments = getattr(self, "_minicpmo45_force_listen_applied_segments", None) + if isinstance(forced_segments, set): + finished = set(finished_req_ids) + completed_segments = {segment for segment in forced_segments if segment[0] in finished} + forced_segments.difference_update(completed_segments) + if hasattr(self.model, "on_requests_finished"): + self.model.on_requests_finished(finished_req_ids) + def sample( self, logits: torch.Tensor, sampling_metadata: SamplingMetadata, ) -> SamplerOutput | None: - # Use model for sampling - return self.model.sample(logits, sampling_metadata) + native_duplex = self._sample_minicpmo45_native_duplex_stage0( + logits, + sampling_metadata, + duplex_rows=getattr(self, "_minicpmo45_active_duplex_rows", None), + ) + if native_duplex is not None: + return native_duplex + if self.model_stage == "tts": + return self.model.sample(logits, sampling_metadata) + return None + + def _sample_minicpmo45_native_duplex_stage0( + self, + logits: torch.Tensor, + sampling_metadata: SamplingMetadata, + *, + duplex_rows: list[int] | None = None, + ) -> SamplerOutput | None: + if self.model_stage != "llm" or logits.ndim != 2 or logits.shape[0] == 0: + return None + token_ids = self._minicpmo45_native_duplex_token_ids() + unit_id = token_ids.get("unit_token_id", -1) + if unit_id < 0: + return None + native_rows = self._minicpmo45_native_duplex_prompt_rows( + sampling_metadata, + unit_id, + logits.shape[0], + duplex_rows=duplex_rows, + ) + if not native_rows or len(native_rows) != logits.shape[0]: + return None + + sampled_ids: list[int] = [] + for row_idx in range(logits.shape[0]): + row_logits = logits[row_idx : row_idx + 1].clone() + sampled = self._sample_minicpmo45_native_duplex_row( + row_logits, + sampling_metadata, + row_idx=row_idx, + token_ids=token_ids, + ) + self._record_minicpmo45_duplex_terminator(row_idx, sampled, token_ids) + sampled_ids.append(sampled) + return SamplerOutput( + sampled_token_ids=torch.tensor(sampled_ids, device=logits.device, dtype=torch.int32).unsqueeze(-1), + logprobs_tensors=None, + ) + + def _sample_minicpmo45_native_duplex_row( + self, + logits: torch.Tensor, + sampling_metadata: SamplingMetadata, + *, + row_idx: int, + token_ids: dict[str, int], + ) -> int: + chunk_eos_id = token_ids.get("chunk_eos_token_id", -1) + generator = getattr(sampling_metadata, "generators", {}).get(row_idx) + output_token_ids = getattr(sampling_metadata, "output_token_ids", None) or [] + raw_recent_tokens = output_token_ids[row_idx] if row_idx < len(output_token_ids) else [] + recent_tokens = [int(token_id) for token_id in raw_recent_tokens if isinstance(token_id, int) and token_id >= 0] + if chunk_eos_id >= 0 and chunk_eos_id < logits.shape[-1]: + max_speak_tokens = int( + getattr( + self, + "max_new_speak_tokens_per_chunk", + MiniCPMO45DuplexPolicy.DEFAULT_MAX_NEW_SPEAK_TOKENS_PER_CHUNK, + ) + or MiniCPMO45DuplexPolicy.DEFAULT_MAX_NEW_SPEAK_TOKENS_PER_CHUNK + ) + request_max_tokens = self._minicpmo45_duplex_row_request_max_tokens(row_idx) + effective_max_speak_tokens = max_speak_tokens + if request_max_tokens is not None: + effective_max_speak_tokens = min(effective_max_speak_tokens, request_max_tokens) + if len(recent_tokens) >= max(1, effective_max_speak_tokens - 1): + return int(chunk_eos_id) + + # Match the released StreamDecoder: first sample the original + # distribution only to preserve the model's own chunk boundary. + # If it does not choose chunk_eos, mask that token before the + # normal text/listen/turn sampling pass below. + if getattr(sampling_metadata, "all_greedy", False): + boundary_sample = int(torch.argmax(logits, dim=-1).item()) + else: + boundary_probs = F.softmax(logits, dim=-1) + boundary_sample = int(torch.multinomial(boundary_probs, num_samples=1, generator=generator).item()) + if boundary_sample == chunk_eos_id: + return int(chunk_eos_id) + + logits = logits.clone() + forbidden = self._minicpmo45_native_forbidden_token_ids(token_ids) + if forbidden: + valid_forbidden = [token_id for token_id in forbidden if 0 <= token_id < logits.shape[-1]] + if valid_forbidden: + logits[:, valid_forbidden] = float("-inf") + + special_ids = self._minicpmo45_native_special_token_ids(token_ids) + state = self._minicpmo45_duplex_state_for_row(row_idx) + generated_text_tokens = getattr(state, "generated_text_tokens", None) + repetition_tokens = generated_text_tokens if generated_text_tokens else recent_tokens + repetition_penalty = 1.05 + if repetition_penalty != 1.0 and repetition_tokens: + history_size = MiniCPMO45DuplexPolicy.REPETITION_HISTORY_SIZE + for token_id in set(repetition_tokens[-history_size:]): + if token_id in special_ids or token_id < 0 or token_id >= logits.shape[-1]: + continue + logits[0, token_id] /= repetition_penalty + + temperature = float(self._sampling_metadata_value(sampling_metadata, "temperature", row_idx, 0.7)) + top_k = int(self._sampling_metadata_value(sampling_metadata, "top_k", row_idx, 100)) + top_p = float(self._sampling_metadata_value(sampling_metadata, "top_p", row_idx, 0.8)) + if getattr(sampling_metadata, "all_greedy", False) or temperature <= 0: + sampled = int(torch.argmax(logits, dim=-1).item()) + sampled = self._maybe_cut_minicpmo45_native_duplex_text_chunk( + sampled, + recent_tokens, + token_ids, + ) + return self._finalize_minicpmo45_native_duplex_sample(row_idx, sampled, token_ids) + + logits = logits / temperature + logits = self._top_k_top_p_filter(logits, top_k=top_k, top_p=top_p) + probs = F.softmax(logits, dim=-1) + sampled = int(torch.multinomial(probs, num_samples=1, generator=generator).item()) + sampled = self._maybe_cut_minicpmo45_native_duplex_text_chunk( + sampled, + recent_tokens, + token_ids, + ) + return self._finalize_minicpmo45_native_duplex_sample( + row_idx, + sampled, + token_ids, + ) + + def _maybe_cut_minicpmo45_native_duplex_text_chunk( + self, + sampled: int, + recent_tokens: list[int], + token_ids: dict[str, int], + ) -> int: + chunk_eos_id = token_ids.get("chunk_eos_token_id", -1) + if chunk_eos_id < 0: + return int(sampled) + special_ids = self._minicpmo45_native_special_token_ids(token_ids) + if sampled in special_ids: + return int(sampled) + max_chars = int( + getattr( + self, + "max_speak_chars_per_chunk", + MiniCPMO45DuplexPolicy.DEFAULT_MAX_SPEAK_CHARS_PER_CHUNK, + ) + or MiniCPMO45DuplexPolicy.DEFAULT_MAX_SPEAK_CHARS_PER_CHUNK + ) + if max_chars <= 0: + return int(sampled) + tokenizer = self._minicpmo45_tokenizer() + decode = getattr(tokenizer, "decode", None) + if not callable(decode): + return int(sampled) + candidate_tokens = self._minicpmo45_current_chunk_tokens(recent_tokens, token_ids) + candidate_tokens.append(int(sampled)) + try: + text = decode(candidate_tokens, skip_special_tokens=True) + except TypeError: + text = decode(candidate_tokens) + except Exception: + return int(sampled) + return int(chunk_eos_id) if isinstance(text, str) and len(text) >= max_chars else int(sampled) + + @staticmethod + def _minicpmo45_current_chunk_tokens( + tokens: list[int], + token_ids: dict[str, int], + ) -> list[int]: + boundaries = { + token_ids.get("listen_token_id", -1), + token_ids.get("chunk_eos_token_id", -1), + token_ids.get("chunk_tts_eos_token_id", -1), + token_ids.get("turn_eos_token_id", -1), + } + start = 0 + for idx, token_id in enumerate(tokens): + if token_id in boundaries: + start = idx + 1 + return list(tokens[start:]) + + def _minicpmo45_duplex_state_for_row(self, row_idx: int): + row_sessions = getattr(self, "_minicpmo45_duplex_row_sessions", None) + session_key = row_sessions.get(row_idx) if isinstance(row_sessions, dict) else None + if not session_key: + return None + helper = getattr(self, "_minicpmo45_duplex_data_plane_helper", None) + sessions = getattr(helper, "sessions", None) if helper is not None else None + return sessions.get(session_key) if isinstance(sessions, dict) else None + + def _minicpmo45_duplex_payload_for_row(self, row_idx: int) -> dict[str, Any] | None: + row_payloads = getattr(self, "_minicpmo45_duplex_row_payloads", None) + payload = row_payloads.get(row_idx) if isinstance(row_payloads, dict) else None + return payload if isinstance(payload, dict) else None + + def _minicpmo45_duplex_row_request_max_tokens(self, row_idx: int) -> int | None: + row_max_tokens = getattr(self, "_minicpmo45_duplex_row_max_tokens", None) + value = row_max_tokens.get(row_idx) if isinstance(row_max_tokens, dict) else None + try: + max_tokens = int(value) + except (TypeError, ValueError): + return None + return max_tokens if max_tokens > 0 else None + + def _finalize_minicpmo45_native_duplex_sample( + self, + row_idx: int, + sampled: int, + token_ids: dict[str, int], + ) -> int: + listen_id = token_ids.get("listen_token_id", -1) + tts_bos_id = token_ids.get("tts_bos_token_id", -1) + state = self._minicpmo45_duplex_state_for_row(row_idx) + payload = self._minicpmo45_duplex_payload_for_row(row_idx) + force_listen = isinstance(payload, dict) and payload.get("force_listen") is True + if ( + sampled == listen_id + and 0 <= tts_bos_id + and state is not None + and not getattr(state, "current_turn_ended", True) + and not force_listen + ): + return int(tts_bos_id) + return int(sampled) + + def _record_minicpmo45_duplex_terminator(self, row_idx: int, sampled: int, token_ids: dict[str, int]) -> None: + """Remember sampled unit state for the next append. + + The scheduler session update discards the final sampled token of a + segment before the next streaming update, but the official duplex + format feeds it (terminator + ) into the KV at every unit + boundary, and the model's listen/speak policy depends on seeing its own + past decisions. Non-terminators clear the turn-ended latch.""" + state = self._minicpmo45_duplex_state_for_row(row_idx) + if state is None: + return + payload = self._minicpmo45_duplex_payload_for_row(row_idx) + force_listen = isinstance(payload, dict) and payload.get("force_listen") is True + listen_id = token_ids.get("listen_token_id", -1) + tts_bos_id = token_ids.get("tts_bos_token_id", -1) + chunk_eos_id = token_ids.get("chunk_eos_token_id", -1) + chunk_tts_eos_id = token_ids.get("chunk_tts_eos_token_id", -1) + turn_eos_id = token_ids.get("turn_eos_token_id", -1) + terminators = {listen_id, chunk_eos_id, chunk_tts_eos_id, turn_eos_id} + if sampled in terminators: + state.pending_terminator_token = int(sampled) + state.last_terminator_token = int(sampled) + if sampled == turn_eos_id or (sampled == listen_id and force_listen): + state.current_turn_ended = True + with suppress(Exception): + state.pending_speech_response_open = False + return + special_ids = self._minicpmo45_native_special_token_ids(token_ids) + if sampled not in special_ids: + generated_text_tokens = getattr(state, "generated_text_tokens", None) + if not isinstance(generated_text_tokens, list): + generated_text_tokens = [] + state.generated_text_tokens = generated_text_tokens + generated_text_tokens.append(int(sampled)) + history_size = MiniCPMO45DuplexPolicy.REPETITION_HISTORY_SIZE + del generated_text_tokens[:-history_size] + if ( + sampled == tts_bos_id + and getattr(state, "current_turn_ended", True) + and getattr(state, "pending_speech_context", False) + ): + with suppress(Exception): + state.pending_speech_response_open = True + elif getattr(state, "pending_speech_response_open", False): + with suppress(Exception): + state.pending_speech_context = False + state.pending_speech_response_open = False + elif getattr(state, "current_turn_ended", True): + with suppress(Exception): + state.pending_speech_context = False + state.pending_terminator_token = None + state.last_terminator_token = None + state.current_turn_ended = False + + def _minicpmo45_tokenizer(self): + if hasattr(self, "_minicpmo45_tokenizer_cache"): + return self._minicpmo45_tokenizer_cache + tokenizer = None + get_tokenizer = getattr(getattr(self, "thinker", None), "get_tokenizer", None) + if callable(get_tokenizer): + tokenizer = get_tokenizer() + if tokenizer is None: + try: + from vllm.tokenizers import cached_tokenizer_from_config + + tokenizer = cached_tokenizer_from_config(self.vllm_config.model_config) + except Exception: + pass + self._minicpmo45_tokenizer_cache = tokenizer + return tokenizer + + def _minicpmo45_native_duplex_token_ids(self) -> dict[str, int]: + cached = getattr(self, "_minicpmo45_native_duplex_token_ids_cache", None) + if isinstance(cached, dict): + return cached + tokenizer = self._minicpmo45_tokenizer() + cached = MiniCPMO45DuplexPolicy.token_ids_from_tokenizer(tokenizer) + self._minicpmo45_native_duplex_token_ids_cache = cached + return cached + + def _minicpmo45_native_duplex_prompt_rows( + self, + sampling_metadata: SamplingMetadata, + unit_id: int, + batch_size: int, + *, + duplex_rows: list[int] | None = None, + ) -> list[int]: + if duplex_rows is not None: + rows: list[int] = [] + for row in duplex_rows: + try: + row_idx = int(row) + except (TypeError, ValueError): + continue + if 0 <= row_idx < batch_size: + rows.append(row_idx) + return rows + + prompt_token_ids = getattr(sampling_metadata, "prompt_token_ids", None) + if prompt_token_ids is None: + return [] + if prompt_token_ids.ndim == 1: + prompt_token_ids = prompt_token_ids.unsqueeze(0) + rows: list[int] = [] + for row_idx in range(min(batch_size, int(prompt_token_ids.shape[0]))): + row = prompt_token_ids[row_idx] + if torch.count_nonzero(row == unit_id).item() >= 2: + rows.append(row_idx) + return rows + + def _minicpmo45_native_forbidden_token_ids(self, token_ids: dict[str, int]) -> list[int]: + tokenizer = self._minicpmo45_tokenizer() + bad_token_ids = getattr(tokenizer, "bad_token_ids", []) if tokenizer is not None else [] + return MiniCPMO45DuplexPolicy.native_forbidden_token_ids(token_ids, bad_token_ids=bad_token_ids) + + def _minicpmo45_native_special_token_ids(self, token_ids: dict[str, int]) -> set[int]: + tokenizer = self._minicpmo45_tokenizer() + return MiniCPMO45DuplexPolicy.native_special_token_ids( + token_ids, + tokenizer_special_ids=getattr(tokenizer, "all_special_ids", []) if tokenizer is not None else [], + ) + + @staticmethod + def _sampling_metadata_value( + sampling_metadata: SamplingMetadata, + name: str, + row_idx: int, + default: float, + ) -> float: + value = getattr(sampling_metadata, name, None) + if value is None: + return default + if isinstance(value, torch.Tensor): + if value.numel() == 0: + return default + if value.ndim == 0: + return float(value.item()) + idx = min(row_idx, int(value.numel()) - 1) + return float(value.reshape(-1)[idx].item()) + try: + return float(value) + except (TypeError, ValueError): + return default + + @staticmethod + def _top_k_top_p_filter(logits: torch.Tensor, *, top_k: int, top_p: float) -> torch.Tensor: + if top_k > 0 and top_k < logits.shape[-1]: + kth = torch.topk(logits, top_k, dim=-1).values[..., -1, None] + logits = logits.masked_fill(logits < kth, float("-inf")) + if 0.0 < top_p < 1.0: + sorted_logits, sorted_indices = torch.sort(logits, descending=True, dim=-1) + cumulative_probs = torch.cumsum(F.softmax(sorted_logits, dim=-1), dim=-1) + sorted_remove = cumulative_probs > top_p + sorted_remove[..., 1:] = sorted_remove[..., :-1].clone() + sorted_remove[..., 0] = False + remove = torch.zeros_like(logits, dtype=torch.bool) + remove.scatter_(dim=-1, index=sorted_indices, src=sorted_remove) + logits = logits.masked_fill(remove, float("-inf")) + return logits def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - """Load weights for the active stage of the omni model.""" + """Load weights for all components of the omni model.""" loaded_weights = set() thinker_weights = [] talker_weights = [] # MiniCPM-o checkpoint prefixes → stage mapping: # thinker: vpm, resampler, llm, apm, audio_projection_layer - # talker: tts (MiniCPMTTS); the Token2Wav vocoder weights load - # separately inside the talker module from the - # ``assets/token2wav`` subdirectory and do not appear - # in this iterator. + # talker: tts (ConditionalChatTTS) for k, v in weights: if k.startswith(("vpm.", "resampler.", "llm.", "apm.", "audio_projection_layer.")): thinker_weights.append((k, v)) diff --git a/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py b/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py index 34a4da1ac..e59742ad7 100644 --- a/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py +++ b/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py @@ -105,6 +105,11 @@ from vllm.multimodal.inputs import ( MultiModalKwargsItems, NestedTensors, ) + +try: + from vllm.multimodal.inputs import ModalityData, MultiModalDataDict +except ImportError: + from vllm.multimodal.parse import ModalityData, MultiModalDataDict from vllm.multimodal.parse import ( AudioItem, AudioProcessorItems, @@ -127,9 +132,15 @@ from vllm.multimodal.processing import ( ) from vllm.sequence import IntermediateTensors +try: + from vllm.transformers_utils.tokenizer import encode_tokens as _vllm_encode_tokens +except ImportError: + _vllm_encode_tokens = None -def encode_tokens(tokenizer, prompt: str) -> list[int]: - """Tokenize ``prompt`` without adding special tokens.""" + +def _encode_tokens(tokenizer: Any, prompt: str) -> list[int]: + if _vllm_encode_tokens is not None: + return _vllm_encode_tokens(tokenizer, prompt) return tokenizer.encode(prompt, add_special_tokens=False) @@ -2561,7 +2572,11 @@ class MiniCPMWhisperEncoder(WhisperEncoder): return_dict=None, past_key_values: EncoderDecoderCache | None = None, use_cache: bool | None = None, - ) -> BaseModelOutputWithPast | tuple: + use_extra_context: bool | None = False, + prefix_extra_frames: int | None = 1, + suffix_extra_frames: int | None = 1, + cnn_min_length: int | None = None, + ) -> BaseModelOutputWithPast | tuple[Any, ...]: r""" Forward pass of the Whisper encoder. @@ -2672,9 +2687,37 @@ class MiniCPMWhisperEncoder(WhisperEncoder): # Ignore copy input_features = input_features.to(dtype=self.conv1.weight.dtype, device=self.conv1.weight.device) + original_length = input_features.shape[2] + padded_for_cnn = False + if cnn_min_length is not None and original_length < cnn_min_length: + padded_features = torch.zeros( + input_features.shape[0], + input_features.shape[1], + cnn_min_length, + dtype=input_features.dtype, + device=input_features.device, + ) + padded_features[:, :, :original_length] = input_features + input_features = padded_features + padded_for_cnn = True + inputs_embeds = nn.functional.gelu(self.conv1(input_features)) inputs_embeds = nn.functional.gelu(self.conv2(inputs_embeds)) + if padded_for_cnn: + actual_cnn_output_length = (original_length + 1) // 2 + inputs_embeds = inputs_embeds[:, :, :actual_cnn_output_length] + + if use_extra_context: + prefix_extra_frames = int(prefix_extra_frames or 0) + suffix_extra_frames = int(suffix_extra_frames or 0) + prefix_to_remove = (prefix_extra_frames + 1) // 2 if prefix_extra_frames > 0 else 0 + suffix_to_remove = (suffix_extra_frames + 1) // 2 if suffix_extra_frames > 0 else 0 + if prefix_to_remove > 0: + inputs_embeds = inputs_embeds[:, :, prefix_to_remove:] + if 0 < suffix_to_remove < inputs_embeds.shape[2]: + inputs_embeds = inputs_embeds[:, :, :-suffix_to_remove] + inputs_embeds = inputs_embeds.permute(0, 2, 1) embed_pos = self.embed_positions.weight @@ -3328,7 +3371,7 @@ class MiniCPMO45OmniLLMMultiModalProcessor(BaseMultiModalProcessor[MiniCPMO45Omn tokenization_kwargs=tokenization_kwargs, ) tokenizer = self.info.get_tokenizer() - prompt_ids = encode_tokens(tokenizer, prompt) + prompt_ids = _encode_tokens(tokenizer, prompt) else: prompt_ids = self._apply_hf_processor_tokens_only(prompt) if use_tts: @@ -3817,8 +3860,18 @@ class MiniCPMO45OmniLLMForConditionalGeneration(nn.Module, SupportsMultiModal, S else: text_config = Qwen2Config.from_dict(config_dict) llm_arch = "Qwen2ForCausalLM" + from vllm_omni.experimental.fullduplex.minicpmo45.compat import ( + patch_minicpmo_remote_config, + ) + + patch_minicpmo_remote_config(text_config) + text_vllm_config = ( + vllm_config.with_hf_config(text_config) + if callable(getattr(vllm_config, "with_hf_config", None)) + else vllm_config + ) self.llm = init_vllm_registered_model( - vllm_config=vllm_config, + vllm_config=text_vllm_config, prefix=maybe_prefix(prefix, "llm"), hf_config=text_config, architectures=[llm_arch], @@ -3846,11 +3899,13 @@ class MiniCPMO45OmniLLMForConditionalGeneration(nn.Module, SupportsMultiModal, S self.audio_avg_pooler = nn.AvgPool1d(config.audio_pool_step, stride=config.audio_pool_step) self.audio_projection_layer = MultiModalProjector(in_dim=audio_output_dim, out_dim=embed_dim) self.audio_encoder_layer = -1 + self.audio_past_key_values = None else: self.apm = None self.audio_avg_pooler = None self.audio_projection_layer = None self.audio_encoder_layer = None + self.audio_past_key_values = None self.mm_token_ids = set[int]() self.make_empty_intermediate_tensors = self.llm.make_empty_intermediate_tensors @@ -4279,6 +4334,100 @@ class MiniCPMO45OmniLLMForConditionalGeneration(nn.Module, SupportsMultiModal, S return final_audio_embeds + def get_audio_embedding_streaming( + self, + data: MiniCPMOAudioFeatureInputs, + use_extra_context: bool = False, + prefix_extra_frames: int = 1, + suffix_extra_frames: int = 1, + cnn_min_length: int | None = None, + ) -> list[list[torch.Tensor]]: + if self.apm is None or self.audio_projection_layer is None or self.audio_avg_pooler is None: + return [] + + wavforms = data["audio_features"] + audio_feature_lens_raw = data["audio_feature_lens"] + if isinstance(wavforms, list): + if not wavforms: + return [] + batch_size = len(wavforms) + channels = wavforms[0].shape[-2] + max_len = max(item.shape[-1] for item in wavforms) + wavforms_tensor = torch.zeros( + (batch_size, channels, max_len), + dtype=wavforms[0].dtype, + device=wavforms[0].device, + ) + for idx, item in enumerate(wavforms): + wavforms_tensor[idx, ..., : item.shape[-1]] = item + wavforms = wavforms_tensor + if isinstance(audio_feature_lens_raw, torch.Tensor): + audio_feature_lens_raw = audio_feature_lens_raw.unbind(0) + if wavforms.shape[0] == 0: + return [] + audio_feature_lens = torch.hstack(audio_feature_lens_raw) + batch_size, _, max_mel_seq_len = wavforms.shape + assert batch_size == 1 + + current_seq_len = (max_mel_seq_len - 1) // 2 + 1 + if use_extra_context: + prefix_to_remove = (int(prefix_extra_frames) + 1) // 2 if prefix_extra_frames > 0 else 0 + suffix_to_remove = (int(suffix_extra_frames) + 1) // 2 if suffix_extra_frames > 0 else 0 + current_seq_len = current_seq_len - prefix_to_remove - suffix_to_remove + if current_seq_len <= 0: + return [] + + if self.audio_past_key_values is not None: + cache_length = self.audio_past_key_values[0][0].shape[2] + apm_max_len = self.apm.embed_positions.weight.shape[0] + if cache_length + current_seq_len >= apm_max_len: + logger.warning( + "audio_past_key_values length %s exceeds %s, reset.", + cache_length + current_seq_len, + apm_max_len, + ) + self.audio_past_key_values = None + + past_len = 0 + if self.audio_past_key_values is not None: + past_len = self.audio_past_key_values[0][0].shape[2] + total_seq_len = past_len + current_seq_len + audio_attention_mask = torch.zeros( + (batch_size, 1, current_seq_len, total_seq_len), + dtype=self.apm.conv1.weight.dtype, + device=wavforms.device, + ) + + audio_outputs = self.apm( + wavforms, + past_key_values=self.audio_past_key_values, + use_cache=True, + output_hidden_states=True, + attention_mask=audio_attention_mask, + use_extra_context=use_extra_context, + prefix_extra_frames=prefix_extra_frames, + suffix_extra_frames=suffix_extra_frames, + cnn_min_length=cnn_min_length, + ) + audio_states = audio_outputs.hidden_states[self.audio_encoder_layer] + self.audio_past_key_values = audio_outputs.past_key_values + + audio_embeds = self.audio_projection_layer(audio_states) + audio_embeds = audio_embeds.transpose(1, 2) + audio_embeds = self.audio_avg_pooler(audio_embeds) + audio_embeds = audio_embeds.transpose(1, 2) + + _, feature_lens_after_pooling = self._get_feat_extract_output_lengths(audio_feature_lens) + final_audio_embeds: list[list[torch.Tensor]] = [] + idx = 0 + for i in range(len(audio_feature_lens_raw)): + target_audio_embeds = [] + for _ in range(len(audio_feature_lens_raw[i])): + target_audio_embeds.append(audio_embeds[idx, : feature_lens_after_pooling[idx], :]) + idx += 1 + final_audio_embeds.append(target_audio_embeds) + return final_audio_embeds + def _process_audio_input( self, audio_input: MiniCPMOAudioInputs, diff --git a/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_tts.py b/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_tts.py index f109eb737..73570ef74 100644 --- a/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_tts.py +++ b/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_tts.py @@ -12,11 +12,19 @@ Pipeline: 5. Run Token2wav(tokens) -> waveform bytes -> numpy array """ +import hashlib import io import logging import os import sys +import tempfile +import threading +import time +from collections import OrderedDict from collections.abc import Iterable +from dataclasses import dataclass +from importlib import import_module +from typing import Any import numpy as np import soundfile as sf @@ -25,14 +33,17 @@ import torch.nn as nn import torch.nn.functional as F from vllm.config import VllmConfig from vllm.model_executor.models.interfaces import SupportsPP +from vllm.v1.outputs import SamplerOutput +from vllm_omni.experimental.fullduplex.engine.intermediate import get_stream_request_key, get_tts_handoff from vllm_omni.model_executor.model_loader.weight_utils import ( download_weights_from_hf_specific, ) from vllm_omni.platforms import current_omni_platform -# Preserve the established external vocoder on CUDA. Ascend uses the in-tree -# adapter because ``stepaudio2-minicpmo`` hard-codes CUDA device placement. +# The external vocoder hard-codes CUDA placement. Ascend uses the in-tree +# adapter, with the external package retained as a fallback for compatibility. +_stepaudio2_import_error: ImportError | None = None if current_omni_platform.is_npu(): try: from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_token2wav import ( @@ -40,28 +51,54 @@ if current_omni_platform.is_npu(): ) _token2wav_backend = "step_audio2_core" - except ImportError: + except ImportError as e: + _stepaudio2_import_error = e try: from stepaudio2 import Token2wav as _Token2wav _token2wav_backend = "stepaudio2_pkg" - except ImportError: + except ImportError as fallback_error: _Token2wav = None _token2wav_backend = None + _stepaudio2_import_error = fallback_error else: try: from stepaudio2 import Token2wav as _Token2wav _token2wav_backend = "stepaudio2_pkg" - except ImportError: + except ImportError as e: _Token2wav = None _token2wav_backend = None + _stepaudio2_import_error = e _stepaudio2_available = _Token2wav is not None logger = logging.getLogger(__name__) +@dataclass(slots=True) +class MiniCPMO45TTSRuntimeConfig: + """Internal MiniCPM-o 4.5 Talker runtime defaults. + + These values are deliberately not process environment knobs. If a value + needs to become user configurable, route it through stage/model config with + an explicit API contract instead of adding another ad hoc env var. + """ + + token2wav_n_timesteps: int = 10 + tts_dtype: torch.dtype = torch.float32 + token2wav_autocast_dtype: torch.dtype | None = None + use_direct_token2wav: bool = True + ref_audio_file_cache_size: int = 16 + max_token_ratio: int = 32 + min_max_new_tokens: int = 256 + hard_max_new_tokens: int = 16384 + min_new_tokens: int = 50 + streaming_generator_chunk: int = 25 + streaming_vocoder_threshold: int = 2500 + streaming_vocoder_chunk: int = 50 + + def _install_torchaudio_soundfile_shim() -> None: """Monkey-patch torchaudio.load to use soundfile instead of the default torchcodec backend, which requires libtorchcodec/ffmpeg shared libs that @@ -94,9 +131,118 @@ def _install_torchaudio_soundfile_shim() -> None: _install_torchaudio_soundfile_shim() +class _TalkerTurnState: + """Per-turn talker continuity for native duplex. + + The official duplex implementation runs ONE TTS stream per spoken turn: + each 1s unit appends its condition to the same KV (carried + past_key_values + text_start_pos) and the token2wav stream caches and + token buffer persist across units, reset only at turn end. Synthesizing + units as independent utterances resets prosody every second and garbles + the reply. + """ + + __slots__ = ( + "past_key_values", + "text_start_pos", + "token2wav_buffer", + "prompt_wav_path", + "temp_prompt_wav_path", + "epoch", + "turn_id", + "pending_text", + "stream_cache", + "hift_cache_dict", + "vocoder_initialized", + ) + + def __init__( + self, + prompt_wav_path, + temp_prompt_wav_path, + *, + epoch: int | None = None, + turn_id: int | None = None, + ): + self.past_key_values = None + self.text_start_pos = 0 + # Official seeds each turn's vocoder buffer with three silence + # tokens so the first synthesized window does not directly abut the + # reference-audio prompt cache (audible ref-voice bleed otherwise). + self.token2wav_buffer: list[int] = [_T2W_SILENCE_TOKEN] * 3 + self.prompt_wav_path = prompt_wav_path + self.temp_prompt_wav_path = temp_prompt_wav_path + self.epoch = epoch + self.turn_id = turn_id + self.pending_text = "" + self.stream_cache = None + self.hift_cache_dict: dict[str, Any] = {} + self.vocoder_initialized = False + + +def _queue_native_duplex_segment_text(state: _TalkerTurnState, text: object) -> None: + if isinstance(text, str) and text: + state.pending_text += text + + +def _drain_native_duplex_emitted_text(state: _TalkerTurnState, *, has_audio: bool) -> str: + if not has_audio: + return "" + text = state.pending_text + state.pending_text = "" + return text + + +_T2W_SILENCE_TOKEN = 4218 +_NATIVE_DUPLEX_UNIT_AUDIO_SAMPLES = 24_000 + + +def _native_duplex_unit_waveform( + waveforms: Iterable[torch.Tensor], + *, + turn_end: bool, + target_samples: int = _NATIVE_DUPLEX_UNIT_AUDIO_SAMPLES, +) -> torch.Tensor | None: + pieces = [torch.as_tensor(waveform).reshape(-1).cpu().contiguous() for waveform in waveforms] + pieces = [piece for piece in pieces if piece.numel() > 0] + if not pieces: + return None + waveform = pieces[0] if len(pieces) == 1 else torch.cat(pieces, dim=0) + if not turn_end and waveform.numel() < target_samples: + waveform = F.pad(waveform, (target_samples - waveform.numel(), 0)) + return waveform + + +def _soundfile_patched_save(orig_save): + def _patched_save(uri, src, sample_rate, **kw): + kw.pop("backend", None) + if hasattr(uri, "write"): + sf.write(uri, src.cpu().numpy().T, sample_rate, format="WAV") + return + return orig_save(uri, src, sample_rate, backend="soundfile", **kw) + + return _patched_save + + +def _torch_clone_recursive(obj): + if isinstance(obj, torch.Tensor): + return obj.clone() + if isinstance(obj, dict): + return {k: _torch_clone_recursive(v) for k, v in obj.items()} + if isinstance(obj, list): + return [_torch_clone_recursive(v) for v in obj] + if isinstance(obj, tuple): + return tuple(_torch_clone_recursive(v) for v in obj) + return obj + + class MiniCPMO45OmniTTSForConditionalGeneration(nn.Module, SupportsPP): """MiniCPM-o 4.5 Talker: MiniCPMTTS + Token2wav in a single forward pass.""" + # llm2tts hands the FULL accumulated condition per handoff (the runner's + # resume-prefill path REPLACES the streaming buffer, so runner-side + # accumulation is lossy); the per-turn state consumes it by cursor. + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): super().__init__() from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni_llm import MiniCPMOConfig @@ -104,40 +250,114 @@ class MiniCPMO45OmniTTSForConditionalGeneration(nn.Module, SupportsPP): config: MiniCPMOConfig = vllm_config.model_config.hf_config self.config = config self.vllm_config = vllm_config + self._runtime_config = MiniCPMO45TTSRuntimeConfig() self.tts = None self.audio_tokenizer = None self._assets_loaded = False - self._model_path = None + self._model_path: str | None = None + self._stream_gens: dict[str, Any] = {} + self._talker_turn_states: dict[str, _TalkerTurnState] = {} + # Consumed-cursor into the accumulated handoff condition for the + # currently open spoken turn. + self._talker_consumed_tokens: dict[str, int] = {} + self._talker_request_keys: dict[str, str] = {} + self._t2w_base_caches: dict[str, tuple[Any, Any]] = {} + self._token2wav_state_lock = threading.RLock() + self._ar_last_chunk_flags: list[bool] = [True] + self._ar_turn_end_flags: list[bool] = [False] + self._ar_last_emitted_text = "" + self._text_tokenizer = None tts_config = getattr(config, "tts_config", None) if tts_config is not None: self._tts_config = tts_config - self._tts_bos_id = getattr(tts_config, "audio_bos_token_id", 151687) - self._text_eos_id = getattr(tts_config, "text_eos_token_id", 151692) + self._tts_bos_id = self._config_token_id(tts_config, "audio_bos_token_id") + self._text_eos_id = self._config_token_id(tts_config, "text_eos_token_id") self._num_audio_tokens = getattr(tts_config, "num_audio_tokens", 6562) self._hidden_size = getattr(tts_config, "hidden_size", 768) self._normalize = getattr(tts_config, "normalize_projected_hidden", True) else: self._tts_config = None + def _tts_runtime_config(self) -> MiniCPMO45TTSRuntimeConfig: + cfg = getattr(self, "_runtime_config", None) + if cfg is None: + cfg = MiniCPMO45TTSRuntimeConfig() + self._runtime_config = cfg + return cfg + + @staticmethod + def _config_token_id(config: Any, attr: str) -> int: + value = getattr(config, attr, None) + if isinstance(value, (list, tuple)): + value = value[0] if value else None + if value is None: + raise ValueError(f"MiniCPM-o 4.5 TTS config missing required {attr}") + return int(value) + + def _get_text_tokenizer(self) -> Any: + tokenizer = getattr(self, "_text_tokenizer", None) + if tokenizer is not None: + return tokenizer + from vllm.transformers_utils.tokenizer import cached_tokenizer_from_config + + tokenizer = cached_tokenizer_from_config(self.vllm_config.model_config) + self._text_tokenizer = tokenizer + return tokenizer + + def _tokenizer_token_id(self, token: str) -> int | None: + tokenizer = self._get_text_tokenizer() + unk_token_id = getattr(tokenizer, "unk_token_id", None) + convert = getattr(tokenizer, "convert_tokens_to_ids", None) + if callable(convert): + value = convert(token) + if isinstance(value, list): + value = value[0] if len(value) == 1 else None + try: + candidate = int(value) + except (TypeError, ValueError): + candidate = None + if candidate is not None and candidate >= 0 and candidate != unk_token_id: + return candidate + encode = getattr(tokenizer, "encode", None) + if callable(encode): + ids = list(encode(token, add_special_tokens=False)) + if len(ids) == 1: + candidate = int(ids[0]) + if candidate >= 0 and candidate != unk_token_id: + return candidate + return None + + def _scheduler_eos_token_id(self) -> int: + eos_raw = getattr(self.config, "eos_token_id", None) + if isinstance(eos_raw, (list, tuple)): + eos_raw = eos_raw[0] if eos_raw else None + if eos_raw is not None: + return int(eos_raw) + eos_id = self._tokenizer_token_id("<|im_end|>") + if eos_id is None: + raise ValueError( + "MiniCPM-o 4.5 TTS scheduler EOS requires config.eos_token_id or tokenizer-defined <|im_end|>" + ) + return eos_id + def _lazy_init_tts(self): if self._assets_loaded or self._tts_config is None: return + self._assets_loaded = True try: model_path = download_weights_from_hf_specific(self.vllm_config.model_config.model, None, ["*"]) self._model_path = model_path - if model_path not in sys.path: sys.path.insert(0, model_path) from transformers import AutoImageProcessor from transformers.dynamic_module_utils import get_class_from_dynamic_module - # openbmb/MiniCPM-o-4_5/processing_minicpmo.py registers via a - # string: AutoImageProcessor.register("MiniCPMVImageProcessor", ...), - # which crashes on transformers>=5 (register reads key.__module__). - # Loading MiniCPMTTS imports that module, so no-op the string form - # (unused by the standalone talker) while it runs, then restore. + # The remote processing module registers an image processor by + # string, which transformers>=5 rejects. The standalone talker does + # not use that registration, so ignore only the string form while + # importing MiniCPMTTS and restore the global method immediately. original_register = AutoImageProcessor.register AutoImageProcessor.register = ( # type: ignore[method-assign] lambda key, *a, **k: None if isinstance(key, str) else original_register(key, *a, **k) @@ -147,28 +367,30 @@ class MiniCPMO45OmniTTSForConditionalGeneration(nn.Module, SupportsPP): finally: AutoImageProcessor.register = original_register # type: ignore[method-assign] - # MiniCPMTTS.__init__ reads `config.top_p / top_k / repetition_penalty` - # directly (modeling_minicpmo.py L4112-4114), but the model repo's - # config.json `tts_config` block does not declare these fields and - # PretrainedConfig in recent transformers no longer surfaces - # generation-style params on `self.config`. Inject the defaults the - # upstream code itself ships with (modeling_minicpmo.py L2212-2214, - # L3132-3133) so attribute access does not raise. - for _attr, _default in (("top_p", 0.8), ("top_k", 100), ("repetition_penalty", 1.02)): - if not hasattr(self._tts_config, _attr): - setattr(self._tts_config, _attr, _default) - - # The copied Hugging Face flash_attention_2 setting is not valid - # for this standalone MiniCPMTTS path. Use PyTorch SDPA on every - # backend until a dedicated flash-attention implementation exists. - self._tts_config.attn_implementation = "sdpa" - prev_dtype = torch.get_default_dtype() torch.set_default_dtype(torch.float32) try: + for name, default in ( + ("top_p", 0.85), + ("top_k", 25), + ("repetition_penalty", 1.05), + ("temperature", 0.8), + ): + if not hasattr(self._tts_config, name): + setattr(self._tts_config, name, default) + self._tts_config.attn_implementation = "sdpa" self.tts_obj = MiniCPMTTS(config=self._tts_config, audio_tokenizer=None) finally: torch.set_default_dtype(prev_dtype) + tts_module = import_module(self.tts_obj.__class__.__module__) + + def get_tts_module_attr(name: str): + return self.tts_obj.generate.__globals__.get(name) or getattr(tts_module, name, None) + + self._tts_sampling_params_cls = get_tts_module_attr("TTSSamplingParams") + self._tts_gen_logits = get_tts_module_attr("gen_logits") + self._tts_parametrize = get_tts_module_attr("P") + self._tts_streaming_generator_cls = get_tts_module_attr("TTSStreamingGenerator") self.emb_text = self.tts_obj.emb_text self.projector_semantic = self.tts_obj.projector_semantic @@ -176,119 +398,914 @@ class MiniCPMO45OmniTTSForConditionalGeneration(nn.Module, SupportsPP): if os.path.isdir(token2wav_dir): if not _stepaudio2_available: raise ImportError( - "MiniCPM-o 4.5 token2wav stage requires the `stepaudio2` Python " - "module (a MiniCPM-o-flavored Token2wav vocoder, NOT the upstream " - "stepfun-ai/Step-Audio2 — the upstream signature does not accept " - "n_timesteps and will fail at __init__). Install via:\n" - " pip install 'vllm-omni[minicpmo]' # recommended, declared as PR extra\n" - "Equivalent direct installs of the same `from stepaudio2 import Token2wav`\n" - "entry point used by openbmb/MiniCPM-o-4_5/modeling_minicpmo.py:\n" - " pip install stepaudio2-minicpmo # bare token2wav package\n" - " pip install 'minicpmo-utils[all]' # MiniCPM-o umbrella (also brings image/video deps)" - ) + "MiniCPM-o 4.5 token2wav stage requires the stepaudio2 package, " + "and all of its runtime dependencies." + ) from _stepaudio2_import_error + self._token2wav_n_timesteps = self._tts_runtime_config().token2wav_n_timesteps prev_dtype2 = torch.get_default_dtype() torch.set_default_dtype(torch.float32) try: - # NB: this must be the MiniCPM-o-flavored Token2wav from - # the `stepaudio2-minicpmo` PyPI package (or the - # `minicpmo-utils[all]` umbrella), not the upstream - # `stepfun-ai/Step-Audio2` repo. The MiniCPM-o variant's - # __init__ accepts n_timesteps; the upstream signature is - # (model_path, float16=False) and will raise - # TypeError on n_timesteps. See ImportError message below - # for installation guidance. - self.audio_tokenizer = _Token2wav(token2wav_dir, float16=False, n_timesteps=10) + self.audio_tokenizer = _Token2wav( + token2wav_dir, + float16=False, + n_timesteps=self._token2wav_n_timesteps, + ) finally: torch.set_default_dtype(prev_dtype2) self.tts_obj.audio_tokenizer = self.audio_tokenizer logger.info( - "Loaded Token2wav from %s (backend=%s)", + "Loaded Token2wav from %s (backend=%s, n_timesteps=%d)", token2wav_dir, _token2wav_backend, + self._token2wav_n_timesteps, ) - # Only mark init as complete after every step succeeds, so a - # partial failure leaves the next call free to retry the full - # init instead of short-circuiting back to a silent empty path. - self._assets_loaded = True except ImportError: # Surface missing dependencies directly so users can act on them # instead of getting a silent None waveform downstream. raise - except Exception: - # Re-raise non-import init failures (bad token2wav assets, missing - # weights, OOM during Token2wav construction, etc.) so the server - # fails loudly at startup / first request instead of returning - # silent empty audio for every subsequent request. - logger.error("Failed to init 4.5 TTS", exc_info=True) - raise + except Exception as e: + logger.error("Failed to init 4.5 TTS: %s", e, exc_info=True) - def generate_speech( - self, - tts_token_ids: torch.Tensor, - tts_hidden_states: torch.Tensor, - ) -> np.ndarray | None: - """Run full 4.5 TTS pipeline using original MiniCPMTTS.generate.""" - self._lazy_init_tts() - if not hasattr(self, "tts_obj") or self.tts_obj is None: - logger.warning("generate_speech: tts_obj not initialized") + def _build_tts_sampling_params(self): + params_cls = getattr(self, "_tts_sampling_params_cls", None) + if params_cls is None or not hasattr(self, "tts_obj"): return None tts = self.tts_obj - device = tts.emb_text.weight.device - # MiniCPMTTS AR backbone uses FlashAttention (fp16/bf16 only). The - # submodule is constructed under float32 default dtype during lazy init, - # so pin the condition embeddings to bfloat16 explicitly rather than - # inheriting the (float32) parameter dtype — a float32 condition breaks - # the CUDA FA2 path and wastes memory on the NPU sdpa path. - ar_dtype = torch.bfloat16 + top_p = getattr(tts, "top_p", getattr(tts.config, "top_p", 0.85)) + top_k = getattr(tts, "top_k", getattr(tts.config, "top_k", 25)) + repetition_penalty = getattr( + tts, + "repetition_penalty", + getattr(tts.config, "repetition_penalty", 1.05), + ) + temperature = getattr(tts.config, "temperature", 0.8) + + return params_cls( + top_p=None if top_p is not None and top_p >= 1.0 else top_p, + top_k=None if top_k is not None and top_k <= 0 else top_k, + repetition_penalty=repetition_penalty, + temperature=temperature, + ) + + def _target_tts_dtype(self) -> torch.dtype: + return self._tts_runtime_config().tts_dtype + + def _token2wav_autocast_context(self): + dtype = self._tts_runtime_config().token2wav_autocast_dtype + device_type = str(current_omni_platform.device_type) + if dtype is None: + return ( + current_omni_platform.create_autocast_context( + device_type=device_type, + dtype=torch.float32, + enabled=False, + ), + "off", + ) + if dtype is torch.bfloat16: + return ( + current_omni_platform.create_autocast_context( + device_type=device_type, + dtype=torch.bfloat16, + enabled=True, + ), + "bf16", + ) + if dtype is torch.float16: + return ( + current_omni_platform.create_autocast_context( + device_type=device_type, + dtype=torch.float16, + enabled=True, + ), + "fp16", + ) + raise ValueError("MiniCPM-o 4.5 token2wav autocast only supports None, bfloat16, or float16") + + def _should_use_direct_token2wav(self) -> bool: + return self._tts_runtime_config().use_direct_token2wav + + def _should_stream_output(self, info: dict[str, Any] | None = None) -> bool: + if isinstance(info, dict): + for key in ("stream_output", "native_duplex"): + value = info.get(key) + if isinstance(value, bool): + return value + return False + + def _token2wav_prompt_cache_key(self, prompt_wav: str | None) -> str | None: + return os.path.abspath(prompt_wav) if prompt_wav else None + + def _reset_token2wav_cache_if_needed(self, prompt_wav: str | None) -> None: + token2wav = self.audio_tokenizer + if token2wav is None: + return + cache_key = self._token2wav_prompt_cache_key(prompt_wav) + if getattr(self, "_token2wav_prompt_cache_id", None) != cache_key: + token2wav.cache = None + self._token2wav_prompt_cache_id = cache_key + + def _normalize_ref_audio_tensor(self, ref_audio) -> np.ndarray | None: + if ref_audio is None: + return None + if isinstance(ref_audio, torch.Tensor): + waveform = ref_audio.detach().float().cpu().numpy() + else: + waveform = np.asarray(ref_audio, dtype=np.float32) + if waveform.ndim > 1: + if waveform.shape[0] <= 2 and waveform.shape[-1] > waveform.shape[0]: + waveform = waveform.mean(axis=0) + else: + waveform = waveform.mean(axis=-1) + waveform = np.asarray(waveform, dtype=np.float32).reshape(-1) + return waveform if waveform.size else None + + def _write_ref_audio_prompt_wav(self, ref_audio, ref_audio_sr: int | None) -> str | None: + waveform = self._normalize_ref_audio_tensor(ref_audio) + if waveform is None: + return None + sample_rate = int(ref_audio_sr or 24000) + cache_size = self._tts_runtime_config().ref_audio_file_cache_size + if cache_size > 0: + digest = hashlib.sha256() + digest.update(str(sample_rate).encode("ascii")) + digest.update(waveform.tobytes()) + cache_key = digest.hexdigest() + cache = getattr(self, "_ref_audio_prompt_files", None) + if cache is None: + cache = OrderedDict() + self._ref_audio_prompt_files = cache + cached_path = cache.get(cache_key) + if cached_path and os.path.exists(cached_path): + cache.move_to_end(cache_key) + return cached_path + + tmp_path = os.path.join(tempfile.gettempdir(), f"minicpmo45_ref_{cache_key[:24]}_{sample_rate}.wav") + if not os.path.exists(tmp_path): + sf.write(tmp_path, waveform, sample_rate, format="WAV") + cache[cache_key] = tmp_path + cache.move_to_end(cache_key) + while len(cache) > cache_size: + _, evicted_path = cache.popitem(last=False) + with self._token2wav_lock(): + self._t2w_base_caches.pop(evicted_path, None) + try: + os.unlink(evicted_path) + except OSError: + pass + return tmp_path + + tmp = tempfile.NamedTemporaryFile(prefix="minicpmo45_ref_", suffix=".wav", delete=False) + tmp_path = tmp.name + tmp.close() + sf.write(tmp_path, waveform, sample_rate, format="WAV") + return tmp_path + + def _is_cached_ref_audio_prompt_wav(self, prompt_wav: str | None) -> bool: + cache = getattr(self, "_ref_audio_prompt_files", None) + return bool(prompt_wav and cache and prompt_wav in cache.values()) + + def _run_token2wav_direct( + self, + generated_speech_tokens: list[int] | torch.Tensor, + prompt_wav: str | None, + ) -> tuple[torch.Tensor, int]: + """Run Token2wav without WAV encode/decode round-tripping. + + ``stepaudio2.Token2wav.__call__`` renders the GPU waveform to a WAV + BytesIO object and the vLLM-Omni adapter immediately decodes it back to + a numpy waveform. The engine already expects a float waveform tensor, + so keep the same flow/HIFT computation and return the waveform directly. + This also makes the configured ``n_timesteps`` apply to one-shot + inference; upstream currently hard-codes ``10`` in ``__call__``. + """ + token2wav = self.audio_tokenizer + if token2wav is None: + raise RuntimeError("Token2wav is not initialized") + required_attrs = ("_prepare_prompt", "flow", "hift") + if any(not hasattr(token2wav, attr) for attr in required_attrs): + raise RuntimeError("Token2wav direct path is incompatible with the installed stepaudio2 package") + + self._reset_token2wav_cache_if_needed(prompt_wav) + if token2wav.cache is None: + token2wav.cache = token2wav._prepare_prompt(prompt_wav) + prompt_speech_tokens, prompt_speech_tokens_lens, spk_emb, prompt_mels, prompt_mels_lens = token2wav.cache + + device = prompt_speech_tokens.device + if isinstance(generated_speech_tokens, torch.Tensor): + generated = generated_speech_tokens + if generated.ndim == 1: + generated = generated.unsqueeze(0) + elif generated.ndim == 3 and generated.shape[-1] == 1: + generated = generated.squeeze(-1) + generated = generated.to(device=device, dtype=torch.int32) + else: + generated = torch.tensor([generated_speech_tokens], dtype=torch.int32, device=device) + generated_lens = torch.tensor([generated.shape[1]], dtype=torch.int32, device=device) + mel = token2wav.flow.inference( + generated, + generated_lens, + prompt_speech_tokens, + prompt_speech_tokens_lens, + prompt_mels, + prompt_mels_lens, + spk_emb, + self._token2wav_n_timesteps, + ) + wav, _ = token2wav.hift(speech_feat=mel) + waveform = wav.squeeze(0).detach().float().reshape(-1).cpu().contiguous() + return waveform, 24000 + + def _build_tts_condition_embeds( + self, + tts_token_ids: torch.Tensor, + tts_hidden_states: torch.Tensor, + ) -> torch.Tensor: + tts = self.tts_obj + device = tts.emb_text.weight.device + dtype = tts.emb_text.weight.dtype llm_embeds = tts.emb_text(tts_token_ids.to(device)) - hidden_embeds = tts.projector_semantic(tts_hidden_states.to(device=device, dtype=ar_dtype)) + hidden_embeds = tts.projector_semantic(tts_hidden_states.to(device=device, dtype=dtype)) if getattr(tts.config, "normalize_projected_hidden", False): hidden_embeds = F.normalize(hidden_embeds, p=2, dim=-1) - tts_embeds = (llm_embeds + hidden_embeds).to(dtype=ar_dtype) + return llm_embeds + hidden_embeds - text_eos = tts.emb_text(torch.tensor([tts.config.text_eos_token_id], device=device, dtype=torch.long)) - audio_bos = tts.emb_text(torch.tensor([tts.audio_bos_token_id], device=device, dtype=torch.long)) - spk_embeds = torch.zeros(0, tts.config.hidden_size, device=device, dtype=ar_dtype) + def _normalize_tts_handoff_tensors( + self, + tts_token_ids: Any, + tts_hidden_states: Any, + ) -> tuple[torch.Tensor, torch.Tensor]: + token_ids = torch.as_tensor(tts_token_ids, dtype=torch.long) + if token_ids.ndim == 0: + token_ids = token_ids.reshape(1) + elif token_ids.ndim > 1: + token_ids = token_ids.reshape(-1) - inputs_embeds = torch.cat([spk_embeds, tts_embeds, text_eos, audio_bos], dim=0).unsqueeze(0) - inputs_embeds = inputs_embeds.to(dtype=ar_dtype) - logger.info("generate_speech: inputs_embeds shape=%s", list(inputs_embeds.shape)) + hidden_states = torch.as_tensor(tts_hidden_states, dtype=torch.float32) + if hidden_states.ndim == 1: + hidden_states = hidden_states.unsqueeze(0) + elif hidden_states.ndim == 3 and hidden_states.shape[0] == 1: + hidden_states = hidden_states.squeeze(0) + elif hidden_states.ndim != 2: + hidden_states = hidden_states.reshape(token_ids.numel(), -1) - # Scale max_new_token with input text length to avoid mid-stream truncation on long - # responses (default 2048 can only cover ~300 text tokens at ~6x audio/text ratio). - # Empirically 511 text tokens → 1951 audio tokens (~3.8x) finishes cleanly, so use 10x - # as a safe upper bound with a floor of 2048 and a hard cap of 16384 to bound latency/mem. - num_text = int(tts_token_ids.shape[-1]) if tts_token_ids.ndim > 0 else 0 - max_new_token = max(2048, min(16384, num_text * 10)) + if hidden_states.shape[0] != token_ids.numel(): + raise ValueError( + "MiniCPM-o 4.5 TTS handoff has mismatched token/hidden lengths: " + f"tokens={token_ids.numel()} hidden_rows={hidden_states.shape[0]}" + ) + return token_ids.contiguous(), hidden_states.contiguous() - eos_token = torch.tensor([tts.config.num_audio_tokens - 1], dtype=torch.long, device=device) - outputs = tts.generate( - inputs_embeds=inputs_embeds, - eos_token=eos_token, - max_new_token=max_new_token, - show_tqdm=False, - ) - generated_tokens = outputs.new_ids.squeeze(-1) - logger.info( - "generate_speech: generated %d audio tokens (cap=%d, text_tokens=%d)", - generated_tokens.shape[-1], - max_new_token, - num_text, + def _resolve_prompt_wav_path(self, ref_audio, ref_audio_sr: int | None) -> tuple[str | None, str | None]: + temp_prompt_wav_path = self._write_ref_audio_prompt_wav(ref_audio, ref_audio_sr) + if temp_prompt_wav_path is not None: + return temp_prompt_wav_path, temp_prompt_wav_path + if (model_path := getattr(self, "_model_path", None)) is not None: + default_ref = os.path.join(model_path, "assets", "HT_ref_audio.wav") + if os.path.exists(default_ref): + return default_ref, None + return None, None + + def _max_tts_tokens_for_text(self, num_text: int) -> tuple[int, int]: + cfg = self._tts_runtime_config() + max_new_token = min( + cfg.hard_max_new_tokens, + max(cfg.min_max_new_tokens, num_text * cfg.max_token_ratio), ) + return cfg.min_new_tokens, max_new_token - if self.audio_tokenizer is None: - logger.warning("No audio_tokenizer") + def _stream_request_key(self, info: dict[str, Any]) -> str: + return get_stream_request_key(info) + + @staticmethod + def _coerce_request_key(value: Any) -> str | None: + if isinstance(value, (list, tuple)): + value = value[0] if value else None + if isinstance(value, bytes): + value = value.decode("utf-8", errors="replace") + if value is None: + return None + text = str(value) + return text if text else None + + @staticmethod + def _coerce_epoch(value: Any) -> int | None: + try: + return int(value) if value is not None else None + except (TypeError, ValueError): return None + _coerce_turn_id = _coerce_epoch + + def _stream_epoch(self, info: dict[str, Any]) -> int | None: + duplex = info.get("duplex") + if isinstance(duplex, dict): + epoch = self._coerce_epoch(duplex.get("epoch")) + if epoch is not None: + return epoch + meta = info.get("meta") + if isinstance(meta, dict): + epoch = self._coerce_epoch(meta.get("epoch")) + if epoch is not None: + return epoch + return self._coerce_epoch(info.get("epoch")) + + def _stream_turn_id(self, info: dict[str, Any]) -> int | None: + duplex = info.get("duplex") + if isinstance(duplex, dict): + turn_id = self._coerce_turn_id(duplex.get("model_turn_id")) + if turn_id is not None: + return turn_id + turn_id = self._coerce_turn_id(duplex.get("turn_id")) + if turn_id is not None: + return turn_id + meta = info.get("meta") + if isinstance(meta, dict): + turn_id = self._coerce_turn_id(meta.get("turn_id")) + if turn_id is not None: + return turn_id + return self._coerce_turn_id(info.get("turn_id")) + + def _remember_talker_request_key(self, info: dict[str, Any], key: str) -> None: + aliases = { + self._coerce_request_key(info.get("request_id")), + self._coerce_request_key(info.get("_omni_req_id")), + } + duplex = info.get("duplex") + if isinstance(duplex, dict): + aliases.add(self._coerce_request_key(duplex.get("request_id"))) + request_keys = getattr(self, "_talker_request_keys", None) + if request_keys is None: + request_keys = {} + self._talker_request_keys = request_keys + for alias in aliases: + if alias and alias != key: + request_keys[alias] = key + + def _empty_audio_chunk(self) -> torch.Tensor: + return torch.zeros((0,), dtype=torch.float32) + + @staticmethod + def _extract_tts_handoff(info: dict[str, Any]) -> tuple[Any, Any]: + return get_tts_handoff(info) + + def _native_duplex_input_ends_turn(self, info: dict[str, Any]) -> bool: + if info.get("native_duplex") is not True: + return False + meta = info.get("meta") + if not isinstance(meta, dict): + meta = {} + if bool(info.get("turn_end") or info.get("end_of_turn") or meta.get("turn_end") or meta.get("end_of_turn")): + return True + turn_eos_id = self._coerce_epoch(meta.get("turn_eos_token_id")) + if turn_eos_id is None: + return False + tts_token_ids, _ = self._extract_tts_handoff(info) + if isinstance(tts_token_ids, torch.Tensor): + return bool((tts_token_ids == turn_eos_id).any().item()) + if isinstance(tts_token_ids, np.ndarray): + return bool(np.any(tts_token_ids == turn_eos_id)) + if isinstance(tts_token_ids, (list, tuple)): + return turn_eos_id in tts_token_ids + return False + + def _t2w_pre_lookahead(self) -> int: + flow = getattr(self.audio_tokenizer, "flow", None) + try: + return int(getattr(flow, "pre_lookahead_len", 3) or 3) + except (TypeError, ValueError): + return 3 + + def _token2wav_lock(self) -> threading.RLock: + lock = getattr(self, "_token2wav_state_lock", None) + if lock is None: + lock = threading.RLock() + self._token2wav_state_lock = lock + return lock + + def _begin_turn_vocoder_cache( + self, + prompt_wav_path: str | None, + *, + state: _TalkerTurnState | None = None, + ) -> None: + """Restore a fresh per-turn clone of the ref-audio vocoder caches.""" import torchaudio - model_path = self._model_path or download_weights_from_hf_specific( - self.vllm_config.model_config.model, None, ["*"] + with self._token2wav_lock(): + cache_key = prompt_wav_path or "" + base = self._t2w_base_caches.get(cache_key) + if base is None: + if prompt_wav_path is None: + base = (None, {}) + else: + _orig_save = torchaudio.save + prev_dtype = torch.get_default_dtype() + torch.set_default_dtype(torch.float32) + try: + torchaudio.save = _soundfile_patched_save(_orig_save) + stream_cache, hift_cache_dict = self.audio_tokenizer.set_stream_cache(prompt_wav_path) + finally: + torch.set_default_dtype(prev_dtype) + torchaudio.save = _orig_save + base = ( + _torch_clone_recursive(stream_cache), + _torch_clone_recursive(hift_cache_dict), + ) + self._t2w_base_caches[cache_key] = base + stream_cache = _torch_clone_recursive(base[0]) + hift_cache_dict = _torch_clone_recursive(base[1]) + if state is None: + self.audio_tokenizer.stream_cache = stream_cache + self.audio_tokenizer.hift_cache_dict = hift_cache_dict + else: + state.stream_cache = stream_cache + state.hift_cache_dict = hift_cache_dict + state.vocoder_initialized = True + self.audio_tokenizer.stream_cache = None + self.audio_tokenizer.hift_cache_dict = {} + + def _t2w_stream_window(self, token_list: list[int], prompt_wav_path: str | None, *, last_chunk: bool): + import torchaudio + + _orig_save = torchaudio.save + prev_dtype = torch.get_default_dtype() + autocast_context, _ = self._token2wav_autocast_context() + torch.set_default_dtype(torch.float32) + try: + torchaudio.save = _soundfile_patched_save(_orig_save) + with autocast_context: + wav_np = self.audio_tokenizer.stream( + token_list, + prompt_wav_path, + last_chunk=bool(last_chunk), + return_waveform=True, + ) + finally: + torch.set_default_dtype(prev_dtype) + torchaudio.save = _orig_save + return torch.as_tensor(np.asarray(wav_np).reshape(-1), dtype=torch.float32).cpu().contiguous() + + def _run_vocoder_window( + self, + state: _TalkerTurnState, + token_list: list[int], + *, + last_chunk: bool, + ) -> torch.Tensor: + with self._token2wav_lock(): + if not state.vocoder_initialized: + self._begin_turn_vocoder_cache(state.prompt_wav_path, state=state) + self.audio_tokenizer.stream_cache = _torch_clone_recursive(state.stream_cache) + self.audio_tokenizer.hift_cache_dict = _torch_clone_recursive(state.hift_cache_dict) + try: + waveform = self._t2w_stream_window( + token_list, + state.prompt_wav_path, + last_chunk=last_chunk, + ) + state.stream_cache = _torch_clone_recursive(self.audio_tokenizer.stream_cache) + state.hift_cache_dict = _torch_clone_recursive(self.audio_tokenizer.hift_cache_dict) + return waveform + finally: + self.audio_tokenizer.stream_cache = None + self.audio_tokenizer.hift_cache_dict = {} + + def _native_duplex_vocode_tokens( + self, + state: _TalkerTurnState, + new_tokens: torch.Tensor, + *, + turn_end: bool, + force_flush: bool, + chunk_size: int, + ) -> list[torch.Tensor]: + """Run Token2wav with the same buffering policy as official duplex.""" + pre_lookahead = self._t2w_pre_lookahead() + token_list = new_tokens.reshape(-1).detach().cpu().tolist() + state.token2wav_buffer.extend(int(t) for t in token_list) + pieces: list[torch.Tensor] = [] + + if force_flush: + while len(state.token2wav_buffer) >= pre_lookahead + 5: + chunk_to_process = min(chunk_size + pre_lookahead, len(state.token2wav_buffer)) + window = state.token2wav_buffer[:chunk_to_process] + pieces.append(self._run_vocoder_window(state, window, last_chunk=False)) + state.token2wav_buffer = state.token2wav_buffer[min(chunk_size, chunk_to_process - pre_lookahead) :] + else: + while len(state.token2wav_buffer) >= chunk_size + pre_lookahead: + window = state.token2wav_buffer[: chunk_size + pre_lookahead] + pieces.append(self._run_vocoder_window(state, window, last_chunk=False)) + state.token2wav_buffer = state.token2wav_buffer[chunk_size:] + + if turn_end and state.token2wav_buffer: + pieces.append( + self._run_vocoder_window( + state, + list(state.token2wav_buffer), + last_chunk=True, + ) + ) + state.token2wav_buffer = [] + + return pieces + + def _close_turn_state( + self, + key: str, + *, + expected_epoch: int | None = None, + expected_turn_id: int | None = None, + ) -> bool: + state = self._talker_turn_states.get(key) + if ( + state is not None + and expected_epoch is not None + and state.epoch is not None + and state.epoch != expected_epoch + ): + return False + if ( + state is not None + and expected_turn_id is not None + and state.turn_id is not None + and state.turn_id != expected_turn_id + ): + return False + state = self._talker_turn_states.pop(key, None) + self._talker_consumed_tokens.pop(key, None) + request_keys = getattr(self, "_talker_request_keys", None) + if isinstance(request_keys, dict): + request_keys.pop(key, None) + for alias, mapped_key in list(request_keys.items()): + if mapped_key == key: + request_keys.pop(alias, None) + if state is None: + return True + if self.audio_tokenizer is not None: + with self._token2wav_lock(): + self.audio_tokenizer.stream_cache = None + self.audio_tokenizer.hift_cache_dict = {} + temp_path = state.temp_prompt_wav_path + if temp_path and not self._is_cached_ref_audio_prompt_wav(temp_path): + with self._token2wav_lock(): + self._t2w_base_caches.pop(temp_path, None) + try: + os.unlink(temp_path) + except OSError: + pass + return True + + @staticmethod + def _stream_identity_relation( + state: _TalkerTurnState | None, + *, + epoch: int | None, + turn_id: int | None, + ) -> str: + if state is None: + return "current" + if epoch is not None and state.epoch is not None: + if epoch < state.epoch: + return "stale" + if epoch > state.epoch: + return "newer" + if turn_id is not None and state.turn_id is not None: + if turn_id < state.turn_id: + return "stale" + if turn_id > state.turn_id: + return "newer" + return "current" + + def _warmup_duplex_vocoder(self) -> None: + """Pre-compile the two token2wav stream() modes used per turn. + + The first stream() call of each mode (mid-turn 25+pre_lookahead + window, and the variable-size last_chunk tail flush) costs ~20s of + one-time compilation; without this warmup the first spoken turn of + the first session stalls for both. Mirrors the official demo's + precompile step. + """ + if getattr(self, "_t2w_warmed", False): + return + self._t2w_warmed = True + try: + self._lazy_init_tts() + if self.audio_tokenizer is None: + return + prompt_wav_path, _ = self._resolve_prompt_wav_path(None, None) + if prompt_wav_path is None: + return + t0 = time.perf_counter() + chunk_size = self._tts_runtime_config().streaming_generator_chunk + pre_lookahead = self._t2w_pre_lookahead() + self._begin_turn_vocoder_cache(prompt_wav_path) + self._t2w_stream_window( + [_T2W_SILENCE_TOKEN] * (chunk_size + pre_lookahead), + prompt_wav_path, + last_chunk=False, + ) + self._begin_turn_vocoder_cache(prompt_wav_path) + self._t2w_stream_window( + [_T2W_SILENCE_TOKEN] * (chunk_size + pre_lookahead), + prompt_wav_path, + last_chunk=True, + ) + self.audio_tokenizer.stream_cache = None + self.audio_tokenizer.hift_cache_dict = {} + logger.info("4.5 Talker duplex vocoder warmup done in %.1fs", time.perf_counter() - t0) + except Exception: + logger.exception("4.5 Talker duplex vocoder warmup failed") + + def _create_native_duplex_stream_gen(self, info: dict[str, Any]): + """Per-segment generator over a persistent per-turn talker stream. + + Mirrors the official duplex talker: each unit calls + ``MiniCPMTTS.generate_chunk`` while carrying ``past_key_values`` and + ``text_start_pos`` across the spoken turn, then feeds the generated + audio tokens through the same per-turn Token2wav buffer/cache. + """ + key = self._stream_request_key(info) + self._remember_talker_request_key(info, key) + stream_epoch = self._stream_epoch(info) + stream_turn_id = self._stream_turn_id(info) + meta_info = info.get("meta") if isinstance(info.get("meta"), dict) else {} + codes_info = info.get("codes") if isinstance(info.get("codes"), dict) else {} + tts_token_ids, tts_hidden_states = self._extract_tts_handoff(info) + + self._lazy_init_tts() + if getattr(self, "tts_obj", None) is None or self.audio_tokenizer is None: + logger.warning("4.5 Talker duplex streaming: TTS runtime unavailable") + yield self._empty_audio_chunk(), True + return + + if isinstance(tts_token_ids, torch.Tensor): + ids_list = tts_token_ids.reshape(-1).tolist() + elif isinstance(tts_token_ids, list): + ids_list = [int(t) for t in tts_token_ids] + else: + ids_list = [] + + state = self._talker_turn_states.get(key) + identity_relation = self._stream_identity_relation( + state, + epoch=stream_epoch, + turn_id=stream_turn_id, ) - default_ref = os.path.join(model_path, "assets", "HT_ref_audio.wav") - prompt_wav_path = default_ref if os.path.exists(default_ref) else None + if identity_relation == "stale": + logger.info( + "4.5 Talker duplex drop stale handoff: key=%s state_epoch=%s state_turn_id=%s " + "handoff_epoch=%s handoff_turn_id=%s", + key, + getattr(state, "epoch", None), + getattr(state, "turn_id", None), + stream_epoch, + stream_turn_id, + ) + yield self._empty_audio_chunk(), True + return + if identity_relation == "newer": + self._close_turn_state(key) + state = None + consumed = self._talker_consumed_tokens.get(key, 0) + if consumed > len(ids_list): + consumed = 0 + pending_ids = ids_list[consumed:] + + turn_eos_raw = meta_info.get("turn_eos_token_id") + try: + turn_eos_id = int(turn_eos_raw) if turn_eos_raw is not None else None + except (TypeError, ValueError): + turn_eos_id = None + explicit_turn_end = bool( + info.get("end_of_turn") or info.get("turn_end") or meta_info.get("end_of_turn") or meta_info.get("turn_end") + ) + turn_end = explicit_turn_end or (turn_eos_id is not None and turn_eos_id in pending_ids) + terminal_only_new_turn = ( + state is None + and turn_end + and turn_eos_id is not None + and bool(pending_ids) + and all(token_id == turn_eos_id for token_id in pending_ids) + ) + if terminal_only_new_turn: + # A model-owned empty turn can be [speak, turn_eos]. There is no + # spoken state to flush, so conditioning TTS on turn_eos alone + # would synthesize an unrelated audio-only response. + yield self._empty_audio_chunk(), True + return + if state is None and not pending_ids: + # No turn open and nothing new to speak: nothing to synthesize. + yield self._empty_audio_chunk(), True + return + + tts = self.tts_obj + if not hasattr(tts.model.config, "rope_theta"): + tts.model.config.rope_theta = 10000.0 + if not callable(getattr(tts, "generate_chunk", None)): + logger.warning("4.5 Talker duplex streaming: MiniCPMTTS.generate_chunk unavailable") + yield self._empty_audio_chunk(), True + return + sampling_params = self._build_tts_sampling_params() + if sampling_params is None: + logger.warning("4.5 Talker duplex streaming: sampling params unavailable") + yield self._empty_audio_chunk(), True + return + chunk_size = self._tts_runtime_config().streaming_generator_chunk + if chunk_size <= 0: + raise ValueError("MiniCPM-o 4.5 TTS streaming generator chunk must be positive") + + if state is None: + ref_audio = codes_info.get("ref", info.get("ref_audio")) + ref_audio_sr = meta_info.get("ref_audio_sr", info.get("ref_audio_sr")) + prompt_wav_path, temp_prompt_wav_path = self._resolve_prompt_wav_path(ref_audio, ref_audio_sr) + if prompt_wav_path is None: + logger.warning("4.5 Talker duplex streaming: no ref_audio prompt; skipping audio synthesis") + yield self._empty_audio_chunk(), True + return + state = _TalkerTurnState( + prompt_wav_path, + temp_prompt_wav_path, + epoch=stream_epoch, + turn_id=stream_turn_id, + ) + self._begin_turn_vocoder_cache(prompt_wav_path, state=state) + self._talker_turn_states[key] = state + + _queue_native_duplex_segment_text( + state, + meta_info.get("native_duplex_segment_text", ""), + ) + + if pending_ids: + pending_hidden = ( + tts_hidden_states[consumed:] + if isinstance(tts_hidden_states, list) + else torch.as_tensor(tts_hidden_states)[consumed:] + ) + cond_ids, cond_hidden = self._normalize_tts_handoff_tensors(pending_ids, pending_hidden) + condition = self._build_tts_condition_embeds(cond_ids, cond_hidden).unsqueeze(0) + else: + emb_dim = int(tts.emb_text.weight.shape[1]) + condition = tts.emb_text.weight.new_zeros((1, 0, emb_dim)) + audio_bos = tts.emb_text( + torch.tensor( + [tts.audio_bos_token_id], + dtype=torch.long, + device=tts.emb_text.weight.device, + ) + ).unsqueeze(0) + condition = torch.cat([condition, audio_bos], dim=1) + max_token_per_chunk = chunk_size + 1 + min_token_per_chunk = 0 if turn_end else max_token_per_chunk + force_flush = False + if state.text_start_pos == 0: + min_token_per_chunk = 0 + force_flush = True + eos_token = torch.tensor( + [tts.config.num_audio_tokens - 1], + dtype=torch.long, + device=tts.emb_text.weight.device, + ) + temperature = torch.tensor( + [float(sampling_params.temperature)], + dtype=torch.float, + device=tts.emb_text.weight.device, + ) + new_tokens, past_key_values = tts.generate_chunk( + inputs_embeds=condition, + temperature=temperature, + repetition_penalty=sampling_params.repetition_penalty, + eos_token=eos_token, + force_no_stop=False, + max_new_token=max_token_per_chunk, + min_new_tokens=min_token_per_chunk, + past_key_values=state.past_key_values, + logits_processors=None, + text_start_pos=state.text_start_pos, + ) + if turn_end: + state.past_key_values = None + state.text_start_pos = 0 + else: + state.past_key_values = past_key_values + state.text_start_pos += int(condition.shape[1]) + int(new_tokens.shape[1]) + waveforms = self._native_duplex_vocode_tokens( + state, + new_tokens, + turn_end=turn_end, + force_flush=force_flush, + chunk_size=chunk_size, + ) + self._talker_consumed_tokens[key] = len(ids_list) + unit_waveform = _native_duplex_unit_waveform(waveforms, turn_end=turn_end) + if unit_waveform is not None: + self._ar_last_emitted_text = _drain_native_duplex_emitted_text( + state, + has_audio=True, + ) + yield unit_waveform, False + if turn_end: + self._close_turn_state(key) + self._ar_last_emitted_text = "" + yield self._empty_audio_chunk(), True + return + self._ar_last_emitted_text = "" + yield self._empty_audio_chunk(), True + + def _create_stream_gen(self, info: dict[str, Any]): + """Yield waveform chunks from MiniCPM-o remote-code TTS streaming. + + This is the real vLLM streaming path: each yielded tensor is returned + through one scheduler step. The older streaming probe still concatenates + chunks inside generate_speech(), so it cannot improve API TTFA. + """ + if info.get("native_duplex") is True: + yield from self._create_native_duplex_stream_gen(info) + return + tts_token_ids, tts_hidden_states = self._extract_tts_handoff(info) + codes_info = info.get("codes") + meta_info = info.get("meta") + if not isinstance(codes_info, dict): + codes_info = {} + if not isinstance(meta_info, dict): + meta_info = {} + + ref_audio = codes_info.get("ref", info.get("ref_audio")) + ref_audio_sr = meta_info.get("ref_audio_sr", info.get("ref_audio_sr")) + + if tts_token_ids is None or tts_hidden_states is None: + logger.warning("4.5 Talker streaming: missing tts_token_ids or tts_hidden_states") + yield self._empty_audio_chunk(), True + return + tts_token_ids, tts_hidden_states = self._normalize_tts_handoff_tensors( + tts_token_ids, + tts_hidden_states, + ) + + self._lazy_init_tts() + if not hasattr(self, "tts_obj") or self.tts_obj is None: + logger.warning("4.5 Talker streaming: tts_obj not initialized") + yield self._empty_audio_chunk(), True + return + if self.audio_tokenizer is None: + logger.warning("4.5 Talker streaming: audio_tokenizer not initialized") + yield self._empty_audio_chunk(), True + return + + generator_cls = getattr(self, "_tts_streaming_generator_cls", None) + if generator_cls is None or self._tts_gen_logits is None: + logger.warning("4.5 Talker streaming: remote-code TTSStreamingGenerator unavailable") + waveform = self.generate_speech( + tts_token_ids, + tts_hidden_states, + ref_audio=ref_audio, + ref_audio_sr=ref_audio_sr, + ) + if waveform is None: + yield self._empty_audio_chunk(), True + else: + yield torch.as_tensor(waveform, dtype=torch.float32).reshape(-1).cpu().contiguous(), True + return + + tts = self.tts_obj + if not hasattr(tts.model.config, "rope_theta"): + tts.model.config.rope_theta = 10000.0 + + tts_embeds = self._build_tts_condition_embeds(tts_token_ids, tts_hidden_states) + num_text = int(tts_token_ids.shape[-1]) if tts_token_ids.ndim > 0 else 0 + min_new_token, max_new_token = self._max_tts_tokens_for_text(num_text) + sampling_params = self._build_tts_sampling_params() + if sampling_params is None: + logger.warning("4.5 Talker streaming: sampling params unavailable") + yield self._empty_audio_chunk(), True + return + + logits_warpers, logits_processors = self._tts_gen_logits( + num_code=tts.config.num_audio_tokens, + repetition_penalty=sampling_params.repetition_penalty, + top_p=sampling_params.top_p, + top_k=sampling_params.top_k, + ) + eos_token = torch.tensor([tts.config.num_audio_tokens - 1], dtype=torch.long, device=tts.emb_text.weight.device) + chunk_size = self._tts_runtime_config().streaming_generator_chunk + if chunk_size <= 0: + raise ValueError("MiniCPM-o 4.5 TTS streaming generator chunk must be positive") + + tts_streaming_generator = generator_cls( + model=tts, + temperature=sampling_params.temperature, + eos_token=eos_token, + chunk_size=chunk_size, + logits_processors=logits_processors, + logits_warpers=logits_warpers, + ) + + prompt_wav_path, temp_prompt_wav_path = self._resolve_prompt_wav_path(ref_audio, ref_audio_sr) + stream_cache = hift_cache_dict = None + import torchaudio _orig_save = torchaudio.save @@ -299,84 +1316,255 @@ class MiniCPMO45OmniTTSForConditionalGeneration(nn.Module, SupportsPP): return return _orig_save(uri, src, sample_rate, backend="soundfile", **kw) - torchaudio.save = _patched_save - prev_dtype = torch.get_default_dtype() - torch.set_default_dtype(torch.float32) + yielded_any = False try: - # Vocoder path is float32; use the platform abstraction because - # torch.amp.autocast validates unsupported device types even when - # autocast is disabled. - autocast_device = device.type if isinstance(device, torch.device) else str(device) - with current_omni_platform.create_autocast_context( - device_type=autocast_device, - dtype=torch.float32, - enabled=False, - ): - token_list = generated_tokens.squeeze(0).tolist() - num_tokens = len(token_list) + torchaudio.save = _patched_save + prev_dtype = torch.get_default_dtype() + torch.set_default_dtype(torch.float32) + try: + stream_cache, hift_cache_dict = self.audio_tokenizer.set_stream_cache(prompt_wav_path) + finally: + torch.set_default_dtype(prev_dtype) + torchaudio.save = _orig_save + self.audio_tokenizer.stream_cache = stream_cache + self.audio_tokenizer.hift_cache_dict = hift_cache_dict + token_iter = tts_streaming_generator.generate_with_buffer( + condition=tts_embeds.unsqueeze(0), + text_finished=True, + max_new_token=max_new_token, + ) + while True: + try: + audio_token_chunk, is_last = next(token_iter) + except StopIteration: + break + if audio_token_chunk is None: + break - # For long outputs, the one-shot vocoder path - # (Token2wav.__call__ -> flow.inference) runs full O(N^2) self- - # attention over all audio tokens and OOMs on a 24GB card once - # N exceeds a few thousand (e.g. 4964 tokens needs ~3GiB for a - # single attention matmul). Switch to the chunked / streaming - # vocoder (set_stream_cache + stream) which truncates the flow - # attention caches to prompt_len + 100 steps on every chunk, - # keeping peak memory bounded regardless of total length. - STREAM_THRESHOLD = int(os.environ.get("MINICPMO45_TTS_STREAM_THRESHOLD", "2500")) # ~100s @ 25Hz - CHUNK_SIZE = int(os.environ.get("MINICPMO45_TTS_STREAM_CHUNK", "50")) # ~2s per chunk - MIN_TAIL = 6 # must exceed flow.pre_lookahead_len (typically 3) + token_list = audio_token_chunk.reshape(-1).detach().cpu().tolist() + if not token_list: + if is_last: + yield self._empty_audio_chunk(), True + yielded_any = True + break + continue - if num_tokens <= STREAM_THRESHOLD: - wav_bytes = self.audio_tokenizer(token_list, prompt_wav_path) - waveform, sr = sf.read(io.BytesIO(wav_bytes)) - waveform = waveform.astype(np.float32) - else: - # Build chunk boundaries, merging a too-small tail into the - # previous chunk so every chunk satisfies MIN_TAIL. - boundaries = [] - i = 0 - while i < num_tokens: - end = min(i + CHUNK_SIZE, num_tokens) - if 0 < num_tokens - end < MIN_TAIL: - end = num_tokens - boundaries.append((i, end)) - i = end - - logger.info( - "generate_speech: streaming vocoder, %d tokens -> %d chunks (chunk=%d)", - num_tokens, - len(boundaries), - CHUNK_SIZE, - ) - - stream_cache, hift_cache_dict = self.audio_tokenizer.set_stream_cache(prompt_wav_path) - self.audio_tokenizer.stream_cache = stream_cache - self.audio_tokenizer.hift_cache_dict = hift_cache_dict - - try: - pieces = [] - for idx, (s, e) in enumerate(boundaries): - is_last = idx == len(boundaries) - 1 - wav_np = self.audio_tokenizer.stream( - token_list[s:e], - prompt_wav_path, - last_chunk=is_last, - return_waveform=True, - ) - pieces.append(np.asarray(wav_np).reshape(-1)) - waveform = np.concatenate(pieces, axis=0).astype(np.float32) - sr = 24000 - finally: - # Free per-request streaming state so the next request starts clean - self.audio_tokenizer.stream_cache = None - self.audio_tokenizer.hift_cache_dict = {} + autocast_context, _ = self._token2wav_autocast_context() + torchaudio.save = _patched_save + prev_dtype = torch.get_default_dtype() + torch.set_default_dtype(torch.float32) + try: + with autocast_context: + wav_np = self.audio_tokenizer.stream( + token_list, + prompt_wav_path, + last_chunk=bool(is_last), + return_waveform=True, + ) + finally: + torch.set_default_dtype(prev_dtype) + torchaudio.save = _orig_save + chunk = torch.as_tensor(np.asarray(wav_np).reshape(-1), dtype=torch.float32).cpu().contiguous() + yielded_any = True + yield chunk, bool(is_last) + if is_last: + break finally: - torch.set_default_dtype(prev_dtype) torchaudio.save = _orig_save + self.audio_tokenizer.stream_cache = None + self.audio_tokenizer.hift_cache_dict = {} + if temp_prompt_wav_path and not self._is_cached_ref_audio_prompt_wav(temp_prompt_wav_path): + try: + os.unlink(temp_prompt_wav_path) + except OSError: + pass - logger.info("generate_speech: waveform %d samples, sr=%d", waveform.shape[0], sr) - return waveform + if not yielded_any: + yield self._empty_audio_chunk(), True + + def _move_tts_modules_to_device(self) -> torch.dtype: + device = current_omni_platform.get_torch_device() + target_dtype = torch.bfloat16 if current_omni_platform.is_npu() else self._target_tts_dtype() + if target_dtype is torch.float32: + self.tts_obj = self.tts_obj.to(device) + logger.info("Moved MiniCPM-o 4.5 TTS object to %s dtype=%s", device, target_dtype) + return target_dtype + + for module_name in ( + "emb_text", + "model", + "projector_spk", + "projector_semantic", + "emb_code", + "head_code", + ): + module = getattr(self.tts_obj, module_name, None) + if module is not None: + module.to(device=device, dtype=target_dtype) + logger.info("Moved MiniCPM-o 4.5 TTS AR modules to %s dtype=%s", device, target_dtype) + return target_dtype + + def generate_speech( + self, + tts_token_ids: torch.Tensor, + tts_hidden_states: torch.Tensor, + ref_audio=None, + ref_audio_sr: int | None = None, + ) -> torch.Tensor | np.ndarray | None: + """Run full 4.5 TTS pipeline using original MiniCPMTTS.generate.""" + self._lazy_init_tts() + if not hasattr(self, "tts_obj") or self.tts_obj is None: + logger.warning("generate_speech: tts_obj not initialized") + return None + + tts = self.tts_obj + device = tts.emb_text.weight.device + dtype = tts.emb_text.weight.dtype + + llm_embeds = tts.emb_text(tts_token_ids.to(device)) + hidden_embeds = tts.projector_semantic(tts_hidden_states.to(device=device, dtype=dtype)) + if getattr(tts.config, "normalize_projected_hidden", False): + hidden_embeds = F.normalize(hidden_embeds, p=2, dim=-1) + tts_embeds = llm_embeds + hidden_embeds + + text_eos = tts.emb_text(torch.tensor([tts.config.text_eos_token_id], device=device, dtype=torch.long)) + audio_bos = tts.emb_text(torch.tensor([tts.audio_bos_token_id], device=device, dtype=torch.long)) + spk_embeds = torch.zeros(0, tts.config.hidden_size, device=device, dtype=tts_embeds.dtype) + + inputs_embeds = torch.cat([spk_embeds, tts_embeds, text_eos, audio_bos], dim=0).unsqueeze(0) + + # Scale max_new_token with input text length. A fixed 2048-token floor + # can turn an EOS miss on a very short response into ~82s of audio and + # ~18s E2E latency. Keep a conservative short-text floor while bounding + # the tail. + num_text = int(tts_token_ids.shape[-1]) if tts_token_ids.ndim > 0 else 0 + min_new_token, max_new_token = self._max_tts_tokens_for_text(num_text) + + eos_token = torch.tensor([tts.config.num_audio_tokens - 1], dtype=torch.long, device=device) + sampling_params = self._build_tts_sampling_params() + generate_kwargs = { + "inputs_embeds": inputs_embeds, + "eos_token": eos_token, + "max_new_token": max_new_token, + "min_new_token": min_new_token, + "show_tqdm": False, + } + if sampling_params is not None: + generate_kwargs["sampling_params"] = sampling_params + + if self.audio_tokenizer is None: + logger.warning("No audio_tokenizer") + return None + + prompt_wav_path, temp_prompt_wav_path = self._resolve_prompt_wav_path(ref_audio, ref_audio_sr) + + try: + outputs = tts.generate(**generate_kwargs) + generated_tokens = outputs.new_ids.squeeze(-1) + + import torchaudio + + _orig_save = torchaudio.save + + def _patched_save(uri, src, sample_rate, **kw): + kw.pop("backend", None) + if hasattr(uri, "write"): + sf.write(uri, src.cpu().numpy().T, sample_rate, format="WAV") + return + return _orig_save(uri, src, sample_rate, backend="soundfile", **kw) + + torchaudio.save = _patched_save + prev_dtype = torch.get_default_dtype() + torch.set_default_dtype(torch.float32) + try: + autocast_context, token2wav_autocast = self._token2wav_autocast_context() + with autocast_context: + num_tokens = int(generated_tokens.shape[-1]) + + # For long outputs, the one-shot vocoder path + # (Token2wav.__call__ -> flow.inference) runs full O(N^2) self- + # attention over all audio tokens and OOMs on a 24GB card once + # N exceeds a few thousand (e.g. 4964 tokens needs ~3GiB for a + # single attention matmul). Switch to the chunked / streaming + # vocoder (set_stream_cache + stream) which truncates the flow + # attention caches to prompt_len + 100 steps on every chunk, + # keeping peak memory bounded regardless of total length. + STREAM_THRESHOLD = self._tts_runtime_config().streaming_vocoder_threshold # ~100s @ 25Hz + CHUNK_SIZE = self._tts_runtime_config().streaming_vocoder_chunk # ~2s per chunk + MIN_TAIL = 6 # must exceed flow.pre_lookahead_len (typically 3) + + if num_tokens <= STREAM_THRESHOLD: + if self._should_use_direct_token2wav() and token2wav_autocast == "off": + try: + waveform, _ = self._run_token2wav_direct(generated_tokens, prompt_wav_path) + except Exception as exc: + logger.warning( + "MiniCPM-o 4.5 direct Token2wav path failed; falling back to WAV path: %s", + exc, + exc_info=True, + ) + token_list = generated_tokens.squeeze(0).tolist() + self._reset_token2wav_cache_if_needed(prompt_wav_path) + wav_bytes = self.audio_tokenizer(token_list, prompt_wav_path) + waveform, _ = sf.read(io.BytesIO(wav_bytes)) + waveform = waveform.astype(np.float32) + else: + token_list = generated_tokens.squeeze(0).tolist() + self._reset_token2wav_cache_if_needed(prompt_wav_path) + wav_bytes = self.audio_tokenizer(token_list, prompt_wav_path) + waveform, _ = sf.read(io.BytesIO(wav_bytes)) + waveform = waveform.astype(np.float32) + else: + token_list = generated_tokens.squeeze(0).tolist() + # Build chunk boundaries, merging a too-small tail into the + # previous chunk so every chunk satisfies MIN_TAIL. + boundaries = [] + i = 0 + while i < num_tokens: + end = min(i + CHUNK_SIZE, num_tokens) + if 0 < num_tokens - end < MIN_TAIL: + end = num_tokens + boundaries.append((i, end)) + i = end + + logger.info( + "generate_speech: streaming vocoder, %d tokens -> %d chunks (chunk=%d)", + num_tokens, + len(boundaries), + CHUNK_SIZE, + ) + + stream_cache, hift_cache_dict = self.audio_tokenizer.set_stream_cache(prompt_wav_path) + self.audio_tokenizer.stream_cache = stream_cache + self.audio_tokenizer.hift_cache_dict = hift_cache_dict + + try: + pieces = [] + for idx, (s, e) in enumerate(boundaries): + is_last = idx == len(boundaries) - 1 + wav_np = self.audio_tokenizer.stream( + token_list[s:e], + prompt_wav_path, + last_chunk=is_last, + return_waveform=True, + ) + pieces.append(np.asarray(wav_np).reshape(-1)) + waveform = np.concatenate(pieces, axis=0).astype(np.float32) + finally: + # Free per-request streaming state so the next request starts clean + self.audio_tokenizer.stream_cache = None + self.audio_tokenizer.hift_cache_dict = {} + finally: + torch.set_default_dtype(prev_dtype) + torchaudio.save = _orig_save + + return waveform + finally: + if temp_prompt_wav_path and not self._is_cached_ref_audio_prompt_wav(temp_prompt_wav_path): + try: + os.unlink(temp_prompt_wav_path) + except OSError: + pass def _generate_tokens(self, inputs_embeds: torch.Tensor, max_new_token: int = 2048) -> torch.Tensor | None: """Autoregressive generation of audio tokens using the TTS LlamaModel.""" @@ -423,30 +1611,6 @@ class MiniCPMO45OmniTTSForConditionalGeneration(nn.Module, SupportsPP): return new_tokens[:, : t + 1 if finished else t, :] - def _dummy_hidden_states( - self, - input_ids: torch.Tensor | None, - positions: torch.Tensor | None, - inputs_embeds: torch.Tensor | None, - ) -> torch.Tensor: - """Shape-correct zero tensor for vllm KV cache profiling. - - vllm's gpu_model_runner._dummy_run takes forward()'s return value as - ``hidden_states`` and does ``hidden_states[logit_indices_device]``; - returning None on the dummy path crashes with - ``TypeError: 'NoneType' object is not subscriptable``. - """ - for ref in (input_ids, positions, inputs_embeds): - if isinstance(ref, torch.Tensor): - num_tokens = int(ref.shape[0]) if ref.ndim >= 1 else 1 - device = ref.device - break - else: - num_tokens = 1 - device = current_omni_platform.get_torch_device() - hidden_size = int(getattr(self, "_hidden_size", 768) or 768) - return torch.zeros((num_tokens, hidden_size), device=device, dtype=torch.bfloat16) - def forward( self, input_ids=None, @@ -458,53 +1622,133 @@ class MiniCPMO45OmniTTSForConditionalGeneration(nn.Module, SupportsPP): ): if additional_information is None: additional_information = {} + if not additional_information: + # Profile/dummy run: use it to pre-compile the per-turn vocoder + # stream modes so the first real spoken turn does not stall. + self._warmup_duplex_vocoder() - tts_token_ids = additional_information.get("tts_token_ids") - tts_hidden_states = additional_information.get("tts_hidden_states") + tts_token_ids, tts_hidden_states = self._extract_tts_handoff(additional_information) tts_text = additional_information.get("llm_output_text", [""]) if isinstance(tts_text, list): tts_text = tts_text[0] if tts_text else "" + codes_info = additional_information.get("codes") + meta_info = additional_information.get("meta") + if not isinstance(codes_info, dict): + codes_info = {} + if not isinstance(meta_info, dict): + meta_info = {} + ref_audio = codes_info.get("ref") + if ref_audio is None: + ref_audio = additional_information.get("ref_audio") + ref_audio_sr = meta_info.get("ref_audio_sr") + if ref_audio_sr is None: + ref_audio_sr = additional_information.get("ref_audio_sr") if tts_token_ids is None or tts_hidden_states is None: - # KV cache profiling / dummy run path — no real TTS input yet. - logger.debug("4.5 Talker: dummy forward (missing tts_token_ids/tts_hidden_states)") - return self._dummy_hidden_states(input_ids, positions, inputs_embeds) + logger.warning("4.5 Talker: missing tts_token_ids or tts_hidden_states") + self._ar_last_chunk_flags = [True] + self._ar_turn_end_flags = [False] + return None, None + tts_token_ids, tts_hidden_states = self._normalize_tts_handoff_tensors( + tts_token_ids, + tts_hidden_states, + ) - logger.info("4.5 Talker: generating speech for %d tokens", tts_token_ids.shape[0]) - waveform = self.generate_speech(tts_token_ids, tts_hidden_states) - # Tuple layout: (mel_spec, waveform). 4.5 talker emits only waveform, - # so mel_spec stays None; the wrapper unpacks in this order and - # packages the waveform into ``multimodal_outputs["model_outputs"]``. + if self._should_stream_output(additional_information): + request_key = self._stream_request_key(additional_information) + input_ends_turn = self._native_duplex_input_ends_turn(additional_information) + if request_key not in self._stream_gens: + self._stream_gens[request_key] = self._create_stream_gen(additional_information) + generator = self._stream_gens[request_key] + try: + waveform_chunk, is_last = next(generator) + except StopIteration: + self._stream_gens.pop(request_key, None) + waveform_chunk = self._empty_audio_chunk() + is_last = True + if is_last: + self._stream_gens.pop(request_key, None) + self._ar_last_chunk_flags = [bool(is_last)] + # A TTS generator also ends at ordinary chunk boundaries. Export + # turn_end only on the terminal output for a condition that + # actually contains the model's <|turn_eos|> decision. + self._ar_turn_end_flags = [bool(is_last and input_ends_turn)] + return None, waveform_chunk.reshape(-1).contiguous() + + self._ar_last_chunk_flags = [True] + self._ar_turn_end_flags = [False] + waveform = self.generate_speech( + tts_token_ids, + tts_hidden_states, + ref_audio=ref_audio, + ref_audio_sr=ref_audio_sr, + ) if waveform is not None: - return None, torch.tensor(waveform, dtype=torch.float32) + waveform_tensor = torch.as_tensor(waveform, dtype=torch.float32).detach() + if waveform_tensor.device.type != "cpu": + waveform_tensor = waveform_tensor.cpu() + return waveform_tensor.reshape(-1).contiguous(), None return None, None def compute_logits(self, hidden_states, *args, **kwargs): - # Placeholder logits: one row per sampled request (the scheduler - # indexes sampled_token_ids by req_index). Hardcoding a single row - # breaks batched/concurrent decoding with IndexError. The values are - # discarded — real output is the waveform via multimodal_outputs. + device = hidden_states.device if isinstance(hidden_states, torch.Tensor) else torch.device("cuda") if isinstance(hidden_states, torch.Tensor): - device = hidden_states.device - num_reqs = hidden_states.shape[0] if hidden_states.ndim >= 1 else 1 + if hidden_states.ndim == 1: + num_rows = 1 + else: + num_rows = max(1, int(hidden_states.shape[0])) else: - device = current_omni_platform.get_torch_device() - num_reqs = 1 - return torch.zeros(num_reqs, 2, device=device) + num_rows = 1 + eos_id = self._scheduler_eos_token_id() + vocab_size = max(int(getattr(self.config, "vocab_size", eos_id + 1) or (eos_id + 1)), eos_id + 1, 3) + safe_id = 1 if eos_id != 1 else 0 + logits = torch.full((num_rows, vocab_size), -1.0e9, dtype=torch.float32, device=device) + flags = self._ar_last_chunk_flags + default_is_last = bool(flags[-1]) if flags else True + for row in range(num_rows): + is_last = bool(flags[row]) if row < len(flags) else default_is_last + if is_last: + logits[row, eos_id] = 1.0e6 + else: + logits[row, safe_id] = 1.0e6 + return logits def sample(self, logits, sampling_metadata): - return None + if logits is None or logits.numel() == 0: + return None + sampled = torch.argmax(logits, dim=-1).to(torch.int32) + return SamplerOutput(sampled_token_ids=sampled.unsqueeze(-1), logprobs_tensors=None) + + def on_requests_finished(self, finished_req_ids: set[str] | list[str]) -> None: + for req_id in finished_req_ids: + request_key = str(req_id) + mapped_key = getattr(self, "_talker_request_keys", {}).pop(request_key, None) + keys = {request_key} + if mapped_key: + keys.add(mapped_key) + for key in list(self._stream_gens): + if key in keys: + gen = self._stream_gens.pop(key, None) + if gen is not None: + try: + gen.close() + except Exception: + logger.exception("MiniCPM-o 4.5 failed to close stream gen for request %s", req_id) + for key in list(self._talker_turn_states): + if key in keys: + self._close_turn_state(key) + for key in list(self._talker_consumed_tokens): + if key in keys: + self._talker_consumed_tokens.pop(key, None) def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]): loaded = set() tts_weights = {} for k, v in weights: if k.startswith("tts."): - tts_weights[k.replace("tts.", "", 1)] = v - # vllm sanity-checks `loaded` against `named_parameters()`. - # The submodule is attached at `self.tts_obj`, not `self.tts`, - # so report the loaded name under the on-module path. - loaded.add(k.replace("tts.", "tts_obj.", 1)) + local_name = k.replace("tts.", "", 1) + tts_weights[local_name] = v + loaded.add(f"tts_obj.{local_name}") if tts_weights and self._tts_config is not None: self._lazy_init_tts() @@ -514,30 +1758,20 @@ class MiniCPMO45OmniTTSForConditionalGeneration(nn.Module, SupportsPP): logger.warning("TTS missing keys (%d): %s", len(missing), missing[:5]) if unexpected: logger.warning("TTS unexpected keys (%d): %s", len(unexpected), unexpected[:5]) - # Move the AR backbone to the active device (cuda / npu / …) and - # cast to bfloat16: MiniCPMTTS AR uses FlashAttention (fp16/bf16 - # only) and is built under a float32 default dtype during lazy - # init, so an uncast float32 backbone breaks CUDA FA2 and wastes - # memory on the NPU sdpa path. Detach the Token2wav vocoder first - # so the cast does not drag it onto the accelerator or downcast - # its float32 flow/HiFT weights: it manages its own device - # placement and may not be an nn.Module. - device = current_omni_platform.get_torch_device() - audio_tok = getattr(self.tts_obj, "audio_tokenizer", None) - if audio_tok is not None: - self.tts_obj.audio_tokenizer = None - try: - self.tts_obj = self.tts_obj.to(device=device, dtype=torch.bfloat16) - finally: - if audio_tok is not None: - self.tts_obj.audio_tokenizer = audio_tok - self.audio_tokenizer = audio_tok + tts_dtype = self._move_tts_modules_to_device() + if ( + not current_omni_platform.is_npu() + and self.audio_tokenizer is not None + and hasattr(self.audio_tokenizer, "to") + ): + self.audio_tokenizer.to("cuda") self.emb_text = self.tts_obj.emb_text self.projector_semantic = self.tts_obj.projector_semantic logger.info( - "Loaded %d TTS weights, moved to %s (bfloat16)", + "Loaded %d TTS weights, moved AR modules to %s dtype=%s", len(tts_weights), - device, + current_omni_platform.get_torch_device(), + tts_dtype, ) return loaded diff --git a/vllm_omni/model_executor/models/minicpmo_4_5/pipeline.py b/vllm_omni/model_executor/models/minicpmo_4_5/pipeline.py index f966555d0..57d8c4614 100644 --- a/vllm_omni/model_executor/models/minicpmo_4_5/pipeline.py +++ b/vllm_omni/model_executor/models/minicpmo_4_5/pipeline.py @@ -24,6 +24,11 @@ MINICPMO_4_5_PIPELINE = PipelineConfig( model_type="minicpmo_4_5", default_deploy_config_name="minicpmo_4_5.yaml", model_arch="MiniCPMO45OmniForConditionalGeneration", + duplex_runtime_extension=("vllm_omni.experimental.fullduplex.minicpmo45.runtime.MiniCPMO45DuplexRuntimeExtension"), + duplex_serving_adapter=( + "vllm_omni.experimental.fullduplex.minicpmo45.serving_adapter.MiniCPMO45ServingRuntimeAdapter" + ), + duplex_control_enabled=True, # MiniCPM-o 4.5's HF config.json reports `model_type="minicpmo"` and # `architectures=["MiniCPMO"]` — both shared verbatim with older MiniCPM-o # 1.0 / 2.6 checkpoints. The only field distinguishing the generations is diff --git a/vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py b/vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py index 799914846..561c6ac8f 100644 --- a/vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py +++ b/vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py @@ -2,37 +2,350 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """Stage input processor for MiniCPM-o 4.5: Thinker (LLM) -> Talker (TTS). -This is the original vLLM-Omni bridge: it converts the thinker stage's -hidden states + token ids into the talker stage's prompt payload. The -talker model itself is adapted from openbmb/MiniCPM-o-4_5 (see the headers -on vllm_omni/model_executor/models/minicpmo_4_5/*.py). +This bridge converts thinker hidden states and token ids into the talker +prompt payload. The talker runs Token2Wav internally; MiniCPM-o 4.5 does not +have an independent code2wav pipeline stage. """ -from typing import Any +import logging +from collections.abc import Callable, Mapping import torch from vllm.inputs import TextPrompt +from vllm_omni.experimental.fullduplex.engine.intermediate import ( + build_duplex_intermediate_buffer, + set_ref_audio, + set_tts_handoff, +) from vllm_omni.inputs.data import OmniTokensPrompt +logger = logging.getLogger(__name__) + + +def _extract_first_audio_ref(multi_modal_data): + if not isinstance(multi_modal_data, dict): + return None + audio_data = multi_modal_data.get("audio") + if audio_data is None: + return None + if isinstance(audio_data, list): + if not audio_data: + return None + audio_data = audio_data[0] + + samples = None + sample_rate = None + if isinstance(audio_data, tuple) and len(audio_data) >= 2: + samples, sample_rate = audio_data[0], audio_data[1] + elif isinstance(audio_data, dict): + sample_rate = audio_data.get("sample_rate") or audio_data.get("sampling_rate") or audio_data.get("sr") + for key in ("audio", "wav", "samples", "array", "waveform"): + if key in audio_data: + samples = audio_data[key] + break + if samples is None or sample_rate is None: + return None + + waveform = torch.as_tensor(samples, dtype=torch.float32) + if waveform.ndim > 1: + if waveform.shape[0] <= 2 and waveform.shape[-1] > waveform.shape[0]: + waveform = waveform.mean(dim=0) + else: + waveform = waveform.mean(dim=-1) + return waveform.reshape(-1).cpu(), int(sample_rate) + + +def _extract_native_runtime_ref_audio(data_plane_metadata): + if not isinstance(data_plane_metadata, dict): + return None + runtime_config = data_plane_metadata.get("runtime_config") + if not isinstance(runtime_config, dict): + return None + from vllm_omni.experimental.fullduplex.minicpmo45.input import decode_native_ref_audio_from_config + + waveform = decode_native_ref_audio_from_config({"extra_body": runtime_config}) + if waveform is None: + return None + sample_rate = runtime_config.get("ref_audio_sample_rate_hz") or 16000 + return torch.as_tensor(waveform, dtype=torch.float32).reshape(-1).cpu(), int(sample_rate) + + +def _coerce_token_id_list(value): + if value is None: + return None + if hasattr(value, "detach"): + value = value.detach().cpu().reshape(-1).tolist() + if isinstance(value, list) and value and isinstance(value[0], list): + value = value[0] + if isinstance(value, tuple): + value = list(value) + if not isinstance(value, list): + return None + out = [] + for item in value: + try: + out.append(int(item)) + except (TypeError, ValueError): + return None + return out + + +def _to_transport_list(value): + if hasattr(value, "detach"): + value = value.detach().cpu() + if isinstance(value, torch.Tensor): + return value.tolist() + return value + + +def _coerce_int(value): + if hasattr(value, "detach"): + flat = value.detach().cpu().reshape(-1) + if flat.numel() == 0: + return None + value = flat[0].item() + try: + return int(value) + except (TypeError, ValueError): + return None + + +def _special_token_ids_from_mm_output(mm_output): + if not isinstance(mm_output, Mapping): + return {} + meta = mm_output.get("meta") + if not isinstance(meta, dict): + meta = {} + special_token_ids = mm_output.get("special_token_ids") + if not isinstance(special_token_ids, dict): + special_token_ids = {} + flat_meta = { + key.removeprefix("meta."): value + for key, value in mm_output.items() + if isinstance(key, str) and key.startswith("meta.") + } + return { + key: value + for key, value in ( + (key, _coerce_int(value)) + for source in (special_token_ids, meta, flat_meta) + for key, value in source.items() + ) + if value is not None and value >= 0 + } + + +def _has_native_duplex_prompt_metadata(mm_output): + if not isinstance(mm_output, Mapping): + return False + return mm_output.get("duplex_prompt_token_ids") is not None or mm_output.get("ids.prompt") is not None + + +def _require_native_tts_boundary_metadata(special_token_ids): + if special_token_ids.get("tts_bos_token_id") is None: + raise ValueError( + "MiniCPM-o native duplex TTS handoff requires tokenizer-derived " + "<|tts_bos|> metadata; refusing to infer special token ids." + ) + + +def _native_tts_boundary_token_ids(special_token_ids): + # Official duplex conditions the talker on mid-unit <|speak|> tokens AND + # the <|turn_eos|> token+hidden (its embedding is the trained stop + # signal); only chunk terminators and framing tokens bound the slice. + return { + token_id + for token_id in ( + special_token_ids.get("tts_eos_token_id"), + special_token_ids.get("tts_pad_token_id"), + special_token_ids.get("listen_token_id"), + special_token_ids.get("chunk_eos_token_id"), + special_token_ids.get("chunk_tts_eos_token_id"), + special_token_ids.get("unit_token_id"), + special_token_ids.get("unit_end_token_id"), + ) + if token_id is not None + } + + +def _native_duplex_segment_output_ids( + output_ids: list[int], + output_text: str, + streaming_context, + *, + request_id: str, +) -> tuple[list[int], str]: + """Slice the cumulative thinker output down to the current segment. + + Tracks how many output tokens were already handed to the talker in the + orchestrator's streaming bridge state. Segment transcript is decoded from + the same token delta, not from a character cursor over cumulative text. + """ + bridge_states = getattr(streaming_context, "bridge_states", None) + if not isinstance(bridge_states, dict): + return output_ids, output_text + state = bridge_states.setdefault("minicpmo45_tts_handoff", {}) + duplex_state = bridge_states.get("duplex") + turn_id = duplex_state.get("model_turn_id", duplex_state.get("turn_id")) if isinstance(duplex_state, dict) else None + if not isinstance(turn_id, int): + turn_id = None + if state.get("request_id") != request_id: + state["request_id"] = request_id + state["sent_output_len"] = 0 + state["sent_output_ids"] = [] + state["acc_tts_ids"] = [] + state["acc_tts_hidden"] = [] + sent_len = state.get("sent_output_len", 0) + prev_output_ids = state.get("sent_output_ids", []) + prev_turn_id = state.get("turn_id") + if not isinstance(sent_len, int) or sent_len < 0 or sent_len > len(output_ids): + # Shrunken cumulative output = epoch reset after barge-in: the talker + # condition history is stale too. + sent_len = 0 + state["acc_tts_ids"] = [] + state["acc_tts_hidden"] = [] + elif sent_len and isinstance(prev_output_ids, list): + prev_prefix = prev_output_ids[:sent_len] + current_prefix = output_ids[:sent_len] + if current_prefix != prev_prefix: + # Some runtimes restart output token ids after a turn boundary while + # others keep returning cumulative ids. Detect restart from the + # token prefix instead of clearing the cursor at turn_eos. + sent_len = 0 + state["acc_tts_ids"] = [] + state["acc_tts_hidden"] = [] + if isinstance(prev_turn_id, int) and isinstance(turn_id, int) and prev_turn_id != turn_id: + # The thinker output list is cumulative across clean turns, so the + # output cursor must stay put. The talker condition is per assistant + # turn, though; carrying it across turn_id boundaries makes stage 1 + # replay the previous turn after its consumed cursor has been reset. + state["acc_tts_ids"] = [] + state["acc_tts_hidden"] = [] + segment_ids = output_ids[sent_len:] + decode_token_ids = getattr(streaming_context, "source_token_decoder", None) + if isinstance(decode_token_ids, Callable): + decode_ids = [int(token_id) for token_id in segment_ids] + try: + try: + segment_text = str(decode_token_ids(decode_ids, skip_special_tokens=True)) + except TypeError: + segment_text = str(decode_token_ids(decode_ids)) + except Exception: + logger.exception("Failed to decode MiniCPM-o duplex token delta for request_id=%s", request_id) + segment_text = "" + elif sent_len == 0: + # Non-orchestrator unit tests and legacy callers may not install a + # decoder. Never slice cumulative text with a stale character cursor. + segment_text = output_text + else: + logger.warning( + "MiniCPM-o native duplex token delta decoder missing for request_id=%s; " + "suppressing cumulative transcript fallback", + request_id, + ) + segment_text = "" + state["sent_output_len"] = len(output_ids) + state["sent_output_ids"] = list(output_ids) + state["turn_id"] = turn_id + return segment_ids, segment_text + + +def _reset_native_tts_handoff(streaming_context) -> None: + bridge_states = getattr(streaming_context, "bridge_states", None) + if not isinstance(bridge_states, dict): + return + state = bridge_states.get("minicpmo45_tts_handoff") + if not isinstance(state, dict): + return + # Keep the output cursor: the thinker can keep reporting cumulative + # output after turn_eos on the same resumable request. If a runtime really + # restarts token ids, _native_duplex_segment_output_ids detects the prefix + # mismatch and resets the slice cursor there. + state["acc_tts_ids"] = [] + state["acc_tts_hidden"] = [] + + +def _native_duplex_data_plane_metadata(streaming_context) -> dict[str, object] | None: + bridge_states = getattr(streaming_context, "bridge_states", None) + if not isinstance(bridge_states, dict): + return None + duplex_state = bridge_states.get("duplex") + if not isinstance(duplex_state, dict): + return None + + metadata: dict[str, object] = {"data_plane": True} + session_id = duplex_state.get("session_id") + if isinstance(session_id, str) and session_id: + metadata["session_id"] = session_id + incarnation = duplex_state.get("incarnation") + if isinstance(incarnation, int): + metadata["incarnation"] = incarnation + epoch = duplex_state.get("epoch") + if isinstance(epoch, int): + metadata["epoch"] = epoch + turn_id = duplex_state.get("model_turn_id", duplex_state.get("turn_id")) + if isinstance(turn_id, int): + metadata["turn_id"] = turn_id + session_config = duplex_state.get("session_config") + if isinstance(session_config, dict): + metadata["session_config"] = dict(session_config) + runtime_config = duplex_state.get("runtime_config") + if isinstance(runtime_config, dict): + metadata["runtime_config"] = dict(runtime_config) + return metadata + + +def _accumulate_native_tts_handoff(streaming_context, new_ids, new_hidden): + """Hand the talker the FULL accumulated condition on every handoff. + + The runner's streaming buffer update is not merge-safe for a resumable + stage-1 request: in-place updates merge sub-keys, but a resume prefill + REPLACES the buffer, silently dropping every earlier segment's tts + tokens/hiddens (observed losing alternating reply segments — the talker + vocalized text it never saw between islands it did). Accumulate here so + the latest handoff always carries the complete history and downstream + replace semantics are lossless; the talker consumes by cursor. + """ + bridge_states = getattr(streaming_context, "bridge_states", None) + if not isinstance(bridge_states, dict): + return new_ids, new_hidden + state = bridge_states.setdefault("minicpmo45_tts_handoff", {}) + acc_ids = state.setdefault("acc_tts_ids", []) + acc_hidden = state.setdefault("acc_tts_hidden", []) + if new_ids: + acc_ids.extend(int(t) for t in new_ids) + if new_hidden: + acc_hidden.extend(new_hidden) + if not acc_ids: + return None, None + return list(acc_ids), list(acc_hidden) + + +def _build_tts_scheduler_prompt_token_ids( + tts_token_ids: torch.Tensor | None, + llm_output_ids: list[int], + prompt_token_ids: list[int], +) -> list[int]: + if tts_token_ids is not None: + ids = _coerce_token_id_list(tts_token_ids) + if ids: + return ids + if llm_output_ids: + return llm_output_ids + if prompt_token_ids: + return prompt_token_ids[-1:] + raise ValueError("MiniCPM-o TTS stage requires at least one scheduler prompt token") + def llm2tts( - source_outputs: list[Any], - prompt: OmniTokensPrompt | TextPrompt | dict | list | None = None, + source_outputs, + prompt: OmniTokensPrompt | TextPrompt = None, requires_multimodal_data: bool = False, - streaming_context: Any | None = None, + _streaming_context=None, ): """Convert thinker stage output to talker stage input for MiniCPMO Omni. - The signature matches the framework's ``custom_process_input_func`` call - convention used by ``StageEngineCoreClientBase.process_engine_inputs``: - - (source_outputs, prompt, requires_multimodal_data, streaming_context) - - ``source_outputs`` is the already-resolved list of upstream engine - outputs (one entry per request), so we do not need to look anything up - via ``stage_list[source_stage_id].engine_outputs``. - Extracts from thinker output: - Full hidden states (prompt + generated) for speaker embedding extraction - Prompt token IDs (for finding spk_bos/spk_eos positions) @@ -44,8 +357,6 @@ def llm2tts( 3. Decode generated text and extract TTS content 4. Run ConditionalChatTTS pipeline """ - del streaming_context # not used by MiniCPM-o 4.5 turn-taking pipeline - if not source_outputs: raise ValueError("source_outputs cannot be empty") @@ -55,75 +366,242 @@ def llm2tts( if not isinstance(prompt, list): prompt = [prompt] - multi_modal_data = { - llm_output.request_id: p.get("multi_modal_data", None) if isinstance(p, dict) else None - for llm_output, p in zip(llm_outputs, prompt) - } + multi_modal_data = {} + for llm_output, p in zip(llm_outputs, prompt): + if isinstance(p, dict): + multi_modal_data[llm_output.request_id] = p.get("multi_modal_data", None) + else: + multi_modal_data[llm_output.request_id] = getattr(p, "multi_modal_data", None) - for i, llm_output in enumerate(llm_outputs): + for llm_output in llm_outputs: output = llm_output.outputs[0] - prompt_token_ids = llm_output.prompt_token_ids - llm_output_ids = output.token_ids + mm_output = output.multimodal_output if isinstance(output.multimodal_output, Mapping) else {} + special_token_ids = _special_token_ids_from_mm_output(mm_output) + prompt_token_ids = ( + _coerce_token_id_list(mm_output.get("duplex_prompt_token_ids")) + or _coerce_token_id_list(mm_output.get("ids.prompt")) + or list(llm_output.prompt_token_ids) + ) + llm_output_ids = getattr(output, "token_ids", None) + if llm_output_ids is None: + llm_output_ids = getattr(output, "cumulative_token_ids", []) + # Always copy: CompletionOutput.token_ids can alias the upstream + # detokenizer's live token list. Forwarding that exact object as the + # talker prompt makes the stage-1 streaming update extend the list + # with itself (state and update bind the same object), doubling the + # thinker's recorded output every segment until the TTS engine input + # buffer overflows. + llm_output_ids = list(llm_output_ids) + thinker_text = getattr(output, "text", "") or "" + if _has_native_duplex_prompt_metadata(mm_output): + # The thinker's resumable duplex request reports cumulative + # output ids/text, but earlier segments are already folded into + # the prompt by the scheduler session update, and the forwarded + # hidden states only cover the current prompt + current segment. + # Hand stage 1 exactly one segment per handoff so token/hidden + # alignment holds, the talker prompt grows linearly with new + # tokens instead of quadratically, and downstream transcripts + # carry per-unit deltas instead of re-sending the whole reply. + llm_output_ids, thinker_text = _native_duplex_segment_output_ids( + llm_output_ids, + thinker_text, + _streaming_context, + request_id=str(llm_output.request_id), + ) prompt_token_ids_len = len(prompt_token_ids) - latent = output.multimodal_output.get("latent", None) + latent = mm_output.get("latent", None) if latent is None: latent = output.hidden_states if hasattr(output, "hidden_states") else None if latent is None: raise ValueError("No latent or hidden_states found in thinker output") - thinker_hidden_states = latent.clone().detach() - - # Split hidden states: prompt portion has speaker embedding, - # generated portion has the text content - prompt_hidden = thinker_hidden_states[:prompt_token_ids_len].to(torch.float32) - - # Extract decoded text from thinker output for TTS text extraction - thinker_text = getattr(output, "text", "") or "" + thinker_hidden_states = latent.detach() + if thinker_hidden_states.ndim == 3 and thinker_hidden_states.shape[0] == 1: + thinker_hidden_states = thinker_hidden_states.squeeze(0) # Build full token sequence and extract TTS region - full_token_ids = list(prompt_token_ids) + ( - list(llm_output_ids) if not isinstance(llm_output_ids, list) else llm_output_ids - ) - full_hidden = thinker_hidden_states.to(torch.float32) + full_token_ids = prompt_token_ids + llm_output_ids - # Detect TTS token IDs (4.5: 151703/151704, 2.6: 151691/151692) - tts_bos_id, tts_eos_id = 151691, 151692 - for _id in [151703, 151704]: - if _id in full_token_ids: - tts_bos_id, tts_eos_id = 151703, 151704 - break + tts_bos_id = special_token_ids.get("tts_bos_token_id") + is_native_duplex_handoff = _has_native_duplex_prompt_metadata(mm_output) + if is_native_duplex_handoff: + _require_native_tts_boundary_metadata(special_token_ids) + tts_end_ids = _native_tts_boundary_token_ids(special_token_ids) - tts_bos_idx = tts_eos_idx = None - for idx_t, tid in enumerate(full_token_ids): - if tid == tts_bos_id: + # Plain-chat (use_tts_template) fallback: non-duplex requests do not + # surface special_token_ids, so use MiniCPM-o 4.5's fixed boundaries. + if tts_bos_id is None and not is_native_duplex_handoff: + tts_bos_id = 151703 + tts_end_ids = set(tts_end_ids) | {151704, 151645} + + tts_bos_idx = None + # For native duplex the resumable prompt folds every earlier unit, so + # a <|tts_bos|> from an already-spoken reply can sit mid-prompt; only + # a boundary folded as the FINAL prompt token (this unit's decision) + # or one inside the current segment may start the slice, or stale + # text would be re-handed to the talker on text-less continuations. + search_start = max(0, prompt_token_ids_len - 1) if is_native_duplex_handoff else 0 + for idx_t in range(search_start, len(full_token_ids)): + if full_token_ids[idx_t] == tts_bos_id: tts_bos_idx = idx_t + 1 - elif tid == tts_eos_id: - tts_eos_idx = idx_t + + tts_eos_idx = None + if tts_bos_idx is not None: + for idx_t in range(tts_bos_idx, len(full_token_ids)): + if full_token_ids[idx_t] in tts_end_ids: + tts_eos_idx = idx_t + break tts_token_ids_slice = tts_hidden_slice = None - if tts_bos_idx is not None and full_hidden.shape[0] > tts_bos_idx: - end_idx = tts_eos_idx if tts_eos_idx is not None else full_hidden.shape[0] + native_segment_end = False + if tts_bos_idx is not None and thinker_hidden_states.shape[0] > tts_bos_idx: + end_idx = tts_eos_idx if tts_eos_idx is not None else thinker_hidden_states.shape[0] + if is_native_duplex_handoff and tts_eos_idx is not None: + boundary_token = full_token_ids[tts_eos_idx] + native_segment_end = boundary_token in { + special_token_ids.get("chunk_eos_token_id"), + special_token_ids.get("chunk_tts_eos_token_id"), + } tts_token_ids_slice = torch.tensor(full_token_ids[tts_bos_idx:end_idx], dtype=torch.long) - tts_hidden_slice = full_hidden[tts_bos_idx:end_idx] + tts_hidden_slice = thinker_hidden_states[tts_bos_idx:end_idx].to(torch.float32).contiguous() + elif is_native_duplex_handoff: + # Official MiniCPM-o duplex does not prefill an assistant + # <|tts_bos|> boundary before generation. A segment delta can + # start with SEVERAL unit decisions (forced/model listens from + # chunks that produced no handoff accumulate ahead of the speak), + # so skip the leading listen run, then the speak decision itself; + # hidden states for TTS start after it and stop at the next + # chunk terminator. + listen_id = special_token_ids.get("listen_token_id") + speak_id = special_token_ids.get("speak_token_id") + out_ids = llm_output_ids + j = 0 + while j < len(out_ids) and out_ids[j] == listen_id: + j += 1 + if j < len(out_ids) and out_ids[j] == speak_id: + out_start = j + 1 + out_end = len(out_ids) + turn_eos_id = special_token_ids.get("turn_eos_token_id") + for idx_t in range(out_start, len(out_ids)): + token_id = out_ids[idx_t] + if turn_eos_id is not None and token_id == turn_eos_id: + # Native duplex trains the talker on the <|turn_eos|> + # embedding itself, but any text sampled after it + # belongs to a stale tail and must not enter TTS. + out_end = idx_t + 1 + break + if token_id in tts_end_ids: + out_end = idx_t + native_segment_end = token_id in { + special_token_ids.get("chunk_eos_token_id"), + special_token_ids.get("chunk_tts_eos_token_id"), + } + break + # Map output indices onto hidden rows by END alignment: the + # leading decision tokens of the delta may ALSO be folded into + # the resumable prompt (they belong to earlier non-forwarded + # segments), so prompt_len + delta over-counts them and + # front-aligned indexing truncates the slice. The hidden + # tensor's last len(out_ids) rows are the delta's rows. + hidden_base = int(thinker_hidden_states.shape[0]) - len(out_ids) + if hidden_base >= 0 and out_end > out_start: + tts_token_ids_slice = torch.tensor(out_ids[out_start:out_end], dtype=torch.long) + tts_hidden_slice = ( + thinker_hidden_states[hidden_base + out_start : hidden_base + out_end] + .to(torch.float32) + .contiguous() + ) + elif j < len(out_ids) and out_ids[j] not in tts_end_ids: + # HF streaming_generate does not require an explicit <|speak|> + # marker. If a unit starts directly with text, the first token + # is fed back into the LLM but is not included in + # total_hidden_in_unit; TTS starts from the following token. + out_start = j + 1 + out_end = len(out_ids) + turn_eos_id = special_token_ids.get("turn_eos_token_id") + for idx_t in range(out_start, len(out_ids)): + token_id = out_ids[idx_t] + if turn_eos_id is not None and token_id == turn_eos_id: + out_end = idx_t + 1 + break + if token_id in tts_end_ids: + out_end = idx_t + native_segment_end = token_id in { + special_token_ids.get("chunk_eos_token_id"), + special_token_ids.get("chunk_tts_eos_token_id"), + } + break + hidden_base = int(thinker_hidden_states.shape[0]) - len(out_ids) + if hidden_base >= 0 and out_end > out_start: + tts_token_ids_slice = torch.tensor(out_ids[out_start:out_end], dtype=torch.long) + tts_hidden_slice = ( + thinker_hidden_states[hidden_base + out_start : hidden_base + out_end] + .to(torch.float32) + .contiguous() + ) + model_intermediate_buffer = build_duplex_intermediate_buffer( + request_id=str(llm_output.request_id), + prompt_token_ids=prompt_token_ids, + output_token_ids=llm_output_ids, + output_text=thinker_text, + stream_output=is_native_duplex_handoff, + native_duplex=is_native_duplex_handoff, + ) + if is_native_duplex_handoff: + turn_eos_id = special_token_ids.get("turn_eos_token_id") + meta = model_intermediate_buffer.setdefault("meta", {}) + data_plane_metadata = _native_duplex_data_plane_metadata(_streaming_context) + if data_plane_metadata is not None: + model_intermediate_buffer["duplex"] = data_plane_metadata + meta["native_duplex_segment_text"] = thinker_text + meta.setdefault("override_keys", []).extend( + [ + "llm_output_text", + ["meta", "native_duplex_segment_text"], + ] + ) + if turn_eos_id is not None: + # The talker detects turn end from <|turn_eos|> inside the + # handed condition (official conditions on its embedding). + meta["turn_eos_token_id"] = int(turn_eos_id) + if native_segment_end: + meta["segment_end"] = True + req_mm_data = multi_modal_data.get(llm_output.request_id) + ref_audio = _extract_first_audio_ref(req_mm_data) + if ref_audio is None: + ref_audio = _extract_native_runtime_ref_audio( + model_intermediate_buffer.get("duplex"), + ) + if ref_audio is not None: + ref_waveform, ref_sr = ref_audio + set_ref_audio(model_intermediate_buffer, _to_transport_list(ref_waveform), ref_sr) + handoff_ids = _coerce_token_id_list(tts_token_ids_slice) if tts_token_ids_slice is not None else None + handoff_hidden = _to_transport_list(tts_hidden_slice) if tts_hidden_slice is not None else None + native_turn_end_handoff = False + if is_native_duplex_handoff: + turn_eos_id = special_token_ids.get("turn_eos_token_id") + native_turn_end_handoff = turn_eos_id is not None and handoff_ids is not None and turn_eos_id in handoff_ids + handoff_ids, handoff_hidden = _accumulate_native_tts_handoff( + _streaming_context, + handoff_ids, + handoff_hidden, + ) + if not handoff_ids: + continue + set_tts_handoff(model_intermediate_buffer, handoff_ids, handoff_hidden) + if native_turn_end_handoff: + _reset_native_tts_handoff(_streaming_context) - additional_information = { - "prompt_embeds": prompt_hidden, - "prompt_token_ids": list(prompt_token_ids), - "llm_output_token_ids": list(llm_output_ids) if not isinstance(llm_output_ids, list) else llm_output_ids, - "llm_output_text": [thinker_text], - } - if tts_token_ids_slice is not None: - additional_information["tts_token_ids"] = tts_token_ids_slice - if tts_hidden_slice is not None: - additional_information["tts_hidden_states"] = tts_hidden_slice - - # Minimal prompt token IDs: the talker's AR framework needs *some* tokens - # to do a single prefill step. We use [BOS, PAD, EOS] as a dummy. + scheduler_prompt_token_ids = _build_tts_scheduler_prompt_token_ids( + tts_token_ids_slice, + llm_output_ids, + prompt_token_ids, + ) tts_inputs.append( OmniTokensPrompt( - prompt_token_ids=[1, 0, 2], - additional_information=additional_information, + prompt_token_ids=scheduler_prompt_token_ids, + model_intermediate_buffer=model_intermediate_buffer, multi_modal_data=( multi_modal_data[llm_output.request_id] if requires_multimodal_data and multi_modal_data.get(llm_output.request_id) is not None @@ -132,5 +610,12 @@ def llm2tts( mm_processor_kwargs=None, ) ) + if native_turn_end_handoff: + bridge_states = getattr(_streaming_context, "bridge_states", None) + duplex_state = bridge_states.get("duplex") if isinstance(bridge_states, dict) else None + if isinstance(duplex_state, dict): + current_model_turn_id = duplex_state.get("model_turn_id", duplex_state.get("turn_id", 0)) + if isinstance(current_model_turn_id, int): + duplex_state["model_turn_id"] = current_model_turn_id + 1 return tts_inputs diff --git a/vllm_omni/outputs/multimodal_accumulation.py b/vllm_omni/outputs/multimodal_accumulation.py new file mode 100644 index 000000000..08b2551bc --- /dev/null +++ b/vllm_omni/outputs/multimodal_accumulation.py @@ -0,0 +1,91 @@ +from typing import Any + +import torch + +from vllm_omni.outputs.mm_outputs import MultimodalPayload +from vllm_omni.outputs.output_modality import DRAINABLE_MODALITIES + +CHUNK_METADATA_KEYS: frozenset[str] = frozenset( + { + "audio_text_total_chars", + "duplex_epoch", + "duplex_turn_id", + "llm_output_text_utf8", + "tts_is_last_chunk", + "turn_end", + } +) + + +def _is_chunk_metadata_key(key: str) -> bool: + if key.startswith("meta."): + return key.split(".", 1)[1] in CHUNK_METADATA_KEYS + return key in CHUNK_METADATA_KEYS + + +def replace_snapshot_keys( + accumulated: MultimodalPayload, + incoming: MultimodalPayload, +) -> None: + """Replace per-chunk metadata with the latest values.""" + for key in (*incoming.tensors, *incoming.metadata): + if _is_chunk_metadata_key(key): + accumulated.tensors.pop(key, None) + accumulated.metadata.pop(key, None) + + +def drain_delta_payload(payload: MultimodalPayload) -> None: + """Remove client-facing delta data while retaining request-level state.""" + for modality_key in DRAINABLE_MODALITIES: + key = str(modality_key) + payload.tensors.pop(key, None) + payload.metadata.pop(key, None) + + for metadata_key in CHUNK_METADATA_KEYS: + flat_key = f"meta.{metadata_key}" + payload.tensors.pop(flat_key, None) + payload.metadata.pop(flat_key, None) + + meta = payload.metadata.get("meta") + if isinstance(meta, dict): + filtered = {key: value for key, value in meta.items() if key not in CHUNK_METADATA_KEYS} + if filtered: + payload.metadata["meta"] = filtered + else: + payload.metadata.pop("meta", None) + + +def _payload_meta_value(payload: MultimodalPayload, key: str) -> Any: + flat_key = f"meta.{key}" + if flat_key in payload.tensors: + return payload.tensors[flat_key] + if flat_key in payload.metadata: + return payload.metadata[flat_key] + meta = payload.metadata.get("meta") + if isinstance(meta, dict): + return meta.get(key) + return None + + +def _last_scalar_int(value: Any) -> int | None: + if value is None: + return None + if isinstance(value, bool | int): + return int(value) + if isinstance(value, list | tuple): + return _last_scalar_int(value[-1]) if value else None + if isinstance(value, torch.Tensor): + try: + if value.numel() == 0: + return None + return int(value.detach().cpu().reshape(-1)[-1].item()) + except (RuntimeError, TypeError, ValueError): + return None + return None + + +def is_non_final_delta_audio_chunk(payload: MultimodalPayload, mm_type: str | None) -> bool: + """Return whether an audio delta explicitly declares more chunks.""" + if str(mm_type or "").lower() != "audio" and "audio" not in payload: + return False + return _last_scalar_int(_payload_meta_value(payload, "tts_is_last_chunk")) == 0 diff --git a/vllm_omni/outputs/output_processor.py b/vllm_omni/outputs/output_processor.py index e39b995b0..ebfe0689a 100644 --- a/vllm_omni/outputs/output_processor.py +++ b/vllm_omni/outputs/output_processor.py @@ -19,15 +19,15 @@ from vllm.v1.metrics.stats import IterationStats, RequestStateStats from vllm_omni.data_entry_keys import unflatten_payload from vllm_omni.outputs import OmniRequestOutput from vllm_omni.outputs.mm_outputs import MultimodalCompletionOutput, MultimodalPayload -from vllm_omni.outputs.output_modality import ( - DRAINABLE_MODALITIES, - OutputModality, - get_accumulation_strategy, +from vllm_omni.outputs.multimodal_accumulation import ( + drain_delta_payload, + is_non_final_delta_audio_chunk, + replace_snapshot_keys, ) +from vllm_omni.outputs.output_modality import OutputModality, get_accumulation_strategy logger = init_logger(__name__) - # --------------------------------------------------------------------------- # Module-level helpers # --------------------------------------------------------------------------- @@ -55,6 +55,14 @@ def _modality_to_type_string(modality: OutputModality) -> str: return "text" +def _mean_time_per_output_token_ms(stats: RequestStateStats) -> float: + output_intervals = stats.num_generation_tokens - 1 + if output_intervals <= 0: + return 0.0 + decode_time_s = max(stats.last_token_ts - stats.first_token_ts, 0.0) + return decode_time_s * 1000.0 / float(output_intervals) + + class OmniRequestState(RequestState): """Request state for omni models, tracking multimodal outputs. @@ -85,7 +93,7 @@ class OmniRequestState(RequestState): def apply_streaming_update(self, update) -> None: super().apply_streaming_update(update) - self.native_text_stats.arrival_time = update.arrival_time + self.native_text_stats = RequestStateStats(arrival_time=float(update.arrival_time or 0.0)) def add_multimodal_tensor(self, payload: Any | None, mm_type: str | None) -> None: """Accumulate a multimodal tensor payload into the request state. @@ -104,6 +112,7 @@ class OmniRequestState(RequestState): incoming = MultimodalPayload.from_raw(payload, modality_key) if incoming is not None: + replace_snapshot_keys(self.mm_accumulated, incoming) self.mm_accumulated = self.mm_accumulated.merged_with(incoming) except (ValueError, TypeError, RuntimeError): logger.exception("Error accumulating multimodal tensor") @@ -170,9 +179,13 @@ class OmniRequestState(RequestState): kv_transfer_params, ) + is_delta = self.output_kind == RequestOutputKind.DELTA + if is_delta and finish_reason is not None and is_non_final_delta_audio_chunk(self.mm_accumulated, self.mm_type): + finish_reason = None + stop_reason = None + finished = finish_reason is not None final_only = self.output_kind == RequestOutputKind.FINAL_ONLY - is_delta = self.output_kind == RequestOutputKind.DELTA if not finished and final_only: return None @@ -280,10 +293,10 @@ class OmniRequestState(RequestState): output.cumulative_text = base_output.cumulative_text # DELTA mode: drain modality keys (e.g. audio) so the next - # step only sees freshly accumulated data for those keys. + # step only sees freshly accumulated data for those keys and + # their client-facing per-chunk metadata. if self.output_kind == RequestOutputKind.DELTA: - for modality_key in DRAINABLE_MODALITIES: - self.mm_accumulated.tensors.pop(modality_key, None) + drain_delta_payload(self.mm_accumulated) return output except (RuntimeError, TypeError, AttributeError): @@ -391,6 +404,7 @@ class MultimodalOutputProcessor(VLLMOutputProcessor): return self._native_text_metrics_by_request.setdefault( request_id, { + "num_generation_tokens": 0, "vllm_ttft_ms": 0.0, "vllm_tpot_ms": 0.0, "vllm_itl_ms": 0.0, @@ -592,6 +606,7 @@ class MultimodalOutputProcessor(VLLMOutputProcessor): req_state.lora_name, ) record = self._native_text_metric_record(req_state.external_req_id) + record["num_generation_tokens"] = int(native_stats.num_generation_tokens) if was_prefilling: record["vllm_ttft_ms"] = max(float(native_stats.first_token_latency) * 1000.0, 0.0) return @@ -601,6 +616,7 @@ class MultimodalOutputProcessor(VLLMOutputProcessor): itls_ms = record.setdefault("vllm_itls_ms", []) itls_ms.append(itl_ms) record["vllm_itl_ms"] = sum(itls_ms) / float(len(itls_ms)) + record["vllm_tpot_ms"] = _mean_time_per_output_token_ms(native_stats) def _update_stats_from_finished( self, diff --git a/vllm_omni/request.py b/vllm_omni/request.py index b094f2d39..2f1fd5379 100644 --- a/vllm_omni/request.py +++ b/vllm_omni/request.py @@ -35,6 +35,7 @@ class OmniRequest(Request): # Optional external request ID for tracking external_req_id: str | None = None, additional_information: AdditionalInformationPayload | None = None, + model_intermediate_buffer: dict | None = None, **kwargs, ): if prompt_embeds is not None: @@ -48,6 +49,8 @@ class OmniRequest(Request): self.external_req_id: str | None = external_req_id # Serialized additional information payload (optional) self.additional_information: AdditionalInformationPayload | None = additional_information + # Runner-owned runtime payload. + self.model_intermediate_buffer: dict | None = model_intermediate_buffer @staticmethod def _maybe_decode_prompt_embeds( @@ -94,6 +97,7 @@ class OmniRequest(Request): trace_headers=request.trace_headers, block_hasher=block_hasher, additional_information=request.additional_information, + model_intermediate_buffer=getattr(request, "model_intermediate_buffer", None), resumable=request.resumable, reasoning_ended=request.reasoning_ended, reasoning_parser_kwargs=request.reasoning_parser_kwargs, @@ -117,6 +121,7 @@ class OmniStreamingUpdate: arrival_time: float sampling_params: SamplingParams | None additional_information: AdditionalInformationPayload | None = None + model_intermediate_buffer: dict | None = None @classmethod def from_request(cls, request: "Request") -> "OmniStreamingUpdate | None": @@ -129,4 +134,5 @@ class OmniStreamingUpdate: arrival_time=request.arrival_time, sampling_params=request.sampling_params, additional_information=request.additional_information, + model_intermediate_buffer=getattr(request, "model_intermediate_buffer", None), ) diff --git a/vllm_omni/utils/mm_outputs.py b/vllm_omni/utils/mm_outputs.py index 189cc00aa..1dc4e37f7 100644 --- a/vllm_omni/utils/mm_outputs.py +++ b/vllm_omni/utils/mm_outputs.py @@ -26,6 +26,17 @@ _CLIENT_MM_ROOT_KEYS: frozenset[str] = frozenset( } ) +_CLIENT_MM_META_KEYS: frozenset[str] = frozenset( + { + "audio_text_total_chars", + "duplex_epoch", + "duplex_turn_id", + "llm_output_text_utf8", + "tts_is_last_chunk", + "turn_end", + } +) + def partition_flat_payload( payload: Mapping[str, object], @@ -39,6 +50,13 @@ def partition_flat_payload( root = key.split(".", 1)[0] if root in _CLIENT_MM_ROOT_KEYS: client_mm[key] = value + elif root == "meta" and "." in key and key.split(".", 1)[1] in _CLIENT_MM_META_KEYS: + # Small final-output metadata needed by serving (for example + # transcript text attached to audio) must ride with client MM + # output. Keep it in inter-stage too so downstream stages that read + # metadata from the full payload are not starved. + inter_stage[key] = value + client_mm[key] = value else: inter_stage[key] = value return inter_stage, client_mm diff --git a/vllm_omni/worker/gpu_ar_model_runner.py b/vllm_omni/worker/gpu_ar_model_runner.py index 67ea34b10..3e1ea6ddb 100644 --- a/vllm_omni/worker/gpu_ar_model_runner.py +++ b/vllm_omni/worker/gpu_ar_model_runner.py @@ -327,6 +327,24 @@ class GPUARModelRunner(OmniGPUModelRunner, OmniConnectorModelRunnerMixin): kv_transfer_manager=self.kv_transfer_manager, ) self._downstream_payload_cache: dict[str, bool] = {} + self._duplex_sampling_hook = None + self._duplex_sampling_hook_resolved = False + + def load_model(self, *args, **kwargs) -> None: + super().load_model(*args, **kwargs) + self._resolve_duplex_sampling_hook(force=True) + + def _resolve_duplex_sampling_hook(self, *, force: bool = False): + if not force and getattr(self, "_duplex_sampling_hook_resolved", False): + return self._duplex_sampling_hook + candidate = getattr(getattr(self, "model", None), "prepare_duplex_sampling", None) + self._duplex_sampling_hook = candidate if callable(candidate) else None + self._duplex_sampling_hook_resolved = True + if self._duplex_sampling_hook is not None and not hasattr(self, "_duplex_sampling_helper"): + from vllm_omni.experimental.fullduplex.model_executor import DuplexSamplingHelper + + self._duplex_sampling_helper = DuplexSamplingHelper() + return self._duplex_sampling_hook def _make_buffer(self, *size, dtype, numpy=True): # Prevent ray from pinning the buffer due to large size @@ -390,9 +408,18 @@ class GPUARModelRunner(OmniGPUModelRunner, OmniConnectorModelRunnerMixin): if getattr(self.model, "skips_model_sampler_output_token_history", False): return sampling_metadata output_token_ids = self._build_model_sampler_output_token_ids() - if output_token_ids == sampling_metadata.output_token_ids: - return sampling_metadata - return replace(sampling_metadata, output_token_ids=output_token_ids) + if output_token_ids != sampling_metadata.output_token_ids: + return replace(sampling_metadata, output_token_ids=output_token_ids) + return sampling_metadata + + def _update_states(self, scheduler_output: SchedulerOutput) -> Callable | None: + deferred_state_corrections_fn = super()._update_states(scheduler_output) + if self._resolve_duplex_sampling_hook() is None: + return deferred_state_corrections_fn + helper = getattr(self, "_duplex_sampling_helper", None) + if helper is not None: + helper.update_states(self, scheduler_output) + return deferred_state_corrections_fn def _request_final_stage_id(self, req_id: str) -> int | None: info = self.model_intermediate_buffer.get(req_id) @@ -519,6 +546,9 @@ class GPUARModelRunner(OmniGPUModelRunner, OmniConnectorModelRunnerMixin): self._downstream_payload_cache.clear() if hasattr(self, "model_intermediate_buffer"): self.model_intermediate_buffer.clear() + duplex_helper = getattr(self, "_duplex_sampling_helper", None) + if duplex_helper is not None: + duplex_helper.clear() # 5. Release all CUDA graphs unconditionally (upstream only does this # on ROCm; on CUDA the graphs are only freed by Python GC during @@ -1430,10 +1460,16 @@ class GPUARModelRunner(OmniGPUModelRunner, OmniConnectorModelRunnerMixin): self.input_batch.idx_mapping_np, self.input_batch.positions[self.input_batch.logits_indices], ) - sampler_output = model_sample( - logits, - self._sampling_metadata_for_model_sampler(sampling_metadata), - ) + prepared_sampling_metadata = self._sampling_metadata_for_model_sampler(sampling_metadata) + prepare_duplex_sampling = self._resolve_duplex_sampling_hook() + if prepare_duplex_sampling is not None: + helper = getattr(self, "_duplex_sampling_helper", None) + rows = helper.rows(self) if helper is not None and helper.active_request_ids else () + if rows or (helper is not None and helper.hook_active): + prepare_duplex_sampling(logits, prepared_sampling_metadata, rows) + if helper is not None: + helper.hook_active = bool(rows) + sampler_output = model_sample(logits, prepared_sampling_metadata) if sampler_output is not None: return sampler_output return self.sampler( diff --git a/vllm_omni/worker/gpu_model_runner.py b/vllm_omni/worker/gpu_model_runner.py index b816a36f7..0e14784ca 100644 --- a/vllm_omni/worker/gpu_model_runner.py +++ b/vllm_omni/worker/gpu_model_runner.py @@ -537,7 +537,7 @@ class OmniGPUModelRunner(GPUModelRunner): for new_req_data in scheduler_output.scheduled_new_reqs: req_id = new_req_data.req_id if req_id in self.requests: - self._update_streaming_input_additional_info(req_id) + self._update_streaming_input_additional_info(new_req_data, req_id) req_state = self._update_streaming_request(req_id, new_req_data) reqs_to_add.append(req_state) continue @@ -598,6 +598,15 @@ class OmniGPUModelRunner(GPUModelRunner): pass except Exception as e: logger.error(f"Error decoding prompt embeds: {e}") + # Direct runner data-plane payloads populate + # model_intermediate_buffer without going through the deprecated + # additional_information request transport. + try: + model_buffer = getattr(new_req_data, "model_intermediate_buffer", None) + if isinstance(model_buffer, dict) and model_buffer: + self._update_intermediate_buffer(req_id, model_buffer) + except Exception as e: + logger.error(f"Error updating model intermediate buffer: {e}") # Decode additional_information payloads (dictionary) try: if getattr(new_req_data, "additional_information", None) is not None: @@ -1260,6 +1269,9 @@ class OmniGPUModelRunner(GPUModelRunner): pe_cpu = self._resolve_prompt_embeds_cpu(getattr(nr, "prompt_embeds", None)) if pe_cpu is not None: setattr(self.requests[req_id], "prompt_embeds_cpu", pe_cpu) + model_buffer = getattr(nr, "model_intermediate_buffer", None) + if isinstance(model_buffer, dict) and model_buffer: + self._update_intermediate_buffer(req_id, model_buffer) info_payload = getattr(nr, "additional_information", None) if info_payload is not None: logger.warning_once( @@ -1466,6 +1478,9 @@ class OmniGPUModelRunner(GPUModelRunner): def _update_additional_information(self, scheduler_output: "SchedulerOutput") -> None: for new_req in scheduler_output.scheduled_new_reqs: + model_buffer = getattr(new_req, "model_intermediate_buffer", None) + if isinstance(model_buffer, dict) and model_buffer: + self._update_intermediate_buffer(new_req.req_id, model_buffer) payload_info = getattr(new_req, "additional_information", None) if isinstance(payload_info, dict): self._update_intermediate_buffer(new_req.req_id, payload_info) @@ -1704,6 +1719,7 @@ class OmniGPUModelRunner(GPUModelRunner): decode_start_offsets.extend(start_offsets_b) decode_batch_items.clear() + preprocess_input_ids = input_ids if input_ids is not None else self.input_ids.gpu[:num_input_tokens] for req_index, req_id in enumerate(self.input_batch.req_ids): req_infos = self.model_intermediate_buffer.get(req_id, {}) @@ -1718,6 +1734,8 @@ class OmniGPUModelRunner(GPUModelRunner): # call the custom process function req_infos["request_id"] = req_id + req_infos["duplex_token_offset"] = int(self.input_batch.num_computed_tokens_cpu[req_index]) + req_infos["duplex_prompt_len"] = len(req_state.prompt_token_ids) if req_state is not None else None prompt_token_ids = getattr(req_state, "prompt_token_ids", ()) if req_state is not None else () prompt_len = len(prompt_token_ids or ()) num_computed_tokens = int(self.input_batch.num_computed_tokens_cpu[req_index]) @@ -1733,14 +1751,15 @@ class OmniGPUModelRunner(GPUModelRunner): embed_slice = inputs_embeds[s:e] if inputs_embeds is not None else None req_input_ids, req_embeds, update_dict = self.model.preprocess( - input_ids=input_ids[s:e], input_embeds=embed_slice, **req_infos + input_ids=preprocess_input_ids[s:e], input_embeds=embed_slice, **req_infos ) if inputs_embeds is None: inputs_embeds = torch.empty( - (input_ids.shape[0], req_embeds.shape[-1]), + (preprocess_input_ids.shape[0], req_embeds.shape[-1]), device=req_embeds.device, dtype=req_embeds.dtype, ) + input_ids = preprocess_input_ids if self.has_talker_mtp and span_len == 1 and not is_prefill: last_talker_hidden, text_step = update_dict.pop("mtp_inputs") @@ -1759,7 +1778,9 @@ class OmniGPUModelRunner(GPUModelRunner): seg_len = min(span_len, req_embeds.shape[0]) inputs_embeds[s : s + seg_len] = req_embeds[:seg_len] if isinstance(req_input_ids, torch.Tensor) and req_input_ids.numel() == seg_len: - input_ids[s : s + seg_len] = req_input_ids + preprocess_input_ids[s : s + seg_len] = req_input_ids + if input_ids is None: + input_ids = preprocess_input_ids flush_decode_batch() @@ -1977,12 +1998,26 @@ class OmniGPUModelRunner(GPUModelRunner): # Backward compatible: mirror to old setattr location setattr(req_state, "additional_information_cpu", existing) - def _update_streaming_input_additional_info(self, req_id): - # For streaming input prefill case only. Set num processed tokens = 0 for new segment input + def _update_streaming_input_additional_info(self, new_req_data, req_id): + # For streaming input prefill case only. Update buffer from last segment input. cached_additional_info = self.model_intermediate_buffer.get(req_id, {}) - if cached_additional_info: - merged_info = dict(cached_additional_info) - merged_info.setdefault("meta", {})["num_processed_tokens"] = 0 - merged_info.setdefault("meta", {})["resumable"] = True - self.model_intermediate_buffer[req_id] = merged_info - setattr(self.requests[req_id], "additional_information_cpu", merged_info) + inc_info = getattr(new_req_data, "model_intermediate_buffer", None) + if not isinstance(inc_info, dict) or not inc_info: + payload_info = getattr(new_req_data, "additional_information", None) + inc_info = deserialize_additional_information(payload_info) + if not isinstance(inc_info, dict) or not inc_info: + return + merged_info = dict(cached_additional_info) if isinstance(cached_additional_info, dict) else {} + for key, value in inc_info.items(): + if isinstance(value, dict): + existing_sub = merged_info.get(key) + merged_sub = dict(existing_sub) if isinstance(existing_sub, dict) else {} + for sk, sv in value.items(): + merged_sub[sk] = sv + merged_info[key] = merged_sub + else: + merged_info[key] = value + merged_info.setdefault("meta", {})["num_processed_tokens"] = 0 + merged_info.setdefault("meta", {})["resumable"] = True + self.model_intermediate_buffer[req_id] = merged_info + setattr(self.requests[req_id], "additional_information_cpu", merged_info) From f72b38184b8ea0a9b9eb7253173634a1a82751ed Mon Sep 17 00:00:00 2001 From: wangyu <53896905+yenuo26@users.noreply.github.com> Date: Thu, 23 Jul 2026 21:47:58 +0800 Subject: [PATCH 073/240] [CI] move Krea-2 expansion to slow and skip failing README snippets (#5347) Signed-off-by: wangyu <410167048@qq.com> --- tests/e2e/offline_inference/test_krea2_expansion.py | 2 +- tests/e2e/online_serving/test_krea2_expansion.py | 2 +- tests/examples/offline_inference/test_text_to_image.py | 3 +++ 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/e2e/offline_inference/test_krea2_expansion.py b/tests/e2e/offline_inference/test_krea2_expansion.py index f00b4bb0b..a8f63032b 100644 --- a/tests/e2e/offline_inference/test_krea2_expansion.py +++ b/tests/e2e/offline_inference/test_krea2_expansion.py @@ -31,7 +31,7 @@ LORA_PROMPT = f"a fox in the snow, {LORA_TRIGGER}" pytestmark = [ pytest.mark.diffusion, - pytest.mark.full_model, + pytest.mark.slow, ] diff --git a/tests/e2e/online_serving/test_krea2_expansion.py b/tests/e2e/online_serving/test_krea2_expansion.py index 10aa520fb..f76a567f4 100644 --- a/tests/e2e/online_serving/test_krea2_expansion.py +++ b/tests/e2e/online_serving/test_krea2_expansion.py @@ -31,7 +31,7 @@ from PIL import Image from tests.helpers.mark import hardware_marks from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler, dummy_messages_from_mix_data -pytestmark = [pytest.mark.diffusion, pytest.mark.full_model] +pytestmark = [pytest.mark.diffusion, pytest.mark.slow] MODEL = os.environ.get("KREA2_MODEL", "krea/Krea-2-Turbo") # vLLM-Omni-compatible PEFT repackaging of krea/Krea-2-LoRA-darkbrush (264 modules, r=alpha=32). diff --git a/tests/examples/offline_inference/test_text_to_image.py b/tests/examples/offline_inference/test_text_to_image.py index 701f1a184..9b5597c3e 100644 --- a/tests/examples/offline_inference/test_text_to_image.py +++ b/tests/examples/offline_inference/test_text_to_image.py @@ -26,6 +26,9 @@ EXAMPLE_OUTPUT_SUBFOLDER = "example_offline_t2i" def _skip_readme_snippet(language: str, code: str, h2_title: str) -> tuple[bool, str]: if h2_title == "Web UI Demo": return True, f"README section '{h2_title}' is intentionally excluded for examples tests" + # Krea-2 Turbo/Raw README snippets currently fail in CI (more_cli_examples_006/007). + if "krea/Krea-2-Turbo" in code or "krea/Krea-2-Raw" in code: + return True, "Krea-2 README snippets are temporarily skipped (CI failures)" return False, "" From e9a06d5bdc5dae249ca0a1c1ce195bc0a974ce76 Mon Sep 17 00:00:00 2001 From: wangyu <53896905+yenuo26@users.noreply.github.com> Date: Thu, 23 Jul 2026 22:15:52 +0800 Subject: [PATCH 074/240] [CI/Bugfix] Fix empty vllm_omni stub breaking stage CLI imports after Docker cache (#5338) (#5341) Signed-off-by: wangyu <410167048@qq.com> --- docker/Dockerfile.ci | 25 +++++++++++-------------- tests/helpers/runtime.py | 5 ++++- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/docker/Dockerfile.ci b/docker/Dockerfile.ci index 3d2876747..4171b063e 100644 --- a/docker/Dockerfile.ci +++ b/docker/Dockerfile.ci @@ -15,10 +15,10 @@ WORKDIR ${APP_DIR} RUN ln -sf /usr/bin/python3 /usr/bin/python -# Create a vllm_omni dir so that we can install the python package -# without failing to write the version files; we'll copy over it -# after installing -RUN mkdir -p vllm_omni +# Placeholder package tree so editable install can register APP_DIR. +# The final stage COPY --link fills this directory with the real sources; +# the editable .pth / egg-link keeps resolving imports there — no PYTHONPATH. +RUN mkdir -p vllm_omni && touch vllm_omni/__init__.py # Install system dependencies RUN apt-get update && \ @@ -32,7 +32,11 @@ RUN apt-get update && \ COPY pyproject.toml setup.py ./ COPY requirements/ requirements/ ENV SETUPTOOLS_SCM_PRETEND_VERSION=0.0.0 -RUN uv pip install --system ".[dev]" +# Editable install: deps + project metadata are cached in this layer, while +# the package itself is linked to APP_DIR (populated later by COPY --link). +# Do NOT use a non-editable install here — that copies an empty stub into +# site-packages and breaks ``import vllm_omni.entrypoints``. +RUN uv pip install --system -e ".[dev]" # The final stage just copies the working dir on top of the # dependencies image. This uses --link so that we can skip @@ -40,17 +44,10 @@ RUN uv pip install --system ".[dev]" # # IMPORTANT: Do not make any modifications that depend on the # file system after the --link copy, otherwise we'd need to pull -# the heavy base image to build it. This is the reason we copy -# link & update the pythonpath to point at the copied code instead -# of running a new pip install on it. +# the heavy base image to build it. Editable install above already +# registered APP_DIR, so we must not re-pip-install after COPY. FROM deps ARG APP_DIR=/workspace/vllm-omni -# Add APP_DIR to the pythonpath, since the entrypoint stub -# that was installed in the earlier step must point at the copied code. -# -# NOTE: this will probably warn about undefined vars, but is expected. -# See: https://github.com/docker/buildx/issues/2751 -ENV PYTHONPATH=${APP_DIR}${PYTHONPATH:+:${PYTHONPATH}} WORKDIR ${APP_DIR} COPY --link . . diff --git a/tests/helpers/runtime.py b/tests/helpers/runtime.py index 7c19c643d..5d1d9a3d1 100644 --- a/tests/helpers/runtime.py +++ b/tests/helpers/runtime.py @@ -638,7 +638,10 @@ class OmniServerStageCli(OmniServer): proc = subprocess.Popen( cmd, env=env, - cwd=os.path.dirname(os.path.dirname(os.path.abspath(__file__))), + # Must be repo root (not tests/): after Docker deps-cache installs an empty + # vllm_omni stub into site-packages, cwd on sys.path[0] is what makes + # ``python -m vllm_omni.entrypoints...`` resolve the real package. + cwd=_omni_subprocess_cwd(), stdout=log_fh, stderr=subprocess.STDOUT, ) From 778959797c5c454f58606dd4b25ff7947d6fd863 Mon Sep 17 00:00:00 2001 From: ZacheryAU <70869627+ZacheryAU@users.noreply.github.com> Date: Thu, 23 Jul 2026 23:55:05 +0800 Subject: [PATCH 075/240] [Refactor] Centralize metrics related helpers to metrics/ (#5168) --- tests/engine/test_orchestrator.py | 5 +- tests/metrics/test_metrics_utils.py | 101 ++++++++++++++ vllm_omni/benchmarks/patch/patch.py | 61 ++++---- vllm_omni/engine/orchestrator.py | 33 +---- vllm_omni/engine/stage_pool.py | 152 ++++++-------------- vllm_omni/metrics/__init__.py | 10 +- vllm_omni/metrics/definitions.py | 110 ++++++++++----- vllm_omni/metrics/modality.py | 58 +------- vllm_omni/metrics/utils.py | 209 +++++++++++++++++++++++++++- 9 files changed, 480 insertions(+), 259 deletions(-) create mode 100644 tests/metrics/test_metrics_utils.py diff --git a/tests/engine/test_orchestrator.py b/tests/engine/test_orchestrator.py index 68d9c2c58..4217e2156 100644 --- a/tests/engine/test_orchestrator.py +++ b/tests/engine/test_orchestrator.py @@ -30,7 +30,6 @@ from vllm_omni.engine.orchestrator import ( Orchestrator, OrchestratorRequestState, _build_terminal_empty_output, - _infer_stage_audio_sample_rate, ) from vllm_omni.engine.stage_pool import StagePool from vllm_omni.experimental.fullduplex.engine.duplex_control_plane import DuplexControlPlane @@ -201,12 +200,12 @@ class FakeStageClient: def test_terminal_empty_audio_output_uses_stage_sample_rate() -> None: final_stage = FakeStageClient(final_output=True, final_output_type="audio") final_stage.sample_rate = 44100 - final_pool = SimpleNamespace(stage_client=final_stage, _stage_vllm_config=None) + final_pool = StagePool(0, [final_stage]) terminal_output = _build_terminal_empty_output( "req-1", final_output_type="audio", - audio_sample_rate=_infer_stage_audio_sample_rate(final_pool), + audio_sample_rate=final_pool._infer_audio_sample_rate(), ) assert terminal_output.outputs[0].multimodal_output["sr"] == 44100 diff --git a/tests/metrics/test_metrics_utils.py b/tests/metrics/test_metrics_utils.py new file mode 100644 index 000000000..544c7819a --- /dev/null +++ b/tests/metrics/test_metrics_utils.py @@ -0,0 +1,101 @@ +from __future__ import annotations + +from types import SimpleNamespace + +import pytest + +from vllm_omni.metrics.utils import ( + count_audio_chunk_frames, + count_audio_frames, + count_image_pixels, + count_tokens_from_outputs, +) + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +@pytest.mark.parametrize( + ("audio_chunk", "expected"), + [ + (SimpleNamespace(shape=(2, 320)), 320), + ([0.0, 0.0, 0.0], 3), + (1.0, 1), + ], +) +def test_count_audio_chunk_frames(audio_chunk: object, expected: int) -> None: + assert count_audio_chunk_frames(audio_chunk) == expected + + +def test_count_audio_frames_sums_audio_chunks() -> None: + mm_out = { + "audio": [ + SimpleNamespace(shape=(1, 120)), + SimpleNamespace(shape=(2, 80)), + ] + } + + assert count_audio_frames(mm_out) == 200 + + +def test_count_audio_frames_supports_model_outputs() -> None: + mm_out = { + "model_outputs": [ + SimpleNamespace(shape=(1, 120)), + SimpleNamespace(shape=(2, 80)), + ] + } + + assert count_audio_frames(mm_out) == 200 + + +@pytest.mark.parametrize("mm_out", [{}, {"audio": None}, {"model_outputs": None}]) +def test_count_audio_frames_returns_zero_without_audio(mm_out: dict[str, object]) -> None: + assert count_audio_frames(mm_out) == 0 + + +@pytest.mark.parametrize( + ("value", "expected"), + [ + (None, 0), + (object(), 0), + (SimpleNamespace(size=(10, 20)), 200), + (SimpleNamespace(shape=(2, 3, 4, 5)), 40), + (SimpleNamespace(shape=(3, 4, 5)), 20), + (SimpleNamespace(shape=(6, 7, 3)), 42), + (SimpleNamespace(shape=(6, 7)), 42), + ], +) +def test_count_image_pixels(value: object, expected: int) -> None: + assert count_image_pixels(value) == expected + + +def test_count_image_pixels_sums_nested_values() -> None: + images = [ + SimpleNamespace(size=(10, 20)), + ( + SimpleNamespace(shape=(3, 4, 5)), + SimpleNamespace(shape=(6, 7, 3)), + ), + ] + + assert count_image_pixels(images) == 262 + + +def test_count_tokens_from_outputs() -> None: + engine_outputs = [ + SimpleNamespace( + outputs=[ + SimpleNamespace(cumulative_token_ids=[1, 2, 3], token_ids=[3]), + SimpleNamespace(cumulative_token_ids=None, token_ids=[4, 5]), + SimpleNamespace(cumulative_token_ids=None, token_ids=None), + ] + ), + SimpleNamespace(outputs=None), + SimpleNamespace( + outputs=[ + SimpleNamespace(cumulative_token_ids=[6], token_ids=[6]), + ] + ), + ] + + assert count_tokens_from_outputs(engine_outputs) == 6 diff --git a/vllm_omni/benchmarks/patch/patch.py b/vllm_omni/benchmarks/patch/patch.py index e4f11b1e7..00af6ee23 100644 --- a/vllm_omni/benchmarks/patch/patch.py +++ b/vllm_omni/benchmarks/patch/patch.py @@ -48,6 +48,7 @@ from vllm_omni.benchmarks.data_modules.seed_tts_dataset import ( from vllm_omni.benchmarks.data_modules.sound_effect_dataset import SoundEffectDataset from vllm_omni.benchmarks.data_modules.ttsd_dataset import TTSDDataset from vllm_omni.metrics import definitions as defs +from vllm_omni.metrics.utils import coerce_positive_int_scalar logger = init_logger(__name__) @@ -524,16 +525,8 @@ def _add_image_edit_extra_body_to_form(form: aiohttp.FormData, extra_body: dict[ form.add_field(key, str(value)) -def _coerce_positive_int(value: Any) -> int | None: - try: - coerced = int(value) - except (TypeError, ValueError): - return None - return coerced if coerced > 0 else None - - def _extract_output_tokens_from_metrics(metrics: dict[str, Any]) -> int | None: - top_level_tokens = _coerce_positive_int(metrics.get(defs.NUM_TOKENS_OUT)) + top_level_tokens = coerce_positive_int_scalar(metrics.get(defs.NUM_TOKENS_OUT)) if top_level_tokens is not None: return top_level_tokens @@ -545,7 +538,7 @@ def _extract_output_tokens_from_metrics(metrics: dict[str, Any]) -> int | None: for info in stage_snapshot.values(): if not isinstance(info, dict): continue - num_tokens_out = _coerce_positive_int(info.get(defs.NUM_TOKENS_OUT)) + num_tokens_out = coerce_positive_int_scalar(info.get(defs.NUM_TOKENS_OUT)) if num_tokens_out is None: continue if info.get("final_output_type") == "text" or info.get("output_unit_type") == "token": @@ -556,9 +549,9 @@ def _extract_output_tokens_from_metrics(metrics: dict[str, Any]) -> int | None: def _apply_usage_to_output(output: MixRequestFuncOutput, usage: dict[str, Any]) -> int | None: """Apply OpenAI ``usage`` fields to the benchmark output.""" - if (pt := _coerce_positive_int(usage.get("prompt_tokens"))) is not None: + if (pt := coerce_positive_int_scalar(usage.get("prompt_tokens"))) is not None: output.prompt_len = pt - completion_tokens = _coerce_positive_int(usage.get("completion_tokens")) + completion_tokens = coerce_positive_int_scalar(usage.get("completion_tokens")) if completion_tokens is not None: output.output_tokens = max(int(output.output_tokens or 0), completion_tokens) return completion_tokens @@ -901,12 +894,22 @@ async def async_request_openai_chat_omni_completions( audio_frames = 0 if response_format == "wav" and wav_pcm_buffer and wav_audio_params is not None: channels, sample_width, frame_rate = wav_audio_params - frame_width = sample_width * channels - if frame_width > 0: - audio_frames = len(wav_pcm_buffer) // frame_width - audio_duration_sec = audio_frames / frame_rate + audio_frames = defs.compute_audio_frames( + len(wav_pcm_buffer), + sample_width=sample_width, + channels=channels, + ) + if audio_frames > 0 and frame_rate > 0: + audio_duration_sec = audio_frames / float(frame_rate) else: - logger.warning("Audio frame width is zero") + logger.warning( + "Unable to derive audio frames/duration from wav pcm " + "(pcm_nbytes=%d, sample_width=%d, channels=%d, sample_rate=%d)", + len(wav_pcm_buffer), + sample_width, + channels, + frame_rate, + ) elif audio_bytes_buffer: try: from vllm.multimodal.audio import get_audio_duration @@ -925,10 +928,8 @@ async def async_request_openai_chat_omni_completions( output.audio_duration = audio_duration_sec output.audio_frames = audio_frames audio_duration = output.audio_duration - if audio_duration > 0: - output.audio_rtf = audio_generate_time / output.audio_duration - else: - output.audio_rtf = 0 + output.audio_rtf = defs.compute_audio_rtf(audio_generate_time, audio_duration) + if audio_duration <= 0: logger.warning("Audio duration is zero") if _seed_tts_capture_pcm_for_wer() and getattr(request_func_input, "seed_tts_row", False): try: @@ -1161,7 +1162,7 @@ async def async_request_openai_audio_speech( # PCM format: 16-bit signed; sample_rate/channels are model-dependent. sample_rate, channels = defs.stream_pcm_format_from_env() - sample_width = 2 # 16-bit = 2 bytes + sample_width = defs.DEFAULT_AUDIO_SAMPLE_WIDTH st = time.perf_counter() output.start_time = st @@ -1190,13 +1191,17 @@ async def async_request_openai_audio_speech( end_time = time.perf_counter() output.latency = end_time - st - total_samples = total_pcm_bytes // (sample_width * channels) - output.audio_duration = total_samples / sample_rate + total_samples = defs.compute_audio_frames( + total_pcm_bytes, + sample_width=sample_width, + channels=channels, + ) output.audio_frames = total_samples - if output.audio_duration > 0: - output.audio_rtf = output.latency / output.audio_duration - else: - output.audio_rtf = 0 + output.audio_duration = ( + float(total_samples) / float(sample_rate) if total_samples > 0 and sample_rate > 0 else 0.0 + ) + output.audio_rtf = defs.compute_audio_rtf(output.latency, output.audio_duration) + if output.audio_duration <= 0: logger.warning("Audio duration is zero") continuity = compute_continuity_stats( diff --git a/vllm_omni/engine/orchestrator.py b/vllm_omni/engine/orchestrator.py index 9372fee28..e10ce8845 100644 --- a/vllm_omni/engine/orchestrator.py +++ b/vllm_omni/engine/orchestrator.py @@ -104,37 +104,6 @@ def _build_terminal_empty_output( ) -def _coerce_int_scalar(value: Any) -> int: - if value is None: - return 0 - if isinstance(value, (list, tuple)): - for item in value: - coerced = _coerce_int_scalar(item) - if coerced > 0: - return coerced - return 0 - if hasattr(value, "item"): - value = value.item() - try: - coerced = int(value) - except (TypeError, ValueError): - return 0 - return coerced if coerced > 0 else 0 - - -def _infer_stage_audio_sample_rate(stage_pool: StagePool, default: int = 24000) -> int: - """Infer the final audio stage sample rate from stage metadata when possible.""" - sample_rate_attrs = ("audio_sample_rate", "sample_rate", "sampling_rate", "output_sample_rate", "sr") - stage_client = getattr(stage_pool, "stage_client", None) - stage_config = getattr(stage_pool, "_stage_vllm_config", None) - for source in (stage_client, stage_config): - for attr in sample_rate_attrs: - sample_rate = _coerce_int_scalar(getattr(source, attr, None)) - if sample_rate > 0: - return sample_rate - return default - - def build_engine_core_request_from_tokens( request_id: str, prompt: dict[str, Any], @@ -1909,7 +1878,7 @@ class Orchestrator: terminal_output = _build_terminal_empty_output( req_id, final_output_type=final_output_type, - audio_sample_rate=_infer_stage_audio_sample_rate(final_pool), + audio_sample_rate=final_pool._infer_audio_sample_rate(), ) submit_ts = _time.time() req_state.stage_submit_ts[final_stage_id] = submit_ts diff --git a/vllm_omni/engine/stage_pool.py b/vllm_omni/engine/stage_pool.py index 4c4d3b7ee..32a35ac8c 100644 --- a/vllm_omni/engine/stage_pool.py +++ b/vllm_omni/engine/stage_pool.py @@ -25,10 +25,18 @@ from vllm_omni.engine.stage_client import ( StagePoolLLMClient, ) from vllm_omni.inputs.data import OmniDiffusionSamplingParams +from vllm_omni.metrics import ( + count_audio_frames, + count_image_pixels, + count_tokens_from_outputs, +) from vllm_omni.metrics import definitions as defs from vllm_omni.metrics.stats import StageRequestStats as StageRequestMetrics from vllm_omni.metrics.stats import StageStats -from vllm_omni.metrics.utils import count_tokens_from_outputs +from vllm_omni.metrics.utils import ( + coerce_positive_int_scalar, + iter_mm_outputs, +) if TYPE_CHECKING: from vllm_omni.engine.orchestrator import OrchestratorRequestState @@ -619,7 +627,7 @@ class StagePool: else 0.0 ) image_pixels = self._count_image_pixels(request_outputs) if output_unit_type == "image" else 0 - num_inference_steps = self._coerce_int_scalar(getattr(sampling_params, "num_inference_steps", None)) + num_inference_steps = coerce_positive_int_scalar(getattr(sampling_params, "num_inference_steps", None)) or 0 denoise_step_latency_ms = ( defs.compute_denoise_step_latency(stage_gen_time_ms, num_inference_steps) if output_unit_type == "image" @@ -741,78 +749,40 @@ class StagePool: def _has_audio_output(self, request_outputs: list[Any]) -> bool: for ro in request_outputs: - for mm_output in self._iter_multimodal_outputs(ro): + for mm_output in iter_mm_outputs(ro): if isinstance(mm_output, Mapping) and mm_output.get("audio") is not None: return True return False - def _collect_audio_metrics(self, request_outputs: list[Any]) -> tuple[int, int, float]: + def _collect_audio_metrics( + self, + request_outputs: list[Any], + *, + use_default_sample_rate: bool = False, + ) -> tuple[int, int, float]: total_frames = 0 sample_rate = 0 for ro in request_outputs: - for mm_output in self._iter_multimodal_outputs(ro): - if not isinstance(mm_output, Mapping): - continue + for mm_output in iter_mm_outputs(ro): if sample_rate <= 0: - sample_rate = self._infer_audio_sample_rate(mm_output) - audio_output = mm_output.get("audio") - if audio_output is None: - continue - items = audio_output if isinstance(audio_output, list) else [audio_output] - for item in items: - total_frames += self._count_audio_frames(item) + sample_rate = self._infer_audio_sample_rate(mm_output, use_default=use_default_sample_rate) + total_frames += count_audio_frames(mm_output) duration_s = (float(total_frames) / float(sample_rate)) if total_frames > 0 and sample_rate > 0 else 0.0 return int(total_frames), int(sample_rate), duration_s - @staticmethod - def _count_audio_frames(audio_output: Any) -> int: - shape = getattr(audio_output, "shape", None) - if shape is not None and len(shape) > 0: - # Audio chunks are concatenated on dim=-1 in the output processor, - # so the frame/sample axis is the last dim (e.g. [channels, frames]). - # Keep this aligned with serving_chat.py: audio tensors are consumed - # as (T,), (C, T), or (B, C, T). Flattening would corrupt - # multi-channel audio. - return int(shape[-1]) - try: - return len(audio_output) - except TypeError: - return 1 - - def _infer_audio_sample_rate(self, mm_output: dict[str, Any]) -> int: - for key in ("audio_sample_rate", "sample_rate", "sampling_rate", "sr"): - rate = self._coerce_int_scalar(mm_output.get(key)) - if rate > 0: - return rate - for attr in ("audio_sample_rate", "sample_rate", "sampling_rate", "output_sample_rate"): - rate = self._coerce_int_scalar(getattr(self.stage_client, attr, None)) - if rate > 0: - return rate - rate = self._coerce_int_scalar(getattr(self._stage_vllm_config, attr, None)) - if rate > 0: - return rate - return 0 - - @classmethod - def _coerce_int_scalar(cls, value: Any) -> int: - if value is None: - return 0 - if isinstance(value, (list, tuple)): - for item in value: - coerced = cls._coerce_int_scalar(item) - if coerced > 0: - return coerced - return 0 - item = getattr(value, "item", None) - if callable(item): - try: - return int(item()) - except Exception: - return 0 - try: - return int(value) - except (TypeError, ValueError): - return 0 + def _infer_audio_sample_rate( + self, + mm_output: dict[str, Any] | None = None, + *, + use_default: bool = True, + ) -> int: + sources: list[Any] = [] + if mm_output is not None: + sources.append(mm_output) + sources.extend([self.stage_client, self._stage_vllm_config]) + if use_default: + return defs.resolve_audio_sample_rate(sources) + return defs.resolve_audio_sample_rate_or_none(sources) or 0 def _has_image_output(self, request_outputs: list[Any]) -> bool: for ro in request_outputs: @@ -834,7 +804,7 @@ class StagePool: def _count_images(self, request_output: Any) -> int: total_images = self._count_value_units(getattr(request_output, "images", None)) - for mm_output in self._iter_multimodal_outputs(request_output): + for mm_output in iter_mm_outputs(request_output): if isinstance(mm_output, Mapping): total_images += self._count_value_units(mm_output.get("image")) total_images += self._count_value_units(mm_output.get("images")) @@ -843,43 +813,17 @@ class StagePool: def _count_image_pixels(self, request_outputs: list[Any]) -> int: total_pixels = 0 for ro in request_outputs: - total_pixels += self._count_image_value_pixels(getattr(ro, "images", None)) - for mm_output in self._iter_multimodal_outputs(ro): + total_pixels += count_image_pixels(getattr(ro, "images", None)) + for mm_output in iter_mm_outputs(ro): if isinstance(mm_output, Mapping): - total_pixels += self._count_image_value_pixels(mm_output.get("image")) - total_pixels += self._count_image_value_pixels(mm_output.get("images")) + total_pixels += count_image_pixels(mm_output.get("image")) + total_pixels += count_image_pixels(mm_output.get("images")) return total_pixels - @classmethod - def _count_image_value_pixels(cls, value: Any) -> int: - if value is None: - return 0 - if isinstance(value, (list, tuple)): - return sum(cls._count_image_value_pixels(item) for item in value) - - size = getattr(value, "size", None) - if isinstance(size, tuple) and len(size) >= 2: - try: - return int(size[0]) * int(size[1]) - except (TypeError, ValueError): - return 0 - - shape = getattr(value, "shape", None) - if shape is None or len(shape) < 2: - return 0 - dims = [int(dim) for dim in shape] - if len(dims) >= 4: - return dims[0] * dims[-2] * dims[-1] - if len(dims) == 3 and dims[0] in (1, 3, 4): - return dims[1] * dims[2] - if len(dims) == 3 and dims[-1] in (1, 3, 4): - return dims[0] * dims[1] - return dims[-2] * dims[-1] - def _count_videos(self, request_output: Any) -> int: total_videos = self._count_video_units(getattr(request_output, "video", None)) total_videos += self._count_video_units(getattr(request_output, "videos", None)) - for mm_output in self._iter_multimodal_outputs(request_output): + for mm_output in iter_mm_outputs(request_output): if isinstance(mm_output, Mapping): total_videos += self._count_video_units(mm_output.get("video")) total_videos += self._count_video_units(mm_output.get("videos")) @@ -904,7 +848,7 @@ class StagePool: custom_output = getattr(request_output, "_custom_output", None) if isinstance(custom_output, dict) and custom_output: return True - for mm_output in self._iter_multimodal_outputs(request_output): + for mm_output in iter_mm_outputs(request_output): if isinstance(mm_output, Mapping) and any(self._is_non_empty_value(value) for value in mm_output.values()): return True for output in getattr(request_output, "outputs", None) or []: @@ -967,23 +911,15 @@ class StagePool: self._output_timestamps_by_request.setdefault(rid, []).append(output_ts) if self._has_non_empty_output(request_output): self._non_empty_first_output_timestamps_by_request.setdefault(rid, output_ts) - audio_frames, audio_sample_rate, _ = self._collect_audio_metrics([request_output]) + audio_frames, audio_sample_rate, _ = self._collect_audio_metrics( + [request_output], + use_default_sample_rate=False, + ) if audio_frames > 0: self._audio_frames_by_request[rid] = self._audio_frames_by_request.get(rid, 0) + audio_frames if self._audio_sample_rate_by_request.get(rid, 0) <= 0 and audio_sample_rate > 0: self._audio_sample_rate_by_request[rid] = audio_sample_rate - def _iter_multimodal_outputs(self, request_output: object) -> list[Mapping[str, Any]]: - multimodal_outputs: list[Mapping[str, Any]] = [] - outer_mm = getattr(request_output, "multimodal_output", None) - if isinstance(outer_mm, Mapping) and outer_mm: - multimodal_outputs.append(outer_mm) - for output in getattr(request_output, "outputs", None) or []: - inner_mm = getattr(output, "multimodal_output", None) - if isinstance(inner_mm, Mapping) and inner_mm: - multimodal_outputs.append(inner_mm) - return multimodal_outputs - # ---- Stage-local admission ---- async def submit_initial( diff --git a/vllm_omni/metrics/__init__.py b/vllm_omni/metrics/__init__.py index 6814a5891..7b6a2979f 100644 --- a/vllm_omni/metrics/__init__.py +++ b/vllm_omni/metrics/__init__.py @@ -1,6 +1,11 @@ from .prometheus import OmniPrometheusMetrics, OmniRequestCounter from .stats import OrchestratorAggregator, StageRequestStats, StageStats -from .utils import count_tokens_from_outputs +from .utils import ( + count_audio_chunk_frames, + count_audio_frames, + count_image_pixels, + count_tokens_from_outputs, +) __all__ = [ "OmniPrometheusMetrics", @@ -8,5 +13,8 @@ __all__ = [ "OrchestratorAggregator", "StageStats", "StageRequestStats", + "count_audio_chunk_frames", + "count_audio_frames", + "count_image_pixels", "count_tokens_from_outputs", ] diff --git a/vllm_omni/metrics/definitions.py b/vllm_omni/metrics/definitions.py index 9660e4f34..23a1364a4 100644 --- a/vllm_omni/metrics/definitions.py +++ b/vllm_omni/metrics/definitions.py @@ -13,7 +13,9 @@ time-bearing metrics use the ``_s`` suffix (values in seconds), counters use import logging import os -from typing import Any +from collections.abc import Mapping, Sequence + +from vllm_omni.metrics.utils import resolve_int_by_sequential_keys # vllm_omni: namespace for omni-specific Prometheus families, distinct from # the upstream vllm:* families. @@ -243,8 +245,20 @@ BYTES_BUCKETS = ( # ============================================================================ -# Audio-continuity defaults +# Audio defaults (shared by server-side observe and bench-side calculation) # ============================================================================ +# Most common across vllm-omni talker variants (cosyvoice3, omnivoice, +# qwen3_tts, mimo_audio). voxcpm2 uses 48000, stable_audio 44100, +# ming_flash 16000 — these models surface a rate at runtime on one of +# ``_SAMPLE_RATE_KEYS``, so DEFAULT_AUDIO_SAMPLE_RATE only applies when no +# usable positive rate is found (and as the bench PCM default when env is unset). +# Streamed OpenAI ``response_format=pcm`` is PCM_16 (s16le); sample width is fixed. +DEFAULT_AUDIO_SAMPLE_RATE = 24000 +DEFAULT_AUDIO_CHANNELS = 1 +DEFAULT_AUDIO_SAMPLE_WIDTH = 2 # PCM s16le bytes per sample +AUDIO_SAMPLE_RATE_ENV = "VLLM_OMNI_BENCH_AUDIO_SAMPLE_RATE" +AUDIO_CHANNELS_ENV = "VLLM_OMNI_BENCH_AUDIO_CHANNELS" + # Default underrun threshold — kept aligned with the bench-side default and # the commonly-cited "audible gap" threshold for streaming TTS. AUDIO_CONTINUITY_DEFAULT_THRESHOLD_S = 0.1 @@ -253,8 +267,8 @@ AUDIO_CONTINUITY_DEFAULT_THRESHOLD_S = 0.1 # ============================================================================ # Formula helpers (shared by server-side observe and bench-side calculation) # ============================================================================ -def compute_audio_rtf(stage_gen_time_s: float, audio_duration_s: float) -> float: - """RTF = stage_gen_time / audio_content_duration. +def compute_audio_rtf(audio_generation_latency_s: float, audio_duration_s: float) -> float: + """RTF = audio_generation_latency / audio_content_duration. SLO red line < 1 — must generate faster than content plays back to stream. Returns 0.0 when audio_duration_s is non-positive (caller decides whether @@ -262,7 +276,30 @@ def compute_audio_rtf(stage_gen_time_s: float, audio_duration_s: float) -> float """ if audio_duration_s <= 0: return 0.0 - return stage_gen_time_s / audio_duration_s + return audio_generation_latency_s / audio_duration_s + + +def compute_audio_frames( + pcm_nbytes: int, + *, + sample_width: int = DEFAULT_AUDIO_SAMPLE_WIDTH, + channels: int = DEFAULT_AUDIO_CHANNELS, +) -> int: + """Convert a PCM byte length to sample/frame count. + + ``pcm_nbytes`` is the raw byte size of interleaved PCM (e.g. + ``total_pcm_bytes`` or ``len(wav_pcm_buffer)``). Defaults match the + streamed s16le mono contract + (``DEFAULT_AUDIO_SAMPLE_WIDTH``, ``DEFAULT_AUDIO_CHANNELS``); + WAV callers should pass header values. + + Returns 0 when the byte count or frame width (``sample_width * channels``) + is non-positive. + """ + frame_width = sample_width * channels + if pcm_nbytes <= 0 or frame_width <= 0: + return 0 + return pcm_nbytes // frame_width def compute_denoise_step_latency(stage_gen_time: float, num_inference_steps: int) -> float: @@ -278,40 +315,45 @@ def compute_denoise_step_latency(stage_gen_time: float, num_inference_steps: int # ============================================================================ # Audio sample-rate resolution # ============================================================================ -# Most common across vllm-omni talker variants (cosyvoice3, omnivoice, -# qwen3_tts, mimo_audio). voxcpm2 uses 48000, stable_audio 44100, -# ming_flash 16000 — these models populate multimodal_output["audio_sample_rate"] -# at runtime so this default only kicks in when the field is missing. -DEFAULT_AUDIO_SAMPLE_RATE = 24000 -DEFAULT_AUDIO_CHANNELS = 1 -AUDIO_SAMPLE_RATE_ENV = "VLLM_OMNI_BENCH_AUDIO_SAMPLE_RATE" -AUDIO_CHANNELS_ENV = "VLLM_OMNI_BENCH_AUDIO_CHANNELS" - _SAMPLE_RATE_KEYS = ("output_sample_rate", "audio_sample_rate", "sample_rate", "sampling_rate", "sr") -def resolve_audio_sample_rate(source: dict[str, Any] | Any | None) -> int: - """Extract audio sample_rate from a dict or config object, with fallbacks. +def resolve_audio_sample_rate_or_none( + source: Sequence[Mapping[str, object] | object | None] | Mapping[str, object] | object | None, +) -> int | None: + """Extract an explicitly configured audio sample rate, or ``None`` when absent. - Tries the same key chain as serving_chat.py's audio response path so - /metrics audio_duration_s = audio_frames / sample_rate stays consistent - with what the OpenAI streaming endpoint reports back to clients. Also - accepts config objects that expose the same values as attributes. - Returns DEFAULT_AUDIO_SAMPLE_RATE when no usable value is present. + ``source`` may be: + + - a Mapping (e.g. ``multimodal_output`` / config dict) + - an object exposing the same fields as attributes + - a Sequence of the above (``str`` / ``bytes`` excluded); the first + positive rate found across items wins """ - if not source: - return DEFAULT_AUDIO_SAMPLE_RATE - for key in _SAMPLE_RATE_KEYS: - raw = source.get(key) if isinstance(source, dict) else getattr(source, key, None) - if raw is None: - continue - try: - value = int(raw) - except (TypeError, ValueError): - continue - if value > 0: - return value - return DEFAULT_AUDIO_SAMPLE_RATE + if isinstance(source, Sequence) and not isinstance(source, (str, bytes, bytearray)): + for item in source: + rate = resolve_int_by_sequential_keys(item, _SAMPLE_RATE_KEYS) + if rate is not None: + return rate + return None + return resolve_int_by_sequential_keys(source, _SAMPLE_RATE_KEYS) + + +def resolve_audio_sample_rate( + source: Sequence[Mapping[str, object] | object | None] | Mapping[str, object] | object | None, + default: int = DEFAULT_AUDIO_SAMPLE_RATE, +) -> int: + """Extract audio sample rate from a dict, config object, or list thereof. + + Delegates to :func:`resolve_audio_sample_rate_or_none` (same key chain and + coerce rules). Returns ``default`` (``DEFAULT_AUDIO_SAMPLE_RATE`` unless + overridden) when no usable positive rate is present. + + Used so /metrics ``audio_duration_s = audio_frames / sample_rate`` stays + consistent with rates surfaced on multimodal outputs and stage configs. + """ + rate = resolve_audio_sample_rate_or_none(source) + return rate if rate is not None else default def stream_pcm_format_from_env( diff --git a/vllm_omni/metrics/modality.py b/vllm_omni/metrics/modality.py index 73d46ee8b..fd455ad36 100644 --- a/vllm_omni/metrics/modality.py +++ b/vllm_omni/metrics/modality.py @@ -15,9 +15,9 @@ Contents: - ``observe_audio_streaming_finalize``: emits ``audio_underrun_s`` + ``audio_continuity_ok_total`` at SSE close using accumulated per-chunk arrival timestamps. -- ``_extract_mm_output`` / ``_count_audio_frames``: shape-tolerant helpers - for the heterogeneous multimodal_output payloads emitted by different - audio pipelines. +- ``extract_mm_output`` / ``count_audio_frames`` (from ``metrics.utils``): + shape-tolerant helpers for the heterogeneous multimodal_output payloads + emitted by different audio pipelines. """ from __future__ import annotations @@ -27,6 +27,7 @@ from typing import Any from prometheus_client import Counter, Histogram from vllm_omni.metrics import definitions as defs +from vllm_omni.metrics.utils import count_audio_frames, extract_mm_output _stage_labels = list(defs.STAGE_LABELS) @@ -139,53 +140,6 @@ class OmniModalityMetrics: ).inc() -def _extract_mm_output(engine_outputs: Any) -> dict[str, Any]: - """Return the multimodal_output dict regardless of where it's nested. - - Three shapes seen in the wild: - * ``engine_outputs.multimodal_output`` — synthesized on OmniRequestOutput - for some pipelines (often empty for AR audio) - * ``engine_outputs.outputs[0].multimodal_output`` — vllm CompletionOutput - nesting (where actual qwen3-omni audio data lives) - * neither — returns ``{}`` - """ - mm = getattr(engine_outputs, "multimodal_output", None) - if isinstance(mm, dict) and mm: - return mm - outs = getattr(engine_outputs, "outputs", None) - if outs: - nested = getattr(outs[0], "multimodal_output", None) - if isinstance(nested, dict): - return nested - return {} - - -def _count_audio_frames(mm_out: dict[str, Any]) -> int: - """Sum the per-tensor sample count of audio chunks in mm_out["audio"]. - - Returns the total number of audio frames (samples) across all chunks. - For multi-dim tensors (e.g. shape [channels, samples]) the last axis is - treated as the sample dim; for 1-D tensors the only axis is the sample - dim; scalars count as 1. - """ - audio_chunks = mm_out.get("audio") if isinstance(mm_out, dict) else None - if audio_chunks is None: - return 0 - chunks = audio_chunks if isinstance(audio_chunks, list) else [audio_chunks] - n = 0 - for t in chunks: - try: - ndim = getattr(t, "ndim", 0) - shape = getattr(t, "shape", None) - if ndim == 0 or shape is None or len(shape) == 0: - n += 1 - else: - n += int(shape[-1]) - except Exception: - continue - return n - - def observe_modality_at_finalize( mod_metrics: OmniModalityMetrics, *, @@ -213,7 +167,7 @@ def observe_modality_at_finalize( stage_label = str(stage_id) replica_label = str(replica_id) gen_time_s = float(getattr(stage_metrics, "stage_gen_time_ms", 0.0)) / 1000.0 - mm_out = _extract_mm_output(engine_outputs) + mm_out = extract_mm_output(engine_outputs) sample_rate = defs.resolve_audio_sample_rate(mm_out) # `stage_metrics.audio_generated_frames` is the legacy per-chunk @@ -222,7 +176,7 @@ def observe_modality_at_finalize( # field lookup in place in case the accumulator gets re-wired upstream. n_frames = int(getattr(stage_metrics, "audio_generated_frames", 0) or 0) if n_frames == 0: - n_frames = _count_audio_frames(mm_out) + n_frames = count_audio_frames(mm_out) mod_metrics.inc_audio_frames(stage_label, replica_label, n_frames) duration_s = n_frames / sample_rate if sample_rate > 0 else 0.0 if duration_s > 0: diff --git a/vllm_omni/metrics/utils.py b/vllm_omni/metrics/utils.py index 2062eb5ba..ef8f69ed1 100644 --- a/vllm_omni/metrics/utils.py +++ b/vllm_omni/metrics/utils.py @@ -1,10 +1,217 @@ -from collections.abc import Callable +from collections.abc import Callable, Mapping, Sequence from dataclasses import fields from typing import Any from prettytable import PrettyTable +def coerce_positive_int_scalar(value: object) -> int | None: + """Coerce a value to a positive int without importing tensor libs. + + Meant to pull positive integers such as sample rate out of the shapes + that show up across omni stage outputs / configs: + + - plain ``int``: ``44100`` + - ``torch.Tensor`` / numpy scalar: ``tensor(44100)`` (via ``.item()``) + - ``list`` / ``tuple`` wrap: ``[tensor(44100)]`` or ``[44100]`` + - Mapping field values: ``{"sr": tensor(44100)}`` → coerce the field + value (key lookup is done by the caller, e.g. + :func:`resolve_int_by_sequential_keys`) + + Returns ``None`` when the value is missing, unparsable, or not positive. + """ + if value is None: + return None + if isinstance(value, (list, tuple)): + for item in value: + coerced = coerce_positive_int_scalar(item) + if coerced is not None: + return coerced + return None + item = getattr(value, "item", None) + if callable(item): + try: + value = item() + except Exception: + return None + try: + value = int(value) + except (TypeError, ValueError): + return None + return value if value > 0 else None + + +def resolve_int_by_sequential_keys( + source: Mapping[str, object] | object | None, + keys: Sequence[str], +) -> int | None: + """Return the first positive int found by trying ``keys`` in order. + + For each key, looks up ``source[key]`` when ``source`` is a Mapping, otherwise + ``getattr(source, key, None)``. Values are coerced via :func:`coerce_positive_int_scalar`. + Returns ``None`` when ``source`` is empty/missing or no key yields a usable int. + """ + if not source: + return None + for key in keys: + raw = source.get(key) if isinstance(source, Mapping) else getattr(source, key, None) + value = coerce_positive_int_scalar(raw) + if value is not None: + return value + return None + + +def extract_mm_output(mm_source: object) -> Mapping[str, Any]: + """Return the first non-empty multimodal_output Mapping on ``mm_source``. + + Lookup order: + 1. ``mm_source.multimodal_output`` — top-level attribute / property + 2. ``mm_source.outputs[0].multimodal_output`` — CompletionOutput nesting + (typical AR audio path) + + Accepts both plain ``dict`` and ``MultimodalPayload`` (a ``Mapping``). + Returns ``{}`` when neither location yields a non-empty Mapping. + + Args: + mm_source: Duck-typed container with optional ``multimodal_output`` and/or + ``outputs`` (e.g. vLLM ``RequestOutput`` or ``OmniRequestOutput``). + + Returns: + The first non-empty multimodal Mapping found, or an empty ``dict``. + """ + mm = getattr(mm_source, "multimodal_output", None) + if isinstance(mm, Mapping) and mm: + return mm + outs = getattr(mm_source, "outputs", None) + if outs: + nested = getattr(outs[0], "multimodal_output", None) + if isinstance(nested, Mapping) and nested: + return nested + return {} + + +def iter_mm_outputs(mm_source: object) -> list[Mapping[str, Any]]: + """Collect all non-empty multimodal_output Mappings from ``mm_source``. + + Lookup order: + 1. ``mm_source.multimodal_output`` — top-level attribute / property + 2. each ``mm_source.outputs[i].multimodal_output`` — every CompletionOutput + entry that carries a non-empty multimodal Mapping + + Accepts both plain ``dict`` and ``MultimodalPayload`` (a ``Mapping``). + Used by stage-pool metrics aggregation that needs to visit every mm payload. + + Args: + mm_source: Duck-typed container with optional ``multimodal_output`` and/or + ``outputs`` (e.g. vLLM ``RequestOutput`` or ``OmniRequestOutput``). + + Returns: + A list of non-empty multimodal Mappings in discovery order. Empty when + none are present. + """ + multimodal_outputs: list[Mapping[str, Any]] = [] + outer_mm = getattr(mm_source, "multimodal_output", None) + if isinstance(outer_mm, Mapping) and outer_mm: + multimodal_outputs.append(outer_mm) + for output in getattr(mm_source, "outputs", None) or []: + inner_mm = getattr(output, "multimodal_output", None) + if isinstance(inner_mm, Mapping) and inner_mm: + multimodal_outputs.append(inner_mm) + return multimodal_outputs + + +def count_audio_chunk_frames(audio_chunk: object) -> int: + """Count frames (samples) in one audio tensor / chunk. + + Audio chunks are concatenated on dim=-1 in the output processor, so the + frame/sample axis is the last dim (e.g. ``[channels, frames]``). Keep this + aligned with serving_chat.py: audio tensors are consumed as ``(T,)``, + ``(C, T)``, or ``(B, C, T)``. Flattening would corrupt multi-channel audio. + + Args: + audio_chunk: A single audio tensor/array-like, or a scalar-like value. + + Returns: + Frame count for this chunk. Uses ``shape[-1]`` when shaped; otherwise + ``len(...)``; scalars / unlenable values count as ``1``. + """ + shape = getattr(audio_chunk, "shape", None) + if shape is not None and len(shape) > 0: + return int(shape[-1]) + try: + return len(audio_chunk) # type: ignore[arg-type] + except TypeError: + return 1 + + +def count_audio_frames(mm_out: Mapping[str, Any]) -> int: + """Sum frame counts over all audio chunks in ``mm_out["audio"]`` or with other related keys. + + For multi-dim tensors (e.g. shape ``[channels, samples]``) the last axis is + the sample dim; for 1-D tensors the only axis is the sample dim; scalars + count as 1. Missing or empty ``audio`` yields ``0``. + + Args: + mm_out: A multimodal_output Mapping (plain ``dict`` or + ``MultimodalPayload``) that may contain an ``audio`` or related key whose value + is one chunk or a list of chunks. + + Returns: + Total audio frames (samples) across all chunks. + """ + audio_chunks = None + if isinstance(mm_out, Mapping): + for key in ("audio", "model_outputs"): + audio_chunks = mm_out.get(key) + if audio_chunks is not None: + break + if audio_chunks is None: + return 0 + chunks = audio_chunks if isinstance(audio_chunks, list) else [audio_chunks] + return sum(count_audio_chunk_frames(chunk) for chunk in chunks) + + +def count_image_pixels(value: object) -> int: + """Count pixels in one image value, or sum over a nested list/tuple. + + Accepts PIL-like objects (``size=(W, H)``), tensors / arrays with a + ``shape`` attribute, and nested ``list`` / ``tuple`` containers. + + Shape heuristics (aligned with StagePool image metrics): + + - ``ndim >= 4`` (e.g. ``BCHW``): ``B * H * W`` via + ``dims[0] * dims[-2] * dims[-1]`` + - ``ndim == 3`` and ``dims[0] in (1, 3, 4)``: CHW → ``H * W`` + - ``ndim == 3`` and ``dims[-1] in (1, 3, 4)``: HWC → ``H * W`` + - otherwise: ``dims[-2] * dims[-1]`` + + Returns ``0`` when ``value`` is missing or cannot be interpreted. + """ + if value is None: + return 0 + if isinstance(value, (list, tuple)): + return sum(count_image_pixels(item) for item in value) + + size = getattr(value, "size", None) + if isinstance(size, tuple) and len(size) >= 2: + try: + return int(size[0]) * int(size[1]) + except (TypeError, ValueError): + return 0 + + shape = getattr(value, "shape", None) + if shape is None or len(shape) < 2: + return 0 + dims = [int(dim) for dim in shape] + if len(dims) >= 4: + return dims[0] * dims[-2] * dims[-1] + if len(dims) == 3 and dims[0] in (1, 3, 4): + return dims[1] * dims[2] + if len(dims) == 3 and dims[-1] in (1, 3, 4): + return dims[0] * dims[1] + return dims[-2] * dims[-1] + + def _build_field_defs( cls: type, exclude: set[str], From 4c23a3164b98fecf63d2d28b87bc4ffad8853bdf Mon Sep 17 00:00:00 2001 From: ZacheryAU <70869627+ZacheryAU@users.noreply.github.com> Date: Fri, 24 Jul 2026 00:00:32 +0800 Subject: [PATCH 076/240] [Refactor] Split vllm_omni/entrypoints/cli/benchmark/serve.py (#5206) --- tests/benchmarks/test_serve_cli.py | 145 +++++++++- vllm_omni/entrypoints/cli/benchmark/base.py | 8 + .../entrypoints/cli/benchmark/cli_args.py | 244 ++++++++++++++++ vllm_omni/entrypoints/cli/benchmark/main.py | 8 + vllm_omni/entrypoints/cli/benchmark/serve.py | 267 ++---------------- 5 files changed, 419 insertions(+), 253 deletions(-) create mode 100644 vllm_omni/entrypoints/cli/benchmark/cli_args.py diff --git a/tests/benchmarks/test_serve_cli.py b/tests/benchmarks/test_serve_cli.py index 6970f6e2f..9ec30639c 100644 --- a/tests/benchmarks/test_serve_cli.py +++ b/tests/benchmarks/test_serve_cli.py @@ -1,3 +1,4 @@ +import argparse import json import os import subprocess @@ -6,10 +7,148 @@ from pathlib import Path import pytest +from vllm_omni.entrypoints.cli.benchmark.cli_args import ( + add_omni_args, + extend_omni_choices, + preprocess_serve_args, + update_omni_help, +) +from vllm_omni.utils.tracking_parser import TrackingArgumentParser + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.benchmark] + + +@pytest.mark.parametrize( + "dataset_name", + [ + "daily-omni", + "seed-tts", + "seed-tts-text", + "seed-tts-design", + "ttsd", + "sound-effect", + ], +) +def test_extend_omni_choices_updates_tracking_parser_shadow(dataset_name: str) -> None: + parser = TrackingArgumentParser() + parser.add_argument("--dataset-name", choices=["random"]) + parser.add_argument("--backend", choices=["openai-chat-omni"]) + + extend_omni_choices(parser) + + args = parser.parse_args( + [ + "--dataset-name", + dataset_name, + "--backend", + "openai-image-edits-omni", + ] + ) + + assert args.dataset_name == dataset_name + assert args.backend == "openai-image-edits-omni" + assert args.explicit_keys == {"dataset_name", "backend"} + + +@pytest.mark.parametrize( + ("argv", "dest", "expected"), + [ + (["--print-stage"], "print_stage", True), + (["--daily-omni-input-mode", "audio"], "daily_omni_input_mode", "audio"), + (["--seed-tts-locale", "zh"], "seed_tts_locale", "zh"), + ], +) +def test_add_omni_args_registers_arguments_on_tracking_parser( + argv: list[str], + dest: str, + expected: object, +) -> None: + parser = TrackingArgumentParser() + + add_omni_args(parser) + + args = parser.parse_args(argv) + assert getattr(args, dest) == expected + assert args.explicit_keys == {dest} + + +def test_add_omni_args_preserves_implicit_defaults() -> None: + parser = TrackingArgumentParser() + + add_omni_args(parser) + + args = parser.parse_args([]) + assert args.print_stage is False + assert args.daily_omni_input_mode == "all" + assert args.seed_tts_locale == "en" + assert args.explicit_keys == set() + + +def test_update_omni_help_updates_upstream_actions() -> None: + parser = TrackingArgumentParser() + parser.add_argument("--percentile-metrics", help="Upstream percentile help.") + parser.add_argument("--random-mm-limit-mm-per-prompt", help="Upstream limit help.") + parser.add_argument("--random-mm-bucket-config", help="Upstream bucket help.") + + update_omni_help(parser) + + actions = {action.dest: action for action in parser._actions} + assert all(metric in actions["percentile_metrics"].help for metric in ("ttfc", "tpop", "audio_rtf")) + assert "probabilities are renormalized" in actions["random_mm_limit_mm_per_prompt"].help + assert "Currently allows for 3 modalities" in actions["random_mm_bucket_config"].help + + +@pytest.mark.parametrize( + ("extra_body", "bot_task", "expected"), + [ + (None, "think", {"bot_task": "think"}), + ({"bot_task": "recaption"}, "think", {"bot_task": "recaption"}), + ({"custom": True}, None, {"custom": True}), + ], +) +def test_preprocess_serve_args_merges_bot_task_without_overriding_extra_body( + extra_body: dict | None, + bot_task: str | None, + expected: dict, +) -> None: + args = argparse.Namespace(extra_body=extra_body, bot_task=bot_task) + + preprocess_serve_args(args) + + assert args.extra_body == expected + + +@pytest.mark.parametrize( + ("argv", "expected_extra_body", "expected_explicit"), + [ + ( + ["--print-stage", "--image-edits-bot-task", "recaption"], + {"bot_task": "recaption"}, + {"print_stage", "bot_task"}, + ), + ( + ["--extra-body", '{"bot_task":"vanilla"}'], + {"bot_task": "vanilla"}, + {"extra_body"}, + ), + ], +) +def test_omni_args_parse_and_preprocess( + argv: list[str], + expected_extra_body: dict, + expected_explicit: set[str], +) -> None: + parser = TrackingArgumentParser() + parser.add_argument("--extra-body", type=json.loads, default=None) + add_omni_args(parser) + + args = parser.parse_args(argv) + preprocess_serve_args(args) + + assert args.extra_body == expected_extra_body + assert args.explicit_keys == expected_explicit + -@pytest.mark.core_model -@pytest.mark.benchmark -@pytest.mark.cpu def test_bench_serve_cli_mocks_http_request(tmp_path: Path): num_prompts = 5 port = 18000 diff --git a/vllm_omni/entrypoints/cli/benchmark/base.py b/vllm_omni/entrypoints/cli/benchmark/base.py index 6a6f97eb1..d35c98787 100644 --- a/vllm_omni/entrypoints/cli/benchmark/base.py +++ b/vllm_omni/entrypoints/cli/benchmark/base.py @@ -1,3 +1,11 @@ +"""Shared interface for vLLM-Omni benchmark CLI subcommands. + +``OmniBenchmarkSubcommandBase`` defines the argument-registration and command +execution hooks required by benchmark implementations. The ``bench`` command +discovers its concrete subclasses and uses these hooks to build and dispatch +nested benchmark subcommands. +""" + import argparse from vllm.entrypoints.cli.types import CLISubcommand diff --git a/vllm_omni/entrypoints/cli/benchmark/cli_args.py b/vllm_omni/entrypoints/cli/benchmark/cli_args.py new file mode 100644 index 000000000..66ece628e --- /dev/null +++ b/vllm_omni/entrypoints/cli/benchmark/cli_args.py @@ -0,0 +1,244 @@ +"""vLLM-Omni extensions for the ``vllm bench serve`` CLI. + +Core functions: + add_omni_args: Register all Omni-specific argument groups by calling the + feature-specific ``add_*_cli_args`` helpers in this module. + extend_omni_choices: Add Omni datasets and backends to choices defined by + the upstream vLLM parser, including its shadow parser. + update_omni_help: Extend upstream help text with Omni-specific behavior. + preprocess_serve_args: Apply transformations that require parsed values + before the serving benchmark starts. + +``OmniBenchmarkServingSubcommand.add_cli_args`` invokes the first three after +upstream vLLM registers its arguments. New feature arguments should normally be +added to the corresponding ``add_*_cli_args`` helper, or to a new helper called +by ``add_omni_args``. +""" + +import argparse + + +def add_multi_stage_cli_args(parser: argparse.ArgumentParser) -> None: + """Add CLI arguments for vLLM-Omni multi-stage benchmarks.""" + group = parser.add_argument_group("vLLM-Omni Multi-stage Benchmark Options") + group.add_argument( + "--print-stage", + action="store_true", + default=False, + help=( + "Print per-stage benchmark metrics for --omni serving when stage metrics are returned by the server. " + "Disabled by default. The latency sections follow --percentile-metrics by modality: " + "ttft/tpot/itl control text stages, ttfc/tpoc/icl control internal stream stages, " + "and tpop controls both text TPOP and internal stream TPOP." + ), + ) + + +def add_diffusion_cli_args(parser: argparse.ArgumentParser) -> None: + """Add CLI arguments for diffusion model benchmarks.""" + group = parser.add_argument_group("Diffusion Models Options") + group.add_argument( + "--image-edits-bot-task", + dest="bot_task", + type=str, + default="think", + help=( + "Default bot_task form field for --backend openai-image-edits-omni " + "(/v1/images/edits). " + 'Use --extra-body \'{"bot_task":"..."}\' to override per run.' + ), + ) + + +def add_daily_omni_cli_args(parser: argparse.ArgumentParser) -> None: + """Add CLI arguments specific to the Daily-Omni dataset.""" + group = parser.add_argument_group("Daily-Omni Dataset Options") + group.add_argument( + "--daily-omni-qa-json", + type=str, + default=None, + help="Path to local upstream qa.json. When set, QA rows are read from this file and " + "the HuggingFace dataset is not loaded (no network). Use with --daily-omni-video-dir " + "for fully offline runs. --dataset-path / Hub split flags are then ignored for QA loading.", + ) + group.add_argument( + "--daily-omni-video-dir", + type=str, + default=None, + help="Root directory of extracted Daily-Omni videos (contents of Videos.tar: " + "each video_id in its own subdir with {video_id}_video.mp4). " + "If omitted, Videos.tar is downloaded from the Hugging Face dataset repo on first multimodal " + "request. " + "When using file URLs, you MUST start the vLLM server with " + "--allowed-local-media-path set to this same directory (or a parent), " + "otherwise requests fail with 'Cannot load local files without " + "--allowed-local-media-path'.", + ) + group.add_argument( + "--daily-omni-inline-local-video", + action="store_true", + default=False, + help="For local videos only: embed MP4 as base64 data URLs in benchmark " + "requests so the server does not need --allowed-local-media-path. " + "Increases request size and client memory; use for small --num-prompts. " + "When using --daily-omni-input-mode audio or all, local WAV files are " + "embedded the same way.", + ) + group.add_argument( + "--daily-omni-input-mode", + type=str, + choices=["all", "visual", "audio"], + default="all", + help="Daily-Omni input protocol (mirrors upstream Lliar-liar/Daily-Omni " + "--input_mode). 'visual': video only (default). 'audio': WAV only, " + "requires {video_id}/{video_id}_audio.wav under --daily-omni-video-dir. " + "'all': video + WAV together. Sets mm_processor_kwargs.use_audio_in_video=false " + "and matches official separate video/audio streams.", + ) + group.add_argument( + "--daily-omni-save-eval-items", + action="store_true", + default=False, + help="Include per-request Daily-Omni accuracy rows (gold/predicted/correct) " + "in the saved JSON under key daily_omni_eval_items. " + "Alternatively set env DAILY_OMNI_SAVE_EVAL_ITEMS=1.", + ) + + +def add_seed_tts_cli_args(parser: argparse.ArgumentParser) -> None: + """Add CLI arguments for Seed-TTS benchmarks.""" + group = parser.add_argument_group("Seed-TTS Dataset Options") + group.add_argument( + "--seed-tts-locale", + type=str, + choices=["en", "zh"], + default="en", + help="Which Seed-TTS split to load: en/meta.lst or zh/meta.lst under the dataset root.", + ) + group.add_argument( + "--seed-tts-root", + type=str, + default=None, + help="Override root directory that contains en/ and zh/ (meta.lst + prompt-wavs). " + "If set, --dataset-path can still name the HF repo for logging; this path is used for files.", + ) + group.add_argument( + "--seed-tts-file-ref-audio", + action="store_true", + default=False, + help="Send ref_audio as file:// URIs (smaller HTTP bodies). Requires the API server " + "to be started with --allowed-local-media-path covering the Seed-TTS dataset root. " + "Default is inline data:audio/wav;base64 so Qwen3-TTS works without that flag.", + ) + group.add_argument( + "--seed-tts-inline-ref-audio", + action="store_true", + default=False, + help=argparse.SUPPRESS, + ) + group.add_argument( + "--seed-tts-system-prompt", + type=str, + default=None, + help="Override chat system message for --backend openai-chat-omni (Qwen3-Omni TTS). " + "Default follows official Qwen3-Omni identity + zero-shot voice-clone instructions.", + ) + group.add_argument( + "--seed-tts-wer-eval", + action="store_true", + default=False, + help="Keep synthesized audio as 24 kHz mono PCM for WER (works with " + "--backend openai-audio-speech or openai-chat-omni). Scoring follows " + "zhaochenyang20/seed-tts-eval (Whisper-large-v3 / Paraformer-zh + jiwer). " + "Sets SEED_TTS_WER_EVAL=1. Install: pip install 'vllm-omni[dev]'. " + "Optional: SEED_TTS_EVAL_DEVICE, SEED_TTS_HF_WHISPER_MODEL.", + ) + group.add_argument( + "--seed-tts-wer-save-items", + action="store_true", + default=False, + help="Include per-utterance ASR rows in the saved JSON under key seed_tts_wer_eval_items. " + "Or set SEED_TTS_WER_SAVE_ITEMS=1.", + ) + + +_OMNI_BENCH_DATASET_CHOICES = ( + "daily-omni", + "seed-tts", + "seed-tts-text", + "seed-tts-design", + "ttsd", + "sound-effect", +) + + +def extend_omni_choices(parser: argparse.ArgumentParser) -> None: + """Extend upstream argument choices with Omni-specific values.""" + parsers = [parser] + shadow = getattr(parser, "_shadow", None) + if shadow is not None: + parsers.append(shadow) + + for current_parser in parsers: + for action in current_parser._actions: + if action.dest == "dataset_name" and action.choices is not None: + extra = [choice for choice in _OMNI_BENCH_DATASET_CHOICES if choice not in action.choices] + if extra: + action.choices = list(action.choices) + extra + if action.dest == "backend" and action.choices is not None: + extra = [choice for choice in ("openai-image-edits-omni",) if choice not in action.choices] + if extra: + action.choices = list(action.choices) + extra + + +def update_omni_help(parser: argparse.ArgumentParser) -> None: + """Update upstream argument help text to describe Omni-specific behavior.""" + for action in parser._actions: + if action.dest == "percentile_metrics": + action.help = ( + "Comma-separated list of selected metrics to report percentiles. " + 'For text metrics, "ttft", "tpot", and "itl" affect the global benchmark and text ' + 'stage metrics. "tpop" also requests text TPOT/TPOP globally and per stage, and internal ' + 'stream TPOP. "ttfc", "tpoc", and "icl" only affect internal stream stage metrics. ' + 'Audio metrics include "audio_ttfp", "audio_rtf", "audio_duration", and "audio_underrun".' + ) + if action.dest == "random_mm_limit_mm_per_prompt": + action.help = ( + "Per-modality hard caps for items attached per request, e.g. " + '\'{"image": 3, "video": 0, "audio": 1}\'. The sampled per-request item ' + "count is clamped to the sum of these limits. When a modality " + "reaches its cap, its buckets are excluded and probabilities are " + "renormalized." + ) + if action.dest == "random_mm_bucket_config": + action.help = ( + "The bucket config is a dictionary mapping a multimodal item" + "sampling configuration to a probability." + "Currently allows for 3 modalities: audio, images and videos. " + "A bucket key is a tuple of (height, width, num_frames)" + "The value is the probability of sampling that specific item. " + "Example: " + "--random-mm-bucket-config " + "{(256, 256, 1): 0.5, (720, 1280, 16): 0.4, (0, 1, 5): 0.10} " + "First item: images with resolution 256x256 w.p. 0.5" + "Second item: videos with resolution 720x1280 and 16 frames " + "Third item: audios with 1s duration and 5 channels w.p. 0.1" + "OBS.: If the probabilities do not sum to 1, they are normalized." + ) + + +def add_omni_args(parser: argparse.ArgumentParser) -> None: + """Register all vLLM-Omni serving benchmark arguments.""" + add_daily_omni_cli_args(parser) + add_seed_tts_cli_args(parser) + add_multi_stage_cli_args(parser) + add_diffusion_cli_args(parser) + + +def preprocess_serve_args(args: argparse.Namespace) -> None: + """Apply serving benchmark CLI transformations after parsing.""" + extra_body = dict(getattr(args, "extra_body", None) or {}) + bot_task = getattr(args, "bot_task", None) + if bot_task is not None: + extra_body.setdefault("bot_task", bot_task) + args.extra_body = extra_body diff --git a/vllm_omni/entrypoints/cli/benchmark/main.py b/vllm_omni/entrypoints/cli/benchmark/main.py index 391def31f..342b4fb4f 100644 --- a/vllm_omni/entrypoints/cli/benchmark/main.py +++ b/vllm_omni/entrypoints/cli/benchmark/main.py @@ -1,3 +1,11 @@ +"""Top-level ``vllm bench`` command integration for vLLM-Omni. + +``OmniBenchmarkSubcommand`` creates the benchmark command group, discovers +``OmniBenchmarkSubcommandBase`` implementations, builds their nested parsers, +and dispatches parsed commands to the selected implementation. ``cmd_init`` +exposes the command to the surrounding vLLM-Omni CLI registration mechanism. +""" + from __future__ import annotations import argparse diff --git a/vllm_omni/entrypoints/cli/benchmark/serve.py b/vllm_omni/entrypoints/cli/benchmark/serve.py index 70e7342c0..5e46aecd2 100644 --- a/vllm_omni/entrypoints/cli/benchmark/serve.py +++ b/vllm_omni/entrypoints/cli/benchmark/serve.py @@ -1,190 +1,25 @@ +"""Online serving benchmark subcommand for vLLM-Omni. + +``OmniBenchmarkServingSubcommand`` starts from vLLM's serving benchmark +arguments, adds Omni-specific options and adaptations, then preprocesses the +parsed namespace and delegates execution to ``vllm_omni.benchmarks.serve``. +This keeps CLI integration separate from the multimodal benchmark runtime. +""" + import argparse -import os from vllm.benchmarks.serve import add_cli_args from vllm_omni.benchmarks.serve import main from vllm_omni.entrypoints.cli.benchmark.base import OmniBenchmarkSubcommandBase - - -def add_daily_omni_cli_args(parser: argparse.ArgumentParser) -> None: - """Add CLI arguments specific to Daily-Omni dataset. - - This function should be called by the CLI entrypoint to add additional - arguments for daily-omni benchmark support. - - Args: - parser: The ArgumentParser instance to extend - """ - # Daily-Omni specific arguments - daily_omni_group = parser.add_argument_group("Daily-Omni Dataset Options") - - daily_omni_group.add_argument( - "--daily-omni-qa-json", - type=str, - default=None, - help="Path to local upstream qa.json. When set, QA rows are read from this file and " - "the HuggingFace dataset is not loaded (no network). Use with --daily-omni-video-dir " - "for fully offline runs. --dataset-path / Hub split flags are then ignored for QA loading.", - ) - daily_omni_group.add_argument( - "--daily-omni-video-dir", - type=str, - default=None, - help="Root directory of extracted Daily-Omni videos (contents of Videos.tar: " - "each video_id in its own subdir with {video_id}_video.mp4). " - "If omitted, Videos.tar is downloaded from the Hugging Face dataset repo on first multimodal " - "request. " - "When using file URLs, you MUST start the vLLM server with " - "--allowed-local-media-path set to this same directory (or a parent), " - "otherwise requests fail with 'Cannot load local files without " - "--allowed-local-media-path'.", - ) - daily_omni_group.add_argument( - "--daily-omni-inline-local-video", - action="store_true", - default=False, - help="For local videos only: embed MP4 as base64 data URLs in benchmark " - "requests so the server does not need --allowed-local-media-path. " - "Increases request size and client memory; use for small --num-prompts. " - "When using --daily-omni-input-mode audio or all, local WAV files are " - "embedded the same way.", - ) - daily_omni_group.add_argument( - "--daily-omni-input-mode", - type=str, - choices=["all", "visual", "audio"], - default="all", - help="Daily-Omni input protocol (mirrors upstream Lliar-liar/Daily-Omni " - "--input_mode). 'visual': video only (default). 'audio': WAV only, " - "requires {video_id}/{video_id}_audio.wav under --daily-omni-video-dir. " - "'all': video + WAV together. Sets mm_processor_kwargs.use_audio_in_video=false " - "and matches official separate video/audio streams.", - ) - daily_omni_group.add_argument( - "--daily-omni-save-eval-items", - action="store_true", - default=False, - help="Include per-request Daily-Omni accuracy rows (gold/predicted/correct) " - "in the saved JSON under key daily_omni_eval_items. " - "Alternatively set env DAILY_OMNI_SAVE_EVAL_ITEMS=1.", - ) - - # Note: --dataset-name daily-omni via get_samples patch; use either Hub (--dataset-path - # liarliar/Daily-Omni) or local --daily-omni-qa-json (offline). - - -def add_seed_tts_cli_args(parser: argparse.ArgumentParser) -> None: - """CLI for Seed-TTS zero-shot TTS benchmark (``--dataset-name seed-tts``).""" - g = parser.add_argument_group("Seed-TTS Dataset Options") - g.add_argument( - "--seed-tts-locale", - type=str, - choices=["en", "zh"], - default="en", - help="Which Seed-TTS split to load: en/meta.lst or zh/meta.lst under the dataset root.", - ) - g.add_argument( - "--seed-tts-root", - type=str, - default=None, - help="Override root directory that contains en/ and zh/ (meta.lst + prompt-wavs). " - "If set, --dataset-path can still name the HF repo for logging; this path is used for files.", - ) - g.add_argument( - "--seed-tts-file-ref-audio", - action="store_true", - default=False, - help="Send ref_audio as file:// URIs (smaller HTTP bodies). Requires the API server " - "to be started with --allowed-local-media-path covering the Seed-TTS dataset root. " - "Default is inline data:audio/wav;base64 so Qwen3-TTS works without that flag.", - ) - g.add_argument( - "--seed-tts-inline-ref-audio", - action="store_true", - default=False, - help=argparse.SUPPRESS, - ) - g.add_argument( - "--seed-tts-system-prompt", - type=str, - default=None, - help="Override chat system message for --backend openai-chat-omni (Qwen3-Omni TTS). " - "Default follows official Qwen3-Omni identity + zero-shot voice-clone instructions.", - ) - g.add_argument( - "--seed-tts-wer-eval", - action="store_true", - default=False, - help="Keep synthesized audio as 24 kHz mono PCM for WER (works with " - "--backend openai-audio-speech or openai-chat-omni). Scoring follows " - "zhaochenyang20/seed-tts-eval (Whisper-large-v3 / Paraformer-zh + jiwer). " - "Sets SEED_TTS_WER_EVAL=1. Install: pip install 'vllm-omni[dev]'. " - "Optional: SEED_TTS_EVAL_DEVICE, SEED_TTS_HF_WHISPER_MODEL.", - ) - g.add_argument( - "--seed-tts-wer-save-items", - action="store_true", - default=False, - help="Include per-utterance ASR rows in the saved JSON under key seed_tts_wer_eval_items. " - "Or set SEED_TTS_WER_SAVE_ITEMS=1.", - ) - - -def add_omni_benchmark_cli_args(parser: argparse.ArgumentParser) -> None: - """Add vLLM-Omni specific serving benchmark options.""" - group = parser.add_argument_group("vLLM-Omni Multi-stage Benchmark Options") - group.add_argument( - "--print-stage", - action="store_true", - default=False, - help=( - "Print per-stage benchmark metrics for --omni serving when stage metrics are returned by the server. " - "Disabled by default. The latency sections follow --percentile-metrics by modality: " - "ttft/tpot/itl control text stages, ttfc/tpoc/icl control internal stream stages, " - "and tpop controls both text TPOP and internal stream TPOP." - ), - ) - group.add_argument( - "--image-edits-bot-task", - type=str, - default="think", - help=( - "Default bot_task form field for --backend openai-image-edits-omni " - '(/v1/images/edits). Use --extra-body \'{"bot_task":"..."}\' to override per run.' - ), - ) - - -_OMNI_BENCH_DATASET_CHOICES = ( - "daily-omni", - "seed-tts", - "seed-tts-text", - "seed-tts-design", - "ttsd", - "sound-effect", +from vllm_omni.entrypoints.cli.benchmark.cli_args import ( + add_omni_args, + extend_omni_choices, + preprocess_serve_args, + update_omni_help, ) -def _extend_omni_dataset_name_choices(parser: argparse.ArgumentParser) -> None: - """Append omni benchmark dataset names to --dataset-name choices. - - TrackingArgumentParser keeps a shadow parser for explicit-arg tracking; both - parsers must list the same choices or parse_args rejects valid omni values. - """ - parsers = [parser] - shadow = getattr(parser, "_shadow", None) - if shadow is not None: - parsers.append(shadow) - - for p in parsers: - for action in p._actions: - if action.dest == "dataset_name" and action.choices is not None: - extra = [c for c in _OMNI_BENCH_DATASET_CHOICES if c not in action.choices] - if extra: - action.choices = list(action.choices) + extra - - class OmniBenchmarkServingSubcommand(OmniBenchmarkSubcommandBase): """The `serve` subcommand for vllm bench.""" @@ -194,79 +29,11 @@ class OmniBenchmarkServingSubcommand(OmniBenchmarkSubcommandBase): @classmethod def add_cli_args(cls, parser: argparse.ArgumentParser) -> None: add_cli_args(parser) - - # Add Daily-Omni specific arguments - add_daily_omni_cli_args(parser) - add_seed_tts_cli_args(parser) - add_omni_benchmark_cli_args(parser) - - for action in parser._actions: - if action.dest == "dataset_name" and action.choices is not None: - extra = [ - c - for c in ( - "daily-omni", - "seed-tts", - "seed-tts-text", - "seed-tts-design", - "ttsd", - "sound-effect", - ) - if c not in action.choices - ] - if extra: - action.choices = list(action.choices) + extra - if action.dest == "backend" and action.choices is not None: - extra = [c for c in ("openai-image-edits-omni",) if c not in action.choices] - if extra: - action.choices = list(action.choices) + extra - _extend_omni_dataset_name_choices(parser) - - # Update help messages for omni-specific features - for action in parser._actions: - if action.dest == "percentile_metrics": - action.help = ( - "Comma-separated list of selected metrics to report percentiles. " - 'For text metrics, "ttft", "tpot", and "itl" affect the global benchmark and text ' - 'stage metrics. "tpop" also requests text TPOT/TPOP globally and per stage, and internal ' - 'stream TPOP. "ttfc", "tpoc", and "icl" only affect internal stream stage metrics. ' - 'Audio metrics include "audio_ttfp", "audio_rtf", "audio_duration", and "audio_underrun".' - ) - if action.dest == "random_mm_limit_mm_per_prompt": - action.help = ( - "Per-modality hard caps for items attached per request, e.g. " - '\'{"image": 3, "video": 0, "audio": 1}\'. The sampled per-request item ' - "count is clamped to the sum of these limits. When a modality " - "reaches its cap, its buckets are excluded and probabilities are " - "renormalized." - ) - if action.dest == "random_mm_bucket_config": - action.help = ( - "The bucket config is a dictionary mapping a multimodal item" - "sampling configuration to a probability." - "Currently allows for 3 modalities: audio, images and videos. " - "A bucket key is a tuple of (height, width, num_frames)" - "The value is the probability of sampling that specific item. " - "Example: " - "--random-mm-bucket-config " - "{(256, 256, 1): 0.5, (720, 1280, 16): 0.4, (0, 1, 5): 0.10} " - "First item: images with resolution 256x256 w.p. 0.5" - "Second item: videos with resolution 720x1280 and 16 frames " - "Third item: audios with 1s duration and 5 channels w.p. 0.1" - "OBS.: If the probabilities do not sum to 1, they are normalized." - ) + add_omni_args(parser) + extend_omni_choices(parser) + update_omni_help(parser) @staticmethod def cmd(args: argparse.Namespace) -> None: - if getattr(args, "daily_omni_save_eval_items", False): - os.environ["DAILY_OMNI_SAVE_EVAL_ITEMS"] = "1" - if getattr(args, "seed_tts_wer_eval", False): - os.environ["SEED_TTS_WER_EVAL"] = "1" - if getattr(args, "seed_tts_wer_save_items", False): - os.environ["SEED_TTS_WER_SAVE_ITEMS"] = "1" - image_edits_bot_task = getattr(args, "image_edits_bot_task", None) - if image_edits_bot_task is not None: - extra_body = dict(getattr(args, "extra_body", None) or {}) - extra_body.setdefault("bot_task", image_edits_bot_task) - args.extra_body = extra_body + preprocess_serve_args(args) main(args) From 0fe5d73a6365893c981398de576e8d20f22f9c19 Mon Sep 17 00:00:00 2001 From: Joshna-Medisetty Date: Thu, 23 Jul 2026 10:17:00 -0700 Subject: [PATCH 077/240] [Bugfix][XPU] Fix free-memory abort and num_speculative_steps crash on XPU (#5330) Signed-off-by: Joshna Medisetty Co-authored-by: Chendi.Xue --- .buildkite/intel/scripts/run-xpu-test.sh | 2 +- tests/e2e/offline_inference/test_qwen3_tts_base.py | 2 +- vllm_omni/platforms/xpu/platform.py | 9 +++++++++ vllm_omni/platforms/xpu/worker/xpu_ar_worker.py | 7 +++++++ vllm_omni/platforms/xpu/worker/xpu_generation_worker.py | 7 +++++++ 5 files changed, 25 insertions(+), 2 deletions(-) diff --git a/.buildkite/intel/scripts/run-xpu-test.sh b/.buildkite/intel/scripts/run-xpu-test.sh index 36860522b..6b40d58fb 100755 --- a/.buildkite/intel/scripts/run-xpu-test.sh +++ b/.buildkite/intel/scripts/run-xpu-test.sh @@ -74,6 +74,6 @@ time timeout -k 30 30m docker run \ cd /workspace/vllm-omni pytest -v -s -m "core_model and xpu and B60" pytest -v -s tests/diffusion/quantization/test_mxfp8_config.py - export VLLM_XPU_USE_SAMPLER_KERNEL=0 # NOTE: Remove this after vLLM v0.21.1 is merged. Fixes Qwen2-5 omni-expansion tests. pytest -v -s -m "advanced_model and xpu and B60" + pytest -v -s -m "omni and xpu and B60" ' diff --git a/tests/e2e/offline_inference/test_qwen3_tts_base.py b/tests/e2e/offline_inference/test_qwen3_tts_base.py index ecd769166..a076526a4 100644 --- a/tests/e2e/offline_inference/test_qwen3_tts_base.py +++ b/tests/e2e/offline_inference/test_qwen3_tts_base.py @@ -68,7 +68,7 @@ def get_prompt(): @pytest.mark.advanced_model @pytest.mark.tts -@hardware_test(res={"cuda": "L4"}, num_cards=1) +@hardware_test(res={"cuda": "L4", "xpu": "B60"}, num_cards=1) @pytest.mark.parametrize("omni_runner", tts_server_params, indirect=True) def test_text_to_audio_001(omni_runner, omni_runner_handler) -> None: """ diff --git a/vllm_omni/platforms/xpu/platform.py b/vllm_omni/platforms/xpu/platform.py index 0bc782e03..8c19868ed 100644 --- a/vllm_omni/platforms/xpu/platform.py +++ b/vllm_omni/platforms/xpu/platform.py @@ -1,6 +1,8 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import os + import torch from vllm.config import VllmConfig from vllm.config.kernel import IrOpPriorityConfig @@ -13,6 +15,13 @@ from vllm_omni.platforms.interface import OmniPlatform, OmniPlatformEnum logger = init_logger(__name__) +# Use the native query; vLLM's XPU custom op reports free=0 in spawned workers. +torch.accelerator.get_memory_info = lambda device=None: torch.xpu.mem_get_info(device) + +# The XPU sampler kernel is broken for omni on the current vLLM; use the native path. +os.environ.setdefault("VLLM_XPU_USE_SAMPLER_KERNEL", "0") + + class XPUOmniPlatform(OmniPlatform, XPUPlatform): """XPU/Intel GPU implementation of OmniPlatform. diff --git a/vllm_omni/platforms/xpu/worker/xpu_ar_worker.py b/vllm_omni/platforms/xpu/worker/xpu_ar_worker.py index ea6349264..9acf869f7 100644 --- a/vllm_omni/platforms/xpu/worker/xpu_ar_worker.py +++ b/vllm_omni/platforms/xpu/worker/xpu_ar_worker.py @@ -1,15 +1,22 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from vllm.logger import init_logger from vllm.v1.worker.xpu_worker import XPUWorker from vllm_omni.platforms.xpu.worker.xpu_ar_model_runner import XPUARModelRunner from vllm_omni.worker.mixins import OmniWorkerMixin +logger = init_logger(__name__) + class XPUARWorker(OmniWorkerMixin, XPUWorker): """XPU AR worker for thinker/talker stages in Omni model.""" def init_device(self): super().init_device() + if self.use_v2_model_runner: + # OMNI: v2 model runner does not yet include omni hooks. + logger.warning("OMNI XPUARWorker forces v1 model runner for omni hooks.") + self.use_v2_model_runner = False self.model_runner: XPUARModelRunner = XPUARModelRunner(self.vllm_config, self.device) diff --git a/vllm_omni/platforms/xpu/worker/xpu_generation_worker.py b/vllm_omni/platforms/xpu/worker/xpu_generation_worker.py index 9c6d530eb..d9bb790d3 100644 --- a/vllm_omni/platforms/xpu/worker/xpu_generation_worker.py +++ b/vllm_omni/platforms/xpu/worker/xpu_generation_worker.py @@ -1,15 +1,22 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from vllm.logger import init_logger from vllm.v1.worker.xpu_worker import XPUWorker from vllm_omni.platforms.xpu.worker.xpu_generation_model_runner import XPUGenerationModelRunner from vllm_omni.worker.mixins import OmniWorkerMixin +logger = init_logger(__name__) + class XPUGenerationWorker(OmniWorkerMixin, XPUWorker): """XPU generation worker for the code2wav (non-AR waveform generation) stage in the Omni model.""" def init_device(self): super().init_device() + if self.use_v2_model_runner: + # OMNI: v2 model runner does not yet include omni hooks. + logger.warning("OMNI XPUGenerationWorker forces v1 model runner for omni hooks.") + self.use_v2_model_runner = False self.model_runner: XPUGenerationModelRunner = XPUGenerationModelRunner(self.vllm_config, self.device) From a61d5c0fd83f6476b84293a192f440297a6a960a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B1=AA=E5=BF=97=E9=B9=8F?= Date: Fri, 24 Jul 2026 07:29:55 +0800 Subject: [PATCH 078/240] [Refector] Remove mammothmodal2-preview example (#5335) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: princepride Signed-off-by: 汪志鹏 --- .../mammothmodal2_preview.md | 35 ------ .../mammothmodal2_preview/README.md | 25 ---- .../run_mammothmoda2_image_summarize.py | 117 ------------------ recipes/MammothModa2/MammothModa2-Preview.md | 47 ++++++- 4 files changed, 46 insertions(+), 178 deletions(-) delete mode 100644 docs/user_guide/examples/offline_inference/mammothmodal2_preview.md delete mode 100644 examples/offline_inference/mammothmodal2_preview/README.md delete mode 100644 examples/offline_inference/mammothmodal2_preview/run_mammothmoda2_image_summarize.py diff --git a/docs/user_guide/examples/offline_inference/mammothmodal2_preview.md b/docs/user_guide/examples/offline_inference/mammothmodal2_preview.md deleted file mode 100644 index 60f1cbe5f..000000000 --- a/docs/user_guide/examples/offline_inference/mammothmodal2_preview.md +++ /dev/null @@ -1,35 +0,0 @@ -# MammothModa2-Preview - -Source . - - -## Run examples (MammothModa2-Preview) - -Download model -```bash -hf download bytedance-research/MammothModa2-Preview --local-dir ./MammothModa2-Preview -``` - -### Text-to-Image (T2I) - -Text-to-image now runs through the shared offline image example -(`examples/offline_inference/text_to_image/text_to_image.py`). See the recipe -`recipes/MammothModa2/MammothModa2-Preview.md` for the full command and the -`extra_body` knobs (`text_guidance_scale`, `cfg_range`, `num_inference_steps`). - -### Image Summary - -```bash -python examples/offline_inference/mammothmodal2_preview/run_mammothmoda2_image_summarize.py \ - --model ./MammothModa2-Preview \ - --deploy-config ./vllm_omni/deploy/mammoth_moda2_ar.yaml \ - --question "Summarize this image." \ - --image ./image.png -``` - -## Example materials - -??? abstract "run_mammothmoda2_image_summarize.py" - ``````py - --8<-- "examples/offline_inference/mammothmodal2_preview/run_mammothmoda2_image_summarize.py" - `````` diff --git a/examples/offline_inference/mammothmodal2_preview/README.md b/examples/offline_inference/mammothmodal2_preview/README.md deleted file mode 100644 index bb68f3845..000000000 --- a/examples/offline_inference/mammothmodal2_preview/README.md +++ /dev/null @@ -1,25 +0,0 @@ -# MammothModa2-Preview - -> Text-to-image (T2I) generation has moved to the shared offline image example. -> See the recipe: [`recipes/MammothModa2/MammothModa2-Preview.md`](../../../recipes/MammothModa2/MammothModa2-Preview.md). -> -> This directory retains the image-understanding (image → text) example below, -> which is outside the scope of the diffusion `model_extras` example migration -> (#4548 / RFC #4539: AR / Omni understanding examples keep dedicated scripts). - -## Run example (MammothModa2-Preview) - -Download model -```bash -hf download bytedance-research/MammothModa2-Preview --local-dir ./MammothModa2-Preview -``` - -### Image Summary - -```bash -python examples/offline_inference/mammothmodal2_preview/run_mammothmoda2_image_summarize.py \ - --model ./MammothModa2-Preview \ - --deploy-config ./vllm_omni/deploy/mammoth_moda2_ar.yaml \ - --question "Summarize this image." \ - --image ./image.png -``` diff --git a/examples/offline_inference/mammothmodal2_preview/run_mammothmoda2_image_summarize.py b/examples/offline_inference/mammothmodal2_preview/run_mammothmoda2_image_summarize.py deleted file mode 100644 index 7a22803c9..000000000 --- a/examples/offline_inference/mammothmodal2_preview/run_mammothmoda2_image_summarize.py +++ /dev/null @@ -1,117 +0,0 @@ -""" -Offline inference example: MammothModa2 image summarization (single AR stage). - -Example: - uv run python examples/offline_inference/mammothmodal2_preview/run_mammothmoda2_image_summarize.py \ - --model path/to/MammothModa2-Preview \ - --deploy-config vllm_omni/deploy/mammoth_moda2_ar.yaml \ - --image /path/to/input.jpg \ - --question "Please summarize the content of this image." -""" - -from __future__ import annotations - -import argparse -import os - -from PIL import Image -from vllm import SamplingParams -from vllm.multimodal.image import convert_image_mode - -from vllm_omni import Omni - -DEFAULT_SYSTEM = "You are a helpful assistant." -DEFAULT_QUESTION = "Please summarize the content of this image." - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description="MammothModa2 image summarization (offline, AR only).") - parser.add_argument("--model", type=str, required=True, help="Path to model directory or model id.") - parser.add_argument("--deploy-config", type=str, required=True, help="Path to deploy config yaml.") - parser.add_argument("--image", type=str, required=True, help="Path to input image.") - parser.add_argument("--question", type=str, default=DEFAULT_QUESTION, help="Question/instruction for the model.") - parser.add_argument("--system", type=str, default=DEFAULT_SYSTEM, help="System prompt.") - parser.add_argument( - "--max-tokens", - type=int, - default=512, - help="Max new tokens to generate.", - ) - parser.add_argument("--temperature", type=float, default=0.2) - parser.add_argument("--top-p", type=float, default=0.9) - parser.add_argument("--seed", type=int, default=42) - parser.add_argument("--trust-remote-code", action="store_true") - parser.add_argument( - "--enable-diffusion-pipeline-profiler", - action="store_true", - help="Enable diffusion pipeline profiler to display stage durations.", - ) - return parser.parse_args() - - -def build_prompt(system: str, question: str) -> str: - return ( - f"<|im_start|>system\n{system}<|im_end|>\n" - "<|im_start|>user\n" - "<|vision_start|><|image_pad|><|vision_end|>" - f"{question}<|im_end|>\n" - "<|im_start|>assistant\n" - ) - - -def main() -> None: - args = parse_args() - - if not os.path.exists(args.image): - raise FileNotFoundError(f"Image file not found: {args.image}") - - pil_image = Image.open(args.image) - image_data = convert_image_mode(pil_image, "RGB") - prompt = build_prompt(args.system, args.question) - - omni = Omni( - model=args.model, - deploy_config=args.deploy_config, - trust_remote_code=args.trust_remote_code, - enable_diffusion_pipeline_profiler=args.enable_diffusion_pipeline_profiler, - ) - try: - sp = SamplingParams( - temperature=float(args.temperature), - top_p=float(args.top_p), - top_k=-1, - max_tokens=int(args.max_tokens), - seed=int(args.seed), - detokenize=True, - ) - # NOTE: omni.generate() returns a Generator[OmniRequestOutput, None, None]. - # Consume it inside the try block so the worker isn't closed early. - outputs = list( - omni.generate( - [ - { - "prompt": prompt, - "multi_modal_data": {"image": image_data}, - "additional_information": {"omni_task": ["chat"]}, - } - ], - [sp], - ) - ) - finally: - omni.close() - - lines: list[str] = [] - for stage_outputs in outputs: - ro = getattr(stage_outputs, "request_output", stage_outputs) - text = ro.outputs[0].text if getattr(ro, "outputs", None) else str(ro) - lines.append(f"request_id: {getattr(ro, 'request_id', 'unknown')}\n") - lines.append("answer:\n") - lines.append(text.strip() + "\n") - lines.append("\n") - - print("\n".join(lines)) - - -if __name__ == "__main__": - main() diff --git a/recipes/MammothModa2/MammothModa2-Preview.md b/recipes/MammothModa2/MammothModa2-Preview.md index 0733b0630..4e4846fba 100644 --- a/recipes/MammothModa2/MammothModa2-Preview.md +++ b/recipes/MammothModa2/MammothModa2-Preview.md @@ -6,7 +6,8 @@ - Vendor: ByteDance Research - Model: `bytedance-research/MammothModa2-Preview` -- Task: Text-to-image generation (AR → DiT two-stage pipeline) +- Task: Text-to-image generation (AR → DiT two-stage pipeline), plus + image-to-text understanding (AR stage) - Mode: Offline inference - Maintainer: Community @@ -108,3 +109,47 @@ exists and is a valid image: ls -lh mammoth_t2i.png python -c "from PIL import Image; print(Image.open('mammoth_t2i.png').size)" ``` + +### Image-to-Text (I2T, simplest) + +For the simplest offline image-understanding run, use a short inline Python +snippet from the repository root: + +```python +from PIL import Image +from vllm import SamplingParams +from vllm.multimodal.image import convert_image_mode +from vllm_omni import Omni + +def main(): + prompt = ( + "<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n" + "<|im_start|>user\n<|vision_start|><|image_pad|><|vision_end|>" + "Summarize this image.<|im_end|>\n" + "<|im_start|>assistant\n" + ) + + omni = Omni( + model="./MammothModa2-Preview", + deploy_config="vllm_omni/deploy/mammoth_moda2_ar.yaml", + ) + try: + outputs = list( + omni.generate( + [{ + "prompt": prompt, + "multi_modal_data": {"image": convert_image_mode(Image.open("./image.png"), "RGB")}, + "additional_information": {"omni_task": ["chat"]}, + }], + [SamplingParams(temperature=0.2, top_p=0.9, top_k=-1, max_tokens=512, seed=42)], + ) + ) + finally: + omni.close() + + ro = getattr(outputs[-1], "request_output", outputs[-1]) + print(ro.outputs[0].text.strip()) + +if __name__ == "__main__": + main() +``` From 6eed6b94344a382ee1a73e55d642a4d67a5d91fa Mon Sep 17 00:00:00 2001 From: wangyu <53896905+yenuo26@users.noreply.github.com> Date: Fri, 24 Jul 2026 10:52:24 +0800 Subject: [PATCH 079/240] [CI/Test] Skip realtime async_chunk WebSocket case blocked by #3907 (#5363) (#5367) Signed-off-by: wangyu <410167048@qq.com> --- .../openai_api/test_qwen3_omni_realtime_websocket.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/entrypoints/openai_api/test_qwen3_omni_realtime_websocket.py b/tests/entrypoints/openai_api/test_qwen3_omni_realtime_websocket.py index 50f8eef91..d935949b3 100644 --- a/tests/entrypoints/openai_api/test_qwen3_omni_realtime_websocket.py +++ b/tests/entrypoints/openai_api/test_qwen3_omni_realtime_websocket.py @@ -234,6 +234,12 @@ class TestQwen3OmniRealtimeWebSocket: @pytest.mark.omni @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2) @pytest.mark.parametrize("omni_server", realtime_async_chunk_server_params, indirect=True) + @pytest.mark.skip( + reason=( + "/v1/realtime rejects async_chunk since #3907 (server closes with 1000 after " + "unsupported error); tracked in #5363" + ) + ) def test_streaming_audio_input_pcm_output_async_chunk(self, omni_server) -> None: """Merge CI: async_chunk on, paced upload, full accuracy check.""" pcm16 = _synthetic_pcm16_input() From acf941da4da8bdca36cf4e8e8ea16d1e2df38e95 Mon Sep 17 00:00:00 2001 From: Diya Peng <50388704+dpeng123@users.noreply.github.com> Date: Fri, 24 Jul 2026 00:00:57 -0700 Subject: [PATCH 080/240] [Diffusion] Add VAE patch parallel support for FLUX.2-dev (#5292) Signed-off-by: dpeng123 --- docs/design/feature/vae_parallel.md | 1 + docs/user_guide/diffusion_features.md | 2 +- .../test_flux2_vae_patch_parallel.py | 210 ++++++++++++++++++ .../test_flux_2_dev_expansion.py | 16 ++ .../autoencoders/autoencoder_kl_flux2.py | 116 ++++++++++ .../diffusion/models/flux2/pipeline_flux2.py | 9 +- 6 files changed, 349 insertions(+), 5 deletions(-) create mode 100644 tests/diffusion/distributed/test_flux2_vae_patch_parallel.py create mode 100644 vllm_omni/diffusion/distributed/autoencoders/autoencoder_kl_flux2.py diff --git a/docs/design/feature/vae_parallel.md b/docs/design/feature/vae_parallel.md index 00f41378a..c465449f8 100644 --- a/docs/design/feature/vae_parallel.md +++ b/docs/design/feature/vae_parallel.md @@ -445,6 +445,7 @@ Complete examples in the codebase: | **Z-Image** | `vllm_omni/diffusion/distributed/autoencoders/autoencoder_kl.py` | ✅ | ❌ | | **Wan2.2** | `vllm_omni/diffusion/distributed/autoencoders/autoencoder_kl_wan.py` | ✅ | ✅ | | **Qwen-Image** | `vllm_omni/diffusion/distributed/autoencoders/autoencoder_kl_qwenimage.py` | ✅ | ❌ | +| **FLUX.2-dev** | `vllm_omni/diffusion/distributed/autoencoders/autoencoder_kl_flux2.py` | ✅ | ✅ | --- diff --git a/docs/user_guide/diffusion_features.md b/docs/user_guide/diffusion_features.md index bf714b3a0..005a0e899 100644 --- a/docs/user_guide/diffusion_features.md +++ b/docs/user_guide/diffusion_features.md @@ -117,7 +117,7 @@ The following tables show which models support each feature: | **FLUX.1-schnell** | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ | ❌ | ❌ | ✅ | ❌ | | **FLUX.2-klein** | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ❌ | ❌ | ✅ | ❌ | | **FLUX.1-Kontext-dev** | ❌ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | -| **FLUX.2-dev** | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | +| **FLUX.2-dev** | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | | **GLM-Image** | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | | **Hidream-I1-Full** | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | | **HunyuanImage3** | ❌ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | diff --git a/tests/diffusion/distributed/test_flux2_vae_patch_parallel.py b/tests/diffusion/distributed/test_flux2_vae_patch_parallel.py new file mode 100644 index 000000000..e861bd61a --- /dev/null +++ b/tests/diffusion/distributed/test_flux2_vae_patch_parallel.py @@ -0,0 +1,210 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +"""FLUX.2-dev VAE patch-parallel correctness tests. + +Compares vae_patch_parallel_size=1 vs 2 on TP=2 with tiling enabled: +- Text-to-image decode path +- Image-to-image encode + decode path (conditioning image) +""" + +import os +from pathlib import Path + +import numpy as np +import pytest +import torch +from PIL import Image + +from tests.helpers.mark import hardware_test +from tests.helpers.runtime import OmniRunner +from vllm_omni.diffusion.data import DiffusionParallelConfig +from vllm_omni.inputs.data import OmniDiffusionSamplingParams +from vllm_omni.outputs import OmniRequestOutput +from vllm_omni.platforms import current_omni_platform + +os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" + +PROMPT = "a photo of a cat sitting on a laptop keyboard" +NEGATIVE_PROMPT = "blurry, low quality" + +VAE_PP_MEAN_THRESHOLD = 3e-2 +VAE_PP_P99_THRESHOLD = 1e-1 + + +def _get_flux2_dev_model() -> str: + return os.environ.get("VLLM_TEST_FLUX2_DEV_MODEL", "black-forest-labs/FLUX.2-dev") + + +def _pil_to_float_rgb_tensor(img: Image.Image) -> torch.Tensor: + arr = np.asarray(img.convert("RGB"), dtype=np.float32) / 255.0 + return torch.from_numpy(arr) + + +def _diff_metrics(a: Image.Image, b: Image.Image) -> tuple[float, float]: + ta = _pil_to_float_rgb_tensor(a) + tb = _pil_to_float_rgb_tensor(b) + assert ta.shape == tb.shape, f"Image shapes differ: {ta.shape} vs {tb.shape}" + abs_diff = torch.abs(ta - tb) + p99_abs_diff = torch.quantile(abs_diff.flatten(), 0.99).item() + return abs_diff.mean().item(), p99_abs_diff + + +def _extract_single_image(outputs) -> Image.Image: + first_output = outputs[0] + assert first_output.final_output_type == "image" + if not hasattr(first_output, "request_output") or not first_output.request_output: + raise ValueError("No request_output found in OmniRequestOutput") + + req_out = first_output.request_output + if not isinstance(req_out, OmniRequestOutput) or not hasattr(req_out, "images"): + raise ValueError("Invalid request_output structure or missing 'images' key") + + images = req_out.images + if images is None or len(images) != 1: + raise ValueError(f"Expected 1 image, got {0 if images is None else len(images)}") + return images[0] + + +def _run_flux2_generate( + *, + tp_size: int, + height: int, + width: int, + num_inference_steps: int, + seed: int, + vae_patch_parallel_size: int, + conditioning_image: Image.Image | None = None, +) -> Image.Image: + current_omni_platform.empty_cache() + + request: dict = {"prompt": PROMPT, "negative_prompt": NEGATIVE_PROMPT} + if conditioning_image is not None: + request["multi_modal_data"] = {"image": conditioning_image} + + with OmniRunner( + _get_flux2_dev_model(), + parallel_config=DiffusionParallelConfig( + tensor_parallel_size=tp_size, + vae_patch_parallel_size=vae_patch_parallel_size, + ), + enable_cpu_offload=True, + vae_use_tiling=True, + ) as runner: + outputs = list( + runner.omni.generate( + request, + OmniDiffusionSamplingParams( + height=height, + width=width, + num_inference_steps=num_inference_steps, + guidance_scale=4.0, + seed=seed, + num_outputs_per_prompt=1, + ), + ) + ) + + return _extract_single_image(outputs) + + +@pytest.mark.advanced_model +@pytest.mark.diffusion +@pytest.mark.parallel +@hardware_test(res={"cuda": "H100"}, num_cards=2) +def test_flux2_vae_patch_parallel_decode_tp2(tmp_path: Path): + if not current_omni_platform.is_available() or current_omni_platform.device_count() < 2: + pytest.skip("FLUX.2-dev VAE patch parallel requires >= 2 devices.") + + height = 1152 + width = 1152 + num_inference_steps = 2 + seed = 42 + + baseline_img = _run_flux2_generate( + tp_size=2, + height=height, + width=width, + num_inference_steps=num_inference_steps, + seed=seed, + vae_patch_parallel_size=1, + ) + pp2_img = _run_flux2_generate( + tp_size=2, + height=height, + width=width, + num_inference_steps=num_inference_steps, + seed=seed, + vae_patch_parallel_size=2, + ) + + baseline_path = tmp_path / "flux2_tp2_vae_pp1.png" + pp2_path = tmp_path / "flux2_tp2_vae_pp2.png" + baseline_img.save(baseline_path) + pp2_img.save(pp2_path) + + assert baseline_img.width == width and baseline_img.height == height + assert pp2_img.width == width and pp2_img.height == height + + mean_abs_diff, p99_abs_diff = _diff_metrics(baseline_img, pp2_img) + print( + "FLUX.2-dev VAE patch parallel decode diff (TP=2, pp=1 vs pp=2): " + f"mean_abs_diff={mean_abs_diff:.6e}, p99_abs_diff={p99_abs_diff:.6e}; " + f"thresholds: mean<={VAE_PP_MEAN_THRESHOLD:.6e}, p99<={VAE_PP_P99_THRESHOLD:.6e}; " + f"pp1_img={baseline_path}, pp2_img={pp2_path}" + ) + assert mean_abs_diff <= VAE_PP_MEAN_THRESHOLD and p99_abs_diff <= VAE_PP_P99_THRESHOLD, ( + f"Image diff exceeded threshold: mean_abs_diff={mean_abs_diff:.6e}, p99_abs_diff={p99_abs_diff:.6e} " + f"(thresholds: mean<={VAE_PP_MEAN_THRESHOLD:.6e}, p99<={VAE_PP_P99_THRESHOLD:.6e})" + ) + + +@pytest.mark.advanced_model +@pytest.mark.diffusion +@pytest.mark.parallel +@hardware_test(res={"cuda": "H100"}, num_cards=2) +def test_flux2_vae_patch_parallel_i2i_tp2(tmp_path: Path): + if not current_omni_platform.is_available() or current_omni_platform.device_count() < 2: + pytest.skip("FLUX.2-dev VAE patch parallel I2I requires >= 2 devices.") + + height = 1152 + width = 1152 + num_inference_steps = 2 + seed = 42 + conditioning_image = Image.new("RGB", (width, height), (120, 80, 40)) + + baseline_img = _run_flux2_generate( + tp_size=2, + height=height, + width=width, + num_inference_steps=num_inference_steps, + seed=seed, + vae_patch_parallel_size=1, + conditioning_image=conditioning_image, + ) + pp2_img = _run_flux2_generate( + tp_size=2, + height=height, + width=width, + num_inference_steps=num_inference_steps, + seed=seed, + vae_patch_parallel_size=2, + conditioning_image=conditioning_image, + ) + + baseline_path = tmp_path / "flux2_i2i_tp2_vae_pp1.png" + pp2_path = tmp_path / "flux2_i2i_tp2_vae_pp2.png" + baseline_img.save(baseline_path) + pp2_img.save(pp2_path) + + mean_abs_diff, p99_abs_diff = _diff_metrics(baseline_img, pp2_img) + print( + "FLUX.2-dev VAE patch parallel I2I diff (TP=2, pp=1 vs pp=2): " + f"mean_abs_diff={mean_abs_diff:.6e}, p99_abs_diff={p99_abs_diff:.6e}; " + f"thresholds: mean<={VAE_PP_MEAN_THRESHOLD:.6e}, p99<={VAE_PP_P99_THRESHOLD:.6e}; " + f"pp1_img={baseline_path}, pp2_img={pp2_path}" + ) + assert mean_abs_diff <= VAE_PP_MEAN_THRESHOLD and p99_abs_diff <= VAE_PP_P99_THRESHOLD, ( + f"Image diff exceeded threshold: mean_abs_diff={mean_abs_diff:.6e}, p99_abs_diff={p99_abs_diff:.6e} " + f"(thresholds: mean<={VAE_PP_MEAN_THRESHOLD:.6e}, p99<={VAE_PP_P99_THRESHOLD:.6e})" + ) diff --git a/tests/e2e/online_serving/test_flux_2_dev_expansion.py b/tests/e2e/online_serving/test_flux_2_dev_expansion.py index 868a1e3bd..9bc3308b8 100644 --- a/tests/e2e/online_serving/test_flux_2_dev_expansion.py +++ b/tests/e2e/online_serving/test_flux_2_dev_expansion.py @@ -6,6 +6,7 @@ Coverage: - Layerwise CPU offload - Ulysses sequence parallelism - Ring sequence parallelism +- VAE patch parallel encode/decode This test verifies that FLUX.2-dev can be launched with CPU offload enabled, accepts text-to-image requests through the OpenAI-compatible API, and returns @@ -90,6 +91,21 @@ def _get_flux_2_dev_feature_cases(model: str): id="ring_2", marks=PARALLEL_FEATURE_MARKS, ), + pytest.param( + OmniServerParams( + model=model, + server_args=[ + "--enable-cpu-offload", + "--tensor-parallel-size", + "2", + "--vae-patch-parallel-size", + "2", + "--vae-use-tiling", + ], + ), + id="vae_patch_parallel_2", + marks=PARALLEL_FEATURE_MARKS, + ), ] diff --git a/vllm_omni/diffusion/distributed/autoencoders/autoencoder_kl_flux2.py b/vllm_omni/diffusion/distributed/autoencoders/autoencoder_kl_flux2.py new file mode 100644 index 000000000..7c0cee0f5 --- /dev/null +++ b/vllm_omni/diffusion/distributed/autoencoders/autoencoder_kl_flux2.py @@ -0,0 +1,116 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from typing import Any + +import torch +from diffusers.models.autoencoders.autoencoder_kl_flux2 import AutoencoderKLFlux2 +from vllm.logger import init_logger + +from vllm_omni.diffusion.distributed.autoencoders.autoencoder_kl import DistributedAutoencoderKL_base +from vllm_omni.diffusion.distributed.autoencoders.distributed_vae_executor import ( + DistributedOperator, + GridSpec, + TileTask, +) + +logger = init_logger(__name__) + + +class DistributedAutoencoderKLFlux2(DistributedAutoencoderKL_base, AutoencoderKLFlux2): + def encode_tile_split(self, x: torch.Tensor) -> tuple[list[TileTask], GridSpec]: + _, _, height, width = x.shape + overlap_size = int(self.tile_sample_min_size * (1 - self.tile_overlap_factor)) + blend_extent = int(self.tile_latent_min_size * self.tile_overlap_factor) + row_limit = self.tile_latent_min_size - blend_extent + + tiletask_list = [] + for i in range(0, height, overlap_size): + for j in range(0, width, overlap_size): + tile = x[:, :, i : i + self.tile_sample_min_size, j : j + self.tile_sample_min_size] + tiletask_list.append( + TileTask( + len(tiletask_list), + (i // overlap_size, j // overlap_size), + tile, + workload=tile.shape[2] * tile.shape[3], + ) + ) + + grid_spec = GridSpec( + split_dims=(2, 3), + grid_shape=(tiletask_list[-1].grid_coord[0] + 1, tiletask_list[-1].grid_coord[1] + 1), + tile_spec={ + "blend_extent": blend_extent, + "row_limit": row_limit, + }, + output_dtype=self.dtype, + ) + return tiletask_list, grid_spec + + def encode_tile_exec(self, task: TileTask) -> torch.Tensor: + tile = self.encoder(task.tensor) + if self.quant_conv is not None: + tile = self.quant_conv(tile) + return tile + + def encode_tile_merge( + self, coord_tensor_map: dict[tuple[int, ...], torch.Tensor], grid_spec: GridSpec + ) -> torch.Tensor: + grid_h, grid_w = grid_spec.grid_shape + result_rows = [] + for i in range(grid_h): + result_row = [] + for j in range(grid_w): + tile = coord_tensor_map[(i, j)] + if i > 0: + tile = self.blend_v(coord_tensor_map[(i - 1, j)], tile, grid_spec.tile_spec["blend_extent"]) + if j > 0: + tile = self.blend_h(coord_tensor_map[(i, j - 1)], tile, grid_spec.tile_spec["blend_extent"]) + result_row.append(tile[:, :, : grid_spec.tile_spec["row_limit"], : grid_spec.tile_spec["row_limit"]]) + result_rows.append(torch.cat(result_row, dim=-1)) + return torch.cat(result_rows, dim=-2) + + def _encode(self, x: torch.Tensor) -> torch.Tensor: + _, _, height, width = x.shape + if self.use_tiling and (width > self.tile_sample_min_size or height > self.tile_sample_min_size): + if self.is_distributed_enabled(): + logger.debug("Encode running with distributed executor") + return self.distributed_executor.execute( + x, + DistributedOperator( + split=self.encode_tile_split, + exec=self.encode_tile_exec, + merge=self.encode_tile_merge, + ), + broadcast_result=True, + ) + return self._tiled_encode(x) + + enc = self.encoder(x) + if self.quant_conv is not None: + enc = self.quant_conv(enc) + return enc + + def decode(self, z: torch.Tensor, return_dict: bool = True, *args: Any, **kwargs: Any): + if not self.is_distributed_enabled(): + return super().decode(z, return_dict=return_dict, *args, **kwargs) + + split, exec_fn, merge = self._strategy_select(z) + + if split is not None: + strategy = "tile" if split == self.tile_split else "patch" + logger.info(f"Decode run with distributed executor, split strategy is {strategy}") + result = self.distributed_executor.execute( + z, + DistributedOperator(split=split, exec=exec_fn, merge=merge), + broadcast_result=True, + ) + if not return_dict: + return (result,) + + from diffusers.models.autoencoders.vae import DecoderOutput + + return DecoderOutput(sample=result) + + return super().decode(z, return_dict=return_dict, *args, **kwargs) diff --git a/vllm_omni/diffusion/models/flux2/pipeline_flux2.py b/vllm_omni/diffusion/models/flux2/pipeline_flux2.py index 351d94f7e..0dbfe0f8c 100644 --- a/vllm_omni/diffusion/models/flux2/pipeline_flux2.py +++ b/vllm_omni/diffusion/models/flux2/pipeline_flux2.py @@ -11,7 +11,7 @@ from typing import Any, ClassVar, cast import numpy as np import PIL.Image import torch -from diffusers import AutoencoderKLFlux2, FlowMatchEulerDiscreteScheduler +from diffusers import FlowMatchEulerDiscreteScheduler from diffusers.image_processor import VaeImageProcessor from diffusers.pipelines.flux2.pipeline_flux2 import UPSAMPLING_MAX_IMAGE_SIZE from diffusers.pipelines.flux2.system_messages import ( @@ -25,6 +25,7 @@ from transformers import AutoConfig, AutoProcessor, PixtralProcessor from vllm.model_executor.models.utils import AutoWeightsLoader from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig +from vllm_omni.diffusion.distributed.autoencoders.autoencoder_kl_flux2 import DistributedAutoencoderKLFlux2 from vllm_omni.diffusion.distributed.cfg_parallel import CFGParallelMixin from vllm_omni.diffusion.distributed.parallel_state import get_classifier_free_guidance_world_size from vllm_omni.diffusion.distributed.utils import get_local_device @@ -416,9 +417,9 @@ class Flux2Pipeline( system_message_t2i=SYSTEM_MESSAGE_UPSAMPLING_T2I, system_message_i2i=SYSTEM_MESSAGE_UPSAMPLING_I2I, ) - self.vae = AutoencoderKLFlux2.from_pretrained(model, subfolder="vae", local_files_only=local_files_only).to( - self._execution_device - ) + self.vae = DistributedAutoencoderKLFlux2.from_pretrained( + model, subfolder="vae", local_files_only=local_files_only + ).to(self._execution_device) transformer_kwargs = get_transformer_config_kwargs(od_config.tf_model_config, Flux2Transformer2DModel) transformer_quant_config = _resolve_component_quant_config(od_config.quantization_config, "transformer") self.transformer = Flux2Transformer2DModel( From ad3a162eb8a843c734b48f8462ecb63abbf4df9b Mon Sep 17 00:00:00 2001 From: rein yang <73573651+R2-Y@users.noreply.github.com> Date: Fri, 24 Jul 2026 17:20:39 +0800 Subject: [PATCH 081/240] [perf] Minicpm o4.5 optimization (#5228) Signed-off-by: R2-Y Co-authored-by: Sy03 <1370724210@qq.com> --- examples/online_serving/minicpmo/README.md | 120 +- .../minicpmo/streaming_chat_completion.py | 71 + recipes/OpenBMB/MiniCPM-o-4_5.md | 218 +- tests/benchmarks/test_serve_cli.py | 23 +- .../sched/test_omni_ar_scheduler_streaming.py | 22 + .../omni_connectors/test_adapter_and_flow.py | 17 + .../test_chunk_transfer_adapter.py | 52 + .../test_omni_connector_configs.py | 25 + .../test_orchestrator_stage_input_bridge.py | 73 + tests/engine/test_stage_engine_core_proc.py | 28 + .../online_serving/test_minicpmo_streaming.py | 29 + .../minicpmo_4_5/test_code2wav_batching.py | 431 ++++ .../models/minicpmo_4_5/test_hf_model_path.py | 74 - .../models/minicpmo_4_5/test_llm2tts.py | 62 +- .../models/minicpmo_4_5/test_pipeline.py | 103 +- .../minicpmo_4_5/test_talker_batching.py | 248 ++ .../test_minicpmo_4_5_async_chunk.py | 182 ++ .../test_minicpmo_4_5_omni.py | 1369 +---------- tests/test_config_factory.py | 32 +- tests/worker/test_gpu_ar_model_runner.py | 1 + tests/worker/test_native_duplex_hooks.py | 363 --- tests/worker/test_omni_gpu_model_runner.py | 27 + vllm_omni/config/stage_config.py | 4 + vllm_omni/core/sched/omni_ar_scheduler.py | 10 +- .../core/sched/omni_generation_scheduler.py | 24 +- vllm_omni/core/sched/omni_scheduler_mixin.py | 28 +- vllm_omni/data_entry_keys.py | 14 + vllm_omni/deploy/minicpmo_4_5.yaml | 57 +- vllm_omni/deploy/minicpmo_4_5_2gpu.yaml | 66 +- vllm_omni/deploy/minicpmo_4_5_3gpu.yaml | 61 +- vllm_omni/deploy/minicpmo_4_5_8x4090.yaml | 45 +- vllm_omni/deploy/minicpmo_4_5_batching.yaml | 88 + vllm_omni/deploy/minicpmo_4_5_duplex.yaml | 22 +- .../distributed/omni_connectors/adapter.py | 16 + .../chunk_transfer_adapter.py | 14 +- .../omni_connectors/utils/config.py | 28 + vllm_omni/engine/orchestrator.py | 14 +- vllm_omni/engine/stage_engine_core_proc.py | 7 + vllm_omni/engine/stage_init_utils.py | 10 + vllm_omni/engine/stage_pool.py | 2 + .../entrypoints/cli/benchmark/cli_args.py | 2 +- .../fullduplex/engine/intermediate.py | 17 +- .../models/minicpmo_4_5/batched_token2wav.py | 400 +++ .../minicpmo_4_5/minicpmo_4_5_code2wav.py | 547 +++++ .../models/minicpmo_4_5/minicpmo_4_5_omni.py | 249 +- .../minicpmo_4_5/minicpmo_4_5_omni_tts.py | 2147 ++++------------- .../models/minicpmo_4_5/pipeline.py | 39 +- vllm_omni/model_executor/models/registry.py | 5 + .../minicpmo_4_5_omni.py | 290 ++- .../npu/worker/npu_ar_model_runner.py | 7 +- vllm_omni/utils/mm_outputs.py | 1 + vllm_omni/worker/gpu_ar_model_runner.py | 22 +- .../worker/gpu_generation_model_runner.py | 8 +- vllm_omni/worker/gpu_model_runner.py | 44 +- 54 files changed, 3861 insertions(+), 3997 deletions(-) create mode 100644 examples/online_serving/minicpmo/streaming_chat_completion.py create mode 100644 tests/engine/test_stage_engine_core_proc.py create mode 100644 tests/examples/online_serving/test_minicpmo_streaming.py create mode 100644 tests/model_executor/models/minicpmo_4_5/test_code2wav_batching.py delete mode 100644 tests/model_executor/models/minicpmo_4_5/test_hf_model_path.py create mode 100644 tests/model_executor/models/minicpmo_4_5/test_talker_batching.py create mode 100644 tests/model_executor/stage_input_processors/test_minicpmo_4_5_async_chunk.py create mode 100644 vllm_omni/deploy/minicpmo_4_5_batching.yaml create mode 100644 vllm_omni/model_executor/models/minicpmo_4_5/batched_token2wav.py create mode 100644 vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_code2wav.py diff --git a/examples/online_serving/minicpmo/README.md b/examples/online_serving/minicpmo/README.md index 11f6f8195..39bbef09b 100644 --- a/examples/online_serving/minicpmo/README.md +++ b/examples/online_serving/minicpmo/README.md @@ -24,18 +24,28 @@ including `librosa`. ## Start the backend server -Pick a deploy config that matches your GPU layout: +The deploy config auto-loads via `--omni`. +The default `vllm_omni/deploy/minicpmo_4_5.yaml` keeps all three stages on +logical device 0 with memory budgets of 65%, 15%, and 15%. For throughput, +`minicpmo_4_5_batching.yaml` gives the Thinker GPU 0 (90%) and colocates the +Talker (55%) and Code2Wav (35%) on GPU 1. Each stage admits at most four +concurrent sequences. -| config | GPUs | TP | Notes | -|---|---|---|---| -| `minicpmo_4_5.yaml` | 1 | 1 | Thinker and talker+t2w co-located on GPU0. | -| `minicpmo_4_5_2gpu.yaml` | 2 | 1 | Thinker on GPU0, talker+t2w on GPU1. | -| `minicpmo_4_5_3gpu.yaml` | 3 | 2 | Thinker 2-way TP on GPU0/1, talker+t2w share GPU2. | -| `minicpmo_4_5_8x4090.yaml` | 8 | 4 | Thinker 4-way TP on GPU0-3, talker+t2w on GPU4. | -| `minicpmo_4_5_3gpu_stage1_replicas.yaml` | 3 | 1 | Experimental validation profile: Thinker on GPU0, two talker+Token2wav replicas on GPU1/2. | -| `minicpmo_4_5_4gpu_stage1_replicas.yaml` | 4 | 1 | Experimental validation profile: Thinker on GPU0, three talker+Token2wav replicas on GPU1/2/3. | -| `minicpmo_4_5_8x4090_stage1_replicas.yaml` | 8 | 4 | Experimental validation profile: Thinker 4-way TP on GPU0-3, four talker+Token2wav replicas on GPU4-7. | -| `minicpmo_4_5_duplex.yaml` | 2 | 1 | Experimental native duplex profile. | +| deploy config | GPUs | Notes | +|---|---|---| +| `minicpmo_4_5.yaml` (default) | 1 | Memory-constrained compatibility layout. | +| `minicpmo_4_5_batching.yaml` | 2 | Recommended continuous-batching layout; Talker and Code2Wav share GPU 1. | +| `minicpmo_4_5_2gpu.yaml` | 2 | Backward-compatible alias for `minicpmo_4_5_batching.yaml`. | +| `minicpmo_4_5_3gpu.yaml` | 3 | One GPU per stage. | +| `minicpmo_4_5_8x4090.yaml` | 8 | Full 8x4090 layout. | +| `minicpmo_4_5_duplex.yaml` | 1 | Experimental native full-duplex overlay. | + +The split pipeline preserves native-duplex epoch/turn identity, segment text, +turn completion, reference voice, and terminal-audio metadata through +Code2Wav. Focused CPU regressions cover this envelope; run the Realtime +scenario below for live barge-in validation on the target GPU. + +Default: ```bash vllm-omni serve openbmb/MiniCPM-o-4_5 \ @@ -49,13 +59,20 @@ For local ModelScope checkpoints, replace `openbmb/MiniCPM-o-4_5` with the checkpoint path. To start the experimental native duplex backend, use `vllm_omni/deploy/minicpmo_4_5_duplex.yaml`. -The `*_stage1_replicas.yaml` files exercise composite Stage1 replica routing -and failure recovery. They are validation profiles, not recommended production -entrypoints. +### Per-stage overrides -## Send chat requests +```bash +vllm serve openbmb/MiniCPM-o-4_5 --omni --trust-remote-code --port 8099 \ + --stage-overrides '{"0": {"gpu_memory_utilization": 0.65}}' +``` -From this directory, run the MiniCPM-specific curl or Python clients: +## Send multimodal requests + +```bash +cd examples/online_serving/minicpmo +``` + +### curl ```bash bash run_curl_multimodal_generation.sh text @@ -64,13 +81,40 @@ bash run_curl_multimodal_generation.sh use_audio '["text"]' python openai_chat_completion_client_for_multimodal_generation.py \ --query-type use_image \ - --host localhost \ + --port 8099 \ + --host localhost + +# Text-only (faster; no <|tts_bos|>) +python openai_chat_completion_client_for_multimodal_generation.py \ + --query-type text \ + --modalities text \ + --prompt "Briefly introduce yourself." +``` + +Streaming text + audio: + +```bash +python streaming_chat_completion.py \ + --base-url http://localhost:8099/v1 \ + --output minicpmo_stream.wav +``` + +The example prints text deltas immediately and joins the independently encoded +24 kHz WAV audio deltas into one valid WAV file. Add `--text-only` to skip +Talker and Code2Wav. + +Shared helpers also work if you pass MiniCPM defaults yourself: + +```bash +python ../openai_chat_completion_client_for_multimodal_generation.py \ + --model openbmb/MiniCPM-o-4_5 \ + --query-type text \ --port 8099 ``` -Speech output requires `chat_template_kwargs.use_tts_template=true`. Put that -field at the request root for curl; the OpenAI Python SDK can merge it from -`extra_body`. +Speech output no longer depends on a MiniCPM-specific default in the generic +serving layer. `chat_template_kwargs.use_tts_template=true` remains an +explicitly supported model option. ## Launch the Gradio demo @@ -86,6 +130,19 @@ python examples/online_serving/minicpmo/gradio_demo.py \ Open `http://:7862` in a browser. +## Daily-Omni accuracy + +Daily-Omni requires one A–D letter. Set +`chat_template_kwargs.enable_thinking=false` explicitly in `--extra-body`; +the generic benchmark CLI does not inject a MiniCPM-specific default. +Leaving reasoning enabled can exhaust the 256-token answer budget inside +`` and make first-letter extraction score reasoning text. + +For the established text benchmark, send +`--extra_body '{"modalities":["text"],"chat_template_kwargs":{"enable_thinking":false}}'`. +Requesting audio also benchmarks Talker and Code2Wav and changes the +assistant template, so it is not an apples-to-apples accuracy run. + ## Run the Realtime duplex CLI demo After the duplex backend is running, stream one WAV through the Realtime @@ -137,3 +194,26 @@ two-response WAV, its `--input-sha256`, and an - [Offline MiniCPM-o inference](../../offline_inference/minicpmo/) - [MiniCPM-o 4.5 recipe](../../../recipes/OpenBMB/MiniCPM-o-4_5.md) + +## Pipeline notes + +- Stage 1 performs request-owned AR continuous batching. Stage 2 keeps + request-owned Flow/HiFT caches and batches exact-shape-compatible chunks. +- Reference audio travels with the first codec chunk; Stage 2 owns its + temporary prompt WAV and evicts prompt features when the request finishes. +- Codec sampling reads the checkpoint `tts_config` (default deterministic + seed 42). Stage-1 YAML sampling parameters govern only the binary + continue/stop token exposed to vLLM. +- `StageRequestStats.batch_size` is request-scoped and does not report the + scheduler's execution batch. +- Stage 0 and Stage 1 use vLLM CUDA Graph capture. Stage 2 remains eager until + a dedicated exact-shape graph wrapper owns static I/O buffers and copies + request cache state outside capture. +- Co-locating all three stages minimizes hardware requirements but makes their + CUDA contexts contend for one GPU. Use the 8x4090 layout or a custom + multi-GPU deploy config when throughput is the primary goal. +- Output audio is base64 WAV in `message.audio.data` (24 kHz mono). +- Offline counterpart: + [`examples/offline_inference/minicpmo/`](../../offline_inference/minicpmo/) +- Recipe: + [`recipes/OpenBMB/MiniCPM-o-4_5.md`](../../../recipes/OpenBMB/MiniCPM-o-4_5.md) diff --git a/examples/online_serving/minicpmo/streaming_chat_completion.py b/examples/online_serving/minicpmo/streaming_chat_completion.py new file mode 100644 index 000000000..b4a4be2f0 --- /dev/null +++ b/examples/online_serving/minicpmo/streaming_chat_completion.py @@ -0,0 +1,71 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Stream MiniCPM-o 4.5 text and audio from /v1/chat/completions.""" + +from __future__ import annotations + +import argparse +import base64 +import io +import wave +from pathlib import Path + +from openai import OpenAI + + +def append_wav_chunk(encoded: str, pcm_parts: list[bytes]) -> tuple[int, int, int]: + """Decode one base64 WAV delta and append its PCM frames.""" + with wave.open(io.BytesIO(base64.b64decode(encoded)), "rb") as wav: + channels = wav.getnchannels() + sample_width = wav.getsampwidth() + sample_rate = wav.getframerate() + pcm_parts.append(wav.readframes(wav.getnframes())) + return channels, sample_width, sample_rate + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--base-url", default="http://localhost:8099/v1") + parser.add_argument("--model", default="openbmb/MiniCPM-o-4_5") + parser.add_argument("--prompt", default="Say hello, then introduce vLLM in one sentence.") + parser.add_argument("--output", type=Path, default=Path("minicpmo_stream.wav")) + parser.add_argument("--text-only", action="store_true") + args = parser.parse_args() + + modalities = ["text"] if args.text_only else ["text", "audio"] + stream = OpenAI(base_url=args.base_url, api_key="EMPTY").chat.completions.create( + model=args.model, + messages=[{"role": "user", "content": args.prompt}], + modalities=modalities, + stream=True, + extra_body={"chat_template_kwargs": {"use_tts_template": not args.text_only}}, + ) + + pcm_parts: list[bytes] = [] + audio_format: tuple[int, int, int] | None = None + for chunk in stream: + for choice in chunk.choices: + content = getattr(choice.delta, "content", None) + if not content: + continue + if getattr(chunk, "modality", None) == "audio": + current_format = append_wav_chunk(content, pcm_parts) + if audio_format is not None and current_format != audio_format: + raise ValueError(f"Audio format changed: {audio_format} -> {current_format}") + audio_format = current_format + elif getattr(chunk, "modality", None) == "text": + print(content, end="", flush=True) + + print() + if pcm_parts and audio_format is not None: + channels, sample_width, sample_rate = audio_format + with wave.open(str(args.output), "wb") as wav: + wav.setnchannels(channels) + wav.setsampwidth(sample_width) + wav.setframerate(sample_rate) + wav.writeframes(b"".join(pcm_parts)) + print(f"Audio saved to {args.output} ({sample_rate} Hz)") + + +if __name__ == "__main__": + main() diff --git a/recipes/OpenBMB/MiniCPM-o-4_5.md b/recipes/OpenBMB/MiniCPM-o-4_5.md index 68e8aac9e..73d0fd5a5 100644 --- a/recipes/OpenBMB/MiniCPM-o-4_5.md +++ b/recipes/OpenBMB/MiniCPM-o-4_5.md @@ -17,22 +17,23 @@ Use this recipe as a known-good starting point for serving `openbmb/MiniCPM-o-4_5` on vLLM-Omni. MiniCPM-o 4.5 is the omni member -of the MiniCPM-o family — it pairs a multimodal-understanding thinker -LLM with a streaming `MiniCPMTTS + Token2Wav` talker so a single +of the MiniCPM-o family — it runs a multimodal thinker, a streaming +MiniCPMTTS codec talker, and a separate batched Code2Wav stage so a single `/v1/chat/completions` call can return text and 24 kHz speech in one -shot. The recipe covers the shipped GPU layouts (single / 2 / 3 / 8 GPUs): - the default co-locates both stages on one GPU, and the larger scale-out - layouts (2 / 3 / 8 GPUs) are selected via `--deploy-config`. +shot. The recommended batching deploy isolates the Thinker on GPU 0 and +co-locates Talker and Code2Wav on GPU 1; 1-GPU, 3-GPU, and 8x4090 layouts are +also provided. ## References - Default deploy configs (auto-loaded by HF `model_type=minicpmo` + `hf_config.version="4.5"`): - - Single-GPU layout (default): + - Default single-GPU compatibility layout (auto-loaded): [`vllm_omni/deploy/minicpmo_4_5.yaml`](../../vllm_omni/deploy/minicpmo_4_5.yaml) - - 2-GPU layout (thinker on GPU 0, talker on GPU 1): - [`vllm_omni/deploy/minicpmo_4_5_2gpu.yaml`](../../vllm_omni/deploy/minicpmo_4_5_2gpu.yaml) - - 3-GPU layout (thinker TP=2): + - Recommended 2-GPU continuous-batching layout: + [`vllm_omni/deploy/minicpmo_4_5_batching.yaml`](../../vllm_omni/deploy/minicpmo_4_5_batching.yaml) + - 2-GPU and 3-GPU layouts: + [`vllm_omni/deploy/minicpmo_4_5_2gpu.yaml`](../../vllm_omni/deploy/minicpmo_4_5_2gpu.yaml), [`vllm_omni/deploy/minicpmo_4_5_3gpu.yaml`](../../vllm_omni/deploy/minicpmo_4_5_3gpu.yaml) - 8x RTX 4090 layout: [`vllm_omni/deploy/minicpmo_4_5_8x4090.yaml`](../../vllm_omni/deploy/minicpmo_4_5_8x4090.yaml) @@ -42,7 +43,7 @@ shot. The recipe covers the shipped GPU layouts (single / 2 / 3 / 8 GPUs): [`examples/offline_inference/minicpmo/`](../../examples/offline_inference/minicpmo/) - Pipeline / talker source: [`vllm_omni/model_executor/models/minicpmo_4_5/`](../../vllm_omni/model_executor/models/minicpmo_4_5/) -- Stage-input processor (thinker → talker bridge): +- Stage-input processors (thinker → talker and talker → Code2Wav): [`vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py`](../../vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py) - Upstream model card: [`openbmb/MiniCPM-o-4_5`](https://huggingface.co/openbmb/MiniCPM-o-4_5) @@ -51,18 +52,25 @@ shot. The recipe covers the shipped GPU layouts (single / 2 / 3 / 8 GPUs): ## Hardware Support -Four GPU layouts ship with default deploy configs. Pick the layout that -matches your hardware and pass it via `--deploy-config`. The default -co-locates the thinker and the talker (`MiniCPMTTS + Token2Wav`, with its -in-process vocoder) on a single GPU; the multi-GPU layouts split the -talker onto its own GPU and scale the thinker out via TP. +Four hardware layouts ship with deploy configs. Every layout uses the +same strict three-stage topology. The Talker emits codec chunks only; +Code2Wav consumes them through a shared-memory async connector. -| Layout | Thinker | Talker + Token2Wav | Typical hardware | -| --- | --- | --- | --- | -| Single-GPU (default) | GPU 0 | GPU 0 | 1x A100/H100/H200 80GB | -| 2-GPU | GPU 0 | GPU 1 | 2x A100/H100/H200 80GB | -| 3-GPU (thinker TP=2) | GPU 0,1 (TP=2) | GPU 2 | 3x mid-tier GPUs | -| 8x RTX 4090 24GB | GPU 0–3 (TP=4) | GPU 4 | 8x RTX 4090 consumer | +| Layout | Thinker | Talker | Code2Wav | Typical hardware | +| --- | --- | --- | --- | --- | +| 1-GPU (default) | GPU 0 | GPU 0 | GPU 0 | 1x large-memory GPU | +| 2-GPU | GPU 0 | GPU 1 | GPU 1 | 2x large-memory GPU | +| 3-GPU | GPU 0 | GPU 1 | GPU 2 | 3x GPU | +| 8x RTX 4090 24GB | GPU 0–3 (TP=4) | GPU 4 | GPU 5 | 8x RTX 4090 consumer | + +### Migration from the fused deployment + +MiniCPM-o 4.5 now requires the three-stage topology: the Talker owns +request-local codec generation and Code2Wav owns waveform state and +reference-voice prompt features. `minicpmo_4_5.yaml` remains the stable +single-GPU entry point; `minicpmo_4_5_batching.yaml` is the recommended +two-GPU profile. The removed fused two-stage implementation is not retained as +a fallback because it would duplicate state machines and correctness paths. ## GPU @@ -70,11 +78,10 @@ talker onto its own GPU and scale the thinker out via TP. The default [`vllm_omni/deploy/minicpmo_4_5.yaml`](../../vllm_omni/deploy/minicpmo_4_5.yaml) -co-locates both stages on GPU 0: the thinker (`~80 %` memory) and the -talker + Token2Wav vocoder (`~10 %` memory, `max_num_seqs: 1`). This is -the recommended starting layout — works on a single 80GB-class GPU -(A100, H100, H200) as long as the thinker model weights and the talker's -in-process vocoder fit together. +co-locates Thinker, codec-only Talker, and Code2Wav on GPU 0. Their +`gpu_memory_utilization` budgets are 0.65, 0.15, and 0.15. This layout +minimizes the GPU count; use a large-memory accelerator and leave the +remaining 5% for runtime overhead. #### Environment @@ -95,6 +102,40 @@ vllm serve openbmb/MiniCPM-o-4_5 --omni \ The deploy config is auto-loaded by the model registry — no `--deploy-config` flag needed for this default single-GPU layout. +For the recommended two-GPU layout, add: + +```bash +--deploy-config vllm_omni/deploy/minicpmo_4_5_batching.yaml +``` + +#### Performance comparison + +Compare text-only and text+audio separately. Text-only isolates Thinker +generation; text+audio also schedules Talker and Code2Wav. The following full +Daily-Omni runs used the same two GPUs, 1197 samples, concurrency 10, and +identical `enable_thinking=false` / `use_tts_template=true` request settings. +The `origin/main` fused Talker ran eager because its graph capture copied an +unpinned CPU metadata tensor. + +| Metric | `origin/main` two-stage | Three-stage batching | +| --- | ---: | ---: | +| Accuracy | 64.83% | 64.83% | +| Throughput | 0.62 req/s | 1.97 req/s | +| Mean E2EL | 16.17 s | 5.07 s | +| Mean serving TTFT | 0.92 s | 1.28 s | +| Mean audio TTFP | 16.17 s | 3.24 s | +| Mean audio RTF | 5.97 | 2.11 | +| Stage 0 mean TPOT / ITL | 8.27 / 8.27 ms | 40.08 / 40.11 ms | +| Stage 0 median TPOT / ITL | 7.23 / 7.24 ms | 7.43 / 7.53 ms | + +The split pipeline improves throughput 3.19x and lowers audio TTFP by 80%. +Isolating the Thinker on GPU 0 also removes the prior single-GPU TPOT +regression: 40.08 ms is slightly better than the pre-rebase report (~44 ms). +Its median TPOT is effectively the same as main; the higher mean is queueing +tail latency because this profile bounds each stage to four sequences while +main's Thinker admits 16. Global TPOT/ITL remains zero when serving emits text +as one aggregated chunk, so the table reports Stage 0 metrics. + #### Verification **Quick smoke test (text-only output)**: @@ -109,10 +150,10 @@ curl http://localhost:8099/v1/chat/completions \ }' ``` -**Text + speech in one response** (the headline 4.5 feature). The TTS -path is gated by a Jinja flag on the chat template. Pass -`use_tts_template=true` via the **top-level** `chat_template_kwargs` -field (curl does not flatten nested `extra_body`): +**Text + speech in one response** (the headline 4.5 feature). The model +bridge conditions the Talker from the generated assistant span, so the +generic serving layer does not inject MiniCPM-specific template defaults. +`use_tts_template=true` remains supported when explicitly requested: ```bash curl http://localhost:8099/v1/chat/completions \ @@ -134,6 +175,17 @@ in another choice's `message.audio.data` (24 kHz mono, see Notes). With `modalities: ["text", "audio"]` you typically get two `choices` entries (one text, one audio). +**Streaming text + speech**: + +```bash +python examples/online_serving/minicpmo/streaming_chat_completion.py \ + --base-url http://localhost:8099/v1 \ + --output minicpmo_stream.wav +``` + +The client prints text deltas as they arrive and reconstructs one valid WAV +from the independently encoded audio deltas. + **Gradio demo (text + image + audio + video UI)**: ```bash @@ -150,68 +202,32 @@ speech output (TTS)"** checkbox on / off. #### Notes -- Memory budget: both stages share GPU 0 — the thinker at - `gpu_memory_utilization: 0.8`, the talker + Token2Wav vocoder at `0.1`. +- Memory budget: Thinker, Talker, and Code2Wav reserve 0.65, 0.15, and + 0.15 of GPU 0. The larger Thinker share protects its multimodal KV cache; + all three model processes still share one CUDA device. - `--trust-remote-code` is required — the HF repo ships a custom `MiniCPMO` config / model class. -- Stage 1 (talker) is hard-capped to `max_num_seqs: 1`: the talker - only consumes `runtime_additional_information[0]`, so any value > 1 - makes concurrent requests share request-0's audio. This is the same - cap baked into the deploy config. - -### 2 x GPU (talker on its own GPU) - -Use -[`vllm_omni/deploy/minicpmo_4_5_2gpu.yaml`](../../vllm_omni/deploy/minicpmo_4_5_2gpu.yaml) -when you have two GPUs and want to give the talker + Token2Wav vocoder a -dedicated card instead of sharing GPU 0 with the thinker. The thinker -runs on GPU 0 (`~90 %` mem, TP=1) and the talker on GPU 1 (`~75 %` mem, -`max_num_seqs: 1`). This relieves the memory pressure of the default -single-GPU co-located layout and is the recommended step up when a -second 80GB-class card is available but full 3-way TP scale-out is not -needed. - -#### Command - -```bash -vllm serve openbmb/MiniCPM-o-4_5 --omni \ - --deploy-config vllm_omni/deploy/minicpmo_4_5_2gpu.yaml \ - --trust-remote-code \ - --host 0.0.0.0 --port 8099 -``` - -Verification and Notes mirror the single-GPU section; the only -difference is that the talker no longer competes with the thinker for -GPU 0 memory. - -### 3 x GPU (thinker TP=2) - -Use -[`vllm_omni/deploy/minicpmo_4_5_3gpu.yaml`](../../vllm_omni/deploy/minicpmo_4_5_3gpu.yaml) -when you have a third GPU available and want the thinker on 2-way -tensor parallel for higher throughput; the talker stays on its own -GPU (talker has its own in-process Token2Wav vocoder, so co-locating -it with the thinker risks OOM under load). - -#### Command - -```bash -vllm serve openbmb/MiniCPM-o-4_5 --omni \ - --deploy-config vllm_omni/deploy/minicpmo_4_5_3gpu.yaml \ - --trust-remote-code \ - --host 0.0.0.0 --port 8099 -``` - -Verification and Notes mirror the single-GPU section; thinker latency -roughly halves under load thanks to TP=2. +- Stage 0 Thinker and Stage 1 Talker enable vLLM CUDA Graphs. Stage 2 remains + eager because its request-owned Flow/HiFT caches and variable chunk/cache + shapes are not yet exposed through a static exact-shape graph wrapper. +- All default stages use `max_num_seqs: 4` to reduce cross-process GPU + contention. Talker AR + state and Code2Wav caches are request-owned; Code2Wav batches only + exact-shape-compatible chunks and does not fall back to serial decode. +- `StageRequestStats.batch_size` is a request-scoped placeholder, not the + scheduler's execution batch. +- Single-GPU co-location trades throughput for hardware density: Stage 0/1 + CUDA Graph replay and eager Stage 2 vocoder kernels compete across three + CUDA contexts. Use the 8x4090 config or a custom multi-GPU mapping for + throughput-sensitive serving. ### 8 x RTX 4090 24GB (consumer-GPU layout) Use [`vllm_omni/deploy/minicpmo_4_5_8x4090.yaml`](../../vllm_omni/deploy/minicpmo_4_5_8x4090.yaml) on an 8x RTX 4090 host. Thinker uses 4-way TP across GPUs 0–3 -(`~85 %` mem each ≈ 20.4 GiB/card), talker + Token2Wav lives on GPU 4 -(`~90 %` mem). GPUs 5–7 are left free. +(`~85 %` mem each ≈ 20.4 GiB/card), Talker uses GPU 4, and Code2Wav +uses GPU 5. GPUs 6–7 are left free. #### Command @@ -233,8 +249,8 @@ vllm serve openbmb/MiniCPM-o-4_5 --omni \ ## Notes (applies to all layouts) -- **Talker dependency**: the `MiniCPM-o 4.5` talker calls - `from stepaudio2 import Token2wav` against the MiniCPM-o-flavored +- **Code2Wav dependency**: Stage 2 loads `Token2wav` from the + MiniCPM-o-flavored vocoder (PyPI package `stepaudio2-minicpmo` — NOT the upstream `stepfun-ai/Step-Audio2`, whose `Token2wav.__init__` signature rejects `n_timesteps`). Install via the published extra: @@ -247,13 +263,20 @@ vllm serve openbmb/MiniCPM-o-4_5 --omni \ missing dep raises `ImportError` at first request with the same install hint instead of silently emitting empty audio. -- **TTS trigger**: speech output requires - `chat_template_kwargs.use_tts_template=true` so the chat template - appends `<|tts_bos|>` before generation. Without it, Stage-1 talker - receives no TTS token span and returns silent audio (not text-only). - For **curl**, put `chat_template_kwargs` at the request root; nested - `extra_body.chat_template_kwargs` is ignored. The OpenAI Python SDK - may use `extra_body` because it flattens those fields into the root. +- **TTS conditioning**: the MiniCPM stage bridge can condition speech from + the generated assistant span without changing shared serving code. + `chat_template_kwargs.use_tts_template=true` remains supported when an + explicit `<|tts_bos|>` boundary is desired. For **curl**, put + `chat_template_kwargs` at the request root; the OpenAI Python SDK may use + `extra_body` because it flattens those fields into the root. + +- **Reference voice**: request audio is carried on the first codec chunk. + Code2Wav owns the temporary prompt WAV and prompt-feature cache, and removes + both when the stream ends. + +- **Talker sampling**: codec-token sampling reads the checkpoint `tts_config` + and defaults to deterministic seed 42. Stage-1 deploy sampling parameters + control only vLLM's binary continue/stop token. - **Output audio**: 24 kHz mono WAV inside the OpenAI-style `message.audio.data` (base64). The Gradio demo's WAV player decodes @@ -266,6 +289,9 @@ vllm serve openbmb/MiniCPM-o-4_5 --omni \ with this recipe's `--deploy-config` will be rejected at startup rather than silently misrouted. -- **Async chunking**: disabled in all three deploy configs - (`async_chunk: false`) — the talker batches a single full thinker - output, not chunks. +- **Async chunking**: enabled in all deploy configs. Talker sends + 25-code chunks with three-code left context to Code2Wav through + `SharedMemoryConnector`; terminal chunks flush held lookahead state. +- **Response choices**: text and audio are separate choices. SDK clients + should select the choice whose `message.audio.data` is populated rather + than assuming `choices[0]` contains audio. diff --git a/tests/benchmarks/test_serve_cli.py b/tests/benchmarks/test_serve_cli.py index 9ec30639c..7dd12d000 100644 --- a/tests/benchmarks/test_serve_cli.py +++ b/tests/benchmarks/test_serve_cli.py @@ -99,19 +99,21 @@ def test_update_omni_help_updates_upstream_actions() -> None: @pytest.mark.parametrize( - ("extra_body", "bot_task", "expected"), + ("backend", "extra_body", "bot_task", "expected"), [ - (None, "think", {"bot_task": "think"}), - ({"bot_task": "recaption"}, "think", {"bot_task": "recaption"}), - ({"custom": True}, None, {"custom": True}), + ("openai-chat-omni", None, "think", {}), + ("openai-image-edits-omni", None, "think", {"bot_task": "think"}), + ("openai-image-edits-omni", {"bot_task": "recaption"}, "think", {"bot_task": "recaption"}), + ("openai-image-edits-omni", {"custom": True}, None, {"custom": True}), ], ) def test_preprocess_serve_args_merges_bot_task_without_overriding_extra_body( + backend: str, extra_body: dict | None, bot_task: str | None, expected: dict, ) -> None: - args = argparse.Namespace(extra_body=extra_body, bot_task=bot_task) + args = argparse.Namespace(backend=backend, extra_body=extra_body, bot_task=bot_task) preprocess_serve_args(args) @@ -122,9 +124,15 @@ def test_preprocess_serve_args_merges_bot_task_without_overriding_extra_body( ("argv", "expected_extra_body", "expected_explicit"), [ ( - ["--print-stage", "--image-edits-bot-task", "recaption"], + [ + "--backend", + "openai-image-edits-omni", + "--print-stage", + "--image-edits-bot-task", + "recaption", + ], {"bot_task": "recaption"}, - {"print_stage", "bot_task"}, + {"backend", "print_stage", "bot_task"}, ), ( ["--extra-body", '{"bot_task":"vanilla"}'], @@ -140,6 +148,7 @@ def test_omni_args_parse_and_preprocess( ) -> None: parser = TrackingArgumentParser() parser.add_argument("--extra-body", type=json.loads, default=None) + parser.add_argument("--backend", default="openai-chat-omni") add_omni_args(parser) args = parser.parse_args(argv) diff --git a/tests/core/sched/test_omni_ar_scheduler_streaming.py b/tests/core/sched/test_omni_ar_scheduler_streaming.py index 41fb414cb..32284eeac 100644 --- a/tests/core/sched/test_omni_ar_scheduler_streaming.py +++ b/tests/core/sched/test_omni_ar_scheduler_streaming.py @@ -26,6 +26,7 @@ def _make_scheduler(*, stage_id: int = 0) -> OmniARScheduler: sched.num_waiting_for_streaming_input = 0 sched.log_stats = False sched.chunk_transfer_adapter = None + sched.skipped_waiting = set() return sched @@ -90,3 +91,24 @@ def test_stage0_streaming_update_keeps_all_computed_tokens_without_placeholder() assert session._output_token_ids == [] assert session.num_prompt_tokens == 8 assert sched._new_prompt_len_snapshot[session.request_id] == 2 + + +def test_explicit_streaming_payload_replaces_placeholder_prompt() -> None: + sched = _make_scheduler(stage_id=1) + sched.chunk_transfer_adapter = SimpleNamespace( + receives_chunks=False, + segment_finished_requests=set(), + ) + session = _make_request() + session.status = RequestStatus.WAITING_FOR_STREAMING_REQ + update = _make_update([10, 20]) + update.additional_information = { + "tts_token_ids": [10, 20], + "meta": {"replace_streaming_prompt": True}, + } + + sched._update_request_as_session(session, update) + + assert session.prompt_token_ids == [10, 20] + assert session.additional_information == update.additional_information + assert session.status == RequestStatus.WAITING diff --git a/tests/distributed/omni_connectors/test_adapter_and_flow.py b/tests/distributed/omni_connectors/test_adapter_and_flow.py index 92dc1d7c0..471d3108a 100644 --- a/tests/distributed/omni_connectors/test_adapter_and_flow.py +++ b/tests/distributed/omni_connectors/test_adapter_and_flow.py @@ -8,6 +8,7 @@ from vllm_omni.distributed.omni_connectors.adapter import try_recv_via_connector from vllm_omni.distributed.omni_connectors.connectors.shm_connector import SharedMemoryConnector from vllm_omni.distributed.omni_connectors.utils.config import ConnectorSpec, OmniTransferConfig from vllm_omni.distributed.omni_connectors.utils.initialization import get_connectors_config_for_stage +from vllm_omni.engine.stage_init_utils import get_stage_connector_spec pytestmark = [pytest.mark.core_model, pytest.mark.cpu] @@ -217,6 +218,22 @@ def test_get_connectors_for_stage(): assert stage_2_config["from_stage_1"]["spec"]["name"] == "C2" +def test_outgoing_only_async_stage_uses_sender_connector_spec(): + config = OmniTransferConfig( + connectors={ + ("1", "2"): ConnectorSpec( + name="SharedMemoryConnector", + extra={"codec_chunk_frames": 25}, + ) + } + ) + + spec = get_stage_connector_spec(config, stage_id=1, async_chunk=True) + + assert spec["name"] == "SharedMemoryConnector" + assert spec["extra"] == {"codec_chunk_frames": 25, "role": "sender"} + + def test_recv_with_missing_metadata(mocker: MockerFixture): """Test recv when queue payload is malformed (missing metadata).""" # Connector expects metadata but task doesn't have it diff --git a/tests/distributed/omni_connectors/test_chunk_transfer_adapter.py b/tests/distributed/omni_connectors/test_chunk_transfer_adapter.py index 76d0cb550..1485ace60 100644 --- a/tests/distributed/omni_connectors/test_chunk_transfer_adapter.py +++ b/tests/distributed/omni_connectors/test_chunk_transfer_adapter.py @@ -13,6 +13,7 @@ from vllm.v1.core.sched.scheduler import Scheduler as VLLMScheduler from vllm.v1.request import RequestStatus from vllm_omni.data_entry_keys import CodesStruct, MetaStruct, OmniPayload, OmniPayloadStruct +from vllm_omni.distributed.omni_connectors.adapter import construct_next_stage_streaming_input_prompt from vllm_omni.distributed.omni_connectors.transfer_adapter.base import OmniTransferAdapterBase from vllm_omni.distributed.omni_connectors.transfer_adapter.chunk_transfer_adapter import ( OmniChunkTransferAdapter, @@ -43,6 +44,33 @@ def _req(req_id: str, status: RequestStatus, external_req_id: str | None = None) ) +def test_streaming_payload_can_replace_placeholder_prompt(mocker: MockerFixture) -> None: + request = SimpleNamespace( + _all_token_ids=[0, 0, 7, 8], + _output_token_ids=[7, 8], + prompt_token_ids=[0, 0], + num_computed_tokens=4, + num_prompt_tokens=2, + update_block_hashes=mocker.Mock(), + ) + payload = { + "ids": {"prompt": [1, 2, 3]}, + "meta": { + "replace_streaming_prompt": True, + "next_stage_prompt_len": 7, + }, + } + + construct_next_stage_streaming_input_prompt(payload, request) + + assert request.prompt_token_ids == [0] * 7 + assert request._all_token_ids == [0] * 7 + assert request._output_token_ids == [] + assert request.num_computed_tokens == 0 + assert request.num_prompt_tokens == 7 + request.update_block_hashes.assert_called_once_with() + + @pytest.fixture def build_adapter(monkeypatch, mocker: MockerFixture): def _build( @@ -81,6 +109,10 @@ def build_adapter(monkeypatch, mocker: MockerFixture): worker_type=model_mode, max_num_seqs=max_num_seqs, active_stream_window=active_stream_window, + stage_connector_config={ + "name": "SharedMemoryConnector", + "extra": connector_extra or {}, + }, ) scheduler_config = SimpleNamespace(max_num_seqs=max_num_seqs) adapter = OmniChunkTransferAdapter( @@ -381,6 +413,26 @@ def test_load_poll_ar_request_additional_information_concats_tensors(build_adapt assert request.additional_information["meta"]["finished"].item() is True +def test_sender_only_adapter_does_not_park_or_clear_requests(build_adapter): + adapter, _ = build_adapter(stage_id=1, connector_extra={"role": "sender"}) + request = _req("req-1", RequestStatus.WAITING) + request.additional_information = {"tts_token_ids": torch.tensor([1])} + waiting_queue = DummyWaitingQueue([request]) + running_queue = [] + + adapter.load_async(request) + adapter.process_pending_chunks( + waiting_queue, + running_queue, + scheduler_requests={request.request_id: request}, + ) + + assert waiting_queue == [request] + assert request.status == RequestStatus.WAITING + assert request.additional_information["tts_token_ids"].item() == 1 + assert adapter._pending_load_reqs == deque() + + def test_process_and_restore_queues(build_adapter): adapter, _ = build_adapter(stage_id=1, max_num_seqs=8) waiting_req = _req("w1", RequestStatus.WAITING) diff --git a/tests/distributed/omni_connectors/test_omni_connector_configs.py b/tests/distributed/omni_connectors/test_omni_connector_configs.py index 82f13c5cd..6048de40a 100644 --- a/tests/distributed/omni_connectors/test_omni_connector_configs.py +++ b/tests/distributed/omni_connectors/test_omni_connector_configs.py @@ -2,10 +2,15 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from pathlib import Path +from types import SimpleNamespace import pytest # Use the new import path for initialization utilities +from vllm_omni.distributed.omni_connectors.utils.config import ( + stage_receives_chunks, + stage_sends_async_output, +) from vllm_omni.distributed.omni_connectors.utils.initialization import load_omni_transfer_config pytestmark = [pytest.mark.core_model, pytest.mark.cpu] @@ -38,6 +43,26 @@ def get_config_files(): config_files = get_config_files() +@pytest.mark.parametrize( + ("role", "stage_id", "receives", "sends"), + [ + ("sender", 0, False, True), + ("receiver", 1, True, False), + (None, 0, True, False), + (None, None, True, True), + ], +) +def test_stage_chunk_direction_helpers(role, stage_id, receives, sends): + extra = {} if role is None else {"role": role} + model_config = SimpleNamespace( + stage_id=stage_id, + stage_connector_config={"extra": extra}, + ) + + assert stage_receives_chunks(model_config) is receives + assert stage_sends_async_output(model_config) is sends + + @pytest.mark.skipif(len(config_files) == 0, reason="No config files found or directory missing") @pytest.mark.parametrize("yaml_file", config_files, ids=lambda p: p.name) def test_load_qwen_yaml_configs(yaml_file): diff --git a/tests/engine/test_orchestrator_stage_input_bridge.py b/tests/engine/test_orchestrator_stage_input_bridge.py index 3abb244eb..e74b9ea18 100644 --- a/tests/engine/test_orchestrator_stage_input_bridge.py +++ b/tests/engine/test_orchestrator_stage_input_bridge.py @@ -6,6 +6,7 @@ from __future__ import annotations import queue from types import SimpleNamespace from typing import Any +from unittest.mock import AsyncMock import janus import pytest @@ -91,6 +92,25 @@ class FakeInputProcessor: ) +class FakePrewarmPool: + stage_type = "llm" + + def __init__(self, role: str) -> None: + self.stage_vllm_config = SimpleNamespace( + model_config=SimpleNamespace( + max_model_len=64, + stage_connector_config={"extra": {"role": role}}, + ) + ) + self.submitted: list[Any] = [] + + async def submit_initial(self, _request_id, _req_state, request, prompt_text=None): + self.submitted.append(request) + + def get_bound_replica_id(self, _request_id): + return 0 + + def _request_output(request_id: str) -> RequestOutput: completion = CompletionOutput( index=0, @@ -168,3 +188,56 @@ async def test_forward_text_prompt_uses_target_stage_input_processor() -> None: submitted_request = stage1.add_request_calls[0][0] assert submitted_request.prompt_token_ids == [101, 102] assert submitted_request.external_req_id == "req-text" + + +@pytest.mark.asyncio +async def test_async_prewarm_skips_outgoing_only_stage() -> None: + orchestrator = object.__new__(Orchestrator) + stage0 = FakePrewarmPool("sender") + stage1 = FakePrewarmPool("sender") + stage2 = FakePrewarmPool("receiver") + orchestrator.stage_pools = [stage0, stage1, stage2] + orchestrator._emit_tx_edge = lambda **_kwargs: None + req_state = OrchestratorRequestState( + request_id="req-prewarm", + prompt={"prompt_token_ids": [1, 2]}, + sampling_params_list=[SamplingParams(max_tokens=1) for _ in range(3)], + final_stage_id=2, + ) + + await orchestrator._prewarm_async_chunk_stages( + "req-prewarm", + SimpleNamespace(prompt_token_ids=[1, 2], resumable=True), + req_state, + ) + + assert stage1.submitted == [] + assert len(stage2.submitted) == 1 + assert 1 not in req_state.stage_submit_ts + assert 2 in req_state.stage_submit_ts + + +@pytest.mark.asyncio +async def test_async_route_forwards_to_outgoing_only_stage() -> None: + orchestrator = object.__new__(Orchestrator) + orchestrator.async_chunk = True + orchestrator._pd_pair = None + orchestrator._cfg_tracker = SimpleNamespace( + is_companion=lambda _request_id: False, + has_companions=lambda _request_id: False, + ) + stage0 = SimpleNamespace(final_output=False) + stage1 = FakePrewarmPool("sender") + orchestrator.stage_pools = [stage0, stage1] + orchestrator._forward_to_next_stage = AsyncMock() + req_state = OrchestratorRequestState( + request_id="req-route", + sampling_params_list=[SamplingParams(max_tokens=1) for _ in range(2)], + final_stage_id=1, + ) + req_state.stage_submit_ts[0] = 1.0 + output = SimpleNamespace(request_id="req-route", finished=True) + + await orchestrator._route_output(0, 0, output, req_state, None) + + orchestrator._forward_to_next_stage.assert_awaited_once() diff --git a/tests/engine/test_stage_engine_core_proc.py b/tests/engine/test_stage_engine_core_proc.py new file mode 100644 index 000000000..a7104b14d --- /dev/null +++ b/tests/engine/test_stage_engine_core_proc.py @@ -0,0 +1,28 @@ +from types import SimpleNamespace +from unittest.mock import patch + +from vllm.v1.engine.core import EngineCoreProc + +from vllm_omni.engine.stage_engine_core_proc import StageEngineCoreProc + + +def test_preprocess_add_request_preserves_omni_fields(): + engine = StageEngineCoreProc.__new__(StageEngineCoreProc) + request = SimpleNamespace( + request_id="internal", + external_req_id="external", + additional_information={"conditioning": "payload"}, + ) + scheduler_request = SimpleNamespace() + + with patch.object( + EngineCoreProc, + "preprocess_add_request", + return_value=(scheduler_request, 3), + ): + result, current_wave = engine.preprocess_add_request(request) + + assert result is scheduler_request + assert current_wave == 3 + assert result.external_req_id == "external" + assert result.additional_information == {"conditioning": "payload"} diff --git a/tests/examples/online_serving/test_minicpmo_streaming.py b/tests/examples/online_serving/test_minicpmo_streaming.py new file mode 100644 index 000000000..90ba992eb --- /dev/null +++ b/tests/examples/online_serving/test_minicpmo_streaming.py @@ -0,0 +1,29 @@ +import base64 +import io +import runpy +import wave +from pathlib import Path + +import pytest + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +def test_append_wav_chunk_extracts_pcm_frames() -> None: + example = Path(__file__).parents[3] / "examples" / "online_serving" / "minicpmo" / "streaming_chat_completion.py" + append_wav_chunk = runpy.run_path(example)["append_wav_chunk"] + encoded_buffer = io.BytesIO() + with wave.open(encoded_buffer, "wb") as wav: + wav.setnchannels(1) + wav.setsampwidth(2) + wav.setframerate(24000) + wav.writeframes(b"\x01\x02\x03\x04") + + pcm_parts: list[bytes] = [] + audio_format = append_wav_chunk( + base64.b64encode(encoded_buffer.getvalue()).decode(), + pcm_parts, + ) + + assert audio_format == (1, 2, 24000) + assert pcm_parts == [b"\x01\x02\x03\x04"] diff --git a/tests/model_executor/models/minicpmo_4_5/test_code2wav_batching.py b/tests/model_executor/models/minicpmo_4_5/test_code2wav_batching.py new file mode 100644 index 000000000..58f3d27b7 --- /dev/null +++ b/tests/model_executor/models/minicpmo_4_5/test_code2wav_batching.py @@ -0,0 +1,431 @@ +from pathlib import Path +from types import SimpleNamespace + +import pytest +import torch +import torch.nn as nn + +from vllm_omni.model_executor.models.minicpmo_4_5.batched_token2wav import ( + BatchedToken2Wav, +) +from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_code2wav import ( + MiniCPMO45Code2Wav, +) + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +class _FakeEncoder(nn.Module): + def __init__(self): + super().__init__() + self.calls: list[int] = [] + + def forward_chunk(self, xs, last_chunk=False, cnn_cache=None, att_cache=None): + batch, length, _ = xs.shape + self.calls.append(batch) + old_length = 0 if att_cache is None else att_cache.shape[3] + output = xs[:, : max(1, length - 1)] + cnn = xs[:, :1, :].transpose(1, 2).contiguous() + marker = xs[:, 0, 0].reshape(1, batch, 1, 1, 1) + att = marker.expand(1, batch, 1, old_length + output.shape[1], 1).clone() + return output, cnn, att + + +class _FakeBlock: + def __init__(self): + conv1 = SimpleNamespace(causal_padding=(1, 0)) + self.conv = SimpleNamespace( + in_channels=1, + out_channels=1, + block=[None, conv1], + ) + self.attn = SimpleNamespace(num_heads=1, head_dim=1) + + +class _FakeEstimator(nn.Module): + def __init__(self): + super().__init__() + self.blocks = [_FakeBlock()] + self.cfg_batches: list[int] = [] + self.speaker_order: list[list[float]] = [] + + def t_embedder(self, time): + return time[:, None] + + def blocks_forward_chunk( + self, + inputs, + time, + mask, + cnn_cache, + att_cache, + cnn_out, + att_out, + ): + del time, mask, cnn_cache, att_cache + self.cfg_batches.append(inputs.shape[0]) + self.speaker_order.append(inputs[:, 2, 0].tolist()) + marker = inputs[:, 1, 0] + cnn_out.copy_(marker.reshape(1, -1, 1, 1).expand_as(cnn_out)) + att_out.copy_(marker.reshape(1, -1, 1, 1, 1).expand_as(att_out)) + return inputs[:, 1:2] + + +class _FakeDecoder(nn.Module): + def __init__(self): + super().__init__() + self.estimator = _FakeEstimator() + self.inference_cfg_rate = 0.7 + self.register_buffer("rand_noise", torch.zeros(1, 1, 100), persistent=False) + + +class _FakeFlow(nn.Module): + def __init__(self): + super().__init__() + self.encoder = _FakeEncoder() + self.encoder_proj = nn.Identity() + self.decoder = _FakeDecoder() + self.spk_embed_affine_layer = nn.Identity() + + def input_embedding(self, tokens): + return tokens.to(torch.float32).unsqueeze(-1) + + +class _FakeHiFT(nn.Module): + def __init__(self): + super().__init__() + self.calls: list[int] = [] + + def forward(self, mel, source): + del source + self.calls.append(mel.shape[0]) + speech = mel[:, 0].repeat_interleave(3, dim=1) + generated_source = speech[:, None] + return speech, generated_source + + +class _FakeToken2Wav: + def __init__(self): + self.flow = _FakeFlow() + self.hift = _FakeHiFT() + self.float16 = False + self.n_timesteps = 2 + self.mel_cache_len = 1 + self.source_cache_len = 2 + self.speech_window = torch.hamming_window(4, periodic=False) + self.prompt_calls = 0 + + def _prepare_prompt(self, prompt_wav): + del prompt_wav + self.prompt_calls += 1 + return ( + torch.tensor([[5, 6]], dtype=torch.long), + torch.tensor([2], dtype=torch.int32), + torch.ones(1, 1), + torch.ones(1, 4, 1), + torch.tensor([4], dtype=torch.int32), + ) + + def stream(self, *args, **kwargs): + raise AssertionError("sequential stream fallback must never be called") + + def __call__(self, *args, **kwargs): + raise AssertionError("sequential __call__ fallback must never be called") + + +def _config(minimum: int = 1): + return SimpleNamespace( + model_config=SimpleNamespace( + model="/fake/model", + stage_connector_config={ + "extra": { + "code2wav_min_batch_size": minimum, + "prompt_cache_id": "shared", + "prompt_wav": "/fake/prompt.wav", + } + }, + ) + ) + + +def _model(): + token2wav = _FakeToken2Wav() + backend = BatchedToken2Wav(token2wav) + model = MiniCPMO45Code2Wav(vllm_config=_config()) + model.backend = backend + return model, token2wav + + +def _info( + request_id: str, + chunk_seq: int, + codes: list[int], + *, + last_chunk: bool = False, + cache_epoch: int = 0, +): + return { + "codes": {"audio": torch.tensor(codes, dtype=torch.long)}, + "meta": { + "request_id": request_id, + "chunk_seq": chunk_seq, + "cache_epoch": cache_epoch, + "last_chunk": last_chunk, + "prompt_cache_id": "shared", + }, + } + + +def _forward(model, infos, placeholder_counts=None, request_ids=None): + placeholder_counts = placeholder_counts or [1] * len(infos) + input_ids = torch.zeros(sum(placeholder_counts), dtype=torch.long) + return model( + input_ids=input_ids, + seq_token_counts=placeholder_counts, + runtime_additional_information=infos, + request_ids=request_ids, + ) + + +def test_adapter_runs_true_batch_cfg_and_splits_request_caches(): + token2wav = _FakeToken2Wav() + adapter = BatchedToken2Wav(token2wav) + prompt = adapter.prepare_prompt("shared", "/fake/prompt.wav") + states = adapter.setup_batch(prompt, 2) + audios, states = adapter.decode_batch( + torch.tensor([[10, 11], [20, 21]]), + prompt, + states, + last_chunk=False, + ) + + assert token2wav.prompt_calls == 1 + assert token2wav.flow.encoder.calls == [2, 2] + assert token2wav.flow.decoder.estimator.cfg_batches == [4, 4, 4, 4] + assert all(order == [1.0, 1.0, 0.0, 0.0] for order in token2wav.flow.decoder.estimator.speaker_order) + assert token2wav.hift.calls == [2] + assert len(audios) == 2 + cache0 = states[0].flow_cache["estimator_cnn_cache"] + cache1 = states[1].flow_cache["estimator_cnn_cache"] + assert cache0.data_ptr() != cache1.data_ptr() + assert cache0[0, 0, 0, 0, 0].item() == 10 + assert cache1[0, 0, 0, 0, 0].item() == 20 + + +def test_estimator_cache_stack_split_round_trip_preserves_cfg_rows(): + token2wav = _FakeToken2Wav() + adapter = BatchedToken2Wav(token2wav) + prompt = adapter.prepare_prompt("shared", "/fake/prompt.wav") + states = adapter.setup_batch(prompt, 2) + _, states = adapter.decode_batch( + torch.tensor([[10, 11], [20, 21]]), + prompt, + states, + last_chunk=False, + ) + + stacked = adapter._stack_flow_cache(states) + assert stacked["estimator_cnn_cache"].shape[2] == 4 + assert stacked["estimator_att_cache"].shape[2] == 4 + restored = adapter._split_flow_cache(stacked, 2) + for original, round_tripped in zip(states, restored, strict=True): + torch.testing.assert_close( + round_tripped["estimator_cnn_cache"], + original.flow_cache["estimator_cnn_cache"], + ) + torch.testing.assert_close( + round_tripped["estimator_att_cache"], + original.flow_cache["estimator_att_cache"], + ) + + +def test_model_preserves_output_slots_and_prefers_runtime_codes(): + model, token2wav = _model() + output = _forward( + model, + [_info("a", 0, [10, 11]), _info("b", 0, [20, 21])], + placeholder_counts=[3, 1], + ) + + audios = output.multimodal_outputs["model_outputs"] + assert len(audios) == 2 + assert len(output.multimodal_outputs["sr"]) == 2 + assert all(sr.item() == 24000 for sr in output.multimodal_outputs["sr"]) + assert all(audio.dtype == torch.float32 for audio in audios) + # Fake CFM uses two Euler steps whose deltas sum to one. Its conditional + # row is mu and its unconditional row is zero, so CFG produces 1.7 * mu. + torch.testing.assert_close(audios[0][0], torch.tensor(1.7 * 10)) + torch.testing.assert_close(audios[1][0], torch.tensor(1.7 * 20)) + assert token2wav.flow.encoder.calls[-1] == 2 + + +def test_mixed_final_exact_buckets_keep_order_and_release_only_final_states(): + model, _ = _model() + _forward( + model, + [_info(name, 0, [index + 1, index + 2]) for index, name in enumerate(("a", "b", "c", "d"))], + ) + output = _forward( + model, + [ + _info("a", 1, [11, 12]), + _info("c", 1, [31, 32, 33], last_chunk=True), + _info("b", 1, [21, 22]), + _info("d", 1, [41, 42, 43], last_chunk=True), + ], + ) + + audios = output.multimodal_outputs["model_outputs"] + window = torch.hamming_window(4, periodic=False) + overlap_scale = 1.7 * (window[0] + window[2]) + expected = torch.tensor([1, 3, 2, 4], dtype=torch.float32) * overlap_scale + actual = torch.stack([audio[0] for audio in audios]) + torch.testing.assert_close(actual, expected) + assert set(model._states) == {"a", "b"} + + +def test_empty_final_sentinel_emits_empty_and_releases_state_without_compute(): + model, token2wav = _model() + _forward(model, [_info("a", 0, [1, 2]), _info("b", 0, [3, 4])]) + hift_calls = list(token2wav.hift.calls) + output = _forward( + model, + [ + _info("a", 1, [], last_chunk=True), + _info("b", 1, [], last_chunk=True), + ], + ) + + assert [audio.numel() for audio in output.multimodal_outputs["model_outputs"]] == [0, 0] + assert model._states == {} + assert token2wav.hift.calls == hift_calls + + +def test_empty_final_ignores_generation_scheduler_placeholder_token(): + model, _ = _model() + _forward(model, [_info("a", 0, [1, 2]), _info("b", 0, [3, 4])]) + infos = [_info("a", 1, [], last_chunk=True), _info("b", 1, [], last_chunk=True)] + for info in infos: + info.pop("codes") + info["meta"]["code_flat_numel"] = 0 + + output = _forward(model, infos, placeholder_counts=[1, 1]) + + assert [audio.numel() for audio in output.multimodal_outputs["model_outputs"]] == [0, 0] + assert model._states == {} + + +@pytest.mark.parametrize( + ("info", "reason"), + [ + (_info("a", 0, [1, 2], cache_epoch=-1), "negative_stream_position"), + (_info("a", 0, [1, 2]), "stale_or_reordered_chunk"), + (_info("a", 2, [1, 2]), "stale_or_reordered_chunk"), + ], +) +def test_stale_epoch_and_reordered_chunks_are_rejected(info, reason): + model, _ = _model() + _forward(model, [_info("a", 0, [1, 2]), _info("b", 0, [3, 4])]) + + with pytest.raises(RuntimeError, match=reason): + _forward(model, [info, _info("b", 1, [3, 4])]) + + +def test_singleton_and_mixed_shape_buckets_use_same_batched_backend_without_fallback(): + model, token2wav = _model() + _forward(model, [_info("a", 0, [1, 2]), _info("b", 0, [3, 4])]) + output = _forward(model, [_info("a", 1, [5, 6]), _info("b", 1, [7, 8, 9])]) + + assert len(output.multimodal_outputs["model_outputs"]) == 2 + # Exact-shape buckets execute independently but both use the same vectorized + # adapter; there is no Token2wav.stream/__call__ fallback. + assert token2wav.hift.calls[-2:] == [1, 1] + + +def test_backend_failure_does_not_commit_any_request_state(monkeypatch): + model, _ = _model() + _forward( + model, + [_info(name, 0, [index + 1, index + 2]) for index, name in enumerate(("a", "b", "c", "d"))], + ) + before = dict(model._states) + original = model.backend.decode_batch + call_count = 0 + + def fail(*args, **kwargs): + nonlocal call_count + call_count += 1 + if call_count == 2: + raise RuntimeError("injected failure") + return original(*args, **kwargs) + + monkeypatch.setattr(model.backend, "decode_batch", fail) + with pytest.raises(RuntimeError, match="injected failure"): + _forward( + model, + [ + _info("a", 1, [5, 6]), + _info("b", 1, [7, 8]), + _info("c", 1, [9, 10, 11]), + _info("d", 1, [12, 13, 14]), + ], + ) + assert call_count == 2 + assert model._states == before + + +def test_cleanup_and_profile_output_are_aligned(): + model, _ = _model() + _forward(model, [_info("a", 0, [1, 2]), _info("b", 0, [3, 4])]) + model.on_requests_finished(["a"]) + assert set(model._states) == {"b"} + + profile = model( + input_ids=torch.zeros(5, dtype=torch.long), + seq_token_counts=[2, 3], + ) + assert [audio.numel() for audio in profile.multimodal_outputs["model_outputs"]] == [0, 0] + assert set(model._states) == {"b"} + + +def test_cleanup_uses_generation_runner_internal_request_ids(): + model, _ = _model() + _forward( + model, + [_info("external-a", 0, [1, 2]), _info("external-b", 0, [3, 4])], + request_ids=["internal-a", "internal-b"], + ) + + model.on_requests_finished(["internal-a"]) + + assert set(model._states) == {"internal-b"} + + +def test_reference_voice_and_duplex_metadata_follow_request_lifecycle(): + model, _ = _model() + first = _info("voice-a", 0, [1, 2]) + first["codes"]["ref"] = torch.linspace(-0.1, 0.1, 160) + first["meta"].update( + ref_audio_sr=16000, + native_duplex_segment_text="hello", + duplex_turn_id=7, + duplex_epoch=3, + ) + first["meta"].pop("prompt_cache_id") + + output = _forward(model, [first]) + prompt_cache_id, prompt_wav = model._owned_prompt_wavs["voice-a"] + assert prompt_cache_id.startswith("voice-a:") + assert Path(prompt_wav).is_file() + assert bytes(output.multimodal_outputs["meta.llm_output_text_utf8"][0].tolist()).decode() == "hello" + assert output.multimodal_outputs["meta.duplex_turn_id"][0].item() == 7 + assert output.multimodal_outputs["meta.duplex_epoch"][0].item() == 3 + + final = _info("voice-a", 1, [3, 4], last_chunk=True) + final["meta"].pop("prompt_cache_id") + output = _forward(model, [final]) + + assert output.multimodal_outputs["meta.tts_is_last_chunk"][0].item() is True + assert "voice-a" not in model._owned_prompt_wavs + assert not Path(prompt_wav).exists() + assert (prompt_cache_id, prompt_wav) not in model.backend._prompt_features diff --git a/tests/model_executor/models/minicpmo_4_5/test_hf_model_path.py b/tests/model_executor/models/minicpmo_4_5/test_hf_model_path.py deleted file mode 100644 index 834b7b0e0..000000000 --- a/tests/model_executor/models/minicpmo_4_5/test_hf_model_path.py +++ /dev/null @@ -1,74 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Regression test for HF model-ID resolution and path caching in MiniCPM-o 4.5 TTS.""" - -from types import SimpleNamespace - -import pytest - -from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni_tts import ( - MiniCPMO45OmniTTSForConditionalGeneration, -) - -pytestmark = [pytest.mark.core_model, pytest.mark.cpu] - - -class _FakeMiniCPMTTS: - def __init__(self, config, audio_tokenizer): - self.config = config - self.audio_tokenizer = audio_tokenizer - self.emb_text = object() - self.projector_semantic = object() - - def generate(self): - raise NotImplementedError - - -def test_hf_model_id_path_caching(mocker, tmp_path): - """Verify the resolved HF path is cached and reused for default ref audio.""" - resolved_path = str(tmp_path / "resolved_hf_model") - assets_dir = tmp_path / "resolved_hf_model" / "assets" - assets_dir.mkdir(parents=True, exist_ok=True) - ref_audio_file = assets_dir / "HT_ref_audio.wav" - ref_audio_file.write_bytes(b"dummy wav content") - - mock_download = mocker.patch( - "vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni_tts.download_weights_from_hf_specific", - return_value=resolved_path, - ) - - mocker.patch( - "transformers.dynamic_module_utils.get_class_from_dynamic_module", - return_value=_FakeMiniCPMTTS, - ) - - # Build fake vllm_config with HF model ID - model_id = "openbmb/MiniCPM-o-4_5" - tts_cfg = SimpleNamespace( - audio_bos_token_id=151687, - text_eos_token_id=151692, - num_audio_tokens=6562, - hidden_size=768, - normalize_projected_hidden=True, - top_p=0.8, - top_k=100, - repetition_penalty=1.02, - attn_implementation="sdpa", - ) - hf_config = SimpleNamespace(tts_config=tts_cfg) - vllm_config = SimpleNamespace( - model_config=SimpleNamespace( - model=model_id, - hf_config=hf_config, - ) - ) - - tts_model = MiniCPMO45OmniTTSForConditionalGeneration(vllm_config=vllm_config) - - tts_model._lazy_init_tts() - tts_model._lazy_init_tts() - - mock_download.assert_called_once_with(model_id, None, ["*"]) - assert tts_model._model_path == resolved_path - assert tts_model._assets_loaded is True - assert tts_model._resolve_prompt_wav_path(None, None) == (str(ref_audio_file), None) diff --git a/tests/model_executor/models/minicpmo_4_5/test_llm2tts.py b/tests/model_executor/models/minicpmo_4_5/test_llm2tts.py index 7ec8e8114..9c4b50967 100644 --- a/tests/model_executor/models/minicpmo_4_5/test_llm2tts.py +++ b/tests/model_executor/models/minicpmo_4_5/test_llm2tts.py @@ -10,7 +10,7 @@ Covers ``vllm_omni.model_executor.stage_input_processors.minicpmo_4_5_omni.llm2t - structured model_intermediate_buffer carries the thinker ids and text - scheduler prompt tokens follow the selected TTS region or output tokens - MiniCPM-o 4.5 TTS region detection on 151703 / 151704 tokens - - No TTS markers present -> no ``tts_token_ids`` / ``tts_hidden_states`` keys + - plain chat without TTS markers conditions on the generated assistant span - prompt arg is normalized to a list and ``multi_modal_data`` is gated by ``requires_multimodal_data`` """ @@ -33,6 +33,7 @@ pytestmark = [pytest.mark.core_model, pytest.mark.cpu] _HIDDEN_DIM = 4 +@pytest.mark.skip(reason="Legacy fused Talker metadata is replaced by separate Talker and Code2Wav stage payloads.") def test_native_duplex_talker_emits_empty_text_metadata_to_clear_previous_segment() -> None: class _Talker: _ar_last_chunk_flags = [True] @@ -91,13 +92,15 @@ def _make_thinker_output( latent: torch.Tensor | None = None, hidden_states: torch.Tensor | None = None, multimodal_output: dict[str, object] | None = None, + request_multimodal_output: dict[str, object] | None = None, ): """Construct a minimal mock of a thinker engine output entry. The real ``llm2tts`` only reads a tight slice of fields: - - top-level: ``request_id``, ``prompt_token_ids``, ``outputs[0]`` - - per-output: ``multimodal_output`` (dict), ``hidden_states`` (opt), - ``text``, ``token_ids`` + - top-level: ``request_id``, ``prompt_token_ids``, optional + ``multimodal_output``, and ``outputs[0]`` + - per-output: fallback ``multimodal_output``, ``hidden_states`` (opt), + ``text``, and ``token_ids`` """ mm_output = dict(multimodal_output or {}) if latent is not None: @@ -109,11 +112,14 @@ def _make_thinker_output( ) if hidden_states is not None: output.hidden_states = hidden_states - return SimpleNamespace( + request_output = SimpleNamespace( request_id=request_id, prompt_token_ids=prompt_token_ids, outputs=[output], ) + if request_multimodal_output is not None: + request_output.multimodal_output = request_multimodal_output + return request_output class TestInputValidation: @@ -131,6 +137,19 @@ class TestInputValidation: class TestBasicShape: + def test_request_level_multimodal_output_feeds_orchestrator_bridge(self) -> None: + hidden = torch.zeros((2, _HIDDEN_DIM)) + thinker_output = _make_thinker_output( + prompt_token_ids=[10], + output_token_ids=[20], + request_multimodal_output={"latent": hidden}, + ) + + out = llm2tts([thinker_output], prompt=None) + + assert out[0]["prompt_token_ids"] == [0, 0, 0] + assert torch.equal(torch.tensor(out[0]["model_intermediate_buffer"]["hidden_states"]["tts"]), hidden[1:]) + def test_returns_one_entry_per_input(self) -> None: hidden = torch.zeros((3, _HIDDEN_DIM)) out = llm2tts( @@ -144,13 +163,13 @@ class TestBasicShape: ) assert len(out) == 2 - def test_talker_scheduler_prompt_uses_output_tokens(self) -> None: + def test_talker_scheduler_prompt_matches_condition_length(self) -> None: hidden = torch.zeros((2, _HIDDEN_DIM)) out = llm2tts( [_make_thinker_output(prompt_token_ids=[10], output_token_ids=[20], hidden_states=hidden)], prompt=None, ) - assert out[0]["prompt_token_ids"] == [20] + assert out[0]["prompt_token_ids"] == [0, 0, 0] assert "stream_output" not in out[0]["model_intermediate_buffer"] def test_model_intermediate_buffer_carries_thinker_outputs(self) -> None: @@ -174,6 +193,24 @@ class TestBasicShape: assert buffer["ids"]["output"] == out_ids assert buffer["llm_output_text"] == ["hello"] + def test_strict_talker_receives_canonical_nested_handoff(self) -> None: + hidden = torch.arange(5 * _HIDDEN_DIM, dtype=torch.float32).reshape(5, _HIDDEN_DIM) + result = llm2tts( + [ + _make_thinker_output( + prompt_token_ids=[10], + output_token_ids=[151703, 20, 21, 151704], + hidden_states=hidden, + ) + ], + prompt=None, + )[0] + + buffer = result["model_intermediate_buffer"] + assert result["prompt_token_ids"] == [0, 0, 0, 0] + assert buffer["ids"]["tts"] == [20, 21] + assert torch.equal(torch.tensor(buffer["hidden_states"]["tts"]), hidden[2:4]) + def test_latent_in_multimodal_output_takes_precedence(self) -> None: # When both ``multimodal_output["latent"]`` and ``hidden_states`` are # present, the latent payload must win (this is the steady-state path @@ -241,13 +278,10 @@ class TestTtsRegionDetection: assert buffer["ids"]["tts"] == [30, 31] assert torch.equal(torch.tensor(buffer["hidden_states"]["tts"]), hidden[3:5]) - def test_no_tts_markers_omits_slice_keys(self) -> None: - # If neither marker pair is present, the bridge should NOT populate - # ``tts_token_ids`` / ``tts_hidden_states`` — the talker should fall - # through to the dummy path. - buffer, _ = self._run([10, 11], [20, 21, 22]) - assert "tts" not in buffer["ids"] - assert "tts" not in buffer.get("hidden_states", {}) + def test_plain_chat_without_tts_markers_uses_assistant_span(self) -> None: + buffer, hidden = self._run([10, 11], [20, 21, 22]) + assert buffer["ids"]["tts"] == [20, 21, 22] + assert torch.equal(torch.tensor(buffer["hidden_states"]["tts"]), hidden[2:5]) class TestPromptAndMultiModal: diff --git a/tests/model_executor/models/minicpmo_4_5/test_pipeline.py b/tests/model_executor/models/minicpmo_4_5/test_pipeline.py index b9c7938f0..1b8bd669f 100644 --- a/tests/model_executor/models/minicpmo_4_5/test_pipeline.py +++ b/tests/model_executor/models/minicpmo_4_5/test_pipeline.py @@ -5,8 +5,8 @@ Covers: - pipeline declared in the central registry - lazy loader returns the expected ``PipelineConfig`` - - 2-stage topology (thinker LLM_AR + talker LLM_AR with audio output) - - stage 1 routes through ``llm2tts`` custom input processor + - split 3-stage topology with no fused compatibility registration + - stage 1 routes through ``llm2tts`` and the async Code2Wav producer - ``hf_architectures`` covers both the shared ``MiniCPMO`` alias and the explicit 4.5 arch - ``hf_config_predicate`` selects MiniCPM-o 4.5 only and rejects 2.6 @@ -15,6 +15,7 @@ Covers: from __future__ import annotations +from pathlib import Path from types import SimpleNamespace import pytest @@ -23,20 +24,29 @@ from vllm_omni.config.pipeline_registry import OMNI_PIPELINES from vllm_omni.config.stage_config import ( PipelineConfig, StageExecutionType, + load_deploy_config, + merge_pipeline_deploy, ) +from vllm_omni.model_executor.models.registry import _OMNI_MODELS pytestmark = [pytest.mark.core_model, pytest.mark.cpu] _PIPELINE_KEY = "minicpmo_4_5" +_DEPLOY_DIR = Path(__file__).resolve().parents[4] / "vllm_omni" / "deploy" +_DEPLOY_LAYOUTS = { + "minicpmo_4_5.yaml": ["0", "0", "0"], + "minicpmo_4_5_batching.yaml": ["0", "1", "1"], + "minicpmo_4_5_2gpu.yaml": ["0", "1", "1"], + "minicpmo_4_5_3gpu.yaml": ["0", "1", "2"], + "minicpmo_4_5_8x4090.yaml": ["0,1,2,3", "4", "5"], +} class TestRegistryDeclaration: def test_declared_in_omni_pipelines(self) -> None: assert _PIPELINE_KEY in OMNI_PIPELINES - - def test_visible_in_central_registry(self) -> None: - assert _PIPELINE_KEY in OMNI_PIPELINES + assert "minicpmo_4_5_fused" not in OMNI_PIPELINES def test_lazy_load_returns_pipeline_config(self) -> None: pipeline = OMNI_PIPELINES[_PIPELINE_KEY] @@ -64,9 +74,9 @@ class TestPipelineTopology: def pipeline(self) -> PipelineConfig: return OMNI_PIPELINES[_PIPELINE_KEY] - def test_two_stages(self, pipeline: PipelineConfig) -> None: - assert len(pipeline.stages) == 2 - assert [s.stage_id for s in pipeline.stages] == [0, 1] + def test_three_stages(self, pipeline: PipelineConfig) -> None: + assert len(pipeline.stages) == 3 + assert [s.stage_id for s in pipeline.stages] == [0, 1, 2] def test_topology_validates(self, pipeline: PipelineConfig) -> None: # ``validate`` returns a list of structural errors; empty == valid. @@ -90,11 +100,15 @@ class TestPipelineTopology: assert talker.execution_type == StageExecutionType.LLM_AR # talker consumes thinker output assert talker.input_sources == (0,) - assert talker.final_output is True - assert talker.final_output_type == "audio" - assert talker.engine_output_type == "audio" + assert talker.final_output is False + assert talker.final_output_type is None + assert talker.engine_output_type == "latent" # scope KV cache / mrope sizing to talker sub-config assert talker.hf_config_name == "tts_config" + assert talker.custom_process_next_stage_input_func is None + assert talker.async_chunk_process_next_stage_input_func == ( + "vllm_omni.model_executor.stage_input_processors.minicpmo_4_5_omni.tts2code2wav_async_chunk" + ) def test_talker_routes_through_llm2tts(self, pipeline: PipelineConfig) -> None: talker = pipeline.get_stage(1) @@ -106,6 +120,73 @@ class TestPipelineTopology: "vllm_omni.model_executor.stage_input_processors.minicpmo_4_5_omni.llm2tts" ) + def test_code2wav_stage(self, pipeline: PipelineConfig) -> None: + code2wav = pipeline.get_stage(2) + assert code2wav is not None + assert code2wav.model_stage == "code2wav" + assert code2wav.execution_type == StageExecutionType.LLM_GENERATION + assert code2wav.input_sources == (1,) + assert code2wav.final_output is True + assert code2wav.final_output_type == "audio" + assert code2wav.engine_output_type == "audio" + assert code2wav.model_arch == "MiniCPMO45Code2Wav" + assert code2wav.sync_process_input_func is None + + +class TestDeployTopology: + @pytest.mark.parametrize(("filename", "devices"), _DEPLOY_LAYOUTS.items()) + def test_deploy_resolves_three_stage_pipeline(self, filename: str, devices: list[str]) -> None: + deploy = load_deploy_config(_DEPLOY_DIR / filename) + pipeline = OMNI_PIPELINES[deploy.pipeline] + stages = merge_pipeline_deploy(pipeline, deploy) + + assert deploy.pipeline == _PIPELINE_KEY + assert deploy.async_chunk is True + assert [stage.stage_id for stage in stages] == [0, 1, 2] + assert [stage.yaml_runtime["devices"] for stage in stages] == devices + assert "additional_config" not in stages[1].yaml_engine_args + assert "skip_mm_profiling" not in stages[0].yaml_engine_args + assert stages[1].yaml_engine_args["skip_mm_profiling"] is True + assert stages[2].yaml_engine_args["skip_mm_profiling"] is True + assert stages[1].yaml_extras["output_connectors"]["to_stage_2"] == "connector_of_shared_memory" + assert stages[2].yaml_extras["input_connectors"]["from_stage_1"] == "connector_of_shared_memory" + connector = deploy.connectors["connector_of_shared_memory"] + assert connector["name"] == "SharedMemoryConnector" + assert connector["extra"]["codec_chunk_frames"] == 25 + assert connector["extra"]["codec_left_context_frames"] == 3 + assert connector["extra"]["connector_get_max_wait_first_chunk"] == 3000 + assert connector["extra"]["connector_get_max_wait"] == 300 + if filename == "minicpmo_4_5.yaml": + assert [stage.yaml_engine_args["max_num_seqs"] for stage in stages] == [4, 4, 4] + assert [stage.yaml_engine_args["gpu_memory_utilization"] for stage in stages] == [ + 0.65, + 0.15, + 0.15, + ] + elif filename in {"minicpmo_4_5_batching.yaml", "minicpmo_4_5_2gpu.yaml"}: + assert [stage.yaml_engine_args["gpu_memory_utilization"] for stage in stages] == [ + 0.9, + 0.55, + 0.35, + ] + + def test_pipeline_exposes_no_full_payload_or_token_placeholder_hooks(self) -> None: + pipeline = OMNI_PIPELINES[_PIPELINE_KEY] + talker = pipeline.get_stage(1) + code2wav = pipeline.get_stage(2) + assert talker is not None + assert code2wav is not None + assert talker.custom_process_next_stage_input_func is None + assert code2wav.sync_process_input_func is None + + +def test_code2wav_model_is_lazily_registered() -> None: + assert _OMNI_MODELS["MiniCPMO45Code2Wav"] == ( + "minicpmo_4_5", + "minicpmo_4_5_code2wav", + "MiniCPMO45Code2Wav", + ) + class TestArchAliases: """``hf_architectures`` must cover both the shared and explicit names.""" diff --git a/tests/model_executor/models/minicpmo_4_5/test_talker_batching.py b/tests/model_executor/models/minicpmo_4_5/test_talker_batching.py new file mode 100644 index 000000000..f0e9169b3 --- /dev/null +++ b/tests/model_executor/models/minicpmo_4_5/test_talker_batching.py @@ -0,0 +1,248 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Request-alignment tests for MiniCPM-o 4.5's native Talker.""" + +from __future__ import annotations + +import pytest +import torch +import torch.nn as nn + +from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni import ( + MiniCPMO45OmniForConditionalGeneration, +) +from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni_tts import ( + MiniCPMO45OmniTTSForConditionalGeneration, + _max_audio_tokens, +) +from vllm_omni.utils.mm_outputs import to_payload_element + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +class _FakeNativeTalker(nn.Module): + has_preprocess = True + + def __init__(self) -> None: + super().__init__() + self.forward_kwargs = None + + def forward(self, **kwargs): + self.forward_kwargs = kwargs + return torch.ones(2, 4) + + +def test_wrapper_always_delegates_talker_to_native_ar_path() -> None: + model = MiniCPMO45OmniForConditionalGeneration.__new__(MiniCPMO45OmniForConditionalGeneration) + nn.Module.__init__(model) + model.model_stage = "tts" + model.talker = _FakeNativeTalker() + + output = model( + input_ids=torch.tensor([1, 2]), + positions=torch.arange(2), + model_intermediate_buffer=[{"request_id": "req"}], + ) + + assert output.shape == (2, 4) + assert model.talker.forward_kwargs["model_intermediate_buffer"][0]["request_id"] == "req" + + +def _make_talker() -> MiniCPMO45OmniTTSForConditionalGeneration: + talker = MiniCPMO45OmniTTSForConditionalGeneration.__new__(MiniCPMO45OmniTTSForConditionalGeneration) + nn.Module.__init__(talker) + talker._num_audio_tokens = 8 + talker._batch_stop_logits = None + talker._request_generators = {} + talker._request_audio_states = {} + talker._deferred_cleanup_ids = set() + return talker + + +def _routed(output, index: int): + return to_payload_element( + output.multimodal_outputs, + index, + index, + index + 1, + seq_len=2, + scheduled_seq_len=2, + ) + + +@pytest.mark.parametrize( + ("condition_tokens", "expected"), + [(3, 64), (100, 1000), (1000, 2048)], +) +def test_audio_token_limit_scales_with_condition_length( + condition_tokens: int, + expected: int, +) -> None: + assert _max_audio_tokens(condition_tokens) == expected + + +def test_talker_emits_request_aligned_codec_deltas_after_compaction(mocker) -> None: + talker = _make_talker() + seen: list[tuple[str, list[float], list[int]]] = [] + + def sample(hidden, history, request_id, step): + assert step == 0 + seen.append((request_id, hidden.reshape(-1).tolist(), history.tolist())) + return torch.tensor(2 if request_id == "req-a" else 3) + + mocker.patch.object(talker, "_sample_audio_code", side_effect=sample) + infos = [ + {"request_id": "req-a", "audio_codes": {"accumulated": torch.tensor([1])}}, + {"request_id": "req-b", "audio_codes": {"accumulated": torch.empty(0, dtype=torch.long)}}, + ] + + output = talker.make_omni_output( + torch.tensor([[1.0, 0.0], [2.0, 0.0], [3.0, 0.0]]), + model_intermediate_buffer=infos, + request_token_spans=[(0, 2), (2, 3)], + ) + + assert seen == [ + ("req-a", [2.0, 0.0], [1]), + ("req-b", [3.0, 0.0], []), + ] + assert infos[0]["audio_codes"]["accumulated"].tolist() == [1, 2] + assert infos[1]["audio_codes"]["accumulated"].tolist() == [3] + assert set(output.multimodal_outputs) == {"codes", "meta"} + assert "model_outputs" not in output.multimodal_outputs + assert "sr" not in output.multimodal_outputs + assert _routed(output, 0)["codes"]["audio"].tolist() == [[2]] + assert _routed(output, 1)["codes"]["audio"].tolist() == [[3]] + assert _routed(output, 0)["meta"]["finished"].item() is False + assert talker.compute_logits(output.text_hidden_states).argmax(dim=-1).tolist() == [0, 0] + + +def test_incomplete_prefill_emits_no_code_and_does_not_advance_state(mocker) -> None: + talker = _make_talker() + sample = mocker.patch.object(talker, "_sample_audio_code", return_value=torch.tensor(2)) + infos = [ + { + "request_id": "req-prefill", + "audio_state": {"step": 0}, + "audio_codes": {"accumulated": torch.empty(0, dtype=torch.long)}, + }, + { + "request_id": "req-decode", + "audio_state": {"step": 4}, + "audio_codes": {"accumulated": torch.tensor([1])}, + }, + ] + + output = talker.make_omni_output( + torch.tensor([[1.0, 0.0], [2.0, 0.0], [3.0, 0.0]]), + model_intermediate_buffer=infos, + request_token_spans=[(0, 2), (2, 3)], + request_sample_eligible=[False, True], + ) + + sample.assert_called_once() + assert sample.call_args.args[2] == "req-decode" + assert infos[0]["audio_state"]["step"] == 0 + assert infos[0]["audio_codes"]["accumulated"].numel() == 0 + assert infos[1]["audio_state"]["step"] == 5 + assert _routed(output, 0)["codes"]["audio"].shape == (0, 1) + assert _routed(output, 1)["codes"]["audio"].tolist() == [[2]] + + +def test_eos_is_terminal_once_and_never_enters_codec_history(mocker) -> None: + talker = _make_talker() + sample = mocker.patch.object(talker, "_sample_audio_code", return_value=torch.tensor(7)) + info = { + "request_id": "req-stop", + "audio_state": {"step": 3}, + "audio_codes": {"accumulated": torch.tensor([4, 5])}, + } + + first = talker.make_omni_output( + torch.ones(1, 2), + model_intermediate_buffer=[info], + request_token_spans=[(0, 1)], + ) + first_logits = talker.compute_logits(first.text_hidden_states) + second = talker.make_omni_output( + torch.ones(1, 2), + model_intermediate_buffer=[info], + request_token_spans=[(0, 1)], + ) + + sample.assert_called_once() + assert info["audio_codes"]["accumulated"].tolist() == [4, 5] + assert first.multimodal_outputs["codes"]["audio"][0].shape == (0, 1) + assert first.multimodal_outputs["meta"]["finished"][0].item() is True + assert second.multimodal_outputs["meta"]["finished"][0].item() is False + assert first_logits.argmax(dim=-1).tolist() == [1] + assert talker.compute_logits(second.text_hidden_states).argmax(dim=-1).tolist() == [1] + + +def test_max_token_terminal_includes_only_new_codec_delta(mocker) -> None: + talker = _make_talker() + mocker.patch.object(talker, "_sample_audio_code", return_value=torch.tensor(3)) + info = { + "request_id": "req-limit", + "audio_state": {"step": 1, "max_tokens": 2}, + "audio_codes": {"accumulated": torch.tensor([4, 5])}, + } + + output = talker.make_omni_output( + torch.ones(1, 2), + model_intermediate_buffer=[info], + request_token_spans=[(0, 1)], + ) + + assert info["audio_codes"]["accumulated"].tolist() == [4, 5, 3] + assert output.multimodal_outputs["codes"]["audio"][0].tolist() == [[3]] + assert output.multimodal_outputs["meta"]["finished"][0].item() is True + assert talker.compute_logits(output.text_hidden_states).argmax(dim=-1).tolist() == [1] + + +def test_request_local_state_survives_missing_runner_buffer_update(mocker) -> None: + talker = _make_talker() + mocker.patch.object(talker, "_sample_audio_code", return_value=torch.tensor(3)) + first_info = { + "request_id": "req-local-state", + "audio_state": {"step": 1, "max_tokens": 3}, + "audio_codes": {"accumulated": torch.tensor([4])}, + } + + talker.make_omni_output( + torch.ones(1, 2), + model_intermediate_buffer=[first_info], + request_token_spans=[(0, 1)], + ) + second = talker.make_omni_output( + torch.ones(1, 2), + model_intermediate_buffer=[{"request_id": "req-local-state"}], + request_token_spans=[(0, 1)], + ) + + assert second.multimodal_outputs["meta"]["finished"][0].item() is True + assert talker._request_audio_states["req-local-state"]["step"] == 3 + + +def test_missing_conditioning_fails_clearly() -> None: + talker = _make_talker() + + with pytest.raises(ValueError, match="tts_token_ids and tts_hidden_states"): + talker.preprocess( + torch.tensor([0]), + None, + _omni_is_prefill=True, + request_id="req-invalid", + ) + + +def test_request_cleanup_evicts_ar_rng_and_decode_state() -> None: + talker = _make_talker() + talker._request_generators["req-done"] = torch.Generator() + talker._request_audio_states["req-done"] = {"step": 1} + + talker.on_requests_finished(["req-done"]) + talker._flush_deferred_cleanup() + + assert "req-done" not in talker._request_generators + assert "req-done" not in talker._request_audio_states diff --git a/tests/model_executor/stage_input_processors/test_minicpmo_4_5_async_chunk.py b/tests/model_executor/stage_input_processors/test_minicpmo_4_5_async_chunk.py new file mode 100644 index 000000000..4f17b2c64 --- /dev/null +++ b/tests/model_executor/stage_input_processors/test_minicpmo_4_5_async_chunk.py @@ -0,0 +1,182 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from collections import defaultdict +from types import SimpleNamespace + +import pytest +import torch +from vllm.v1.request import RequestStatus + +from vllm_omni.model_executor.stage_input_processors.minicpmo_4_5_omni import ( + tts2code2wav_async_chunk, +) + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +def _manager(): + return SimpleNamespace( + connector=SimpleNamespace(config={"extra": {"codec_chunk_frames": 25, "codec_left_context_frames": 3}}), + code_prompt_token_ids=defaultdict(list), + request_payload={}, + put_req_chunk=defaultdict(int), + ) + + +def _request(external_id: str, internal_id: str | None = None): + request = SimpleNamespace( + external_req_id=external_id, + request_id=internal_id or external_id, + status=RequestStatus.RUNNING, + ) + request.is_finished = lambda: RequestStatus.is_finished(request.status) + return request + + +def _delta(*codes: int): + return { + "codes": {"audio": torch.tensor(codes, dtype=torch.long).reshape(-1, 1)}, + "meta": {"finished": torch.tensor(False)}, + } + + +def _codes(payload) -> list[int]: + assert payload.codes is not None + assert isinstance(payload.codes.audio, torch.Tensor) + assert payload.codes.audio.dtype == torch.long + assert payload.codes.audio.ndim == 1 + return payload.codes.audio.tolist() + + +@pytest.mark.parametrize(("count", "emitted"), [(24, False), (25, True), (26, True)]) +def test_first_chunk_threshold_is_25_generated_codes(count: int, emitted: bool) -> None: + manager = _manager() + payload = tts2code2wav_async_chunk( + transfer_manager=manager, + multimodal_output=_delta(*range(count)), + request=_request("req"), + is_finished=False, + ) + + assert (payload is not None) is emitted + if payload is not None: + assert _codes(payload) == [4218, 4218, 4218, *range(25)] + assert payload.meta.chunk_seq == 0 + assert payload.meta.code_flat_numel == 28 + + +def test_steady_chunk_has_three_code_overlap_and_25_new_codes() -> None: + manager = _manager() + request = _request("req") + + first = tts2code2wav_async_chunk(manager, _delta(*range(25)), request, False) + manager.put_req_chunk["req"] += 1 + steady = tts2code2wav_async_chunk(manager, _delta(*range(25, 50)), request, False) + + assert first is not None + assert steady is not None + assert _codes(steady) == [22, 23, 24, *range(25, 50)] + assert steady.meta.chunk_seq == 1 + + +def test_exact_boundary_final_flushes_held_lookahead() -> None: + manager = _manager() + request = _request("req") + + assert tts2code2wav_async_chunk(manager, _delta(*range(25)), request, False) is not None + manager.put_req_chunk["req"] += 1 + final = tts2code2wav_async_chunk(manager, None, request, True) + + assert final is not None + assert _codes(final) == [22, 23, 24] + assert final.meta.chunk_seq == 1 + assert final.meta.code_flat_numel == 3 + assert final.meta.last_chunk is True + assert final.meta.finished.item() is True + + +def test_short_final_flushes_silence_prefix_and_tail() -> None: + manager = _manager() + final = tts2code2wav_async_chunk(manager, _delta(*range(7)), _request("req"), True) + + assert final is not None + assert _codes(final) == [4218, 4218, 4218, *range(7)] + assert final.meta.last_chunk is True + assert final.meta.finished.item() is True + + +def test_first_chunk_forwards_reference_voice_and_duplex_identity() -> None: + manager = _manager() + request = _request("req") + request.additional_information = { + "codes": {"ref": [0.1, -0.1]}, + "meta": { + "ref_audio_sr": 16000, + "native_duplex_segment_text": "hello", + "segment_end": True, + "turn_end": True, + }, + "duplex": {"epoch": 3, "model_turn_id": 7}, + } + + payload = tts2code2wav_async_chunk(manager, _delta(*range(7)), request, True) + + assert payload is not None + assert payload.codes.ref.tolist() == pytest.approx([0.1, -0.1]) + assert payload.meta.ref_audio_sr == 16000 + assert payload.meta.native_duplex_segment_text == "hello" + assert payload.meta.duplex_epoch == 3 + assert payload.meta.duplex_turn_id == 7 + assert payload.meta.segment_end is True + assert payload.meta.turn_end is True + + +def test_empty_final_releases_wait_gate_once() -> None: + manager = _manager() + request = _request("req") + + final = tts2code2wav_async_chunk(manager, None, request, True) + duplicate = tts2code2wav_async_chunk(manager, None, request, True) + + assert final is not None + assert _codes(final) == [] + assert final.meta.chunk_seq == 0 + assert final.meta.request_id == "req" + assert final.meta.cache_epoch == 0 + assert final.meta.last_chunk is True + assert duplicate is None + + +def test_staggered_requests_keep_accumulators_isolated() -> None: + manager = _manager() + req_a = _request("a") + req_b = _request("b") + + assert tts2code2wav_async_chunk(manager, _delta(*range(24)), req_a, False) is None + out_b = tts2code2wav_async_chunk(manager, _delta(*range(100, 125)), req_b, False) + out_a = tts2code2wav_async_chunk(manager, _delta(24), req_a, False) + + assert out_b is not None + assert out_a is not None + assert _codes(out_b) == [4218, 4218, 4218, *range(100, 125)] + assert _codes(out_a) == [4218, 4218, 4218, *range(25)] + assert out_a.meta.request_id == "a" + assert out_b.meta.request_id == "b" + + +def test_cancel_drops_epoch_state_and_stale_request_cannot_publish() -> None: + manager = _manager() + stale = _request("req", "internal-0") + + assert tts2code2wav_async_chunk(manager, _delta(*range(10)), stale, False) is None + stale.status = RequestStatus.FINISHED_ABORTED + assert tts2code2wav_async_chunk(manager, None, stale, True) is None + assert tts2code2wav_async_chunk(manager, _delta(*range(25)), stale, False) is None + + replacement = _request("req", "internal-1") + payload = tts2code2wav_async_chunk(manager, _delta(*range(25)), replacement, False) + + assert payload is not None + assert payload.meta.cache_epoch == 1 + assert _codes(payload) == [4218, 4218, 4218, *range(25)] diff --git a/tests/model_executor/stage_input_processors/test_minicpmo_4_5_omni.py b/tests/model_executor/stage_input_processors/test_minicpmo_4_5_omni.py index 453b3a595..c8381d881 100644 --- a/tests/model_executor/stage_input_processors/test_minicpmo_4_5_omni.py +++ b/tests/model_executor/stage_input_processors/test_minicpmo_4_5_omni.py @@ -1,75 +1,39 @@ -import base64 -import os -import struct from types import SimpleNamespace import pytest import torch -from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni_tts import ( - MiniCPMO45OmniTTSForConditionalGeneration, - _drain_native_duplex_emitted_text, - _queue_native_duplex_segment_text, - _TalkerTurnState, -) from vllm_omni.model_executor.stage_input_processors.minicpmo_4_5_omni import ( _extract_first_audio_ref, - _native_duplex_segment_output_ids, llm2tts, ) pytestmark = [pytest.mark.core_model, pytest.mark.cpu] -def test_native_duplex_transcript_waits_for_buffered_vocoder_audio(): - state = _TalkerTurnState(None, None, turn_id=1) - - _queue_native_duplex_segment_text(state, "好") - assert _drain_native_duplex_emitted_text(state, has_audio=False) == "" - - _queue_native_duplex_segment_text(state, "的,刚") - assert _drain_native_duplex_emitted_text(state, has_audio=True) == "好的,刚" - assert _drain_native_duplex_emitted_text(state, has_audio=True) == "" +def _output( + *, + prompt_ids: list[int], + output_ids: list[int], + latent: torch.Tensor, + multimodal_output: dict | None = None, + token_list: list[int] | None = None, +): + mm_output = dict(multimodal_output or {}) + mm_output["latent"] = latent + completion = SimpleNamespace( + token_ids=output_ids if token_list is None else token_list, + text="hello", + multimodal_output=mm_output, + ) + return SimpleNamespace( + request_id="req-1", + prompt_token_ids=prompt_ids, + outputs=[completion], + ) -def test_native_duplex_token2wav_cache_isolated_across_interleaved_sessions(): - model = MiniCPMO45OmniTTSForConditionalGeneration.__new__(MiniCPMO45OmniTTSForConditionalGeneration) - model.audio_tokenizer = SimpleNamespace(stream_cache=None, hift_cache_dict={}) - observed = [] - - def stream_window(token_list, _prompt, *, last_chunk): - del token_list, last_chunk - marker = model.audio_tokenizer.stream_cache["marker"] - observed.append(marker) - prefix = marker[0] - index = int(marker[1:]) + 1 - model.audio_tokenizer.stream_cache = {"marker": f"{prefix}{index}"} - model.audio_tokenizer.hift_cache_dict = {"marker": f"{prefix}{index}"} - return torch.ones(1) - - model._t2w_stream_window = stream_window - a = _TalkerTurnState(None, None) - b = _TalkerTurnState(None, None) - a.stream_cache = {"marker": "A0"} - a.hift_cache_dict = {"marker": "A0"} - a.vocoder_initialized = True - b.stream_cache = {"marker": "B0"} - b.hift_cache_dict = {"marker": "B0"} - b.vocoder_initialized = True - - model._run_vocoder_window(a, [1], last_chunk=False) - model._run_vocoder_window(b, [1], last_chunk=False) - model._run_vocoder_window(a, [1], last_chunk=False) - model._run_vocoder_window(b, [1], last_chunk=True) - - assert observed == ["A0", "B0", "A1", "B1"] - assert a.stream_cache == {"marker": "A2"} - assert b.stream_cache == {"marker": "B2"} - assert model.audio_tokenizer.stream_cache is None - assert model.audio_tokenizer.hift_cache_dict == {} - - -def test_extract_first_audio_ref_accepts_dict_stereo_audio(): +def test_extract_first_audio_ref_accepts_dict_stereo_audio() -> None: ref = _extract_first_audio_ref( { "audio": { @@ -85,1263 +49,116 @@ def test_extract_first_audio_ref_accepts_dict_stereo_audio(): assert torch.allclose(waveform, torch.tensor([1.5, 3.5, 5.5])) -def test_llm2tts_carries_request_ref_audio_to_talker_payload(): - latent = torch.arange(20, dtype=torch.float16).reshape(5, 4) - output = SimpleNamespace( - token_ids=[11, 12, 9002], - text="hello", +def test_plain_chat_handoff_owns_talker_prompt_contract() -> None: + prompt_ids = [101, 102] + output_ids = [11, 12] + latent = torch.arange(16, dtype=torch.float32).reshape(4, 4) + + converted = llm2tts( + [_output(prompt_ids=prompt_ids, output_ids=output_ids, latent=latent)], + prompt=[{}], + )[0] + + info = converted["model_intermediate_buffer"] + assert info["ids"]["tts"] == output_ids + assert torch.equal(torch.tensor(info["hidden_states"]["tts"]), latent[2:4]) + assert converted["prompt_token_ids"] == [0, 0, 0, 0] + assert info["meta"]["replace_streaming_prompt"] is True + assert info["meta"]["next_stage_prompt_len"] == 4 + + +def test_llm2tts_carries_request_ref_audio() -> None: + latent = torch.arange(20, dtype=torch.float32).reshape(5, 4) + source = _output( + prompt_ids=[101, 9001], + output_ids=[11, 12, 9002], + latent=latent, multimodal_output={ - "latent": latent, "meta": { "tts_bos_token_id": 9001, "tts_eos_token_id": 9002, - }, + } }, ) - llm_output = SimpleNamespace( - request_id="req-1", - prompt_token_ids=[101, 9001], - outputs=[output], - ) ref_waveform = torch.tensor([0.1, 0.2, 0.3]) converted = llm2tts( - [llm_output], - prompt=[ - { - "multi_modal_data": { - "audio": (ref_waveform, 22050), - } - } - ], - ) + [source], + prompt=[{"multi_modal_data": {"audio": (ref_waveform, 22050)}}], + )[0] - assert "additional_information" not in converted[0] - info = converted[0]["model_intermediate_buffer"] + info = converted["model_intermediate_buffer"] assert info["codes"]["ref"] == ref_waveform.tolist() assert info["meta"]["ref_audio_sr"] == 22050 assert info["ids"]["tts"] == [11, 12] - assert torch.equal(torch.as_tensor(info["hidden_states"]["tts"]), latent[2:4].float()) - assert "tts_token_ids" not in info - assert "tts_hidden_states" not in info -def test_llm2tts_uses_duplex_prompt_token_ids_for_tts_boundary(): - latent = torch.arange(24, dtype=torch.float16).reshape(6, 4) - output = SimpleNamespace( - token_ids=[11, 12, 9002], - text="hello", +def test_native_duplex_speak_segment_reaches_split_talker() -> None: + prompt_ids = [101, 102] + output_ids = [9304, 21, 22, 9308] + latent = torch.arange(24, dtype=torch.float32).reshape(6, 4) + source = _output( + prompt_ids=prompt_ids, + output_ids=output_ids, + latent=latent, multimodal_output={ - "latent": latent, - "duplex_prompt_token_ids": [101, 102, 9001], - "meta": { - "tts_bos_token_id": 9001, - "tts_eos_token_id": 9002, - }, - }, - ) - llm_output = SimpleNamespace( - request_id="duplex-req", - prompt_token_ids=[0, 0, 0], - outputs=[output], - ) - - converted = llm2tts([llm_output], prompt=[{}]) - - assert "additional_information" not in converted[0] - info = converted[0]["model_intermediate_buffer"] - assert info["prompt_token_ids"] == [101, 102, 9001] - assert converted[0]["prompt_token_ids"] == [11, 12] - assert info["ids"]["tts"] == [11, 12] - assert torch.equal(torch.as_tensor(info["hidden_states"]["tts"]), latent[3:5].float()) - assert "tts_token_ids" not in info - assert "tts_hidden_states" not in info - - -def test_llm2tts_accepts_flat_tokenizer_derived_special_token_metadata(): - latent = torch.arange(24, dtype=torch.float16).reshape(6, 4) - output = SimpleNamespace( - token_ids=[11, 12, 9302], - text="hello", - multimodal_output={ - "latent": latent, - "duplex_prompt_token_ids": [101, 102, 9301], - "meta.tts_bos_token_id": torch.tensor([9301]), - "meta.tts_eos_token_id": torch.tensor([9302]), - "meta.listen_token_id": torch.tensor([9303]), - "meta.speak_token_id": torch.tensor([9304]), - "meta.chunk_eos_token_id": torch.tensor([9305]), - "meta.chunk_tts_eos_token_id": torch.tensor([9306]), - "meta.turn_eos_token_id": torch.tensor([9307]), - }, - ) - llm_output = SimpleNamespace( - request_id="duplex-flat-meta", - prompt_token_ids=[0, 0, 0], - outputs=[output], - ) - - converted = llm2tts([llm_output], prompt=[{}]) - - assert "additional_information" not in converted[0] - info = converted[0]["model_intermediate_buffer"] - assert converted[0]["prompt_token_ids"] == [11, 12] - assert info["ids"]["tts"] == [11, 12] - assert info["stream_output"] is True - assert info["native_duplex"] is True - assert torch.equal(torch.as_tensor(info["hidden_states"]["tts"]), latent[3:5].float()) - assert "tts_token_ids" not in info - assert "tts_hidden_states" not in info - - -def test_llm2tts_stops_at_tokenizer_derived_duplex_state_tokens(): - latent = torch.arange(32, dtype=torch.float16).reshape(8, 4) - output = SimpleNamespace( - token_ids=[11, 12, 9303, 13, 9302], - text="hello", - multimodal_output={ - "latent": latent, - "duplex_prompt_token_ids": [101, 102, 9301], + "duplex_prompt_token_ids": prompt_ids, "meta": { "tts_bos_token_id": 9301, "tts_eos_token_id": 9302, "listen_token_id": 9303, "speak_token_id": 9304, - "tts_pad_token_id": 9305, - "unit_token_id": 9306, - "unit_end_token_id": 9307, "chunk_eos_token_id": 9308, "chunk_tts_eos_token_id": 9309, "turn_eos_token_id": 9310, }, }, ) - llm_output = SimpleNamespace( - request_id="duplex-state-boundary", - prompt_token_ids=[0, 0, 0], - outputs=[output], - ) - - converted = llm2tts([llm_output], prompt=[{}]) - - info = converted[0]["model_intermediate_buffer"] - assert converted[0]["prompt_token_ids"] == [11, 12] - assert info["ids"]["tts"] == [11, 12] - assert torch.equal(torch.as_tensor(info["hidden_states"]["tts"]), latent[3:5].float()) - assert "tts_token_ids" not in info - assert "tts_hidden_states" not in info - - -def test_llm2tts_native_duplex_uses_speak_region_without_tts_bos_prompt(): - latent = torch.arange(36, dtype=torch.float16).reshape(9, 4) - output = SimpleNamespace( - token_ids=[9304, 11, 12, 9308, 13], - text="hello", - multimodal_output={ - "latent": latent, - "duplex_prompt_token_ids": [101, 102, 9306, 9306], - "meta": { - "tts_bos_token_id": 9301, - "tts_eos_token_id": 9302, - "listen_token_id": 9303, - "speak_token_id": 9304, - "tts_pad_token_id": 9305, - "unit_token_id": 9306, - "unit_end_token_id": 9307, - "chunk_eos_token_id": 9308, - "chunk_tts_eos_token_id": 9309, - "turn_eos_token_id": 9310, - }, - }, - ) - llm_output = SimpleNamespace( - request_id="duplex-speak-region", - prompt_token_ids=[0, 0, 0, 0], - outputs=[output], - ) - - converted = llm2tts([llm_output], prompt=[{}]) - - info = converted[0]["model_intermediate_buffer"] - assert converted[0]["prompt_token_ids"] == [11, 12] - assert info["ids"]["tts"] == [11, 12] - assert torch.equal(torch.as_tensor(info["hidden_states"]["tts"]), latent[5:7].float()) - assert "tts_token_ids" not in info - assert "tts_hidden_states" not in info - - -def test_llm2tts_native_duplex_uses_plain_text_region_without_speak_marker(): - latent = torch.arange(32, dtype=torch.float16).reshape(8, 4) - output = SimpleNamespace( - token_ids=[11, 12, 13, 9308], - text="hello", - multimodal_output={ - "latent": latent, - "duplex_prompt_token_ids": [101, 102, 9306, 9306], - "meta": { - "tts_bos_token_id": 9301, - "tts_eos_token_id": 9302, - "listen_token_id": 9303, - "speak_token_id": 9304, - "tts_pad_token_id": 9305, - "unit_token_id": 9306, - "unit_end_token_id": 9307, - "chunk_eos_token_id": 9308, - "chunk_tts_eos_token_id": 9309, - "turn_eos_token_id": 9310, - }, - }, - ) - llm_output = SimpleNamespace( - request_id="duplex-plain-text-region", - prompt_token_ids=[0, 0, 0, 0], - outputs=[output], - ) - - converted = llm2tts([llm_output], prompt=[{}]) - - info = converted[0]["model_intermediate_buffer"] - assert converted[0]["prompt_token_ids"] == [12, 13] - assert info["ids"]["tts"] == [12, 13] - assert torch.equal(torch.as_tensor(info["hidden_states"]["tts"]), latent[5:7].float()) - - -def test_llm2tts_native_duplex_conditions_on_turn_eos_and_midunit_speak(): - latent = torch.arange(36, dtype=torch.float16).reshape(9, 4) - # Official duplex includes mid-unit <|speak|> tokens AND the <|turn_eos|> - # token+hidden in the talker condition (its embedding is the trained - # stop signal); only chunk terminators bound the slice. - output = SimpleNamespace( - token_ids=[9304, 11, 9304, 12, 9310, 9308], - text="hello", - multimodal_output={ - "latent": latent, - "duplex_prompt_token_ids": [101, 102, 9306], - "meta": { - "tts_bos_token_id": 9301, - "tts_eos_token_id": 9302, - "listen_token_id": 9303, - "speak_token_id": 9304, - "tts_pad_token_id": 9305, - "unit_token_id": 9306, - "unit_end_token_id": 9307, - "chunk_eos_token_id": 9308, - "chunk_tts_eos_token_id": 9309, - "turn_eos_token_id": 9310, - }, - }, - ) - llm_output = SimpleNamespace( - request_id="duplex-turn-eos", - prompt_token_ids=[0, 0, 0], - outputs=[output], - ) - - converted = llm2tts([llm_output], prompt=[{}]) - - info = converted[0]["model_intermediate_buffer"] - assert info["ids"]["tts"] == [11, 9304, 12, 9310] - assert torch.equal(torch.as_tensor(info["hidden_states"]["tts"]), latent[4:8].float()) - assert info["meta"]["turn_eos_token_id"] == 9310 - - -def test_llm2tts_native_duplex_ignores_stale_tts_bos_inside_folded_prompt(): - latent = torch.arange(36, dtype=torch.float16).reshape(9, 4) - # A continuation unit: the resumable prompt has an EARLIER reply's - # <|tts_bos|> folded mid-prompt, and the model decided to LISTEN (no new - # speak text). The stale boundary must not re-slice already-spoken text. - output = SimpleNamespace( - token_ids=[9303, 9309], - text="", - multimodal_output={ - "latent": latent, - "duplex_prompt_token_ids": [101, 9301, 41, 42, 9306, 9306], - "meta": { - "tts_bos_token_id": 9301, - "tts_eos_token_id": 9302, - "listen_token_id": 9303, - "speak_token_id": 9304, - "tts_pad_token_id": 9305, - "unit_token_id": 9306, - "unit_end_token_id": 9307, - "chunk_eos_token_id": 9308, - "chunk_tts_eos_token_id": 9309, - "turn_eos_token_id": 9310, - }, - }, - ) - llm_output = SimpleNamespace( - request_id="duplex-stale-bos", - prompt_token_ids=[0, 0, 0, 0, 0, 0], - outputs=[output], - ) - - converted = llm2tts([llm_output], prompt=[{}]) - - assert converted == [] - - -def test_llm2tts_rejects_native_duplex_output_without_special_token_metadata(): - latent = torch.arange(16, dtype=torch.float16).reshape(4, 4) - output = SimpleNamespace( - token_ids=[11, 12], - text="hello", - multimodal_output={ - "latent": latent, - "duplex_prompt_token_ids": [101, 102, 9301], - }, - ) - llm_output = SimpleNamespace( - request_id="duplex-missing-meta", - prompt_token_ids=[0, 0, 0], - outputs=[output], - ) - - with pytest.raises(ValueError, match=r"<\|tts_bos\|>"): - llm2tts([llm_output], prompt=[{}]) - - -def test_minicpmo_talker_normalizes_list_handoff_payload(monkeypatch): - model = MiniCPMO45OmniTTSForConditionalGeneration.__new__(MiniCPMO45OmniTTSForConditionalGeneration) - captured = {} - - monkeypatch.setattr(model, "_should_stream_output", lambda *args, **kwargs: False) - - def fake_generate_speech(tts_token_ids, tts_hidden_states, **kwargs): - captured["tts_token_ids"] = tts_token_ids - captured["tts_hidden_states"] = tts_hidden_states - return torch.tensor([0.0, 0.1], dtype=torch.float32) - - monkeypatch.setattr(model, "generate_speech", fake_generate_speech) - - waveform, mel = model.forward( - additional_information={ - "tts_token_ids": [11, 12], - "tts_hidden_states": [[0.1, 0.2], [0.3, 0.4]], - } - ) - - assert mel is None - assert torch.equal(waveform, torch.tensor([0.0, 0.1], dtype=torch.float32)) - assert torch.equal(captured["tts_token_ids"], torch.tensor([11, 12])) - assert captured["tts_hidden_states"].dtype == torch.float32 - assert captured["tts_hidden_states"].shape == (2, 2) - - -def test_minicpmo_talker_accepts_structured_omni_payload_handoff(monkeypatch): - model = MiniCPMO45OmniTTSForConditionalGeneration.__new__(MiniCPMO45OmniTTSForConditionalGeneration) - captured = {} - - monkeypatch.setattr(model, "_should_stream_output", lambda *args, **kwargs: False) - - def fake_generate_speech(tts_token_ids, tts_hidden_states, **kwargs): - captured["tts_token_ids"] = tts_token_ids - captured["tts_hidden_states"] = tts_hidden_states - return torch.tensor([0.0, 0.1], dtype=torch.float32) - - monkeypatch.setattr(model, "generate_speech", fake_generate_speech) - - waveform, mel = model.forward( - additional_information={ - "ids": {"tts": [11, 12]}, - "hidden_states": {"tts": torch.tensor([[0.1, 0.2], [0.3, 0.4]])}, - } - ) - - assert mel is None - assert torch.equal(waveform, torch.tensor([0.0, 0.1], dtype=torch.float32)) - assert torch.equal(captured["tts_token_ids"], torch.tensor([11, 12])) - assert captured["tts_hidden_states"].dtype == torch.float32 - assert captured["tts_hidden_states"].shape == (2, 2) - - -def test_token2wav_prompt_cache_resets_when_reference_changes(): - model = MiniCPMO45OmniTTSForConditionalGeneration.__new__(MiniCPMO45OmniTTSForConditionalGeneration) - model.audio_tokenizer = SimpleNamespace(cache="old") - - model._reset_token2wav_cache_if_needed("/tmp/ref_a.wav") - assert model.audio_tokenizer.cache is None - - model.audio_tokenizer.cache = "prepared" - model._reset_token2wav_cache_if_needed("/tmp/ref_a.wav") - assert model.audio_tokenizer.cache == "prepared" - - model._reset_token2wav_cache_if_needed("/tmp/ref_b.wav") - assert model.audio_tokenizer.cache is None - - -def test_request_ref_audio_prompt_wav_uses_content_cache(): - model = MiniCPMO45OmniTTSForConditionalGeneration.__new__(MiniCPMO45OmniTTSForConditionalGeneration) - ref_audio = torch.tensor([0.0, 0.1, 0.2], dtype=torch.float32) - - path1 = model._write_ref_audio_prompt_wav(ref_audio, 16000) - path2 = model._write_ref_audio_prompt_wav(ref_audio.clone(), 16000) - - try: - assert path1 == path2 - assert os.path.exists(path1) - assert model._is_cached_ref_audio_prompt_wav(path1) - finally: - for path in set(getattr(model, "_ref_audio_prompt_files", {}).values()): - try: - os.unlink(path) - except OSError: - pass - - -def test_native_duplex_talker_uses_generate_chunk_continuation(monkeypatch): - class _FakeTTS: - audio_bos_token_id = 88 - - def __init__(self): - self.config = SimpleNamespace(num_audio_tokens=128) - self.model = SimpleNamespace(config=SimpleNamespace()) - self.emb_text = torch.nn.Embedding(256, 4) - self.calls = [] - self._responses = [ - (torch.tensor([[[10], [11], [12]]]), "kv1"), - (torch.tensor([[[20]]]), "kv2"), - ] - - def generate_chunk(self, **kwargs): - self.calls.append(kwargs) - return self._responses[len(self.calls) - 1] - - model = MiniCPMO45OmniTTSForConditionalGeneration.__new__(MiniCPMO45OmniTTSForConditionalGeneration) - tts = _FakeTTS() - windows = [] - model.tts_obj = tts - model.audio_tokenizer = SimpleNamespace() - model._talker_turn_states = {} - model._talker_consumed_tokens = {} - - monkeypatch.setattr(model, "_lazy_init_tts", lambda: None) - monkeypatch.setattr( - model, "_build_tts_sampling_params", lambda: SimpleNamespace(temperature=0.8, repetition_penalty=1.05) - ) - monkeypatch.setattr(model, "_tts_runtime_config", lambda: SimpleNamespace(streaming_generator_chunk=2)) - monkeypatch.setattr(model, "_resolve_prompt_wav_path", lambda ref, sr: ("/tmp/ref.wav", None)) - monkeypatch.setattr(model, "_begin_turn_vocoder_cache", lambda prompt, **kwargs: None) - monkeypatch.setattr(model, "_t2w_pre_lookahead", lambda: 1) - monkeypatch.setattr( - model, - "_build_tts_condition_embeds", - lambda ids, hidden: torch.ones((ids.numel(), 4), dtype=torch.float32), - ) - - def fake_stream_window(token_list, prompt_wav_path, *, last_chunk): - windows.append((list(token_list), last_chunk)) - return torch.tensor([float(len(token_list))]) - - monkeypatch.setattr(model, "_t2w_stream_window", fake_stream_window) - - base_info = { - "request_id": "duplex-generate-chunk", - "meta": {"turn_eos_token_id": 9310}, - "ids": {"tts": [21]}, - "hidden_states": {"tts": [[0.1, 0.2, 0.3, 0.4]]}, - "codes": {"ref": None}, - "native_duplex": True, - } - list(model._create_native_duplex_stream_gen(base_info)) - - turn_end_info = { - **base_info, - "ids": {"tts": [21, 9310]}, - "hidden_states": {"tts": [[0.1, 0.2, 0.3, 0.4], [0.5, 0.6, 0.7, 0.8]]}, - } - list(model._create_native_duplex_stream_gen(turn_end_info)) - - assert len(tts.calls) == 2 - first, second = tts.calls - assert first["past_key_values"] is None - assert first["text_start_pos"] == 0 - assert first["min_new_tokens"] == 0 - assert first["max_new_token"] == 3 - assert torch.equal(first["eos_token"], torch.tensor([127])) - assert float(first["temperature"].item()) == pytest.approx(0.8) - assert first["repetition_penalty"] == pytest.approx(1.05) - - assert second["past_key_values"] == "kv1" - assert second["text_start_pos"] == 5 - assert second["min_new_tokens"] == 0 - assert second["max_new_token"] == 3 - assert "duplex-generate-chunk" not in model._talker_turn_states - assert windows[-1] == ([20], True) - - -def test_minicpmo_native_duplex_talker_turn_end_metadata_flushes_tail(monkeypatch): - from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni_tts import ( - MiniCPMO45OmniTTSForConditionalGeneration, - ) - - class _FakeEmbText: - weight = torch.ones((200, 4), dtype=torch.float32) - - def __call__(self, ids): - return torch.ones((ids.numel(), 4), dtype=torch.float32) - - class _FakeTTS: - audio_bos_token_id = 1 - config = SimpleNamespace(num_audio_tokens=128) - emb_text = _FakeEmbText() - model = SimpleNamespace(config=SimpleNamespace(rope_theta=10000.0)) - - def generate_chunk(self, **kwargs): - del kwargs - return torch.tensor([[[33]]]), None - - model = MiniCPMO45OmniTTSForConditionalGeneration.__new__(MiniCPMO45OmniTTSForConditionalGeneration) - model.tts_obj = _FakeTTS() - model.audio_tokenizer = SimpleNamespace() - model._talker_turn_states = {} - model._talker_consumed_tokens = {} - - monkeypatch.setattr(model, "_lazy_init_tts", lambda: None) - monkeypatch.setattr( - model, "_build_tts_sampling_params", lambda: SimpleNamespace(temperature=0.8, repetition_penalty=1.05) - ) - monkeypatch.setattr(model, "_tts_runtime_config", lambda: SimpleNamespace(streaming_generator_chunk=25)) - monkeypatch.setattr(model, "_resolve_prompt_wav_path", lambda ref, sr: ("/tmp/ref.wav", None)) - monkeypatch.setattr(model, "_begin_turn_vocoder_cache", lambda prompt, **kwargs: None) - monkeypatch.setattr(model, "_t2w_pre_lookahead", lambda: 5) - monkeypatch.setattr( - model, - "_build_tts_condition_embeds", - lambda ids, hidden: torch.ones((ids.numel(), 4), dtype=torch.float32), - ) - - windows = [] - - def fake_stream_window(token_list, prompt_wav_path, *, last_chunk): - del prompt_wav_path - windows.append((list(token_list), last_chunk)) - return torch.tensor([float(len(token_list))]) - - monkeypatch.setattr(model, "_t2w_stream_window", fake_stream_window) - - info = { - "request_id": "duplex-metadata-tail-flush", - "meta": {"turn_eos_token_id": 9310}, - "ids": {"tts": [21]}, - "hidden_states": {"tts": [[0.1, 0.2, 0.3, 0.4]]}, - "codes": {"ref": None}, - "end_of_turn": True, - "native_duplex": True, - } - - list(model._create_native_duplex_stream_gen(info)) - - assert windows[-1][1] is True - assert "duplex-metadata-tail-flush" not in model._talker_turn_states - assert "duplex-metadata-tail-flush" not in model._talker_consumed_tokens - - -def test_minicpmo_native_duplex_terminal_only_new_turn_does_not_start_tts(monkeypatch): - from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni_tts import ( - MiniCPMO45OmniTTSForConditionalGeneration, - ) - - class _FakeEmbText: - weight = torch.ones((200, 4), dtype=torch.float32) - - def __call__(self, ids): - return torch.ones((ids.numel(), 4), dtype=torch.float32) - - class _FakeTTS: - audio_bos_token_id = 1 - config = SimpleNamespace(num_audio_tokens=128) - emb_text = _FakeEmbText() - model = SimpleNamespace(config=SimpleNamespace(rope_theta=10000.0)) - - def __init__(self): - self.calls = 0 - - def generate_chunk(self, **kwargs): - del kwargs - self.calls += 1 - return torch.tensor([[[33]]]), None - - model = MiniCPMO45OmniTTSForConditionalGeneration.__new__(MiniCPMO45OmniTTSForConditionalGeneration) - tts = _FakeTTS() - model.tts_obj = tts - model.audio_tokenizer = SimpleNamespace() - model._talker_turn_states = {} - model._talker_consumed_tokens = {} - - monkeypatch.setattr(model, "_lazy_init_tts", lambda: None) - monkeypatch.setattr( - model, "_build_tts_sampling_params", lambda: SimpleNamespace(temperature=0.8, repetition_penalty=1.05) - ) - monkeypatch.setattr(model, "_tts_runtime_config", lambda: SimpleNamespace(streaming_generator_chunk=25)) - monkeypatch.setattr(model, "_resolve_prompt_wav_path", lambda ref, sr: ("/tmp/ref.wav", None)) - monkeypatch.setattr(model, "_begin_turn_vocoder_cache", lambda prompt, **kwargs: None) - monkeypatch.setattr(model, "_t2w_pre_lookahead", lambda: 5) - monkeypatch.setattr( - model, - "_build_tts_condition_embeds", - lambda ids, hidden: torch.ones((ids.numel(), 4), dtype=torch.float32), - ) - monkeypatch.setattr( - model, - "_t2w_stream_window", - lambda token_list, prompt_wav_path, *, last_chunk: torch.tensor([float(len(token_list))]), - ) - - outputs = list( - model._create_native_duplex_stream_gen( - { - "request_id": "duplex-terminal-only", - "duplex": {"epoch": 0, "turn_id": 1}, - "meta": {"turn_eos_token_id": 9310}, - "ids": {"tts": [9310]}, - "hidden_states": {"tts": [[0.1, 0.2, 0.3, 0.4]]}, - "codes": {"ref": None}, - "native_duplex": True, + context = SimpleNamespace( + bridge_states={ + "duplex": { + "epoch": 3, + "model_turn_id": 7, } - ) - ) - - assert tts.calls == 0 - assert len(outputs) == 1 - assert outputs[0][0].numel() == 0 - assert outputs[0][1] is True - assert model._talker_turn_states == {} - assert model._talker_consumed_tokens == {} - - -def test_minicpmo_native_duplex_talker_finished_request_closes_session_keyed_turn_state(): - from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni_tts import ( - MiniCPMO45OmniTTSForConditionalGeneration, - _TalkerTurnState, - ) - - model = MiniCPMO45OmniTTSForConditionalGeneration.__new__(MiniCPMO45OmniTTSForConditionalGeneration) - model.audio_tokenizer = SimpleNamespace(stream_cache={"stale": True}, hift_cache_dict={"stale": True}) - model._stream_gens = {} - model._talker_turn_states = { - "sid-stage1": _TalkerTurnState(prompt_wav_path=None, temp_prompt_wav_path=None), - } - model._talker_consumed_tokens = {"sid-stage1": 12} - model._talker_request_keys = {"req-stage1": "sid-stage1"} - - model.on_requests_finished({"req-stage1"}) - - assert "sid-stage1" not in model._talker_turn_states - assert "sid-stage1" not in model._talker_consumed_tokens - assert "req-stage1" not in model._talker_request_keys - assert model.audio_tokenizer.stream_cache is None - assert model.audio_tokenizer.hift_cache_dict == {} - - -def test_minicpmo_native_duplex_talker_new_turn_reopens_session_keyed_state(monkeypatch): - from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni_tts import ( - MiniCPMO45OmniTTSForConditionalGeneration, - _TalkerTurnState, - ) - - class _FakeTTS: - audio_bos_token_id = 88 - - def __init__(self): - self.config = SimpleNamespace(num_audio_tokens=128) - self.model = SimpleNamespace(config=SimpleNamespace()) - self.emb_text = torch.nn.Embedding(256, 4) - self.calls = [] - - def generate_chunk(self, **kwargs): - self.calls.append(kwargs) - return torch.tensor([[[10]]]), "new-kv" - - model = MiniCPMO45OmniTTSForConditionalGeneration.__new__(MiniCPMO45OmniTTSForConditionalGeneration) - tts = _FakeTTS() - model.tts_obj = tts - model.audio_tokenizer = SimpleNamespace(stream_cache={"old": True}, hift_cache_dict={"old": True}) - model._talker_turn_states = { - "sid-stage1": _TalkerTurnState( - prompt_wav_path=None, - temp_prompt_wav_path=None, - epoch=0, - turn_id=1, - ), - } - model._talker_turn_states["sid-stage1"].past_key_values = "old-kv" - model._talker_consumed_tokens = {"sid-stage1": 9} - model._talker_request_keys = {} - - monkeypatch.setattr(model, "_lazy_init_tts", lambda: None) - monkeypatch.setattr( - model, "_build_tts_sampling_params", lambda: SimpleNamespace(temperature=0.8, repetition_penalty=1.05) - ) - monkeypatch.setattr(model, "_tts_runtime_config", lambda: SimpleNamespace(streaming_generator_chunk=25)) - monkeypatch.setattr(model, "_resolve_prompt_wav_path", lambda ref, sr: ("/tmp/ref.wav", None)) - monkeypatch.setattr(model, "_begin_turn_vocoder_cache", lambda prompt, **kwargs: None) - monkeypatch.setattr(model, "_t2w_pre_lookahead", lambda: 5) - monkeypatch.setattr( - model, - "_build_tts_condition_embeds", - lambda ids, hidden: torch.ones((ids.numel(), 4), dtype=torch.float32), - ) - monkeypatch.setattr( - model, - "_t2w_stream_window", - lambda token_list, prompt_wav_path, *, last_chunk: torch.tensor([float(len(token_list))]), - ) - - info = { - "request_id": "sid-stage1", - "duplex": {"epoch": 0, "turn_id": 2}, - "meta": {"turn_eos_token_id": 9310}, - "ids": {"tts": [21]}, - "hidden_states": {"tts": [[0.1, 0.2, 0.3, 0.4]]}, - "codes": {"ref": None}, - "native_duplex": True, - } - - list(model._create_native_duplex_stream_gen(info)) - - assert tts.calls[0]["past_key_values"] is None - assert tts.calls[0]["text_start_pos"] == 0 - assert model._talker_consumed_tokens["sid-stage1"] == 1 - assert model._talker_turn_states["sid-stage1"].turn_id == 2 - - -def test_minicpmo_native_duplex_talker_segment_end_preserves_token2wav_stream_until_turn_end(monkeypatch): - from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni_tts import ( - MiniCPMO45OmniTTSForConditionalGeneration, - ) - - class _FakeEmbText: - weight = torch.ones((200, 4), dtype=torch.float32) - - def __call__(self, ids): - return torch.ones((ids.numel(), 4), dtype=torch.float32) - - class _FakeTTS: - audio_bos_token_id = 1 - config = SimpleNamespace(num_audio_tokens=128) - emb_text = _FakeEmbText() - model = SimpleNamespace(config=SimpleNamespace(rope_theta=10000.0)) - - def generate_chunk(self, **kwargs): - del kwargs - return torch.tensor([[[33]]]), "kv" - - model = MiniCPMO45OmniTTSForConditionalGeneration.__new__(MiniCPMO45OmniTTSForConditionalGeneration) - model.tts_obj = _FakeTTS() - model.audio_tokenizer = SimpleNamespace() - model._talker_turn_states = {} - model._talker_consumed_tokens = {} - - monkeypatch.setattr(model, "_lazy_init_tts", lambda: None) - monkeypatch.setattr( - model, "_build_tts_sampling_params", lambda: SimpleNamespace(temperature=0.8, repetition_penalty=1.05) - ) - monkeypatch.setattr(model, "_tts_runtime_config", lambda: SimpleNamespace(streaming_generator_chunk=25)) - monkeypatch.setattr(model, "_resolve_prompt_wav_path", lambda ref, sr: ("/tmp/ref.wav", None)) - monkeypatch.setattr(model, "_begin_turn_vocoder_cache", lambda prompt, **kwargs: None) - monkeypatch.setattr(model, "_t2w_pre_lookahead", lambda: 5) - monkeypatch.setattr( - model, - "_build_tts_condition_embeds", - lambda ids, hidden: torch.ones((ids.numel(), 4), dtype=torch.float32), - ) - - windows = [] - - def fake_stream_window(token_list, prompt_wav_path, *, last_chunk): - del prompt_wav_path - windows.append((list(token_list), last_chunk)) - return torch.tensor([float(len(token_list))]) - - monkeypatch.setattr(model, "_t2w_stream_window", fake_stream_window) - - info = { - "request_id": "duplex-segment-tail-flush", - "meta": {"turn_eos_token_id": 9310, "segment_end": True}, - "ids": {"tts": [21]}, - "hidden_states": {"tts": [[0.1, 0.2, 0.3, 0.4]]}, - "codes": {"ref": None}, - "native_duplex": True, - } - - list(model._create_native_duplex_stream_gen(info)) - - state = model._talker_turn_states["duplex-segment-tail-flush"] - assert windows == [] - assert state.token2wav_buffer == [4218, 4218, 4218, 33] - - turn_end_info = { - **info, - "ids": {"tts": [21, 9310]}, - "hidden_states": { - "tts": [ - [0.1, 0.2, 0.3, 0.4], - [0.5, 0.6, 0.7, 0.8], - ] - }, - "meta": {"turn_eos_token_id": 9310, "turn_end": True}, - } - list(model._create_native_duplex_stream_gen(turn_end_info)) - - assert windows == [([4218, 4218, 4218, 33, 33], True)] - assert "duplex-segment-tail-flush" not in model._talker_turn_states - - -def test_tts_scheduler_eos_uses_tokenizer_im_end_when_config_has_no_eos(): - class _Tokenizer: - unk_token_id = 0 - - def convert_tokens_to_ids(self, token): - return {"<|im_end|>": 77}.get(token, self.unk_token_id) - - def encode(self, text, add_special_tokens=False): - del add_special_tokens - return [77] if text == "<|im_end|>" else [self.unk_token_id] - - model = MiniCPMO45OmniTTSForConditionalGeneration.__new__(MiniCPMO45OmniTTSForConditionalGeneration) - model.config = SimpleNamespace(eos_token_id=None, vocab_size=100) - model.vllm_config = SimpleNamespace(model_config=SimpleNamespace()) - model._text_tokenizer = _Tokenizer() - model._ar_last_chunk_flags = [True] - - logits = model.compute_logits(torch.zeros(1, 4)) - - assert int(torch.argmax(logits[0]).item()) == 77 - - -def test_tts_streaming_non_final_chunk_does_not_mark_extra_rows_eos(): - class _Tokenizer: - unk_token_id = 0 - - def convert_tokens_to_ids(self, token): - return {"<|im_end|>": 77}.get(token, self.unk_token_id) - - def encode(self, text, add_special_tokens=False): - del add_special_tokens - return [77] if text == "<|im_end|>" else [self.unk_token_id] - - model = MiniCPMO45OmniTTSForConditionalGeneration.__new__(MiniCPMO45OmniTTSForConditionalGeneration) - model.config = SimpleNamespace(eos_token_id=None, vocab_size=100) - model.vllm_config = SimpleNamespace(model_config=SimpleNamespace()) - model._text_tokenizer = _Tokenizer() - model._ar_last_chunk_flags = [False] - - logits = model.compute_logits(torch.zeros(3, 4)) - sampled = torch.argmax(logits, dim=-1).tolist() - - assert sampled == [1, 1, 1] - - -def test_native_duplex_turn_end_detection_is_not_segment_end_detection(): - model = MiniCPMO45OmniTTSForConditionalGeneration.__new__(MiniCPMO45OmniTTSForConditionalGeneration) - turn_eos_id = 9310 - - assert model._native_duplex_input_ends_turn( - { - "native_duplex": True, - "ids": {"tts": [9304, 42, turn_eos_id]}, - "meta": {"turn_eos_token_id": turn_eos_id}, - } - ) - assert not model._native_duplex_input_ends_turn( - { - "native_duplex": True, - "ids": {"tts": [9304, 42, 9308]}, - "meta": { - "turn_eos_token_id": turn_eos_id, - "segment_end": True, - }, } ) + converted = llm2tts([source], prompt=[{}], _streaming_context=context)[0] -def _native_duplex_meta(): - return { - "tts_bos_token_id": 9301, - "tts_eos_token_id": 9302, - "listen_token_id": 9303, - "speak_token_id": 9304, - "tts_pad_token_id": 9305, - "unit_token_id": 9306, - "unit_end_token_id": 9307, - "chunk_eos_token_id": 9308, - "chunk_tts_eos_token_id": 9309, - "turn_eos_token_id": 9310, - } + info = converted["model_intermediate_buffer"] + assert info["native_duplex"] is True + assert info["ids"]["tts"] == [21, 22] + assert converted["prompt_token_ids"] == [0, 0, 0, 0] + assert info["meta"]["replace_streaming_prompt"] is True + assert info["meta"]["segment_end"] is True + assert info["duplex"]["epoch"] == 3 + assert info["duplex"]["turn_id"] == 7 -def _native_duplex_handoff(request_id, prompt_ids, cumulative_output_ids, *, text="hello"): - latent = torch.arange( - (len(prompt_ids) + len(cumulative_output_ids)) * 4, - dtype=torch.float16, - ).reshape(-1, 4) - output = SimpleNamespace( - token_ids=cumulative_output_ids, - text=text, - multimodal_output={ - "latent": latent, - "duplex_prompt_token_ids": list(prompt_ids), - "meta": _native_duplex_meta(), - }, - ) - return SimpleNamespace( - request_id=request_id, - prompt_token_ids=[0, 0, 0], - outputs=[output], +def test_native_duplex_requires_tokenizer_boundary_metadata() -> None: + latent = torch.zeros((3, 4)) + source = _output( + prompt_ids=[101], + output_ids=[21, 22], + latent=latent, + multimodal_output={"duplex_prompt_token_ids": [101]}, ) - -def _install_native_duplex_decoder(streaming_context, token_text): - def _decode(token_ids): - return "".join(token_text.get(int(token_id), "") for token_id in token_ids) - - streaming_context.source_token_decoder = _decode - return streaming_context + with pytest.raises(ValueError, match="tokenizer-derived.*metadata"): + llm2tts([source], prompt=[{}], _streaming_context=SimpleNamespace(bridge_states={})) -def test_native_duplex_segment_text_comes_from_token_delta_decode(): - streaming_context = _install_native_duplex_decoder( - SimpleNamespace(bridge_states={}), - { - 21: "今早", - 22: "吃东。", - 31: "西了吗?", - }, +def test_llm2tts_does_not_alias_live_thinker_token_list() -> None: + live_tokens = [11, 12] + latent = torch.zeros((3, 4)) + source = _output( + prompt_ids=[101], + output_ids=list(live_tokens), + latent=latent, + token_list=live_tokens, ) - ids1, text1 = _native_duplex_segment_output_ids( - [9304, 21, 22, 9308], - "今早吃东。", - streaming_context, - request_id="duplex-token-text", - ) - assert ids1 == [9304, 21, 22, 9308] - assert text1 == "今早吃东。" + converted = llm2tts([source], prompt=[{}])[0] + live_tokens.append(13) - ids2, text2 = _native_duplex_segment_output_ids( - [9304, 21, 22, 9308, 9304, 31, 9308], - "今早吃东。西了吗?如需陪聊,", - streaming_context, - request_id="duplex-token-text", - ) - assert ids2 == [9304, 31, 9308] - assert text2 == "西了吗?" - - -def test_native_duplex_segment_restart_does_not_reuse_character_cursor(): - streaming_context = _install_native_duplex_decoder( - SimpleNamespace(bridge_states={}), - { - 21: "你好", - 31: "你好", - 32: ",有什么想聊的吗?", - }, - ) - - _native_duplex_segment_output_ids( - [9304, 21, 9310], - "你好,第一轮尾巴", - streaming_context, - request_id="duplex-restart-text", - ) - - ids2, text2 = _native_duplex_segment_output_ids( - [9304, 31, 32, 9310], - "你好,第一轮尾巴你好,有什么想聊的吗?", - streaming_context, - request_id="duplex-restart-text", - ) - assert ids2 == [9304, 31, 32, 9310] - assert text2 == "你好,有什么想聊的吗?" - - -def test_native_duplex_segment_turn_id_change_keeps_cumulative_token_cursor(): - streaming_context = _install_native_duplex_decoder( - SimpleNamespace(bridge_states={"duplex": {"turn_id": 1}}), - { - 21: "你好", - 32: ",有什么想聊的吗?", - }, - ) - - ids1, text1 = _native_duplex_segment_output_ids( - [9304, 21], - "你好", - streaming_context, - request_id="duplex-turn-id-reset", - ) - assert ids1 == [9304, 21] - assert text1 == "你好" - - streaming_context.bridge_states["duplex"]["turn_id"] = 2 - ids2, text2 = _native_duplex_segment_output_ids( - [9304, 21, 32], - "你好,有什么想聊的吗?", - streaming_context, - request_id="duplex-turn-id-reset", - ) - assert ids2 == [32] - assert text2 == ",有什么想聊的吗?" - - -def test_llm2tts_native_duplex_hands_off_segment_deltas(): - """Each duplex handoff must carry only the current segment's tokens. - - The thinker's resumable request reports cumulative output ids while - earlier segments are already folded into the prompt, so re-forwarding - the full list misaligns the hidden states and grows the talker prompt - without bound. - """ - streaming_context = SimpleNamespace(bridge_states={}) - - # Segment 1: speak + two text tokens + chunk_eos. - seg1_output = [9304, 21, 22, 9308] - handoff1 = _native_duplex_handoff("duplex-delta", [101, 102], list(seg1_output)) - converted1 = llm2tts([handoff1], prompt=[{}], _streaming_context=streaming_context) - info1 = converted1[0]["model_intermediate_buffer"] - assert info1["ids"]["output"] == seg1_output - assert info1["llm_output_text"] == ["hello"] - assert info1["meta"]["native_duplex_segment_text"] == "hello" - assert "llm_output_text" in info1["meta"]["override_keys"] - assert ["meta", "native_duplex_segment_text"] in info1["meta"]["override_keys"] - assert converted1[0]["prompt_token_ids"] == [21, 22] - assert info1["meta"]["segment_end"] is True - - # Segment 2: prompt now contains the folded segment-1 tokens; the - # cumulative output list still carries segment 1 ahead of segment 2. - seg2_output = [9304, 31, 9308] - prompt2 = [101, 102, *seg1_output, 555] - handoff2 = _native_duplex_handoff("duplex-delta", prompt2, [*seg1_output, *seg2_output]) - converted2 = llm2tts([handoff2], prompt=[{}], _streaming_context=streaming_context) - info2 = converted2[0]["model_intermediate_buffer"] - assert info2["ids"]["output"] == seg2_output - # Cumulative text "hello" was fully delivered with segment 1; only the - # delta (empty here) rides along with segment 2. - assert info2["llm_output_text"] == [""] - assert info2["meta"]["native_duplex_segment_text"] == "" - assert converted2[0]["prompt_token_ids"] == [31] - assert info2["meta"]["segment_end"] is True - - -def test_llm2tts_native_duplex_marks_talker_handoff_as_data_plane(): - streaming_context = SimpleNamespace( - bridge_states={ - "duplex": { - "session_id": "sid-stage1-stop", - "incarnation": 2, - "epoch": 0, - "turn_id": 7, - "session_config": { - "extra_body": { - "duplex_stage_sampling_params": { - "1": {"stop_token_ids": [151645]}, - }, - }, - }, - }, - }, - ) - handoff = _native_duplex_handoff("duplex-stage1-stop", [101, 102], [9304, 21, 9308]) - - converted = llm2tts([handoff], prompt=[{}], _streaming_context=streaming_context) - - duplex = converted[0]["model_intermediate_buffer"]["duplex"] - assert duplex["data_plane"] is True - assert duplex["session_id"] == "sid-stage1-stop" - assert duplex["incarnation"] == 2 - assert duplex["epoch"] == 0 - assert duplex["turn_id"] == 7 - assert duplex["session_config"]["extra_body"]["duplex_stage_sampling_params"]["1"]["stop_token_ids"] == [151645] - - -def test_llm2tts_native_duplex_carries_runtime_ref_audio_to_talker_payload(): - ref_audio = base64.b64encode(struct.pack("<3f", 0.1, 0.2, 0.3)).decode("ascii") - streaming_context = SimpleNamespace( - bridge_states={ - "duplex": { - "session_id": "sid-runtime-ref", - "runtime_config": { - "ref_audio_data": ref_audio, - "ref_audio_format": "pcm_f32le", - "ref_audio_sample_rate_hz": 16000, - }, - }, - }, - ) - handoff = _native_duplex_handoff("duplex-runtime-ref", [101, 102], [9304, 21, 9308]) - - converted = llm2tts([handoff], prompt=[{}], _streaming_context=streaming_context) - - info = converted[0]["model_intermediate_buffer"] - assert torch.allclose(torch.tensor(info["codes"]["ref"]), torch.tensor([0.1, 0.2, 0.3])) - assert info["meta"]["ref_audio_sr"] == 16000 - assert info["duplex"]["runtime_config"]["ref_audio_data"] == ref_audio - - -def test_llm2tts_native_duplex_advances_model_turn_only_after_turn_eos(): - streaming_context = SimpleNamespace( - bridge_states={ - "duplex": { - "session_id": "sid-model-turn", - "epoch": 0, - "turn_id": 9, - "model_turn_id": 0, - }, - } - ) - - first = _native_duplex_handoff( - "duplex-model-turn", - [101], - [9304, 21, 9310], - text="first", - ) - converted_first = llm2tts([first], prompt=[{}], _streaming_context=streaming_context) - - first_duplex = converted_first[0]["model_intermediate_buffer"]["duplex"] - assert first_duplex["turn_id"] == 0 - assert streaming_context.bridge_states["duplex"]["model_turn_id"] == 1 - - second = _native_duplex_handoff( - "duplex-model-turn", - [101, 9304, 21, 9310, 555], - [9304, 21, 9310, 9304, 22, 9308], - text="second", - ) - converted_second = llm2tts([second], prompt=[{}], _streaming_context=streaming_context) - - second_duplex = converted_second[0]["model_intermediate_buffer"]["duplex"] - assert second_duplex["turn_id"] == 1 - assert streaming_context.bridge_states["duplex"]["model_turn_id"] == 1 - - -def test_llm2tts_native_duplex_accumulates_tts_condition_across_handoffs(): - """Every handoff must carry the FULL accumulated tts condition. - - The runner's resume-prefill path REPLACES the streaming buffer (only - in-place updates merge), so per-segment tts payloads were silently lost - for alternating segments; the talker then vocalized text it never saw. - Handing the complete history per handoff makes replacement lossless. - """ - streaming_context = SimpleNamespace(bridge_states={}) - - seg1_output = [9304, 21, 22, 9308] - handoff1 = _native_duplex_handoff("duplex-acc", [101, 102], list(seg1_output)) - converted1 = llm2tts([handoff1], prompt=[{}], _streaming_context=streaming_context) - info1 = converted1[0]["model_intermediate_buffer"] - assert info1["ids"]["tts"] == [21, 22] - assert len(info1["hidden_states"]["tts"]) == 2 - - seg2_output = [9304, 31, 9308] - prompt2 = [101, 102, *seg1_output, 555] - handoff2 = _native_duplex_handoff("duplex-acc", prompt2, [*seg1_output, *seg2_output]) - converted2 = llm2tts([handoff2], prompt=[{}], _streaming_context=streaming_context) - info2 = converted2[0]["model_intermediate_buffer"] - assert info2["ids"]["tts"] == [21, 22, 31] - assert len(info2["hidden_states"]["tts"]) == 3 - - -def test_llm2tts_native_duplex_preserves_text_cursor_across_turn_reset(): - streaming_context = _install_native_duplex_decoder( - SimpleNamespace(bridge_states={}), - { - 21: "hello", - 31: "hello", - }, - ) - - seg1_output = [9304, 21, 9310] - handoff1 = _native_duplex_handoff("duplex-turn-text", [101, 102], seg1_output, text="hello") - converted1 = llm2tts([handoff1], prompt=[{}], _streaming_context=streaming_context) - info1 = converted1[0]["model_intermediate_buffer"] - assert info1["meta"]["native_duplex_segment_text"] == "hello" - - seg2_output = [9304, 31, 9310] - handoff2 = _native_duplex_handoff( - "duplex-turn-text", - [101, 102, *seg1_output], - seg2_output, - text="hellohello", - ) - converted2 = llm2tts([handoff2], prompt=[{}], _streaming_context=streaming_context) - info2 = converted2[0]["model_intermediate_buffer"] - assert info2["ids"]["tts"] == [31, 9310] - assert info2["meta"]["native_duplex_segment_text"] == "hello" - - -def test_llm2tts_native_duplex_preserves_output_cursor_across_turn_reset(): - streaming_context = _install_native_duplex_decoder( - SimpleNamespace(bridge_states={}), - { - 21: "hello", - 31: "hello", - }, - ) - - seg1_output = [9304, 21, 9310] - handoff1 = _native_duplex_handoff("duplex-turn-output", [101, 102], seg1_output, text="hello") - converted1 = llm2tts([handoff1], prompt=[{}], _streaming_context=streaming_context) - info1 = converted1[0]["model_intermediate_buffer"] - assert info1["ids"]["output"] == seg1_output - assert info1["ids"]["tts"] == [21, 9310] - - seg2_output = [9304, 31, 9310] - handoff2 = _native_duplex_handoff( - "duplex-turn-output", - [101, 102, *seg1_output], - [*seg1_output, *seg2_output], - text="hellohello", - ) - converted2 = llm2tts([handoff2], prompt=[{}], _streaming_context=streaming_context) - info2 = converted2[0]["model_intermediate_buffer"] - assert info2["ids"]["output"] == seg2_output - assert info2["ids"]["tts"] == [31, 9310] - assert info2["meta"]["native_duplex_segment_text"] == "hello" - - -def test_llm2tts_native_duplex_resets_tts_condition_on_new_turn_without_resetting_output_cursor(): - streaming_context = _install_native_duplex_decoder( - SimpleNamespace(bridge_states={"duplex": {"turn_id": 0}}), - { - 21: "first", - 31: "second", - }, - ) - - seg1_output = [9304, 21, 9308] - handoff1 = _native_duplex_handoff("duplex-turn-condition", [101, 102], seg1_output, text="first") - converted1 = llm2tts([handoff1], prompt=[{}], _streaming_context=streaming_context) - info1 = converted1[0]["model_intermediate_buffer"] - assert info1["ids"]["output"] == seg1_output - assert info1["ids"]["tts"] == [21] - - streaming_context.bridge_states["duplex"]["turn_id"] = 1 - seg2_output = [9304, 31, 9308] - handoff2 = _native_duplex_handoff( - "duplex-turn-condition", - [101, 102, *seg1_output], - [*seg1_output, *seg2_output], - text="firstsecond", - ) - converted2 = llm2tts([handoff2], prompt=[{}], _streaming_context=streaming_context) - info2 = converted2[0]["model_intermediate_buffer"] - assert info2["ids"]["output"] == seg2_output - assert info2["ids"]["tts"] == [31] - assert info2["meta"]["native_duplex_segment_text"] == "second" - - -def test_llm2tts_never_aliases_thinker_token_list(): - """The talker prompt must never be the thinker's live token list object. - - CompletionOutput.token_ids can alias the upstream detokenizer's internal - list; forwarding that object lets the stage-1 streaming update extend the - list with itself, doubling the recorded output every segment. - """ - live_token_list = [9304, 21, 22, 9308] - handoff = _native_duplex_handoff("duplex-alias", [101, 102], live_token_list) - converted = llm2tts([handoff], prompt=[{}]) - scheduler_prompt = converted[0]["prompt_token_ids"] - assert scheduler_prompt is not live_token_list - scheduler_prompt.extend(scheduler_prompt) - assert live_token_list == [9304, 21, 22, 9308] + assert converted["model_intermediate_buffer"]["ids"]["output"] == [11, 12] diff --git a/tests/test_config_factory.py b/tests/test_config_factory.py index 13a491896..10f423c84 100644 --- a/tests/test_config_factory.py +++ b/tests/test_config_factory.py @@ -1129,23 +1129,22 @@ class TestDeployConfigLoading: stages = merge_pipeline_deploy(pipeline, deploy) assert deploy.session_mode == "duplex" - assert deploy.async_chunk is False + assert deploy.async_chunk is True assert deploy.active_stream_window == 1 - assert [stage.session_mode for stage in stages] == ["duplex", "duplex"] - assert [stage.to_omegaconf().session_mode for stage in stages] == ["duplex", "duplex"] - assert [stage.yaml_engine_args["async_scheduling"] for stage in stages] == [False, False] + assert [stage.session_mode for stage in stages] == ["duplex", "duplex", "duplex"] + assert [stage.to_omegaconf().session_mode for stage in stages] == ["duplex", "duplex", "duplex"] + assert [stage.yaml_engine_args["async_scheduling"] for stage in stages] == [False, False, False] assert all("Async" not in (stage.scheduler_cls or "") for stage in stages) assert deploy.stages[1].enforce_eager is False - assert deploy.stages[1].compilation_config == {"cudagraph_mode": "PIECEWISE"} - assert stages[1].yaml_extras["default_sampling_params"]["max_tokens"] == 4096 - assert stages[1].yaml_extras["default_sampling_params"]["stop_token_ids"] == [151645] + assert stages[1].yaml_extras["default_sampling_params"]["max_tokens"] == 2048 + assert stages[1].yaml_extras["default_sampling_params"]["stop_token_ids"] == [1] @pytest.mark.parametrize( - ("filename", "stage0_devices", "stage1_devices", "stage1_replicas"), + ("filename", "stage0_devices", "stage1_devices", "stage2_devices", "stage1_replicas"), [ - ("minicpmo_4_5_3gpu_stage1_replicas.yaml", "0", "1,2", 2), - ("minicpmo_4_5_4gpu_stage1_replicas.yaml", "0", "1,2,3", 3), - ("minicpmo_4_5_8x4090_stage1_replicas.yaml", "0,1,2,3", "4,5,6,7", 4), + ("minicpmo_4_5_3gpu_stage1_replicas.yaml", "0", "1,2", "0", 2), + ("minicpmo_4_5_4gpu_stage1_replicas.yaml", "0", "1,2,3", "0", 3), + ("minicpmo_4_5_8x4090_stage1_replicas.yaml", "0,1,2,3", "4,5,6,7", "5", 4), ], ) def test_load_minicpmo_replica_deploy_configs( @@ -1153,6 +1152,7 @@ class TestDeployConfigLoading: filename: str, stage0_devices: str, stage1_devices: str, + stage2_devices: str, stage1_replicas: int, ): deploy_path = Path(__file__).parent.parent / "vllm_omni" / "deploy" / filename @@ -1164,11 +1164,15 @@ class TestDeployConfigLoading: stages = merge_pipeline_deploy(pipeline, deploy) assert deploy.session_mode == "duplex" - assert [stage.yaml_runtime["devices"] for stage in stages] == [stage0_devices, stage1_devices] + assert [stage.yaml_runtime["devices"] for stage in stages] == [ + stage0_devices, + stage1_devices, + stage2_devices, + ] assert stages[0].yaml_runtime["num_replicas"] == 1 assert stages[1].yaml_runtime["num_replicas"] == stage1_replicas - assert all(stage.yaml_engine_args["async_scheduling"] is False for stage in stages) - assert all("Async" not in (stage.scheduler_cls or "") for stage in stages) + assert all(stage.yaml_engine_args["async_scheduling"] is False for stage in stages[:2]) + assert all("Async" not in (stage.scheduler_cls or "") for stage in stages[:2]) def test_custom_voice_dir_is_pipeline_wide_engine_arg(self, tmp_path): deploy_path = tmp_path / "qwen3_tts_custom_voice.yaml" diff --git a/tests/worker/test_gpu_ar_model_runner.py b/tests/worker/test_gpu_ar_model_runner.py index 90cf3afe1..83d270da4 100644 --- a/tests/worker/test_gpu_ar_model_runner.py +++ b/tests/worker/test_gpu_ar_model_runner.py @@ -663,6 +663,7 @@ def test_sample_tokens_tail_only_prefix_cache_uses_staged_cpu_hidden_states(monk # inter_stage_outputs mirrors multimodal_outputs (PR #4792). assert output.inter_stage_outputs is not None assert output.multimodal_outputs is not None + assert output.inter_stage_outputs is output.multimodal_outputs assert torch.equal(output.inter_stage_outputs[0]["hidden"], output.multimodal_outputs[0]["hidden"]) assert torch.equal(output.inter_stage_outputs[1]["hidden"], output.multimodal_outputs[1]["hidden"]) assert torch.equal(output.multimodal_outputs[0]["hidden"], torch.tensor([[1.0, 10.0]])) diff --git a/tests/worker/test_native_duplex_hooks.py b/tests/worker/test_native_duplex_hooks.py index a5a81a1de..661656b7e 100644 --- a/tests/worker/test_native_duplex_hooks.py +++ b/tests/worker/test_native_duplex_hooks.py @@ -1,8 +1,6 @@ from __future__ import annotations import inspect -import os -import threading from types import SimpleNamespace import numpy as np @@ -544,220 +542,6 @@ def test_minicpmo_stage0_rejects_invalid_resolved_ref_audio(): ) -def test_minicpmo_tts_native_duplex_exports_segment_text_not_accumulated_condition_text(): - from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni import ( - MiniCPMO45OmniForConditionalGeneration, - ) - - class _Talker: - _ar_last_chunk_flags = [False] - _ar_last_emitted_text = "你好,你有什莫想聊的吗?" - - def __call__(self, **kwargs): - del kwargs - return None, torch.zeros(8, dtype=torch.float32) - - model = MiniCPMO45OmniForConditionalGeneration.__new__(MiniCPMO45OmniForConditionalGeneration) - model.model_stage = "tts" - model.config = SimpleNamespace(hidden_size=4) - model.talker = _Talker() - - output = model.forward( - input_ids=torch.zeros(1, dtype=torch.long), - positions=torch.zeros(1, dtype=torch.long), - runtime_additional_information=[ - { - "native_duplex": True, - "llm_output_text": ["你好,你有什莫想聊的吗?你好,你有什莫想聊的吗?"], - "meta": { - "native_duplex_segment_text": "你好,你有什莫想聊的吗?", - }, - } - ], - ) - - text_bytes = output.multimodal_outputs["meta.llm_output_text_utf8"].detach().cpu().tolist() - assert bytes(text_bytes).decode("utf-8") == "你好,你有什莫想聊的吗?" - assert int(output.multimodal_outputs["meta.audio_text_total_chars"].item()) == len("你好,你有什莫想聊的吗?") - - -def test_minicpmo_tts_native_duplex_exports_model_turn_end_metadata(): - from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni import ( - MiniCPMO45OmniForConditionalGeneration, - ) - - class _Talker: - _ar_last_chunk_flags = [True] - _ar_turn_end_flags = [True] - - def __call__(self, **kwargs): - del kwargs - return None, torch.zeros(0, dtype=torch.float32) - - model = MiniCPMO45OmniForConditionalGeneration.__new__(MiniCPMO45OmniForConditionalGeneration) - model.model_stage = "tts" - model.config = SimpleNamespace(hidden_size=4) - model.talker = _Talker() - - output = model.forward( - input_ids=torch.zeros(1, dtype=torch.long), - positions=torch.zeros(1, dtype=torch.long), - runtime_additional_information=[ - { - "native_duplex": True, - "meta": {"native_duplex_segment_text": ""}, - } - ], - ) - - assert int(output.multimodal_outputs["meta.turn_end"].item()) == 1 - - -def test_minicpmo_tts_routes_batched_requests_and_terminal_flags_independently(): - from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni import ( - MiniCPMO45OmniForConditionalGeneration, - ) - - class _Talker: - def __init__(self): - self.calls = [] - self._ar_last_chunk_flags = [True] - self._ar_turn_end_flags = [False] - self._ar_last_emitted_text = "" - - def __call__(self, **kwargs): - info = kwargs["additional_information"] - request_id = info["global_request_id"][0] - self.calls.append(request_id) - is_last = request_id == "req-b" - self._ar_last_chunk_flags = [is_last] - self._ar_turn_end_flags = [is_last] - self._ar_last_emitted_text = request_id - value = 1.0 if request_id == "req-a" else 2.0 - return None, torch.full((2,), value, dtype=torch.float32) - - model = MiniCPMO45OmniForConditionalGeneration.__new__(MiniCPMO45OmniForConditionalGeneration) - model.model_stage = "tts" - model.config = SimpleNamespace(hidden_size=4) - model.talker = _Talker() - - output = model.forward( - input_ids=torch.zeros(3, dtype=torch.long), - positions=torch.zeros(3, dtype=torch.long), - runtime_additional_information=[ - { - "global_request_id": ["req-a"], - "native_duplex": True, - "duplex": {"epoch": 3, "turn_id": 7}, - "meta": {"native_duplex_segment_text": "first"}, - }, - { - "global_request_id": ["req-idle"], - "native_duplex": True, - "duplex": {"epoch": 3, "turn_id": 9}, - "meta": {"native_duplex_segment_text": "idle"}, - }, - { - "global_request_id": ["req-b"], - "native_duplex": True, - "duplex": {"epoch": 4, "turn_id": 8}, - "meta": {"native_duplex_segment_text": "second"}, - }, - ], - request_token_spans=[(0, 2), (2, 2), (2, 3)], - ) - - assert model.talker.calls == ["req-a", "req-b"] - assert model.talker._ar_last_chunk_flags == [False, False, True] - assert model.talker._ar_turn_end_flags == [False, False, True] - assert output.multimodal_outputs["meta.req_id"] == ["req-a", "req-b"] - assert output.multimodal_outputs["meta.sparse_audio"] == ["1"] - assert [waveform.tolist() for waveform in output.multimodal_outputs["model_outputs"]] == [ - [1.0, 1.0], - [2.0, 2.0], - ] - assert [int(value.item()) for value in output.multimodal_outputs["meta.tts_is_last_chunk"]] == [0, 1] - assert [int(value.item()) for value in output.multimodal_outputs["meta.turn_end"]] == [0, 1] - assert [int(value.item()) for value in output.multimodal_outputs["meta.duplex_epoch"]] == [3, 4] - assert [int(value.item()) for value in output.multimodal_outputs["meta.duplex_turn_id"]] == [7, 8] - - -@pytest.mark.parametrize( - ("runtime_info", "request_token_spans", "expected_rows", "expected_ids"), - [ - ( - [ - { - "global_request_id": ["req-a"], - "native_duplex": True, - "duplex": {"epoch": 3, "turn_id": 7}, - "meta": {"native_duplex_segment_text": "first"}, - } - ], - [(0, 3)], - [3], - ["req-a"], - ), - ( - [ - { - "global_request_id": ["req-a"], - "native_duplex": True, - "duplex": {"epoch": 3, "turn_id": 7}, - "meta": {"native_duplex_segment_text": "first"}, - }, - { - "global_request_id": ["req-b"], - "native_duplex": True, - "duplex": {"epoch": 4, "turn_id": 8}, - "meta": {"native_duplex_segment_text": "second"}, - }, - ], - [(0, 2), (2, 3)], - [2, 1], - ["req-a", "req-b"], - ), - ], - ids=["single-request", "batched-requests"], -) -def test_minicpmo_tts_ignores_cuda_graph_padding_after_request_spans( - runtime_info, - request_token_spans, - expected_rows, - expected_ids, -): - from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni import ( - MiniCPMO45OmniForConditionalGeneration, - ) - - class _Talker: - def __init__(self): - self.input_rows = [] - self._ar_last_chunk_flags = [True] - self._ar_turn_end_flags = [False] - self._ar_last_emitted_text = "" - - def __call__(self, **kwargs): - self.input_rows.append(int(kwargs["input_ids"].shape[0])) - return None, torch.ones(2, dtype=torch.float32) - - model = MiniCPMO45OmniForConditionalGeneration.__new__(MiniCPMO45OmniForConditionalGeneration) - model.model_stage = "tts" - model.config = SimpleNamespace(hidden_size=4) - model.talker = _Talker() - - output = model.forward( - input_ids=torch.zeros(4, dtype=torch.long), - positions=torch.zeros(4, dtype=torch.long), - runtime_additional_information=runtime_info, - request_token_spans=request_token_spans, - ) - - assert model.talker.input_rows == expected_rows - assert output.text_hidden_states.shape == (4, 4) - assert output.multimodal_outputs["meta.req_id"] == expected_ids - - def test_minicpmo_stage0_special_token_ids_are_tokenizer_derived(): from vllm_omni.experimental.fullduplex.minicpmo45.stage0 import ( MiniCPMO45Stage0DuplexRuntime, @@ -2427,150 +2211,3 @@ def test_minicpmo_stage0_session_context_includes_resolved_ref_audio(): assert state.context_token_ids == [1, 2, 3, 151683, 151683, 4, 5] assert len(state.context_embeds) == 6 - - -def test_minicpmo_tts_ref_audio_lru_evicts_matching_vocoder_base_cache(tmp_path): - from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni_tts import ( - MiniCPMO45OmniTTSForConditionalGeneration, - MiniCPMO45TTSRuntimeConfig, - ) - - model = MiniCPMO45OmniTTSForConditionalGeneration.__new__(MiniCPMO45OmniTTSForConditionalGeneration) - model._runtime_config = MiniCPMO45TTSRuntimeConfig(ref_audio_file_cache_size=1) - model._t2w_base_caches = {} - model._token2wav_state_lock = threading.RLock() - first_path = model._write_ref_audio_prompt_wav(np.array([0.1, -0.1], dtype=np.float32), 16_000) - assert first_path is not None - model._t2w_base_caches[first_path] = (torch.tensor([1]), {"cache": torch.tensor([2])}) - - second_path = model._write_ref_audio_prompt_wav(np.array([0.2, -0.2], dtype=np.float32), 16_000) - - assert second_path is not None - assert second_path != first_path - assert first_path not in model._t2w_base_caches - assert not os.path.exists(first_path) - - -def test_minicpmo_tts_prompt_wav_path_does_not_default_to_model_asset(tmp_path): - from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni_tts import ( - MiniCPMO45OmniTTSForConditionalGeneration, - ) - - model_dir = tmp_path / "MiniCPM-o-4_5" - assets_dir = model_dir / "assets" - assets_dir.mkdir(parents=True) - (assets_dir / "HT_ref_audio.wav").write_bytes(b"placeholder") - - model = MiniCPMO45OmniTTSForConditionalGeneration.__new__(MiniCPMO45OmniTTSForConditionalGeneration) - model.vllm_config = SimpleNamespace(model_config=SimpleNamespace(model=str(model_dir))) - model._write_ref_audio_prompt_wav = lambda ref_audio, ref_audio_sr: None - - assert model._resolve_prompt_wav_path(None, None) == (None, None) - - -def test_minicpmo_tts_stream_turn_id_prefers_duplex_identity(): - from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni_tts import ( - MiniCPMO45OmniTTSForConditionalGeneration, - ) - - model = MiniCPMO45OmniTTSForConditionalGeneration.__new__(MiniCPMO45OmniTTSForConditionalGeneration) - - assert ( - model._stream_turn_id( - { - "duplex": {"turn_id": 0, "model_turn_id": 4}, - "meta": {"turn_id": 3}, - } - ) - == 4 - ) - assert ( - model._stream_turn_id( - { - "duplex": {"turn_id": 0}, - "meta": {"turn_id": 3}, - } - ) - == 0 - ) - assert model._stream_turn_id({"meta": {"turn_id": 3}, "turn_id": 2}) == 3 - assert model._stream_turn_id({"turn_id": 2}) == 2 - - -def test_minicpmo_tts_no_ref_audio_initializes_empty_vocoder_cache(): - from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni_tts import ( - MiniCPMO45OmniTTSForConditionalGeneration, - _TalkerTurnState, - ) - - class AudioTokenizer: - def set_stream_cache(self, prompt_wav_path): - raise AssertionError(f"unexpected reference-audio cache load: {prompt_wav_path!r}") - - model = MiniCPMO45OmniTTSForConditionalGeneration.__new__(MiniCPMO45OmniTTSForConditionalGeneration) - model._t2w_base_caches = {} - model._token2wav_state_lock = threading.RLock() - model.audio_tokenizer = AudioTokenizer() - state = _TalkerTurnState(None, None) - - model._begin_turn_vocoder_cache(None, state=state) - - assert state.stream_cache is None - assert state.hift_cache_dict == {} - assert state.vocoder_initialized is True - assert model._t2w_base_caches[""] == (None, {}) - - -def test_minicpmo_tts_native_duplex_merges_and_left_pads_nonfinal_unit_audio(): - from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni_tts import ( - _native_duplex_unit_waveform, - ) - - waveform = _native_duplex_unit_waveform( - [torch.tensor([1.0, 2.0]), torch.tensor([3.0])], - turn_end=False, - target_samples=6, - ) - - assert waveform is not None - assert waveform.tolist() == [0.0, 0.0, 0.0, 1.0, 2.0, 3.0] - - -def test_minicpmo_tts_native_duplex_does_not_pad_final_unit_audio(): - from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni_tts import ( - _native_duplex_unit_waveform, - ) - - waveform = _native_duplex_unit_waveform( - [torch.tensor([1.0, 2.0]), torch.tensor([3.0])], - turn_end=True, - target_samples=6, - ) - - assert waveform is not None - assert waveform.tolist() == [1.0, 2.0, 3.0] - - -def test_minicpmo_tts_turn_cleanup_removes_deleted_ref_audio_vocoder_cache(tmp_path): - from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni_tts import ( - MiniCPMO45OmniTTSForConditionalGeneration, - _TalkerTurnState, - ) - - prompt_path = tmp_path / "session-ref.wav" - prompt_path.write_bytes(b"wav") - model = MiniCPMO45OmniTTSForConditionalGeneration.__new__(MiniCPMO45OmniTTSForConditionalGeneration) - model._talker_turn_states = { - "request": _TalkerTurnState(str(prompt_path), str(prompt_path)), - } - model._talker_consumed_tokens = {"request": 1} - model._talker_request_keys = {} - model._t2w_base_caches = { - str(prompt_path): (torch.tensor([1]), {"cache": torch.tensor([2])}), - } - model._token2wav_state_lock = threading.RLock() - model.audio_tokenizer = None - - assert model._close_turn_state("request") is True - assert str(prompt_path) not in model._t2w_base_caches - assert not prompt_path.exists() diff --git a/tests/worker/test_omni_gpu_model_runner.py b/tests/worker/test_omni_gpu_model_runner.py index cff36b9f6..701e3b589 100644 --- a/tests/worker/test_omni_gpu_model_runner.py +++ b/tests/worker/test_omni_gpu_model_runner.py @@ -440,6 +440,33 @@ def test_update_intermediate_buffer_accumulates(): assert torch.allclose(buf["b"], torch.tensor([2.0])) +def test_update_additional_information_deserializes_new_request_payload(): + from vllm_omni.engine.serialization import serialize_additional_information + + runner = _make_runner(req_ids=("r1",), hidden_size=4) + conditioning = { + "tts_token_ids": torch.tensor([1, 2]), + "tts_hidden_states": torch.ones(2, 4), + } + scheduler_output = SimpleNamespace( + scheduled_new_reqs=[ + SimpleNamespace( + req_id="r1", + additional_information=serialize_additional_information(conditioning), + ) + ], + scheduled_cached_reqs=SimpleNamespace(), + ) + + OmniGPUModelRunner._update_additional_information(runner, scheduler_output) + + assert torch.equal(runner.model_intermediate_buffer["r1"]["tts_token_ids"], conditioning["tts_token_ids"]) + assert torch.equal( + runner.model_intermediate_buffer["r1"]["tts_hidden_states"], + conditioning["tts_hidden_states"], + ) + + def test_update_intermediate_buffer_skips_empty_update(): """Validate that an empty update dict is a no-op.""" runner = _make_runner(req_ids=("r1",), hidden_size=4) diff --git a/vllm_omni/config/stage_config.py b/vllm_omni/config/stage_config.py index 5e16140b8..2a8efe824 100644 --- a/vllm_omni/config/stage_config.py +++ b/vllm_omni/config/stage_config.py @@ -921,6 +921,10 @@ def merge_pipeline_deploy( stage_type, worker_type = _resolve_execution_mode(ps.execution_type) input_proc, next_stage_proc = _select_processor_funcs(ps, deploy.async_chunk) engine_args = _build_engine_args(ps, ds, pipeline, deploy, next_stage_proc) + # Downstream stages may share a multimodal wrapper class without owning + # an encoder. Do not make vLLM profile dummy multimodal inputs for them. + if not ps.requires_multimodal_data: + engine_args.setdefault("skip_mm_profiling", True) sched_cls = _resolve_scheduler( ps.execution_type, engine_args.get("async_scheduling", True), diff --git a/vllm_omni/core/sched/omni_ar_scheduler.py b/vllm_omni/core/sched/omni_ar_scheduler.py index e88730353..93654f27f 100644 --- a/vllm_omni/core/sched/omni_ar_scheduler.py +++ b/vllm_omni/core/sched/omni_ar_scheduler.py @@ -700,9 +700,10 @@ class OmniARScheduler(OmniSchedulerMixin, VLLMScheduler): session.num_computed_tokens -= session.num_output_placeholders session.num_output_placeholders = 0 session.spec_token_ids = [] - if self.chunk_transfer_adapter: + stage_id = self.vllm_config.model_config.stage_id + if self.chunk_transfer_adapter and self.chunk_transfer_adapter.receives_chunks: self.chunk_transfer_adapter.requests_num_chunks_sent.pop(session.external_req_id, None) - if self.vllm_config.model_config.stage_id != 0: + if stage_id != 0: # Downstream async-chunk stages receive real payloads from the # connector. This update only resumes polling for the next segment. self.chunk_transfer_adapter.segment_finished_requests.discard(session.request_id) @@ -720,6 +721,11 @@ class OmniARScheduler(OmniSchedulerMixin, VLLMScheduler): if self.log_stats: session.record_event(EngineCoreEventType.QUEUED) return + update_info = getattr(update, "additional_information", None) + update_meta = update_info.get("meta") if isinstance(update_info, dict) else None + if isinstance(update_meta, dict) and update_meta.get("replace_streaming_prompt") is True: + self._replace_streaming_session(session, update) + return super()._update_request_as_session(session, update) if hasattr(update, "model_intermediate_buffer"): session.model_intermediate_buffer = update.model_intermediate_buffer diff --git a/vllm_omni/core/sched/omni_generation_scheduler.py b/vllm_omni/core/sched/omni_generation_scheduler.py index c6f82ec23..3882fd42d 100644 --- a/vllm_omni/core/sched/omni_generation_scheduler.py +++ b/vllm_omni/core/sched/omni_generation_scheduler.py @@ -685,26 +685,4 @@ class OmniGenerationScheduler(OmniSchedulerMixin, VLLMScheduler): Do not expend prompt id using update. """ - if self.chunk_transfer_adapter: - self.chunk_transfer_adapter.segment_finished_requests.discard(session.request_id) - session._output_token_ids.clear() - session._all_token_ids.clear() - new_prompt = update.prompt_token_ids or () - session._all_token_ids.extend(new_prompt) - session.num_computed_tokens = 0 - session.prompt_token_ids = update.prompt_token_ids or () - session.additional_information = update.additional_information or None - # Update block hashes for the new tokens. - session.update_block_hashes() - session.num_prompt_tokens = len(session.prompt_token_ids) - session.arrival_time = update.arrival_time - session.sampling_params = update.sampling_params - if session.status == RequestStatus.WAITING_FOR_STREAMING_REQ: - self.num_waiting_for_streaming_input -= 1 - session.status = RequestStatus.WAITING - if session in self.skipped_waiting: - self.skipped_waiting.remove_requests((session,)) - self._enqueue_waiting_request(session) - - if self.log_stats: - session.record_event(EngineCoreEventType.QUEUED) + self._replace_streaming_session(session, update) diff --git a/vllm_omni/core/sched/omni_scheduler_mixin.py b/vllm_omni/core/sched/omni_scheduler_mixin.py index 4eaa07cbb..3df47a357 100644 --- a/vllm_omni/core/sched/omni_scheduler_mixin.py +++ b/vllm_omni/core/sched/omni_scheduler_mixin.py @@ -7,8 +7,9 @@ from typing import Any from vllm.logger import init_logger from vllm.v1.core.sched.output import SchedulerOutput +from vllm.v1.engine import EngineCoreEventType from vllm.v1.metrics.stats import SchedulerStats -from vllm.v1.request import RequestStatus +from vllm.v1.request import Request, RequestStatus, StreamingUpdate from vllm_omni.core.sched.output import OmniChunkRecvHandle, OmniSchedulerOutput @@ -50,6 +51,31 @@ class OmniSchedulerMixin: if input_coordinator is not None: input_coordinator.free_finished_request(request_id) + def _replace_streaming_session(self, session: Request, update: StreamingUpdate) -> None: + """Replace a downstream stage's placeholder with its next payload.""" + adapter = getattr(self, "chunk_transfer_adapter", None) + if adapter is not None: + adapter.segment_finished_requests.discard(session.request_id) + session._output_token_ids.clear() + session._all_token_ids.clear() + new_prompt = update.prompt_token_ids or () + session._all_token_ids.extend(new_prompt) + session.num_computed_tokens = 0 + session.prompt_token_ids = new_prompt + session.additional_information = update.additional_information or None + session.update_block_hashes() + session.num_prompt_tokens = len(new_prompt) + session.arrival_time = update.arrival_time + session.sampling_params = update.sampling_params + if session.status == RequestStatus.WAITING_FOR_STREAMING_REQ: + self.num_waiting_for_streaming_input -= 1 + session.status = RequestStatus.WAITING + if session in self.skipped_waiting: + self.skipped_waiting.remove_requests((session,)) + self._enqueue_waiting_request(session) + if self.log_stats: + session.record_event(EngineCoreEventType.QUEUED) + def _consume_pending_connector_output(self, model_mode: str) -> None: """Drain ``self._latest_omni_connector_output`` into the coordinator. diff --git a/vllm_omni/data_entry_keys.py b/vllm_omni/data_entry_keys.py index eec3e5c33..db2cf15ea 100644 --- a/vllm_omni/data_entry_keys.py +++ b/vllm_omni/data_entry_keys.py @@ -61,12 +61,20 @@ class OmniPayloadMeta(TypedDict, total=False): finished: torch.Tensor is_segment_finished: torch.Tensor stream_finished: torch.Tensor + request_id: str + chunk_seq: int + cache_epoch: int + codec_chunk_frames: int + codec_left_context_frames: int + code_flat_numel: int + last_chunk: bool req_id: list[str] left_context_size: int right_holdback_size: int override_keys: list[tuple[str, str]] num_processed_tokens: int next_stage_prompt_len: int + replace_streaming_prompt: bool ar_width: int eol_token_id: int visual_token_start_id: int @@ -82,6 +90,7 @@ class OmniPayloadMeta(TypedDict, total=False): ref_context_request_id: str ref_context_included: bool talker_prefill_offset: int + omni_final_stage_id: int class OmniPayload(TypedDict, total=False): @@ -150,12 +159,17 @@ class MetaStruct(_StructBase): finished: torch.Tensor | None = None is_segment_finished: torch.Tensor | None = None stream_finished: torch.Tensor | None = None + request_id: str | None = None + chunk_seq: int | None = None + cache_epoch: int | None = None + last_chunk: bool | None = None req_id: list[str] | None = None left_context_size: int | None = None right_holdback_size: int | None = None override_keys: list[tuple[str, str]] | None = None num_processed_tokens: int | None = None next_stage_prompt_len: int | None = None + replace_streaming_prompt: bool | None = None ar_width: int | None = None eol_token_id: int | None = None visual_token_start_id: int | None = None diff --git a/vllm_omni/deploy/minicpmo_4_5.yaml b/vllm_omni/deploy/minicpmo_4_5.yaml index d191dddcc..da84667af 100644 --- a/vllm_omni/deploy/minicpmo_4_5.yaml +++ b/vllm_omni/deploy/minicpmo_4_5.yaml @@ -1,20 +1,24 @@ -# MiniCPM-o 4.5 default deploy: single-GPU layout. -# -# Both stages are co-located on GPU 0: -# Stage 0 (thinker): multimodal understanding, ~80% mem. -# Stage 1 (talker) : MiniCPMTTS + in-process Token2Wav vocoder, ~10% mem -# (talker LM plus the vocoder that emits the waveform). -# -# Other GPU layouts: minicpmo_4_5_3gpu.yaml, minicpmo_4_5_8x4090.yaml. -async_chunk: false +# MiniCPM-o 4.5 three-stage deploy on one large-memory GPU. +pipeline: minicpmo_4_5 +async_chunk: true trust_remote_code: true enable_prefix_caching: false +connectors: + connector_of_shared_memory: + name: SharedMemoryConnector + extra: + connector_get_sleep_s: 0.01 + connector_get_max_wait_first_chunk: 3000 + connector_get_max_wait: 300 + codec_chunk_frames: 25 + codec_left_context_frames: 3 + stages: - stage_id: 0 - max_num_seqs: 16 - gpu_memory_utilization: 0.8 + max_num_seqs: 4 + gpu_memory_utilization: 0.65 enforce_eager: false tensor_parallel_size: 1 max_num_batched_tokens: 16384 @@ -31,21 +35,42 @@ stages: repetition_penalty: 1.1 - stage_id: 1 - # Talker only consumes runtime_additional_information[0]; any value > 1 - # makes concurrent requests share request-0's audio (mm_outputs fallback). - max_num_seqs: 1 - gpu_memory_utilization: 0.1 + max_num_seqs: 4 + gpu_memory_utilization: 0.15 enforce_eager: false max_num_batched_tokens: 8192 devices: "0" + output_connectors: + to_stage_2: connector_of_shared_memory + # Codec sampling comes from the checkpoint's tts_config. These parameters + # govern only vLLM's binary continue/stop token. default_sampling_params: temperature: 0.0 top_p: 1.0 top_k: -1 - max_tokens: 1 + min_tokens: 50 + max_tokens: 2048 + stop_token_ids: [1] seed: 42 detokenize: false + - stage_id: 2 + max_num_seqs: 4 + gpu_memory_utilization: 0.15 + enforce_eager: true + enable_chunked_prefill: false + max_num_batched_tokens: 65536 + max_model_len: 65536 + devices: "0" + input_connectors: + from_stage_1: connector_of_shared_memory + default_sampling_params: + temperature: 0.0 + top_p: 1.0 + top_k: -1 + max_tokens: 65536 + detokenize: true + platforms: npu: stages: diff --git a/vllm_omni/deploy/minicpmo_4_5_2gpu.yaml b/vllm_omni/deploy/minicpmo_4_5_2gpu.yaml index 7a9c400cc..eddfd0877 100644 --- a/vllm_omni/deploy/minicpmo_4_5_2gpu.yaml +++ b/vllm_omni/deploy/minicpmo_4_5_2gpu.yaml @@ -1,64 +1,2 @@ -# MiniCPM-o 4.5 deploy: 2-GPU layout (thinker on GPU 0, talker on GPU 1). -# -# Stage 0 (thinker): GPU 0 — multimodal understanding, ~90% mem (thinker -# gets a dedicated card, so it can take more of -# the GPU than the single-GPU co-located layout). -# Stage 1 (talker) : GPU 1 — MiniCPMTTS + Token2Wav, ~75% mem -# (covers both the talker LM and the in-process -# Token2Wav vocoder that emits the waveform). -# -# Use this when you have two GPUs and want the talker + Token2Wav vocoder -# on its own card instead of sharing GPU 0 with the thinker (the default -# single-GPU layout co-locates both and can OOM under load). -pipeline: minicpmo_4_5 -async_chunk: false - -trust_remote_code: true -enable_prefix_caching: false - -stages: - - stage_id: 0 - max_num_seqs: 16 - gpu_memory_utilization: 0.9 - enforce_eager: false - tensor_parallel_size: 1 - max_num_batched_tokens: 16384 - devices: "0" - limit_mm_per_prompt: - video: 1 - default_sampling_params: - temperature: 0.0 - top_p: 1.0 - top_k: -1 - max_tokens: 2048 - seed: 42 - detokenize: true - repetition_penalty: 1.1 - - - stage_id: 1 - # Talker only consumes runtime_additional_information[0]; any value > 1 - # makes concurrent requests share request-0's audio (mm_outputs fallback). - max_num_seqs: 1 - gpu_memory_utilization: 0.75 - enforce_eager: false - max_num_batched_tokens: 8192 - devices: "1" - default_sampling_params: - temperature: 0.0 - top_p: 1.0 - top_k: -1 - max_tokens: 1 - seed: 42 - detokenize: false - -platforms: - npu: - stages: - - stage_id: 0 - max_num_batched_tokens: 8192 - compilation_config: - cudagraph_mode: PIECEWISE - - stage_id: 1 - max_num_batched_tokens: 8192 - compilation_config: - cudagraph_mode: PIECEWISE +# Backward-compatible name for the canonical two-GPU batching profile. +base_config: minicpmo_4_5_batching.yaml diff --git a/vllm_omni/deploy/minicpmo_4_5_3gpu.yaml b/vllm_omni/deploy/minicpmo_4_5_3gpu.yaml index 69f77eb6d..9254d3c70 100644 --- a/vllm_omni/deploy/minicpmo_4_5_3gpu.yaml +++ b/vllm_omni/deploy/minicpmo_4_5_3gpu.yaml @@ -1,23 +1,31 @@ -# MiniCPM-o 4.5 deploy: 3-GPU layout (thinker TP=2 + talker on GPU 2). +# MiniCPM-o 4.5 three-stage deploy with one GPU per stage. # -# Stage 0 (thinker): GPU 0,1 — 2-way tensor parallel, ~70% mem each. -# Stage 1 (talker) : GPU 2 — MiniCPMTTS + Token2Wav, ~75% mem -# (covers both the talker LM and the in-process -# Token2Wav vocoder that emits the waveform). +# This replaces the former fused two-stage layout: Thinker, Talker, and +# Code2Wav run independently on GPUs 0, 1, and 2. pipeline: minicpmo_4_5 -async_chunk: false +async_chunk: true trust_remote_code: true enable_prefix_caching: false +connectors: + connector_of_shared_memory: + name: SharedMemoryConnector + extra: + connector_get_sleep_s: 0.01 + connector_get_max_wait_first_chunk: 3000 + connector_get_max_wait: 300 + codec_chunk_frames: 25 + codec_left_context_frames: 3 + stages: - stage_id: 0 - max_num_seqs: 16 - gpu_memory_utilization: 0.7 + max_num_seqs: 4 + gpu_memory_utilization: 0.9 enforce_eager: false - tensor_parallel_size: 2 + tensor_parallel_size: 1 max_num_batched_tokens: 16384 - devices: "0,1" + devices: "0" limit_mm_per_prompt: video: 1 default_sampling_params: @@ -30,19 +38,40 @@ stages: repetition_penalty: 1.1 - stage_id: 1 - # Talker only consumes runtime_additional_information[0]; any value > 1 - # makes concurrent requests share request-0's audio (mm_outputs fallback). - max_num_seqs: 1 - gpu_memory_utilization: 0.75 + max_num_seqs: 4 + gpu_memory_utilization: 0.9 enforce_eager: false max_num_batched_tokens: 8192 - devices: "2" + devices: "1" + output_connectors: + to_stage_2: connector_of_shared_memory + # vLLM samples the binary continue/stop row; codec sampling comes from + # the checkpoint's tts_config and currently uses a deterministic seed. default_sampling_params: temperature: 0.0 top_p: 1.0 top_k: -1 - max_tokens: 1 + min_tokens: 50 + max_tokens: 2048 + stop_token_ids: [1] seed: 42 detokenize: false + - stage_id: 2 + max_num_seqs: 4 + gpu_memory_utilization: 0.9 + enforce_eager: true + enable_chunked_prefill: false + max_num_batched_tokens: 65536 + max_model_len: 65536 + devices: "2" + input_connectors: + from_stage_1: connector_of_shared_memory + default_sampling_params: + temperature: 0.0 + top_p: 1.0 + top_k: -1 + max_tokens: 65536 + detokenize: true + platforms: {} diff --git a/vllm_omni/deploy/minicpmo_4_5_8x4090.yaml b/vllm_omni/deploy/minicpmo_4_5_8x4090.yaml index dd249c70a..1b38059a3 100644 --- a/vllm_omni/deploy/minicpmo_4_5_8x4090.yaml +++ b/vllm_omni/deploy/minicpmo_4_5_8x4090.yaml @@ -4,10 +4,8 @@ # thinker weights take ~17.5 GB per card # on 4090, leaving ~3 GB for KV cache / # activations). -# Stage 1 (talker) : GPU 4 — MiniCPMTTS + Token2Wav, ~90% mem -# (covers both the talker LM and the -# in-process Token2Wav vocoder that emits -# the waveform). +# Stage 1 (talker) : GPU 4 — MiniCPMTTS codec generation, ~90% mem. +# Stage 2 (code2wav): GPU 5 — codec-to-waveform generation, ~90% mem. # # max_model_len is pinned to 4096 because the upstream config.json # advertises max_position_embeddings=40960 (tuned for H20 141GB); on a @@ -16,11 +14,21 @@ # this thinker model + vision/audio encoders + Whisper preallocation, # so 4096 is the empirically-verified safe ceiling. pipeline: minicpmo_4_5 -async_chunk: false +async_chunk: true trust_remote_code: true enable_prefix_caching: false +connectors: + connector_of_shared_memory: + name: SharedMemoryConnector + extra: + connector_get_sleep_s: 0.01 + connector_get_max_wait_first_chunk: 3000 + connector_get_max_wait: 300 + codec_chunk_frames: 25 + codec_left_context_frames: 3 + stages: - stage_id: 0 max_num_seqs: 1 @@ -42,19 +50,40 @@ stages: repetition_penalty: 1.1 - stage_id: 1 - # Talker only consumes runtime_additional_information[0]; any value > 1 - # makes concurrent requests share request-0's audio (mm_outputs fallback). max_num_seqs: 1 gpu_memory_utilization: 0.9 enforce_eager: true max_num_batched_tokens: 8192 devices: "4" + output_connectors: + to_stage_2: connector_of_shared_memory + # vLLM samples the binary continue/stop row; codec sampling comes from + # the checkpoint's tts_config and currently uses a deterministic seed. default_sampling_params: temperature: 0.0 top_p: 1.0 top_k: -1 - max_tokens: 1 + min_tokens: 50 + max_tokens: 2048 + stop_token_ids: [1] seed: 42 detokenize: false + - stage_id: 2 + max_num_seqs: 1 + gpu_memory_utilization: 0.9 + enforce_eager: true + enable_chunked_prefill: false + max_num_batched_tokens: 65536 + max_model_len: 65536 + devices: "5" + input_connectors: + from_stage_1: connector_of_shared_memory + default_sampling_params: + temperature: 0.0 + top_p: 1.0 + top_k: -1 + max_tokens: 65536 + detokenize: true + platforms: {} diff --git a/vllm_omni/deploy/minicpmo_4_5_batching.yaml b/vllm_omni/deploy/minicpmo_4_5_batching.yaml new file mode 100644 index 000000000..75693feb7 --- /dev/null +++ b/vllm_omni/deploy/minicpmo_4_5_batching.yaml @@ -0,0 +1,88 @@ +# MiniCPM-o 4.5 two-GPU continuous-batching layout. +# +# Thinker owns GPU 0. Talker and Code2Wav share GPU 1, keeping text decoding +# isolated from codec/vocoder contention while preserving stage-2 batching. +pipeline: minicpmo_4_5 +async_chunk: true + +trust_remote_code: true +enable_prefix_caching: false + +connectors: + connector_of_shared_memory: + name: SharedMemoryConnector + extra: + connector_get_sleep_s: 0.01 + connector_get_max_wait_first_chunk: 3000 + connector_get_max_wait: 300 + codec_chunk_frames: 25 + codec_left_context_frames: 3 + +stages: + - stage_id: 0 + max_num_seqs: 4 + gpu_memory_utilization: 0.9 + enforce_eager: false + tensor_parallel_size: 1 + max_num_batched_tokens: 16384 + devices: "0" + limit_mm_per_prompt: + video: 1 + default_sampling_params: + temperature: 0.0 + top_p: 1.0 + top_k: -1 + max_tokens: 2048 + seed: 42 + detokenize: true + repetition_penalty: 1.1 + + - stage_id: 1 + max_num_seqs: 4 + gpu_memory_utilization: 0.55 + enforce_eager: false + max_num_batched_tokens: 8192 + devices: "1" + output_connectors: + to_stage_2: connector_of_shared_memory + # Codec sampling is model-owned by the checkpoint's tts_config (defaults: + # T=0.8, top-k=25, top-p=0.85, repetition penalty=1.05, seed=42). + # These parameters only govern vLLM's binary continue/stop token. + default_sampling_params: + temperature: 0.0 + top_p: 1.0 + top_k: -1 + min_tokens: 50 + max_tokens: 2048 + stop_token_ids: [1] + seed: 42 + detokenize: false + + - stage_id: 2 + max_num_seqs: 4 + gpu_memory_utilization: 0.35 + enforce_eager: true + enable_chunked_prefill: false + max_num_batched_tokens: 65536 + max_model_len: 65536 + devices: "1" + input_connectors: + from_stage_1: connector_of_shared_memory + default_sampling_params: + temperature: 0.0 + top_p: 1.0 + top_k: -1 + max_tokens: 65536 + detokenize: true + +platforms: + npu: + stages: + - stage_id: 0 + max_num_batched_tokens: 8192 + compilation_config: + cudagraph_mode: PIECEWISE + - stage_id: 1 + max_num_batched_tokens: 8192 + compilation_config: + cudagraph_mode: PIECEWISE diff --git a/vllm_omni/deploy/minicpmo_4_5_duplex.yaml b/vllm_omni/deploy/minicpmo_4_5_duplex.yaml index c227f2817..e58c515c7 100644 --- a/vllm_omni/deploy/minicpmo_4_5_duplex.yaml +++ b/vllm_omni/deploy/minicpmo_4_5_duplex.yaml @@ -1,9 +1,9 @@ # MiniCPM-o 4.5 native duplex deployment. # -# The registered minicpmo_4_5 pipeline owns the fixed Stage0 -> Stage1 -# topology and llm2tts bridge. This overlay only changes session lifetime and -# the Stage1 token budget needed to drain streaming audio segments. -base_config: minicpmo_4_5.yaml +# The registered minicpmo_4_5 pipeline owns the fixed +# Thinker -> Talker -> Code2Wav topology. This overlay changes session +# lifetime and bounds all three stages to two live duplex sessions. +base_config: minicpmo_4_5_batching.yaml pipeline: minicpmo_4_5 session_mode: duplex active_stream_window: 1 @@ -27,10 +27,12 @@ stages: - stage_id: 1 max_num_seqs: 2 async_scheduling: false - # The streaming talker advances Python-owned per-request generators on - # every step, so only fixed-topology model regions are graph-capturable. - compilation_config: - cudagraph_mode: PIECEWISE + # The split Talker exposes a binary continue/stop vocabulary; native + # segment and turn boundaries travel as explicit output metadata. default_sampling_params: - max_tokens: 4096 - stop_token_ids: [151645] + max_tokens: 2048 + stop_token_ids: [1] + + - stage_id: 2 + max_num_seqs: 2 + async_scheduling: false diff --git a/vllm_omni/distributed/omni_connectors/adapter.py b/vllm_omni/distributed/omni_connectors/adapter.py index db253ebad..c124e8e43 100644 --- a/vllm_omni/distributed/omni_connectors/adapter.py +++ b/vllm_omni/distributed/omni_connectors/adapter.py @@ -229,6 +229,22 @@ def construct_next_stage_streaming_input_prompt(payload_data: dict[str, Any], re extended prompt without discarding prior computed state. """ ids = payload_data.get("ids", {}) + meta = payload_data.get("meta", {}) + replace_prompt = isinstance(meta, dict) and meta.get("replace_streaming_prompt") is True + condition_len = meta.get("next_stage_prompt_len") if replace_prompt else None + if isinstance(condition_len, int) and condition_len > 0: + # Some downstream stages consume complete, independently conditioned + # segments instead of extending an existing KV prefix. The producer + # declares that transport behavior explicitly in payload metadata. + new_prompt = [0] * condition_len + request._output_token_ids.clear() + request._all_token_ids.clear() + request._all_token_ids.extend(new_prompt) + request.prompt_token_ids = new_prompt + request.num_computed_tokens = 0 + request.num_prompt_tokens = condition_len + request.update_block_hashes() + return prompt_token_ids = ids.get("prompt", None) if not prompt_token_ids: return diff --git a/vllm_omni/distributed/omni_connectors/transfer_adapter/chunk_transfer_adapter.py b/vllm_omni/distributed/omni_connectors/transfer_adapter/chunk_transfer_adapter.py index 26be84012..ea3642c8d 100644 --- a/vllm_omni/distributed/omni_connectors/transfer_adapter/chunk_transfer_adapter.py +++ b/vllm_omni/distributed/omni_connectors/transfer_adapter/chunk_transfer_adapter.py @@ -13,7 +13,7 @@ from vllm_omni.data_entry_keys import MetaStruct, OmniPayloadStruct, unflatten_p from ..adapter import construct_next_stage_streaming_input_prompt from ..factory import OmniConnectorFactory -from ..utils.config import ConnectorSpec +from ..utils.config import ConnectorSpec, stage_receives_chunks from ..utils.logging import get_connector_logger from .base import OmniTransferAdapterBase @@ -56,6 +56,7 @@ class OmniChunkTransferAdapter(OmniTransferAdapterBase): self._active_window, ) self.connector = self.create_connector(model_config) + self.receives_chunks = stage_receives_chunks(model_config) super().__init__(model_config) self.model_mode = getattr(model_config, "worker_type", None) or "ar" # State specific to Chunk management @@ -135,7 +136,7 @@ class OmniChunkTransferAdapter(OmniTransferAdapterBase): """ stage_id = self.connector.stage_id - if stage_id == 0: + if stage_id == 0 or not self.receives_chunks: return if not hasattr(request, "additional_information"): request.additional_information = None @@ -224,7 +225,8 @@ class OmniChunkTransferAdapter(OmniTransferAdapterBase): payload_segment_finished = self._is_truthy_scalar(meta.get("is_segment_finished")) if self.model_mode == "ar": request.additional_information = payload_data - if chunk_id > 0 and request.resumable: + replace_prompt = meta.get("replace_streaming_prompt") is True + if getattr(request, "resumable", False) and (chunk_id > 0 or replace_prompt): # For new streaming input segment, we should update prompt from payload construct_next_stage_streaming_input_prompt(payload_data, request) @@ -464,6 +466,8 @@ class OmniChunkTransferAdapter(OmniTransferAdapterBase): callers that don't track aborts may omit it to keep the prior (unguarded) behaviour. """ + if not self.receives_chunks: + return if self.connector.stage_id == 0: return @@ -628,6 +632,8 @@ class OmniChunkTransferAdapter(OmniTransferAdapterBase): every parked request is restored unconditionally (the pre-purge behavior). """ + if not self.receives_chunks: + return if scheduler_requests is not None: self._purge_untracked_chunk_requests(self.waiting_for_chunk_waiting_requests, scheduler_requests) self._purge_untracked_chunk_requests(self.waiting_for_chunk_running_requests, scheduler_requests) @@ -659,6 +665,8 @@ class OmniChunkTransferAdapter(OmniTransferAdapterBase): Add additional info for cached requests and clean up ready chunks from scheduler output. """ + if not self.receives_chunks: + return stage_id = self.connector.stage_id if stage_id == 0: diff --git a/vllm_omni/distributed/omni_connectors/utils/config.py b/vllm_omni/distributed/omni_connectors/utils/config.py index ee049904f..1a6d395c2 100644 --- a/vllm_omni/distributed/omni_connectors/utils/config.py +++ b/vllm_omni/distributed/omni_connectors/utils/config.py @@ -17,6 +17,34 @@ TRANSFER_ENGINE_CONNECTOR_NAMES = frozenset( ) +def get_stage_connector_role(model_config: Any) -> str | None: + """Return the configured stage connector direction, if explicit.""" + connector_config = getattr(model_config, "stage_connector_config", None) + if isinstance(connector_config, dict): + extra = connector_config.get("extra") + else: + extra = getattr(connector_config, "extra", None) + if isinstance(extra, dict): + role = extra.get("role") + return role if isinstance(role, str) else None + return None + + +def stage_receives_chunks(model_config: Any) -> bool: + """Whether connector chunks, rather than the orchestrator, feed a stage.""" + return get_stage_connector_role(model_config) != "sender" + + +def stage_sends_async_output(model_config: Any) -> bool: + """Whether async output should be partitioned for connector transport.""" + role = get_stage_connector_role(model_config) + if role is not None: + return role == "sender" + # Preserve legacy partitioning while keeping stage-0 orchestrator bridges + # on the normal RequestOutput path. + return getattr(model_config, "stage_id", None) != 0 + + @dataclass class ConnectorSpec: """Specification for a connector instance.""" diff --git a/vllm_omni/engine/orchestrator.py b/vllm_omni/engine/orchestrator.py index e10ce8845..e37111063 100644 --- a/vllm_omni/engine/orchestrator.py +++ b/vllm_omni/engine/orchestrator.py @@ -29,6 +29,7 @@ from vllm.v1.engine.exceptions import EngineDeadError from vllm.v1.metrics.stats import IterationStats from vllm_omni.config.stage_config import DuplexSessionRuntimeConfig +from vllm_omni.distributed.omni_connectors.utils.config import stage_receives_chunks from vllm_omni.engine import OmniEngineCoreRequest from vllm_omni.engine.cfg_companion_tracker import CfgCompanionTracker from vllm_omni.engine.membership_controller import MembershipController @@ -1243,7 +1244,7 @@ class Orchestrator: if ( (finished or (req_state.streaming.enabled and req_state.streaming.segment_finished)) and stage_id < req_state.final_stage_id - and not self.async_chunk + and (not self.async_chunk or not self._stage_receives_async_chunks(stage_id + 1)) and (not self._next_stage_already_submitted(stage_id, req_state) or req_state.streaming.enabled) ): if ( @@ -1291,6 +1292,12 @@ class Orchestrator: def _next_stage_already_submitted(self, stage_id: int, req_state: OrchestratorRequestState) -> bool: return (stage_id + 1) in req_state.stage_submit_ts + def _stage_receives_async_chunks(self, stage_id: int) -> bool: + """Whether a stage's connector supplies its runtime inputs.""" + pool = self.stage_pools[stage_id] + model_config = getattr(pool.stage_vllm_config, "model_config", None) + return stage_receives_chunks(model_config) + def _get_stage_input_processor(self, stage_id: int) -> Any: processor = self._stage_input_processors.get(stage_id) if processor is None: @@ -1963,6 +1970,11 @@ class Orchestrator: for next_stage_id in range(1, req_state.final_stage_id + 1): next_pool = self.stage_pools[next_stage_id] params = req_state.sampling_params_list[next_stage_id] + if not self._stage_receives_async_chunks(next_stage_id): + # Outgoing-only stages receive their first real input from the + # orchestrator. Pre-submitting a placeholder lets it race and + # execute before that conditioning payload arrives. + continue req_state.stage_submit_ts[next_stage_id] = _time.time() _t_submit_start = _time.perf_counter() diff --git a/vllm_omni/engine/stage_engine_core_proc.py b/vllm_omni/engine/stage_engine_core_proc.py index e3f75d9e3..c2989179c 100644 --- a/vllm_omni/engine/stage_engine_core_proc.py +++ b/vllm_omni/engine/stage_engine_core_proc.py @@ -51,6 +51,13 @@ class StageEngineCoreProc(EngineCoreProc): ``EngineCoreProc.run_engine_core()``. """ + def preprocess_add_request(self, request: OmniEngineCoreRequest) -> tuple[Any, int]: + """Preserve omni payloads when vLLM builds its scheduler request.""" + scheduler_request, current_wave = super().preprocess_add_request(request) + scheduler_request.additional_information = request.additional_information + scheduler_request.external_req_id = getattr(request, "external_req_id", request.request_id) + return scheduler_request, current_wave + @staticmethod def run_stage_core( *args: Any, diff --git a/vllm_omni/engine/stage_init_utils.py b/vllm_omni/engine/stage_init_utils.py index d741e9038..9861d7b38 100644 --- a/vllm_omni/engine/stage_init_utils.py +++ b/vllm_omni/engine/stage_init_utils.py @@ -1063,6 +1063,16 @@ def get_stage_connector_spec( stage_connectors_cfg = get_stage_connector_config(omni_transfer_config, stage_id) for cfg in stage_connectors_cfg.values(): return dict(cfg.get("spec", {})) + + # A stage can be an async producer without consuming chunks itself. Keep + # its connector for save_async(), but mark it sender-only so the scheduler + # does not park normal orchestrator-provided inputs waiting for a chunk. + target_stage = str(stage_id) + for (from_stage, _to_stage), spec in getattr(omni_transfer_config, "connectors", {}).items(): + if from_stage == target_stage: + extra = dict(spec.extra or {}) + extra.setdefault("role", "sender") + return {"name": spec.name, "extra": extra} return {} diff --git a/vllm_omni/engine/stage_pool.py b/vllm_omni/engine/stage_pool.py index 32a35ac8c..fe4ec5a78 100644 --- a/vllm_omni/engine/stage_pool.py +++ b/vllm_omni/engine/stage_pool.py @@ -669,6 +669,8 @@ class StagePool: num_tokens_out=num_tokens_out, stage_gen_time_ms=stage_gen_time_ms, batch_id=batch_id, + # This event summarizes one completed request. Execution batching + # happens inside the model runner and is not observable here. batch_size=1, replica_id=replica_id, rx_decode_time_ms=0.0, diff --git a/vllm_omni/entrypoints/cli/benchmark/cli_args.py b/vllm_omni/entrypoints/cli/benchmark/cli_args.py index 66ece628e..7f7671496 100644 --- a/vllm_omni/entrypoints/cli/benchmark/cli_args.py +++ b/vllm_omni/entrypoints/cli/benchmark/cli_args.py @@ -239,6 +239,6 @@ def preprocess_serve_args(args: argparse.Namespace) -> None: """Apply serving benchmark CLI transformations after parsing.""" extra_body = dict(getattr(args, "extra_body", None) or {}) bot_task = getattr(args, "bot_task", None) - if bot_task is not None: + if getattr(args, "backend", None) == "openai-image-edits-omni" and bot_task is not None: extra_body.setdefault("bot_task", bot_task) args.extra_body = extra_body diff --git a/vllm_omni/experimental/fullduplex/engine/intermediate.py b/vllm_omni/experimental/fullduplex/engine/intermediate.py index 07c6ab668..d85d8f712 100644 --- a/vllm_omni/experimental/fullduplex/engine/intermediate.py +++ b/vllm_omni/experimental/fullduplex/engine/intermediate.py @@ -65,7 +65,7 @@ def set_ref_audio(buffer: dict[str, object], waveform: object, sample_rate_hz: i def set_tts_handoff(buffer: dict[str, object], token_ids: object | None, hidden_states: object | None) -> None: - """Store the generic AR-stage token/hidden-state handoff consumed by TTS.""" + """Store the AR-to-TTS handoff used by the full-duplex stage bridge.""" if token_ids is not None: buffer.setdefault("ids", {})["tts"] = token_ids if hidden_states is not None: @@ -73,16 +73,15 @@ def set_tts_handoff(buffer: dict[str, object], token_ids: object | None, hidden_ def get_tts_handoff(info: dict[str, object]) -> tuple[object | None, object | None]: - """Read the generic AR-stage to TTS handoff, including legacy flat keys.""" + """Read the canonical handoff, including the legacy flat aliases.""" ids_info = info.get("ids") hidden_info = info.get("hidden_states") - tts_token_ids = ids_info.get("tts") if isinstance(ids_info, dict) else None - tts_hidden_states = hidden_info.get("tts") if isinstance(hidden_info, dict) else None - if tts_token_ids is None: - tts_token_ids = info.get("tts_token_ids") - if tts_hidden_states is None: - tts_hidden_states = info.get("tts_hidden_states") - return tts_token_ids, tts_hidden_states + token_ids = ids_info.get("tts") if isinstance(ids_info, dict) else None + hidden_states = hidden_info.get("tts") if isinstance(hidden_info, dict) else None + return ( + info.get("tts_token_ids") if token_ids is None else token_ids, + info.get("tts_hidden_states") if hidden_states is None else hidden_states, + ) def get_stream_request_key(info: dict[str, object]) -> str: diff --git a/vllm_omni/model_executor/models/minicpmo_4_5/batched_token2wav.py b/vllm_omni/model_executor/models/minicpmo_4_5/batched_token2wav.py new file mode 100644 index 000000000..34cc9a733 --- /dev/null +++ b/vllm_omni/model_executor/models/minicpmo_4_5/batched_token2wav.py @@ -0,0 +1,400 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Strict, state-explicit batching for MiniCPM-o 4.5 Token2wav.""" + +from __future__ import annotations + +from contextlib import nullcontext +from dataclasses import dataclass +from typing import Any + +import torch +import torch.nn as nn +import torch.nn.functional as F + +_SILENCE_TOKEN = 4218 + + +def tensor_signature(value: torch.Tensor) -> tuple[tuple[int, ...], str, str]: + return tuple(value.shape), str(value.dtype), value.device.type + + +def state_shape_signature(state: BatchedToken2WavState) -> tuple[Any, ...]: + flow = tuple((name, tensor_signature(state.flow_cache[name])) for name in sorted(state.flow_cache)) + hift = tuple((name, tensor_signature(state.hift_cache[name])) for name in sorted(state.hift_cache)) + return flow, hift + + +@dataclass(frozen=True) +class PromptFeatures: + speech_tokens: torch.Tensor + speaker_embedding: torch.Tensor + mels: torch.Tensor + + +@dataclass(frozen=True) +class BatchedToken2WavState: + flow_cache: dict[str, torch.Tensor] + hift_cache: dict[str, torch.Tensor] + + +class BatchedToken2Wav(nn.Module): + """Drive Token2wav's modules with dynamically-sized, request-owned caches. + + This class intentionally never calls ``Token2wav.stream`` or + ``Token2wav.__call__``. The upstream object is used only as a one-time + asset loader and prompt feature extractor. + """ + + def __init__(self, token2wav: Any): + super().__init__() + self._token2wav = token2wav + self.flow = token2wav.flow + self.hift = token2wav.hift + self.float16 = bool(token2wav.float16) + self.n_timesteps = int(token2wav.n_timesteps) + self.mel_cache_len = int(token2wav.mel_cache_len) + self.source_cache_len = int(token2wav.source_cache_len) + self.register_buffer( + "speech_window", + token2wav.speech_window.detach().clone(), + persistent=False, + ) + self._prompt_features: dict[tuple[str, str], PromptFeatures] = {} + + def prepare_prompt(self, prompt_cache_id: str, prompt_wav: str) -> PromptFeatures: + cache_key = (prompt_cache_id, prompt_wav) + cached = self._prompt_features.get(cache_key) + if cached is None: + # The generation runner may wrap model.forward in bf16 autocast, + # and vLLM constructs the model under a bf16 default dtype, while + # S3Tokenizer prompt extraction uses fp32 convolution weights. + previous_dtype = torch.get_default_dtype() + try: + torch.set_default_dtype(torch.float32) + with torch.amp.autocast("cuda", enabled=False): + values = self._token2wav._prepare_prompt(prompt_wav) + finally: + torch.set_default_dtype(previous_dtype) + cached = PromptFeatures( + speech_tokens=values[0], + speaker_embedding=values[2], + mels=values[3], + ) + self._prompt_features[cache_key] = cached + return cached + + def evict_prompt(self, prompt_cache_id: str, prompt_wav: str) -> None: + """Release request-owned prompt features after stream completion.""" + self._prompt_features.pop((prompt_cache_id, prompt_wav), None) + + @staticmethod + def _repeat_prompt(features: PromptFeatures, batch_size: int) -> tuple[torch.Tensor, ...]: + return ( + features.speech_tokens.expand(batch_size, -1), + features.speaker_embedding.expand(batch_size, -1), + features.mels.expand(batch_size, -1, -1), + ) + + def _autocast(self, device: torch.device): + if device.type != "cuda": + return nullcontext() + if not self.float16: + return torch.amp.autocast("cuda", enabled=False) + return torch.amp.autocast( + "cuda", + dtype=torch.float16, + ) + + def _encode_chunk( + self, + tokens: torch.Tensor, + *, + last_chunk: bool, + cnn_cache: torch.Tensor | None, + att_cache: torch.Tensor | None, + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + embedded = self.flow.input_embedding(tokens) + hidden, new_cnn, new_att = self.flow.encoder.forward_chunk( + xs=embedded, + last_chunk=last_chunk, + cnn_cache=cnn_cache, + att_cache=att_cache, + ) + return self.flow.encoder_proj(hidden), new_cnn, new_att + + @staticmethod + def _estimator_buffers( + estimator: nn.Module, + x: torch.Tensor, + old_att: torch.Tensor | None, + ) -> tuple[torch.Tensor, torch.Tensor]: + blocks = estimator.blocks + depth = len(blocks) + batch_size = int(x.shape[0]) + chunk_size = int(x.shape[2]) + old_att_len = int(old_att.shape[3]) if old_att is not None else 0 + block0 = blocks[0] + cnn_channels = int(block0.conv.in_channels + block0.conv.out_channels) + cnn_width = int(block0.conv.block[1].causal_padding[0]) + heads = int(block0.attn.num_heads) + att_width = int(block0.attn.head_dim * 2) + cnn = x.new_empty((depth, batch_size, cnn_channels, cnn_width)) + att = x.new_empty((depth, batch_size, heads, old_att_len + chunk_size, att_width)) + return cnn, att + + def _estimator_step( + self, + estimator: nn.Module, + *, + x: torch.Tensor, + mu: torch.Tensor, + time: torch.Tensor, + speakers: torch.Tensor, + cond: torch.Tensor, + cnn_cache: torch.Tensor | None, + att_cache: torch.Tensor | None, + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + time_embedding = estimator.t_embedder(time).unsqueeze(1) + width = int(x.shape[-1]) + speaker_features = speakers.unsqueeze(-1).expand(-1, -1, width) + estimator_input = torch.cat((x, mu, speaker_features, cond), dim=1) + cnn_out, att_out = self._estimator_buffers(estimator, estimator_input, att_cache) + old_cnn: Any = cnn_cache if cnn_cache is not None else [None] * len(estimator.blocks) + old_att: Any = att_cache if att_cache is not None else [None] * len(estimator.blocks) + result = estimator.blocks_forward_chunk( + estimator_input, + time_embedding, + None, + old_cnn, + old_att, + cnn_out, + att_out, + ) + return result, cnn_out, att_out + + def _decode_cfm( + self, + mu: torch.Tensor, + speakers: torch.Tensor, + cond: torch.Tensor, + *, + cnn_cache: torch.Tensor | None, + att_cache: torch.Tensor | None, + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + decoder = self.flow.decoder + estimator = decoder.estimator + batch_size = int(mu.shape[0]) + offset = int(att_cache.shape[4]) if att_cache is not None else 0 + end = offset + int(mu.shape[2]) + if end > int(decoder.rand_noise.shape[2]): + raise RuntimeError( + "MiniCPMO45Code2WavBatchError " + f'{{"reason":"noise_capacity","required":{end},' + f'"available":{int(decoder.rand_noise.shape[2])}}}' + ) + x = decoder.rand_noise[:, :, offset:end].expand(batch_size, -1, -1).clone() + timeline = torch.linspace( + 0, + 1, + self.n_timesteps + 1, + device=mu.device, + dtype=mu.dtype, + ) + timeline = 1 - torch.cos(timeline * 0.5 * torch.pi) + time = timeline[0].expand(batch_size) + mu_cfg = torch.cat((mu, torch.zeros_like(mu)), dim=0) + speakers_cfg = torch.cat((speakers, torch.zeros_like(speakers)), dim=0) + cond_cfg = torch.cat((cond, torch.zeros_like(cond)), dim=0) + next_cnn: list[torch.Tensor] = [] + next_att: list[torch.Tensor] = [] + dt = timeline[1] - timeline[0] + for step in range(self.n_timesteps): + old_cnn = cnn_cache[step] if cnn_cache is not None else None + old_att = att_cache[step] if att_cache is not None else None + estimate, step_cnn, step_att = self._estimator_step( + estimator, + x=torch.cat((x, x), dim=0), + mu=mu_cfg, + time=torch.cat((time, time), dim=0), + speakers=speakers_cfg, + cond=cond_cfg, + cnn_cache=old_cnn, + att_cache=old_att, + ) + conditional, unconditional = estimate.split(batch_size, dim=0) + velocity = (1.0 + decoder.inference_cfg_rate) * conditional - decoder.inference_cfg_rate * unconditional + x = x + dt * velocity + time = time + dt + if step + 1 < self.n_timesteps: + dt = timeline[step + 2] - time[0] + next_cnn.append(step_cnn) + next_att.append(step_att) + return x, torch.stack(next_cnn), torch.stack(next_att) + + @staticmethod + def _split_flow_cache(cache: dict[str, torch.Tensor], batch_size: int) -> list[dict[str, torch.Tensor]]: + result: list[dict[str, torch.Tensor]] = [] + for row in range(batch_size): + result.append( + { + "conformer_cnn_cache": cache["conformer_cnn_cache"][row : row + 1].detach().clone(), + "conformer_att_cache": cache["conformer_att_cache"][:, row : row + 1].detach().clone(), + "estimator_cnn_cache": torch.cat( + ( + cache["estimator_cnn_cache"][:, :, row : row + 1], + cache["estimator_cnn_cache"][:, :, batch_size + row : batch_size + row + 1], + ), + dim=2, + ).detach(), + "estimator_att_cache": torch.cat( + ( + cache["estimator_att_cache"][:, :, row : row + 1], + cache["estimator_att_cache"][:, :, batch_size + row : batch_size + row + 1], + ), + dim=2, + ).detach(), + } + ) + return result + + @staticmethod + def _stack_flow_cache(states: list[BatchedToken2WavState]) -> dict[str, torch.Tensor]: + flows = [state.flow_cache for state in states] + conditional_cnn = [flow["estimator_cnn_cache"][:, :, 0:1] for flow in flows] + unconditional_cnn = [flow["estimator_cnn_cache"][:, :, 1:2] for flow in flows] + conditional_att = [flow["estimator_att_cache"][:, :, 0:1] for flow in flows] + unconditional_att = [flow["estimator_att_cache"][:, :, 1:2] for flow in flows] + return { + "conformer_cnn_cache": torch.cat([flow["conformer_cnn_cache"] for flow in flows], dim=0), + "conformer_att_cache": torch.cat([flow["conformer_att_cache"] for flow in flows], dim=1), + "estimator_cnn_cache": torch.cat((*conditional_cnn, *unconditional_cnn), dim=2), + "estimator_att_cache": torch.cat((*conditional_att, *unconditional_att), dim=2), + } + + def setup_batch( + self, + features: PromptFeatures, + batch_size: int, + ) -> list[BatchedToken2WavState]: + prompt_tokens, speakers, prompt_mels = self._repeat_prompt(features, batch_size) + lookahead = prompt_tokens.new_full((batch_size, 3), _SILENCE_TOKEN) + with self._autocast(prompt_tokens.device): + hidden, conformer_cnn, conformer_att = self._encode_chunk( + torch.cat((prompt_tokens, lookahead), dim=1), + last_chunk=False, + cnn_cache=None, + att_cache=None, + ) + projected_speakers = self.flow.spk_embed_affine_layer(F.normalize(speakers, dim=1)) + _, estimator_cnn, estimator_att = self._decode_cfm( + hidden.transpose(1, 2).contiguous(), + projected_speakers, + prompt_mels.transpose(1, 2).contiguous(), + cnn_cache=None, + att_cache=None, + ) + flow_cache = { + "conformer_cnn_cache": conformer_cnn, + "conformer_att_cache": conformer_att, + "estimator_cnn_cache": estimator_cnn, + "estimator_att_cache": estimator_att, + } + split = self._split_flow_cache(flow_cache, batch_size) + mel_channels = int(prompt_mels.shape[2]) + return [ + BatchedToken2WavState( + flow_cache=row, + hift_cache={ + "mel": prompt_mels.new_zeros((1, mel_channels, 0)), + "source": prompt_mels.new_zeros((1, 1, 0)), + "speech": prompt_mels.new_zeros((1, 0)), + }, + ) + for row in split + ] + + @staticmethod + def _fade_in_out( + speech: torch.Tensor, + previous: torch.Tensor, + window: torch.Tensor, + ) -> torch.Tensor: + overlap = int(window.shape[0] // 2) + result = speech.clone() + result[..., :overlap] = result[..., :overlap] * window[:overlap] + previous[..., -overlap:] * window[overlap:] + return result + + def decode_batch( + self, + tokens: torch.Tensor, + features: PromptFeatures, + states: list[BatchedToken2WavState], + *, + last_chunk: bool, + ) -> tuple[list[torch.Tensor], list[BatchedToken2WavState]]: + batch_size = int(tokens.shape[0]) + if batch_size != len(states): + raise ValueError(f"tokens batch {batch_size} != state batch {len(states)}") + flow_cache = self._stack_flow_cache(states) + speakers = features.speaker_embedding.expand(batch_size, -1) + with self._autocast(tokens.device): + hidden, conformer_cnn, conformer_att = self._encode_chunk( + tokens, + last_chunk=last_chunk, + cnn_cache=flow_cache["conformer_cnn_cache"], + att_cache=flow_cache["conformer_att_cache"], + ) + projected_speakers = self.flow.spk_embed_affine_layer(F.normalize(speakers, dim=1)) + cond = torch.zeros_like(hidden).transpose(1, 2).contiguous() + chunk_mel, estimator_cnn, estimator_att = self._decode_cfm( + hidden.transpose(1, 2).contiguous(), + projected_speakers, + cond, + cnn_cache=flow_cache["estimator_cnn_cache"], + att_cache=flow_cache["estimator_att_cache"], + ) + + prompt_len = int(features.mels.shape[1]) + if estimator_att.shape[4] > prompt_len + 100: + estimator_att = torch.cat( + (estimator_att[..., :prompt_len, :], estimator_att[..., -100:, :]), + dim=4, + ) + if conformer_att.shape[3] > prompt_len + 100: + conformer_att = torch.cat( + (conformer_att[..., :prompt_len, :], conformer_att[..., -100:, :]), + dim=3, + ) + new_flow = self._split_flow_cache( + { + "conformer_cnn_cache": conformer_cnn, + "conformer_att_cache": conformer_att, + "estimator_cnn_cache": estimator_cnn, + "estimator_att_cache": estimator_att, + }, + batch_size, + ) + old_mel = torch.cat([state.hift_cache["mel"] for state in states], dim=0) + old_source = torch.cat([state.hift_cache["source"] for state in states], dim=0) + old_speech = torch.cat([state.hift_cache["speech"] for state in states], dim=0) + mel = torch.cat((old_mel, chunk_mel), dim=2) + speech, source = self.hift(mel, old_source) + if old_speech.shape[-1] > 0: + window = self.speech_window.to(device=speech.device, dtype=speech.dtype) + speech = self._fade_in_out(speech, old_speech, window) + next_hift = { + "mel": mel[..., -self.mel_cache_len :].detach(), + "source": source[..., -self.source_cache_len :].detach(), + "speech": speech[..., -self.source_cache_len :].detach(), + } + emitted = speech if last_chunk else speech[..., : -self.source_cache_len] + next_states = [ + BatchedToken2WavState( + flow_cache=new_flow[row], + hift_cache={name: value[row : row + 1].detach().clone() for name, value in next_hift.items()}, + ) + for row in range(batch_size) + ] + audios = [emitted[row].reshape(-1).to(dtype=torch.float32) for row in range(batch_size)] + return audios, next_states diff --git a/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_code2wav.py b/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_code2wav.py new file mode 100644 index 000000000..93bd98e76 --- /dev/null +++ b/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_code2wav.py @@ -0,0 +1,547 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Strict batched codec-to-waveform stage for MiniCPM-o 4.5.""" + +from __future__ import annotations + +import hashlib +import json +import os +import tempfile +from collections.abc import Iterable, Mapping, Sequence +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import soundfile as sf +import torch +import torch.nn as nn +from vllm.config import VllmConfig + +from vllm_omni.model_executor.models.output_templates import OmniOutput + +from .batched_token2wav import ( + BatchedToken2Wav, + BatchedToken2WavState, + state_shape_signature, +) + + +def _batch_error(reason: str, **details: Any) -> RuntimeError: + payload = {"reason": reason, **details} + return RuntimeError(f"MiniCPMO45Code2WavBatchError {json.dumps(payload, sort_keys=True)}") + + +def _scalar(value: Any, default: Any = None) -> Any: + if isinstance(value, torch.Tensor): + return value.reshape(-1)[0].item() if value.numel() else default + if isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)): + return _scalar(value[0], default) if value else default + return default if value is None else value + + +def _codec_tensor(value: Any, fallback: torch.Tensor) -> torch.Tensor: + if isinstance(value, torch.Tensor): + return value.reshape(-1).to(device=fallback.device, dtype=torch.long) + if isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)): + return torch.as_tensor(value, device=fallback.device, dtype=torch.long).reshape(-1) + return fallback.reshape(-1).to(dtype=torch.long) + + +@dataclass(frozen=True) +class _RequestState: + cache_epoch: int + chunk_seq: int + prompt_cache_id: str + prompt_wav: str + token2wav: BatchedToken2WavState + + +@dataclass(frozen=True) +class _WorkItem: + output_index: int + state_id: str + request_id: str + cache_epoch: int + chunk_seq: int + prompt_cache_id: str + prompt_wav: str + last_chunk: bool + tokens: torch.Tensor + previous: _RequestState | None + segment_text: str + duplex_turn_id: int | None + duplex_epoch: int | None + segment_end: bool + turn_end: bool + + +class MiniCPMO45Code2Wav(nn.Module): + """LLM_GENERATION model that admits only true exact-shape GPU batches.""" + + input_modalities = "audio" + have_multimodal_outputs = True + enable_update_additional_information = True + requires_raw_input_tokens = True + requires_request_ids = True + has_preprocess = False + has_postprocess = False + + def __init__( + self, + *, + vllm_config: VllmConfig, + prefix: str = "", + ): + super().__init__() + del prefix + self.vllm_config = vllm_config + self.model_path = str(vllm_config.model_config.model) + self.backend: BatchedToken2Wav | None = None + self._states: dict[str, _RequestState] = {} + self._owned_prompt_wavs: dict[str, tuple[str, str]] = {} + extra = self._extra_config() + self._min_batch_size = int(extra.get("code2wav_min_batch_size", 1)) + if self._min_batch_size < 1: + raise ValueError("MiniCPM-o Code2Wav code2wav_min_batch_size must be >= 1") + self._default_prompt_id = str(extra.get("prompt_cache_id", "HT_ref_audio")) + self._default_prompt_wav = str( + extra.get( + "prompt_wav", + Path(self.model_path) / "assets" / "HT_ref_audio.wav", + ) + ) + + def _extra_config(self) -> dict[str, Any]: + model_config = getattr(self.vllm_config, "model_config", None) + connector = getattr(model_config, "stage_connector_config", None) + if isinstance(connector, Mapping): + extra = connector.get("extra", connector) + else: + extra = getattr(connector, "extra", None) + return dict(extra) if isinstance(extra, Mapping) else {} + + def _release_prompt(self, state_id: str) -> None: + owned = self._owned_prompt_wavs.pop(state_id, None) + if owned is None: + return + prompt_cache_id, prompt_wav = owned + if self.backend is not None: + self.backend.evict_prompt(prompt_cache_id, prompt_wav) + try: + os.unlink(prompt_wav) + except FileNotFoundError: + pass + + def _write_reference_prompt( + self, + state_id: str, + reference: object, + sample_rate: object, + ) -> tuple[str, str]: + waveform = torch.as_tensor(reference, dtype=torch.float32).reshape(-1).cpu() + if waveform.numel() == 0: + return self._default_prompt_id, self._default_prompt_wav + sample_rate_hz = int(_scalar(sample_rate, 24000)) + if sample_rate_hz <= 0: + raise _batch_error( + "invalid_ref_audio_sample_rate", + request_id=state_id, + sample_rate=sample_rate_hz, + ) + digest = hashlib.sha256() + digest.update(str(sample_rate_hz).encode("ascii")) + digest.update(waveform.numpy().tobytes()) + prompt_cache_id = f"{state_id}:{digest.hexdigest()}" + current = self._owned_prompt_wavs.get(state_id) + if current is not None and current[0] == prompt_cache_id and os.path.exists(current[1]): + return current + + self._release_prompt(state_id) + handle = tempfile.NamedTemporaryFile(prefix="minicpmo45-ref-", suffix=".wav", delete=False) + prompt_wav = handle.name + handle.close() + try: + sf.write(prompt_wav, waveform.numpy(), sample_rate_hz, format="WAV") + except Exception: + try: + os.unlink(prompt_wav) + except FileNotFoundError: + pass + raise + self._owned_prompt_wavs[state_id] = (prompt_cache_id, prompt_wav) + return prompt_cache_id, prompt_wav + + def embed_input_ids(self, input_ids: torch.Tensor, **_: Any) -> torch.Tensor: + return torch.zeros((input_ids.numel(), 1), device=input_ids.device, dtype=torch.float32) + + def compute_logits(self, hidden_states: Any, sampling_metadata: Any = None) -> None: + return None + + @staticmethod + def _split_segments(input_ids: torch.Tensor, counts: Any) -> list[torch.Tensor]: + flat = input_ids.reshape(-1) + if counts is None: + return [flat] + if not isinstance(counts, Sequence) or isinstance(counts, (str, bytes, bytearray)): + raise _batch_error("invalid_seq_token_counts", value_type=type(counts).__name__) + normalized = [int(value) for value in counts] + if any(value < 0 for value in normalized): + raise _batch_error("negative_seq_token_count", counts=normalized) + if sum(normalized) != int(flat.numel()): + raise _batch_error( + "seq_token_count_mismatch", + counts=normalized, + total=int(flat.numel()), + ) + return list(torch.split(flat, normalized)) + + def _parse_item( + self, + index: int, + state_id: str, + segment: torch.Tensor, + info: Mapping[str, Any], + ) -> _WorkItem: + meta = info.get("meta") + if not isinstance(meta, Mapping): + meta = info + request_id = str(_scalar(meta.get("request_id"), _scalar(info.get("request_id"), ""))) + if not request_id: + raise _batch_error("missing_request_id", output_index=index) + cache_epoch = int(_scalar(meta.get("cache_epoch"), 0)) + chunk_seq = int(_scalar(meta.get("chunk_seq"), 0)) + if cache_epoch < 0 or chunk_seq < 0: + raise _batch_error( + "negative_stream_position", + request_id=request_id, + cache_epoch=cache_epoch, + chunk_seq=chunk_seq, + ) + last_chunk = bool(_scalar(meta.get("last_chunk"), False)) + codes = info.get("codes") + audio = codes.get("audio") if isinstance(codes, Mapping) else None + reference = codes.get("ref") if isinstance(codes, Mapping) else None + tokens = _codec_tensor(audio, segment) + if last_chunk and int(_scalar(meta.get("code_flat_numel"), tokens.numel())) == 0: + # The generation scheduler reserves one placeholder token for an + # empty terminal chunk. The producer's explicit length is the + # authority, so do not decode that placeholder as codec data. + tokens = segment.new_empty(0, dtype=torch.long) + previous = self._states.get(state_id) + if previous is None: + if chunk_seq != 0: + raise _batch_error( + "missing_state_for_chunk", + request_id=request_id, + cache_epoch=cache_epoch, + chunk_seq=chunk_seq, + ) + elif cache_epoch < previous.cache_epoch: + raise _batch_error( + "stale_cache_epoch", + request_id=request_id, + expected=previous.cache_epoch, + actual=cache_epoch, + ) + elif cache_epoch > previous.cache_epoch: + if chunk_seq != 0: + raise _batch_error( + "new_epoch_requires_first_chunk", + request_id=request_id, + cache_epoch=cache_epoch, + chunk_seq=chunk_seq, + ) + self._release_prompt(state_id) + previous = None + elif chunk_seq != previous.chunk_seq + 1: + raise _batch_error( + "stale_or_reordered_chunk", + request_id=request_id, + expected=previous.chunk_seq + 1, + actual=chunk_seq, + ) + + explicit_prompt_id = _scalar(meta.get("prompt_cache_id")) + explicit_prompt_wav = _scalar(meta.get("prompt_wav")) + if previous is not None and explicit_prompt_id is None and explicit_prompt_wav is None: + prompt_cache_id = previous.prompt_cache_id + prompt_wav = previous.prompt_wav + elif reference is not None: + prompt_cache_id, prompt_wav = self._write_reference_prompt( + state_id, + reference, + meta.get("ref_audio_sr"), + ) + else: + prompt_cache_id = str(explicit_prompt_id or self._default_prompt_id) + prompt_wav = str(explicit_prompt_wav or self._default_prompt_wav) + + if previous is not None and prompt_cache_id != previous.prompt_cache_id: + raise _batch_error( + "prompt_changed_midstream", + request_id=request_id, + expected=previous.prompt_cache_id, + actual=prompt_cache_id, + ) + if previous is not None and prompt_wav != previous.prompt_wav: + raise _batch_error( + "prompt_changed_midstream", + request_id=request_id, + expected=previous.prompt_wav, + actual=prompt_wav, + ) + duplex_turn_id = _scalar(meta.get("duplex_turn_id")) + duplex_epoch = _scalar(meta.get("duplex_epoch")) + return _WorkItem( + output_index=index, + state_id=state_id, + request_id=request_id, + cache_epoch=cache_epoch, + chunk_seq=chunk_seq, + prompt_cache_id=prompt_cache_id, + prompt_wav=prompt_wav, + last_chunk=last_chunk, + tokens=tokens, + previous=previous, + segment_text=str(_scalar(meta.get("native_duplex_segment_text"), "")), + duplex_turn_id=int(duplex_turn_id) if duplex_turn_id is not None else None, + duplex_epoch=int(duplex_epoch) if duplex_epoch is not None else None, + segment_end=bool(_scalar(meta.get("segment_end"), False)), + turn_end=bool(_scalar(meta.get("turn_end"), False)), + ) + + @staticmethod + def _bucket_key(item: _WorkItem) -> tuple[Any, ...]: + cache_signature: Any + if item.previous is None: + cache_signature = ("uninitialized",) + else: + cache_signature = state_shape_signature(item.previous.token2wav) + return ( + item.prompt_cache_id, + item.prompt_wav, + int(item.tokens.numel()), + cache_signature, + item.last_chunk, + item.cache_epoch, + ) + + @torch.inference_mode() + def forward( + self, + input_ids: torch.Tensor | None = None, + positions: torch.Tensor | None = None, + intermediate_tensors: Any = None, + inputs_embeds: torch.Tensor | None = None, + runtime_additional_information: list[dict[str, Any]] | None = None, + **kwargs: Any, + ) -> OmniOutput: + del positions, intermediate_tensors, inputs_embeds + ids = input_ids if isinstance(input_ids, torch.Tensor) else torch.empty(0, dtype=torch.long) + segments = self._split_segments(ids, kwargs.get("seq_token_counts")) + empty = torch.empty(0, dtype=torch.float32, device=ids.device) + sample_rate = torch.tensor(24000, dtype=torch.int32) + if not runtime_additional_information: + count = len(segments) + return OmniOutput( + text_hidden_states=None, + multimodal_outputs={ + "model_outputs": [empty for _ in range(count)], + "sr": [sample_rate for _ in range(count)], + }, + ) + if len(runtime_additional_information) != len(segments): + raise _batch_error( + "runtime_info_count_mismatch", + segments=len(segments), + runtime_infos=len(runtime_additional_information), + ) + if self.backend is None: + raise _batch_error("backend_not_loaded") + + state_ids = kwargs.get("request_ids") + if state_ids is None: + state_ids = [] + for index, info in enumerate(runtime_additional_information): + if not isinstance(info, Mapping): + state_ids.append(str(index)) + continue + meta = info.get("meta") + source = meta if isinstance(meta, Mapping) else info + state_ids.append(str(_scalar(source.get("request_id"), index))) + if len(state_ids) != len(segments): + raise _batch_error( + "request_id_count_mismatch", + segments=len(segments), + request_ids=len(state_ids), + ) + items: list[_WorkItem] = [] + for index, (state_id, segment, info) in enumerate( + zip(state_ids, segments, runtime_additional_information, strict=True) + ): + if not isinstance(info, Mapping): + raise _batch_error( + "invalid_runtime_info", + output_index=index, + value_type=type(info).__name__, + ) + items.append(self._parse_item(index, str(state_id), segment, info)) + state_ids = [item.state_id for item in items] + if len(state_ids) != len(set(state_ids)): + raise _batch_error("duplicate_request_in_forward", request_ids=state_ids) + outputs = [empty for _ in segments] + sentinels = [item for item in items if item.last_chunk and item.tokens.numel() == 0] + compute_items = [item for item in items if item.tokens.numel() > 0] + invalid_empty = [item.request_id for item in items if not item.last_chunk and item.tokens.numel() == 0] + if invalid_empty: + raise _batch_error("empty_nonfinal_chunk", request_ids=invalid_empty) + + buckets: dict[tuple[Any, ...], list[_WorkItem]] = {} + for item in compute_items: + buckets.setdefault(self._bucket_key(item), []).append(item) + undersized = [ + { + "size": len(bucket), + "request_ids": [item.request_id for item in bucket], + "codec_len": int(bucket[0].tokens.numel()), + } + for bucket in buckets.values() + if len(bucket) < self._min_batch_size + ] + if undersized: + raise _batch_error( + "exact_shape_bucket_below_minimum", + minimum=self._min_batch_size, + buckets=undersized, + ) + + pending: dict[str, _RequestState | None] = {item.state_id: None for item in sentinels} + for bucket in buckets.values(): + batch_size = len(bucket) + try: + features = self.backend.prepare_prompt( + bucket[0].prompt_cache_id, + bucket[0].prompt_wav, + ) + if bucket[0].previous is None: + states = self.backend.setup_batch(features, batch_size) + else: + states = [item.previous.token2wav for item in bucket if item.previous is not None] + tokens = torch.stack([item.tokens for item in bucket], dim=0) + audios, next_states = self.backend.decode_batch( + tokens, + features, + states, + last_chunk=bucket[0].last_chunk, + ) + except Exception as exc: + if isinstance(exc, RuntimeError) and str(exc).startswith("MiniCPMO45Code2WavBatchError "): + raise + raise _batch_error( + "backend_unsupported_or_failed", + request_ids=[item.request_id for item in bucket], + error_type=type(exc).__name__, + error=str(exc), + ) from exc + if len(audios) != batch_size or len(next_states) != batch_size: + raise _batch_error( + "backend_result_size_mismatch", + expected=batch_size, + audios=len(audios), + states=len(next_states), + ) + for item, audio, next_state in zip(bucket, audios, next_states, strict=True): + outputs[item.output_index] = audio.reshape(-1).to(dtype=torch.float32) + pending[item.state_id] = ( + None + if item.last_chunk + else _RequestState( + cache_epoch=item.cache_epoch, + chunk_seq=item.chunk_seq, + prompt_cache_id=item.prompt_cache_id, + prompt_wav=item.prompt_wav, + token2wav=next_state, + ) + ) + + for request_id, state in pending.items(): + if state is None: + self._states.pop(request_id, None) + self._release_prompt(request_id) + else: + self._states[request_id] = state + sample_rate_tensor = torch.as_tensor(sample_rate, dtype=torch.int32) + return OmniOutput( + text_hidden_states=None, + multimodal_outputs={ + "model_outputs": outputs, + "sr": [sample_rate_tensor.clone() for _ in outputs], + "meta.llm_output_text_utf8": [ + torch.tensor(list(item.segment_text.encode("utf-8")), dtype=torch.uint8) for item in items + ], + "meta.tts_is_last_chunk": [torch.tensor(item.last_chunk, dtype=torch.bool) for item in items], + "meta.segment_end": [torch.tensor(item.segment_end, dtype=torch.bool) for item in items], + "meta.turn_end": [torch.tensor(item.turn_end, dtype=torch.bool) for item in items], + "meta.duplex_turn_id": [ + torch.tensor(item.duplex_turn_id, dtype=torch.int64) + if item.duplex_turn_id is not None + else torch.empty(0, dtype=torch.int64) + for item in items + ], + "meta.duplex_epoch": [ + torch.tensor(item.duplex_epoch, dtype=torch.int64) + if item.duplex_epoch is not None + else torch.empty(0, dtype=torch.int64) + for item in items + ], + }, + ) + + def on_requests_finished(self, finished_req_ids: set[str] | list[str]) -> None: + for request_id in finished_req_ids: + state_id = str(request_id) + self._states.pop(state_id, None) + self._release_prompt(state_id) + + def make_omni_output(self, model_outputs: Any, **_: Any) -> OmniOutput: + if isinstance(model_outputs, OmniOutput): + return model_outputs + if isinstance(model_outputs, tuple) and len(model_outputs) == len(OmniOutput._fields): + return OmniOutput(*model_outputs) + raise TypeError(f"MiniCPMO45Code2Wav expected OmniOutput, got {type(model_outputs).__name__}") + + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: + for _ in weights: + pass + if self.backend is not None: + return {name for name, _ in self.named_parameters()} + from stepaudio2.token2wav import Token2wav + + extra = self._extra_config() + prompt_path = Path(self._default_prompt_wav) + if not prompt_path.is_file(): + raise FileNotFoundError(f"MiniCPM-o Code2Wav prompt audio not found: {prompt_path}") + token2wav_path = Path(self.model_path) / "assets" / "token2wav" + if not token2wav_path.is_dir(): + raise FileNotFoundError(f"MiniCPM-o Code2Wav assets not found: {token2wav_path}") + use_float16 = bool(extra.get("token2wav_float16", False)) + previous_dtype = torch.get_default_dtype() + try: + # vLLM constructs bf16 models under a bf16 default-dtype context. + # Token2wav contains fp32-only S3Tokenizer/HiFT modules, so build + # its independent assets in their native precision. + torch.set_default_dtype(torch.float32) + token2wav = Token2wav( + str(token2wav_path), + float16=use_float16, + n_timesteps=int(extra.get("token2wav_n_timesteps", 10)), + ) + finally: + torch.set_default_dtype(previous_dtype) + self.backend = BatchedToken2Wav(token2wav) + # Token2wav loads flow.pt and hift.pt inside its constructor instead of + # from the parent MiniCPM checkpoint iterator. Report those registered + # parameters as initialized so vLLM's strict loader audit does not + # misclassify the independently loaded Stage-2 weights as missing. + return {name for name, _ in self.named_parameters()} diff --git a/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni.py b/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni.py index ba51c9cfc..c9083c261 100644 --- a/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni.py +++ b/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni.py @@ -34,7 +34,6 @@ from vllm.sequence import IntermediateTensors from vllm.v1.outputs import SamplerOutput from vllm.v1.sample.metadata import SamplingMetadata -from vllm_omni.experimental.fullduplex.engine.intermediate import get_stream_request_key from vllm_omni.experimental.fullduplex.minicpmo45.policy import MiniCPMO45DuplexPolicy from vllm_omni.experimental.fullduplex.model_executor import DuplexSamplingRow from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni_llm import ( @@ -58,9 +57,11 @@ logger = init_logger(__name__) class MiniCPMO45OmniForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP, SupportsMRoPE): """MiniCPM-o 4.5 Omni model for conditional generation. - This model has two pipeline stages: - - llm: multimodal thinker and text generation - - tts: talker generation followed by its built-in Token2Wav vocoder + Three-stage pipeline: + - thinker (model_stage="llm"): image / video / audio encoders + 3D + resampler + the omni LLM that emits text + hidden states. + - talker (model_stage="tts"): native continuous MiniCPMTTS AR that emits + codec-token deltas for the separate Code2Wav stage. """ @classmethod @@ -106,7 +107,7 @@ class MiniCPMO45OmniForConditionalGeneration(nn.Module, SupportsMultiModal, Supp elif self.model_stage == "tts": self.thinker = None - # Initialize talker model (LLM generation) + # The Talker is always the runner-owned continuous codec producer. self.talker = init_vllm_registered_model( vllm_config=vllm_config, prefix=maybe_prefix(prefix, "talker"), @@ -125,12 +126,18 @@ class MiniCPMO45OmniForConditionalGeneration(nn.Module, SupportsMultiModal, Supp self.make_empty_intermediate_tensors = ( (self.thinker.make_empty_intermediate_tensors) if self.model_stage == "llm" and self.thinker is not None + else self.talker.make_empty_intermediate_tensors + if self.talker is not None else lambda: None ) self._language_model_names = ["model"] self.prefer_model_sampler = self.model_stage in {"llm", "tts"} - self.has_preprocess = self.model_stage == "llm" + # Both AR stages require model-specific embeddings. The Thinker uses + # preprocess for duplex audio, while the Talker converts the + # tts_token_ids/tts_hidden_states handoff into its conditioning + # embeddings and initializes request-local codec generation state. + self.has_preprocess = self.model_stage in {"llm", "tts"} @cached_property def sampler(self): @@ -299,6 +306,8 @@ class MiniCPMO45OmniForConditionalGeneration(nn.Module, SupportsMultiModal, Supp and sampler. This hook only turns the current duplex audio append into the prompt embeddings consumed by the normal runner forward. """ + if self.model_stage == "tts": + return self.talker.preprocess(input_ids=input_ids, input_embeds=input_embeds, **kwargs) if self.model_stage != "llm": embeds = input_embeds if input_embeds is not None else self.get_input_embeddings(input_ids) return input_ids, embeds, {} @@ -474,174 +483,6 @@ class MiniCPMO45OmniForConditionalGeneration(nn.Module, SupportsMultiModal, Supp """vLLM V1 encoder profiling calls this; the inherited Protocol stub returns None.""" return self.get_multimodal_embeddings(**kwargs) - def _run_tts_request( - self, - *, - input_ids: torch.Tensor | None, - positions: torch.Tensor | None, - inputs_embeds: torch.Tensor | None, - talker_info: dict[str, Any], - device: torch.device, - ) -> tuple[dict[str, Any] | None, bool, bool]: - meta_info = talker_info.get("meta") if isinstance(talker_info.get("meta"), dict) else {} - if talker_info.get("native_duplex") is True: - # ids/hidden_states may be accumulated for the talker KV stream, - # but displayed transcript belongs to the current thinker segment. - tts_text = meta_info.get("native_duplex_segment_text", "") - else: - tts_text = talker_info.get("llm_output_text", "") - if isinstance(tts_text, list): - tts_text = ( - tts_text[-1] - if talker_info.get("native_duplex") is True and tts_text - else (tts_text[0] if tts_text else "") - ) - if not isinstance(tts_text, str): - tts_text = "" - - with torch.inference_mode(): - talker_result = self.talker( - input_ids=input_ids, - positions=positions, - inputs_embeds=inputs_embeds, - additional_information=talker_info, - ) - chunk_flags = getattr(self.talker, "_ar_last_chunk_flags", [True]) - chunk_is_last = bool(chunk_flags[-1]) if chunk_flags else True - turn_end_flags = getattr(self.talker, "_ar_turn_end_flags", [False]) - turn_ended = bool(turn_end_flags[-1]) if turn_end_flags else False - if talker_info.get("native_duplex") is True: - emitted_text = getattr(self.talker, "_ar_last_emitted_text", "") - tts_text = emitted_text if isinstance(emitted_text, str) else "" - - if not (isinstance(talker_result, tuple) and len(talker_result) == 2): - return None, chunk_is_last, turn_ended - - mel_spec, waveform = talker_result - mm_out: dict[str, Any] = {} - duplex_info = talker_info.get("duplex") if isinstance(talker_info.get("duplex"), dict) else {} - if talker_info.get("native_duplex") is True: - turn_id = duplex_info.get("turn_id") - if isinstance(turn_id, int): - mm_out["meta.duplex_turn_id"] = torch.tensor([turn_id], dtype=torch.int32, device=device) - epoch = duplex_info.get("epoch") - if isinstance(epoch, int): - mm_out["meta.duplex_epoch"] = torch.tensor([epoch], dtype=torch.int32, device=device) - mm_out["meta.tts_is_last_chunk"] = torch.tensor( - [int(chunk_is_last)], - dtype=torch.int32, - device=device, - ) - if talker_info.get("native_duplex") is True: - mm_out["meta.turn_end"] = torch.tensor( - [int(turn_ended)], - dtype=torch.int32, - device=device, - ) - if tts_text or talker_info.get("native_duplex") is True: - mm_out["meta.llm_output_text_utf8"] = torch.tensor( - list(tts_text.encode("utf-8")), - dtype=torch.uint8, - device=device, - ) - mm_out["meta.audio_text_total_chars"] = torch.tensor( - [len(tts_text)], - dtype=torch.int32, - device=device, - ) - if mel_spec is not None: - mm_out["mel_spec"] = [mel_spec] - if waveform is not None: - mm_out["model_outputs"] = [waveform] - elif mel_spec is not None: - mm_out["model_outputs"] = [mel_spec] - return mm_out, chunk_is_last, turn_ended - - @staticmethod - def _slice_tts_request_rows(value: torch.Tensor | None, start: int, end: int) -> torch.Tensor | None: - return value[start:end] if isinstance(value, torch.Tensor) else value - - def _run_batched_tts_requests( - self, - *, - input_ids: torch.Tensor | None, - positions: torch.Tensor | None, - inputs_embeds: torch.Tensor | None, - runtime_info: list[object], - request_token_spans: object, - num_tokens: int, - hidden_dim: int, - device: torch.device, - ) -> OmniOutput: - if not isinstance(request_token_spans, list) or len(request_token_spans) != len(runtime_info): - raise RuntimeError( - "MiniCPM-o 4.5 batched TTS requires one request_token_spans entry " - f"per request; got {request_token_spans!r} for {len(runtime_info)} requests" - ) - - request_ids: list[str] = [] - request_outputs: list[dict[str, Any]] = [] - row_chunk_flags = [True] * num_tokens - row_turn_end_flags = [False] * num_tokens - previous_end = 0 - for index, raw_info in enumerate(runtime_info): - if not isinstance(raw_info, dict): - raise RuntimeError(f"MiniCPM-o 4.5 TTS request metadata at index {index} is not a dict") - span = request_token_spans[index] - if not ( - isinstance(span, (list, tuple)) and len(span) == 2 and all(isinstance(value, int) for value in span) - ): - raise RuntimeError(f"Invalid MiniCPM-o 4.5 TTS token span at index {index}: {span!r}") - start, end = span - if start != previous_end or start < 0 or end < start or end > num_tokens: - raise RuntimeError( - f"Invalid MiniCPM-o 4.5 TTS token span at index {index}: " - f"expected start {previous_end} and 0 <= start <= end <= {num_tokens}, got {span!r}" - ) - previous_end = end - if start == end: - continue - mm_out, chunk_is_last, turn_ended = self._run_tts_request( - input_ids=self._slice_tts_request_rows(input_ids, start, end), - positions=self._slice_tts_request_rows(positions, start, end), - inputs_embeds=self._slice_tts_request_rows(inputs_embeds, start, end), - talker_info=raw_info, - device=device, - ) - row_chunk_flags[start:end] = [chunk_is_last] * (end - start) - row_turn_end_flags[start:end] = [turn_ended] * (end - start) - if mm_out is not None: - request_ids.append(get_stream_request_key(raw_info)) - request_outputs.append(mm_out) - # vLLM pads the flattened token tensor to a CUDA Graph capture size. - # The spans describe only scheduled request rows, so an uncovered tail - # is graph padding rather than a missing request. Contiguity checks - # above still reject gaps between real requests. - - self.talker._ar_last_chunk_flags = row_chunk_flags - self.talker._ar_turn_end_flags = row_turn_end_flags - dummy_hidden = torch.zeros(num_tokens, hidden_dim, device=device) - if not request_outputs: - return OmniOutput(text_hidden_states=dummy_hidden, multimodal_outputs=None) - - sparse_output: dict[str, Any] = { - "meta.req_id": request_ids, - "meta.sparse_audio": ["1"], - } - output_keys = set().union(*(output.keys() for output in request_outputs)) - for key in output_keys: - values: list[Any] = [] - for output in request_outputs: - value = output.get(key) - if isinstance(value, list) and len(value) == 1: - value = value[0] - if value is None: - dtype = torch.uint8 if key == "meta.llm_output_text_utf8" else torch.float32 - value = torch.empty(0, dtype=dtype, device=device) - values.append(value) - sparse_output[key] = values - return OmniOutput(text_hidden_states=dummy_hidden, multimodal_outputs=sparse_output) - def forward( self, input_ids: torch.Tensor, @@ -658,8 +499,9 @@ class MiniCPMO45OmniForConditionalGeneration(nn.Module, SupportsMultiModal, Supp Forward pass for MiniCPM-o Omni model. Workflow: - 1) LLM: multimodal thinker → hidden states and text tokens - 2) TTS: talker + Token2Wav → speech waveform + 1) Thinker (model_stage="llm"): Image / video / audio encoders + + 3D resampler + omni LLM → text + hidden states. + 2) Talker (model_stage="tts"): native MiniCPMTTS AR → codec deltas. """ if self.model_stage == "llm": # Normalize to batched inputs if caller provides 1D/2D unbatched tensors @@ -778,57 +620,24 @@ class MiniCPMO45OmniForConditionalGeneration(nn.Module, SupportsMultiModal, Supp multimodal_outputs=multimodal_outputs, ) - # Talker stage: runs TTS generation and its built-in Token2Wav vocoder. + # Talker stage: runner-owned native AR only. Waveform generation belongs + # to the separate Code2Wav stage. if self.model_stage == "tts": - if input_ids is not None: - num_tokens = input_ids.shape[0] - device = input_ids.device - elif inputs_embeds is not None: - num_tokens = inputs_embeds.shape[0] - device = inputs_embeds.device - else: - num_tokens = 1 - device = current_omni_platform.get_torch_device() - hidden_dim = self.config.hidden_size if hasattr(self.config, "hidden_size") else 2560 - - # Profile/dummy run: both input_ids and inputs_embeds are None. - # Note: SupportsMultiModal preprocessing converts input_ids to - # inputs_embeds, so input_ids=None alone does NOT indicate a dummy run. - if input_ids is None and inputs_embeds is None: - dummy_hidden = torch.zeros(num_tokens, hidden_dim, device=device) - return OmniOutput(text_hidden_states=dummy_hidden, multimodal_outputs=None) - - runtime_info = kwargs.get("runtime_additional_information") - request_token_spans = kwargs.get("request_token_spans") - if isinstance(runtime_info, list) and (len(runtime_info) > 1 or request_token_spans is not None): - return self._run_batched_tts_requests( - input_ids=input_ids, - positions=positions, - inputs_embeds=inputs_embeds, - runtime_info=runtime_info, - request_token_spans=request_token_spans, - num_tokens=num_tokens, - hidden_dim=hidden_dim, - device=device, - ) - talker_info = {} - if runtime_info and isinstance(runtime_info, list) and len(runtime_info) > 0: - talker_info = runtime_info[0] if isinstance(runtime_info[0], dict) else {} - dummy_hidden = torch.zeros(num_tokens, hidden_dim, device=device) - mm_out, _, _ = self._run_tts_request( + return self.talker( input_ids=input_ids, positions=positions, + intermediate_tensors=intermediate_tensors, inputs_embeds=inputs_embeds, - talker_info=talker_info, - device=device, + **kwargs, ) - if mm_out is not None: - return OmniOutput(text_hidden_states=dummy_hidden, multimodal_outputs=mm_out) - - return OmniOutput(text_hidden_states=dummy_hidden, multimodal_outputs=None) raise ValueError(f"Unsupported model stage: {self.model_stage}") + def make_omni_output(self, model_outputs, **kwargs): + if self.model_stage != "tts": + return model_outputs + return self.talker.make_omni_output(model_outputs, **kwargs) + def compute_logits(self, hidden_states: torch.Tensor | OmniOutput) -> torch.Tensor | None: # Handle OmniOutput type if isinstance(hidden_states, OmniOutput): @@ -1261,7 +1070,7 @@ class MiniCPMO45OmniForConditionalGeneration(nn.Module, SupportsMultiModal, Supp # MiniCPM-o checkpoint prefixes → stage mapping: # thinker: vpm, resampler, llm, apm, audio_projection_layer - # talker: tts (ConditionalChatTTS) + # talker: tts (native MiniCPMTTS AR codec producer) for k, v in weights: if k.startswith(("vpm.", "resampler.", "llm.", "apm.", "audio_projection_layer.")): thinker_weights.append((k, v)) diff --git a/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_tts.py b/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_tts.py index 73570ef74..09bbb2771 100644 --- a/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_tts.py +++ b/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_tts.py @@ -2,246 +2,129 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project # Adapted from: # https://huggingface.co/openbmb/MiniCPM-o-4_5/blob/main/modeling_minicpmo.py -"""MiniCPM-o 4.5 Talker + Token2Wav: MiniCPMTTS with hidden_text_merge condition. +"""MiniCPM-o 4.5 native autoregressive Talker. Pipeline: 1. Receive thinker hidden_states + full token IDs via additional_information 2. Extract tts_bos..tts_eos region 3. Build condition: emb_text(tokens) + projector_semantic(hidden) (hidden_text_merge) - 4. Run MiniCPMTTS.generate() -> discrete audio tokens - 5. Run Token2wav(tokens) -> waveform bytes -> numpy array + 4. Continuously generate request-aligned discrete audio-code deltas """ -import hashlib -import io -import logging -import os -import sys -import tempfile -import threading -import time -from collections import OrderedDict from collections.abc import Iterable -from dataclasses import dataclass -from importlib import import_module from typing import Any -import numpy as np -import soundfile as sf import torch import torch.nn as nn import torch.nn.functional as F +from transformers import LlamaConfig from vllm.config import VllmConfig from vllm.model_executor.models.interfaces import SupportsPP -from vllm.v1.outputs import SamplerOutput +from vllm.model_executor.models.llama import LlamaModel +from vllm.model_executor.models.utils import maybe_prefix +from vllm.v1.sample.sampler import Sampler -from vllm_omni.experimental.fullduplex.engine.intermediate import get_stream_request_key, get_tts_handoff -from vllm_omni.model_executor.model_loader.weight_utils import ( - download_weights_from_hf_specific, -) +from vllm_omni.experimental.fullduplex.engine.intermediate import get_tts_handoff +from vllm_omni.model_executor.models.output_templates import OmniOutput from vllm_omni.platforms import current_omni_platform -# The external vocoder hard-codes CUDA placement. Ascend uses the in-tree -# adapter, with the external package retained as a fallback for compatibility. -_stepaudio2_import_error: ImportError | None = None -if current_omni_platform.is_npu(): - try: - from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_token2wav import ( - MiniCPMO45Token2wav as _Token2wav, - ) - - _token2wav_backend = "step_audio2_core" - except ImportError as e: - _stepaudio2_import_error = e - try: - from stepaudio2 import Token2wav as _Token2wav - - _token2wav_backend = "stepaudio2_pkg" - except ImportError as fallback_error: - _Token2wav = None - _token2wav_backend = None - _stepaudio2_import_error = fallback_error -else: - try: - from stepaudio2 import Token2wav as _Token2wav - - _token2wav_backend = "stepaudio2_pkg" - except ImportError as e: - _Token2wav = None - _token2wav_backend = None - _stepaudio2_import_error = e - -_stepaudio2_available = _Token2wav is not None - -logger = logging.getLogger(__name__) +_REPETITION_WINDOW = 16 +_MIN_AUDIO_TOKENS = 64 +_MAX_AUDIO_TOKENS = 2048 +_AUDIO_TOKENS_PER_TEXT_TOKEN = 10 +# Codec-token sampling happens inside the model; vLLM sampling parameters +# only choose the Talker's binary continue/stop row. +_CODEC_SEED = 42 +_CODEC_TEMPERATURE = 0.8 +_CODEC_TOP_K = 25 +_CODEC_TOP_P = 0.85 +_CODEC_REPETITION_PENALTY = 1.05 +_CODEC_MIN_TOKENS = 50 -@dataclass(slots=True) -class MiniCPMO45TTSRuntimeConfig: - """Internal MiniCPM-o 4.5 Talker runtime defaults. +def _max_audio_tokens(condition_tokens: int) -> int: + """Bound codec generation with a conservative text-length estimate. - These values are deliberately not process environment knobs. If a value - needs to become user configurable, route it through stage/model config with - an explicit API contract instead of adding another ad hoc env var. + EOS is masked for the first 50 steps, so a direct ``text_tokens * 10`` + limit can terminate short responses before EOS is eligible. The 2048 + ceiling matches the checkpoint's native generation default and keeps the + sequence within the Talker's 4096-position context. """ - - token2wav_n_timesteps: int = 10 - tts_dtype: torch.dtype = torch.float32 - token2wav_autocast_dtype: torch.dtype | None = None - use_direct_token2wav: bool = True - ref_audio_file_cache_size: int = 16 - max_token_ratio: int = 32 - min_max_new_tokens: int = 256 - hard_max_new_tokens: int = 16384 - min_new_tokens: int = 50 - streaming_generator_chunk: int = 25 - streaming_vocoder_threshold: int = 2500 - streaming_vocoder_chunk: int = 50 - - -def _install_torchaudio_soundfile_shim() -> None: - """Monkey-patch torchaudio.load to use soundfile instead of the default - torchcodec backend, which requires libtorchcodec/ffmpeg shared libs that - may be missing on the deployment machine.""" - try: - import torchaudio - - if getattr(torchaudio, "_soundfile_shim_installed", False): - return - _orig_load = torchaudio.load - - def _patched_load(uri, *args, **kwargs): - try: - return _orig_load(uri, *args, **kwargs) - except Exception: - import numpy as _np - import soundfile as _sf - - data, sr = _sf.read(uri, dtype="float32", always_2d=True) - wav = torch.from_numpy(_np.ascontiguousarray(data.T)) - return wav, sr - - torchaudio.load = _patched_load - torchaudio._soundfile_shim_installed = True - logger.info("Installed torchaudio.load soundfile shim") - except Exception as _e: - logger.warning("Could not install torchaudio shim: %s", _e) - - -_install_torchaudio_soundfile_shim() - - -class _TalkerTurnState: - """Per-turn talker continuity for native duplex. - - The official duplex implementation runs ONE TTS stream per spoken turn: - each 1s unit appends its condition to the same KV (carried - past_key_values + text_start_pos) and the token2wav stream caches and - token buffer persist across units, reset only at turn end. Synthesizing - units as independent utterances resets prosody every second and garbles - the reply. - """ - - __slots__ = ( - "past_key_values", - "text_start_pos", - "token2wav_buffer", - "prompt_wav_path", - "temp_prompt_wav_path", - "epoch", - "turn_id", - "pending_text", - "stream_cache", - "hift_cache_dict", - "vocoder_initialized", + return max( + _MIN_AUDIO_TOKENS, + min(_MAX_AUDIO_TOKENS, condition_tokens * _AUDIO_TOKENS_PER_TEXT_TOKEN), ) - def __init__( - self, - prompt_wav_path, - temp_prompt_wav_path, - *, - epoch: int | None = None, - turn_id: int | None = None, - ): - self.past_key_values = None - self.text_start_pos = 0 - # Official seeds each turn's vocoder buffer with three silence - # tokens so the first synthesized window does not directly abut the - # reference-audio prompt cache (audible ref-voice bleed otherwise). - self.token2wav_buffer: list[int] = [_T2W_SILENCE_TOKEN] * 3 - self.prompt_wav_path = prompt_wav_path - self.temp_prompt_wav_path = temp_prompt_wav_path - self.epoch = epoch - self.turn_id = turn_id - self.pending_text = "" - self.stream_cache = None - self.hift_cache_dict: dict[str, Any] = {} - self.vocoder_initialized = False + +def _restore_weight_norm_weight(weight_g: torch.Tensor, weight_v: torch.Tensor) -> torch.Tensor: + """Materialize ``weight_norm(..., dim=0)`` checkpoint parameters.""" + row_norm = torch.linalg.vector_norm( + weight_v.float(), + dim=1, + keepdim=True, + ).clamp_min(torch.finfo(torch.float32).tiny) + return weight_v * (weight_g.to(dtype=weight_v.dtype) / row_norm.to(dtype=weight_v.dtype)) -def _queue_native_duplex_segment_text(state: _TalkerTurnState, text: object) -> None: - if isinstance(text, str) and text: - state.pending_text += text - - -def _drain_native_duplex_emitted_text(state: _TalkerTurnState, *, has_audio: bool) -> str: - if not has_audio: - return "" - text = state.pending_text - state.pending_text = "" - return text - - -_T2W_SILENCE_TOKEN = 4218 -_NATIVE_DUPLEX_UNIT_AUDIO_SAMPLES = 24_000 - - -def _native_duplex_unit_waveform( - waveforms: Iterable[torch.Tensor], +def _apply_repetition_penalty( + logits: torch.Tensor, + history: torch.Tensor, *, - turn_end: bool, - target_samples: int = _NATIVE_DUPLEX_UNIT_AUDIO_SAMPLES, -) -> torch.Tensor | None: - pieces = [torch.as_tensor(waveform).reshape(-1).cpu().contiguous() for waveform in waveforms] - pieces = [piece for piece in pieces if piece.numel() > 0] - if not pieces: - return None - waveform = pieces[0] if len(pieces) == 1 else torch.cat(pieces, dim=0) - if not turn_end and waveform.numel() < target_samples: - waveform = F.pad(waveform, (target_samples - waveform.numel(), 0)) - return waveform + penalty: float, + window_size: int, +) -> torch.Tensor: + """Match MiniCPMTTS' frequency-aware repetition penalty.""" + if penalty == 1.0 or history.numel() == 0: + return logits + recent = history.reshape(-1)[-window_size:].to(device=logits.device, dtype=torch.long) + frequencies = torch.bincount(recent, minlength=logits.shape[-1]).to(dtype=logits.dtype) + alpha = torch.pow(torch.as_tensor(penalty, device=logits.device, dtype=logits.dtype), frequencies) + return torch.where(logits < 0, logits * alpha, logits / alpha) -def _soundfile_patched_save(orig_save): - def _patched_save(uri, src, sample_rate, **kw): - kw.pop("backend", None) - if hasattr(uri, "write"): - sf.write(uri, src.cpu().numpy().T, sample_rate, format="WAV") - return - return orig_save(uri, src, sample_rate, backend="soundfile", **kw) - - return _patched_save +def _apply_top_k_top_p( + logits: torch.Tensor, + *, + top_k: int | None, + top_p: float | None, + min_tokens_to_keep: int = 3, +) -> torch.Tensor: + """Apply the same candidate floors as the upstream Transformers warpers.""" + filtered = logits.clone() + vocab_size = filtered.shape[-1] + # MiniCPM-o's gen_logits() appends TopPLogitsWarper before + # TopKLogitsWarper. The order is observable for fixed-seed sampling. + if top_p is not None and 0.0 < top_p < 1.0: + sorted_logits, sorted_indices = torch.sort(filtered, descending=False, dim=-1) + cumulative_probs = torch.softmax(sorted_logits, dim=-1).cumsum(dim=-1) + remove = cumulative_probs <= (1.0 - float(top_p)) + remove[..., -min_tokens_to_keep:] = False + remove = remove.scatter(-1, sorted_indices, remove) + filtered.masked_fill_(remove, float("-inf")) + if top_k is not None and top_k > 0: + keep = min(vocab_size, max(int(top_k), min_tokens_to_keep)) + threshold = torch.topk(filtered, keep, dim=-1).values[..., -1, None] + filtered.masked_fill_(filtered < threshold, float("-inf")) + return filtered -def _torch_clone_recursive(obj): - if isinstance(obj, torch.Tensor): - return obj.clone() - if isinstance(obj, dict): - return {k: _torch_clone_recursive(v) for k, v in obj.items()} - if isinstance(obj, list): - return [_torch_clone_recursive(v) for v in obj] - if isinstance(obj, tuple): - return tuple(_torch_clone_recursive(v) for v in obj) - return obj +class _MiniCPMTTSProjector(nn.Module): + """Checkpoint-compatible hidden-state projector used by MiniCPMTTS.""" + + def __init__(self, input_size: int, hidden_size: int): + super().__init__() + self.linear1 = nn.Linear(input_size, hidden_size, bias=True) + self.relu = nn.ReLU() + self.linear2 = nn.Linear(hidden_size, hidden_size, bias=True) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + return self.linear2(self.relu(self.linear1(hidden_states))) class MiniCPMO45OmniTTSForConditionalGeneration(nn.Module, SupportsPP): - """MiniCPM-o 4.5 Talker: MiniCPMTTS + Token2wav in a single forward pass.""" + """Runner-owned MiniCPM-o 4.5 Talker that emits codec tokens only.""" - # llm2tts hands the FULL accumulated condition per handoff (the runner's - # resume-prefill path REPLACES the streaming buffer, so runner-side - # accumulation is lossy); the per-turn state consumes it by cursor. + requires_request_sample_eligibility = True def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): super().__init__() @@ -250,1366 +133,360 @@ class MiniCPMO45OmniTTSForConditionalGeneration(nn.Module, SupportsPP): config: MiniCPMOConfig = vllm_config.model_config.hf_config self.config = config self.vllm_config = vllm_config - self._runtime_config = MiniCPMO45TTSRuntimeConfig() - - self.tts = None - self.audio_tokenizer = None - self._assets_loaded = False - self._model_path: str | None = None - self._stream_gens: dict[str, Any] = {} - self._talker_turn_states: dict[str, _TalkerTurnState] = {} - # Consumed-cursor into the accumulated handoff condition for the - # currently open spoken turn. - self._talker_consumed_tokens: dict[str, int] = {} - self._talker_request_keys: dict[str, str] = {} - self._t2w_base_caches: dict[str, tuple[Any, Any]] = {} - self._token2wav_state_lock = threading.RLock() - self._ar_last_chunk_flags: list[bool] = [True] - self._ar_turn_end_flags: list[bool] = [False] - self._ar_last_emitted_text = "" - self._text_tokenizer = None + self._batch_stop_logits: torch.Tensor | None = None + self._request_generators: dict[str, torch.Generator] = {} + self._request_audio_states: dict[str, dict[str, Any]] = {} + self._deferred_cleanup_ids: set[str] = set() tts_config = getattr(config, "tts_config", None) + if tts_config is None and getattr(config, "model_type", None) == "minicpmtts": + tts_config = config if tts_config is not None: self._tts_config = tts_config - self._tts_bos_id = self._config_token_id(tts_config, "audio_bos_token_id") - self._text_eos_id = self._config_token_id(tts_config, "text_eos_token_id") + self._tts_bos_id = getattr(tts_config, "audio_bos_token_id", 151687) + self._text_eos_id = getattr(tts_config, "text_eos_token_id", 151692) self._num_audio_tokens = getattr(tts_config, "num_audio_tokens", 6562) self._hidden_size = getattr(tts_config, "hidden_size", 768) self._normalize = getattr(tts_config, "normalize_projected_hidden", True) + self._codec_seed = int(getattr(tts_config, "seed", _CODEC_SEED)) + self._codec_temperature = float(getattr(tts_config, "temperature", _CODEC_TEMPERATURE)) + self._codec_top_k = int(getattr(tts_config, "top_k", _CODEC_TOP_K)) + self._codec_top_p = float(getattr(tts_config, "top_p", _CODEC_TOP_P)) + self._codec_repetition_penalty = float(getattr(tts_config, "repetition_penalty", _CODEC_REPETITION_PENALTY)) + self._codec_min_tokens = int(getattr(tts_config, "min_new_tokens", _CODEC_MIN_TOKENS)) else: self._tts_config = None - def _tts_runtime_config(self) -> MiniCPMO45TTSRuntimeConfig: - cfg = getattr(self, "_runtime_config", None) - if cfg is None: - cfg = MiniCPMO45TTSRuntimeConfig() - self._runtime_config = cfg - return cfg + self.has_preprocess = True + self.has_postprocess = False + self.gpu_resident_buffer_keys: set[tuple[str, str]] = { + ("audio_codes", "current"), + ("audio_codes", "accumulated"), + } + self._init_native_talker(prefix) - @staticmethod - def _config_token_id(config: Any, attr: str) -> int: - value = getattr(config, attr, None) - if isinstance(value, (list, tuple)): - value = value[0] if value else None - if value is None: - raise ValueError(f"MiniCPM-o 4.5 TTS config missing required {attr}") - return int(value) - - def _get_text_tokenizer(self) -> Any: - tokenizer = getattr(self, "_text_tokenizer", None) - if tokenizer is not None: - return tokenizer - from vllm.transformers_utils.tokenizer import cached_tokenizer_from_config - - tokenizer = cached_tokenizer_from_config(self.vllm_config.model_config) - self._text_tokenizer = tokenizer - return tokenizer - - def _tokenizer_token_id(self, token: str) -> int | None: - tokenizer = self._get_text_tokenizer() - unk_token_id = getattr(tokenizer, "unk_token_id", None) - convert = getattr(tokenizer, "convert_tokens_to_ids", None) - if callable(convert): - value = convert(token) - if isinstance(value, list): - value = value[0] if len(value) == 1 else None - try: - candidate = int(value) - except (TypeError, ValueError): - candidate = None - if candidate is not None and candidate >= 0 and candidate != unk_token_id: - return candidate - encode = getattr(tokenizer, "encode", None) - if callable(encode): - ids = list(encode(token, add_special_tokens=False)) - if len(ids) == 1: - candidate = int(ids[0]) - if candidate >= 0 and candidate != unk_token_id: - return candidate - return None - - def _scheduler_eos_token_id(self) -> int: - eos_raw = getattr(self.config, "eos_token_id", None) - if isinstance(eos_raw, (list, tuple)): - eos_raw = eos_raw[0] if eos_raw else None - if eos_raw is not None: - return int(eos_raw) - eos_id = self._tokenizer_token_id("<|im_end|>") - if eos_id is None: + def _init_native_talker(self, prefix: str) -> None: + if self._tts_config is None: + raise ValueError("MiniCPM-o continuous Talker requires tts_config") + cfg = self._tts_config + if int(getattr(cfg, "num_vq", 1)) != 1: raise ValueError( - "MiniCPM-o 4.5 TTS scheduler EOS requires config.eos_token_id or tokenizer-defined <|im_end|>" + "MiniCPM-o continuous Talker currently requires num_vq=1; " + f"checkpoint reports {getattr(cfg, 'num_vq', None)}" ) - return eos_id - - def _lazy_init_tts(self): - if self._assets_loaded or self._tts_config is None: - return - self._assets_loaded = True - try: - model_path = download_weights_from_hf_specific(self.vllm_config.model_config.model, None, ["*"]) - self._model_path = model_path - if model_path not in sys.path: - sys.path.insert(0, model_path) - from transformers import AutoImageProcessor - from transformers.dynamic_module_utils import get_class_from_dynamic_module - - # The remote processing module registers an image processor by - # string, which transformers>=5 rejects. The standalone talker does - # not use that registration, so ignore only the string form while - # importing MiniCPMTTS and restore the global method immediately. - original_register = AutoImageProcessor.register - AutoImageProcessor.register = ( # type: ignore[method-assign] - lambda key, *a, **k: None if isinstance(key, str) else original_register(key, *a, **k) - ) - try: - MiniCPMTTS = get_class_from_dynamic_module("modeling_minicpmo.MiniCPMTTS", model_path) - finally: - AutoImageProcessor.register = original_register # type: ignore[method-assign] - - prev_dtype = torch.get_default_dtype() - torch.set_default_dtype(torch.float32) - try: - for name, default in ( - ("top_p", 0.85), - ("top_k", 25), - ("repetition_penalty", 1.05), - ("temperature", 0.8), - ): - if not hasattr(self._tts_config, name): - setattr(self._tts_config, name, default) - self._tts_config.attn_implementation = "sdpa" - self.tts_obj = MiniCPMTTS(config=self._tts_config, audio_tokenizer=None) - finally: - torch.set_default_dtype(prev_dtype) - tts_module = import_module(self.tts_obj.__class__.__module__) - - def get_tts_module_attr(name: str): - return self.tts_obj.generate.__globals__.get(name) or getattr(tts_module, name, None) - - self._tts_sampling_params_cls = get_tts_module_attr("TTSSamplingParams") - self._tts_gen_logits = get_tts_module_attr("gen_logits") - self._tts_parametrize = get_tts_module_attr("P") - self._tts_streaming_generator_cls = get_tts_module_attr("TTSStreamingGenerator") - self.emb_text = self.tts_obj.emb_text - self.projector_semantic = self.tts_obj.projector_semantic - - token2wav_dir = os.path.join(model_path, "assets", "token2wav") - if os.path.isdir(token2wav_dir): - if not _stepaudio2_available: - raise ImportError( - "MiniCPM-o 4.5 token2wav stage requires the stepaudio2 package, " - "and all of its runtime dependencies." - ) from _stepaudio2_import_error - self._token2wav_n_timesteps = self._tts_runtime_config().token2wav_n_timesteps - prev_dtype2 = torch.get_default_dtype() - torch.set_default_dtype(torch.float32) - try: - self.audio_tokenizer = _Token2wav( - token2wav_dir, - float16=False, - n_timesteps=self._token2wav_n_timesteps, - ) - finally: - torch.set_default_dtype(prev_dtype2) - self.tts_obj.audio_tokenizer = self.audio_tokenizer - logger.info( - "Loaded Token2wav from %s (backend=%s, n_timesteps=%d)", - token2wav_dir, - _token2wav_backend, - self._token2wav_n_timesteps, - ) - except ImportError: - # Surface missing dependencies directly so users can act on them - # instead of getting a silent None waveform downstream. - raise - except Exception as e: - logger.error("Failed to init 4.5 TTS: %s", e, exc_info=True) - - def _build_tts_sampling_params(self): - params_cls = getattr(self, "_tts_sampling_params_cls", None) - if params_cls is None or not hasattr(self, "tts_obj"): - return None - - tts = self.tts_obj - - top_p = getattr(tts, "top_p", getattr(tts.config, "top_p", 0.85)) - top_k = getattr(tts, "top_k", getattr(tts.config, "top_k", 25)) - repetition_penalty = getattr( - tts, - "repetition_penalty", - getattr(tts.config, "repetition_penalty", 1.05), + llama_config = LlamaConfig( + vocab_size=32000, + hidden_size=int(cfg.hidden_size), + intermediate_size=int(cfg.intermediate_size), + num_hidden_layers=int(cfg.num_hidden_layers), + num_attention_heads=int(cfg.num_attention_heads), + num_key_value_heads=int(cfg.num_key_value_heads), + hidden_act=getattr(cfg, "hidden_act", "silu"), + max_position_embeddings=int(cfg.max_position_embeddings), + rms_norm_eps=float(getattr(cfg, "rms_norm_eps", 1e-6)), + tie_word_embeddings=False, ) - temperature = getattr(tts.config, "temperature", 0.8) - - return params_cls( - top_p=None if top_p is not None and top_p >= 1.0 else top_p, - top_k=None if top_k is not None and top_k <= 0 else top_k, - repetition_penalty=repetition_penalty, - temperature=temperature, + talker_config = self.vllm_config.with_hf_config(llama_config, architectures=["LlamaForCausalLM"]) + talker_config.model_config.hf_text_config = llama_config + self.tts_model = LlamaModel( + vllm_config=talker_config, + prefix=maybe_prefix(prefix, "tts_obj.model"), ) - - def _target_tts_dtype(self) -> torch.dtype: - return self._tts_runtime_config().tts_dtype - - def _token2wav_autocast_context(self): - dtype = self._tts_runtime_config().token2wav_autocast_dtype - device_type = str(current_omni_platform.device_type) - if dtype is None: - return ( - current_omni_platform.create_autocast_context( - device_type=device_type, - dtype=torch.float32, - enabled=False, - ), - "off", - ) - if dtype is torch.bfloat16: - return ( - current_omni_platform.create_autocast_context( - device_type=device_type, - dtype=torch.bfloat16, - enabled=True, - ), - "bf16", - ) - if dtype is torch.float16: - return ( - current_omni_platform.create_autocast_context( - device_type=device_type, - dtype=torch.float16, - enabled=True, - ), - "fp16", - ) - raise ValueError("MiniCPM-o 4.5 token2wav autocast only supports None, bfloat16, or float16") - - def _should_use_direct_token2wav(self) -> bool: - return self._tts_runtime_config().use_direct_token2wav - - def _should_stream_output(self, info: dict[str, Any] | None = None) -> bool: - if isinstance(info, dict): - for key in ("stream_output", "native_duplex"): - value = info.get(key) - if isinstance(value, bool): - return value - return False - - def _token2wav_prompt_cache_key(self, prompt_wav: str | None) -> str | None: - return os.path.abspath(prompt_wav) if prompt_wav else None - - def _reset_token2wav_cache_if_needed(self, prompt_wav: str | None) -> None: - token2wav = self.audio_tokenizer - if token2wav is None: - return - cache_key = self._token2wav_prompt_cache_key(prompt_wav) - if getattr(self, "_token2wav_prompt_cache_id", None) != cache_key: - token2wav.cache = None - self._token2wav_prompt_cache_id = cache_key - - def _normalize_ref_audio_tensor(self, ref_audio) -> np.ndarray | None: - if ref_audio is None: - return None - if isinstance(ref_audio, torch.Tensor): - waveform = ref_audio.detach().float().cpu().numpy() - else: - waveform = np.asarray(ref_audio, dtype=np.float32) - if waveform.ndim > 1: - if waveform.shape[0] <= 2 and waveform.shape[-1] > waveform.shape[0]: - waveform = waveform.mean(axis=0) - else: - waveform = waveform.mean(axis=-1) - waveform = np.asarray(waveform, dtype=np.float32).reshape(-1) - return waveform if waveform.size else None - - def _write_ref_audio_prompt_wav(self, ref_audio, ref_audio_sr: int | None) -> str | None: - waveform = self._normalize_ref_audio_tensor(ref_audio) - if waveform is None: - return None - sample_rate = int(ref_audio_sr or 24000) - cache_size = self._tts_runtime_config().ref_audio_file_cache_size - if cache_size > 0: - digest = hashlib.sha256() - digest.update(str(sample_rate).encode("ascii")) - digest.update(waveform.tobytes()) - cache_key = digest.hexdigest() - cache = getattr(self, "_ref_audio_prompt_files", None) - if cache is None: - cache = OrderedDict() - self._ref_audio_prompt_files = cache - cached_path = cache.get(cache_key) - if cached_path and os.path.exists(cached_path): - cache.move_to_end(cache_key) - return cached_path - - tmp_path = os.path.join(tempfile.gettempdir(), f"minicpmo45_ref_{cache_key[:24]}_{sample_rate}.wav") - if not os.path.exists(tmp_path): - sf.write(tmp_path, waveform, sample_rate, format="WAV") - cache[cache_key] = tmp_path - cache.move_to_end(cache_key) - while len(cache) > cache_size: - _, evicted_path = cache.popitem(last=False) - with self._token2wav_lock(): - self._t2w_base_caches.pop(evicted_path, None) - try: - os.unlink(evicted_path) - except OSError: - pass - return tmp_path - - tmp = tempfile.NamedTemporaryFile(prefix="minicpmo45_ref_", suffix=".wav", delete=False) - tmp_path = tmp.name - tmp.close() - sf.write(tmp_path, waveform, sample_rate, format="WAV") - return tmp_path - - def _is_cached_ref_audio_prompt_wav(self, prompt_wav: str | None) -> bool: - cache = getattr(self, "_ref_audio_prompt_files", None) - return bool(prompt_wav and cache and prompt_wav in cache.values()) - - def _run_token2wav_direct( - self, - generated_speech_tokens: list[int] | torch.Tensor, - prompt_wav: str | None, - ) -> tuple[torch.Tensor, int]: - """Run Token2wav without WAV encode/decode round-tripping. - - ``stepaudio2.Token2wav.__call__`` renders the GPU waveform to a WAV - BytesIO object and the vLLM-Omni adapter immediately decodes it back to - a numpy waveform. The engine already expects a float waveform tensor, - so keep the same flow/HIFT computation and return the waveform directly. - This also makes the configured ``n_timesteps`` apply to one-shot - inference; upstream currently hard-codes ``10`` in ``__call__``. - """ - token2wav = self.audio_tokenizer - if token2wav is None: - raise RuntimeError("Token2wav is not initialized") - required_attrs = ("_prepare_prompt", "flow", "hift") - if any(not hasattr(token2wav, attr) for attr in required_attrs): - raise RuntimeError("Token2wav direct path is incompatible with the installed stepaudio2 package") - - self._reset_token2wav_cache_if_needed(prompt_wav) - if token2wav.cache is None: - token2wav.cache = token2wav._prepare_prompt(prompt_wav) - prompt_speech_tokens, prompt_speech_tokens_lens, spk_emb, prompt_mels, prompt_mels_lens = token2wav.cache - - device = prompt_speech_tokens.device - if isinstance(generated_speech_tokens, torch.Tensor): - generated = generated_speech_tokens - if generated.ndim == 1: - generated = generated.unsqueeze(0) - elif generated.ndim == 3 and generated.shape[-1] == 1: - generated = generated.squeeze(-1) - generated = generated.to(device=device, dtype=torch.int32) - else: - generated = torch.tensor([generated_speech_tokens], dtype=torch.int32, device=device) - generated_lens = torch.tensor([generated.shape[1]], dtype=torch.int32, device=device) - mel = token2wav.flow.inference( - generated, - generated_lens, - prompt_speech_tokens, - prompt_speech_tokens_lens, - prompt_mels, - prompt_mels_lens, - spk_emb, - self._token2wav_n_timesteps, + self.emb_text = nn.Embedding(int(cfg.num_text_tokens), int(cfg.hidden_size)) + self.projector_semantic = _MiniCPMTTSProjector(int(cfg.llm_dim), int(cfg.hidden_size)) + self.emb_code = nn.ModuleList( + [nn.Embedding(int(cfg.num_audio_tokens), int(cfg.hidden_size)) for _ in range(int(cfg.num_vq))] ) - wav, _ = token2wav.hift(speech_feat=mel) - waveform = wav.squeeze(0).detach().float().reshape(-1).cpu().contiguous() - return waveform, 24000 + self.head_code = nn.ModuleList( + [nn.Linear(int(cfg.hidden_size), int(cfg.num_audio_tokens), bias=False) for _ in range(int(cfg.num_vq))] + ) + self.make_empty_intermediate_tensors = self.tts_model.make_empty_intermediate_tensors - def _build_tts_condition_embeds( + def _build_condition_embeddings( self, tts_token_ids: torch.Tensor, tts_hidden_states: torch.Tensor, ) -> torch.Tensor: - tts = self.tts_obj - device = tts.emb_text.weight.device - dtype = tts.emb_text.weight.dtype - llm_embeds = tts.emb_text(tts_token_ids.to(device)) - hidden_embeds = tts.projector_semantic(tts_hidden_states.to(device=device, dtype=dtype)) - if getattr(tts.config, "normalize_projected_hidden", False): - hidden_embeds = F.normalize(hidden_embeds, p=2, dim=-1) - return llm_embeds + hidden_embeds - - def _normalize_tts_handoff_tensors( - self, - tts_token_ids: Any, - tts_hidden_states: Any, - ) -> tuple[torch.Tensor, torch.Tensor]: - token_ids = torch.as_tensor(tts_token_ids, dtype=torch.long) - if token_ids.ndim == 0: - token_ids = token_ids.reshape(1) - elif token_ids.ndim > 1: - token_ids = token_ids.reshape(-1) - - hidden_states = torch.as_tensor(tts_hidden_states, dtype=torch.float32) - if hidden_states.ndim == 1: - hidden_states = hidden_states.unsqueeze(0) - elif hidden_states.ndim == 3 and hidden_states.shape[0] == 1: - hidden_states = hidden_states.squeeze(0) - elif hidden_states.ndim != 2: - hidden_states = hidden_states.reshape(token_ids.numel(), -1) - - if hidden_states.shape[0] != token_ids.numel(): + device = self.emb_text.weight.device + dtype = self.emb_text.weight.dtype + token_ids = tts_token_ids.to(device=device, dtype=torch.long).reshape(-1) + hidden = tts_hidden_states.to(device=device, dtype=dtype) + if hidden.shape[0] != token_ids.shape[0] and token_ids.shape[0] != 1: raise ValueError( - "MiniCPM-o 4.5 TTS handoff has mismatched token/hidden lengths: " - f"tokens={token_ids.numel()} hidden_rows={hidden_states.shape[0]}" + "MiniCPM-o Talker condition length mismatch: " + f"token_ids={token_ids.shape[0]} hidden_states={hidden.shape[0]}" ) - return token_ids.contiguous(), hidden_states.contiguous() - - def _resolve_prompt_wav_path(self, ref_audio, ref_audio_sr: int | None) -> tuple[str | None, str | None]: - temp_prompt_wav_path = self._write_ref_audio_prompt_wav(ref_audio, ref_audio_sr) - if temp_prompt_wav_path is not None: - return temp_prompt_wav_path, temp_prompt_wav_path - if (model_path := getattr(self, "_model_path", None)) is not None: - default_ref = os.path.join(model_path, "assets", "HT_ref_audio.wav") - if os.path.exists(default_ref): - return default_ref, None - return None, None - - def _max_tts_tokens_for_text(self, num_text: int) -> tuple[int, int]: - cfg = self._tts_runtime_config() - max_new_token = min( - cfg.hard_max_new_tokens, - max(cfg.min_max_new_tokens, num_text * cfg.max_token_ratio), - ) - return cfg.min_new_tokens, max_new_token - - def _stream_request_key(self, info: dict[str, Any]) -> str: - return get_stream_request_key(info) - - @staticmethod - def _coerce_request_key(value: Any) -> str | None: - if isinstance(value, (list, tuple)): - value = value[0] if value else None - if isinstance(value, bytes): - value = value.decode("utf-8", errors="replace") - if value is None: - return None - text = str(value) - return text if text else None - - @staticmethod - def _coerce_epoch(value: Any) -> int | None: - try: - return int(value) if value is not None else None - except (TypeError, ValueError): - return None - - _coerce_turn_id = _coerce_epoch - - def _stream_epoch(self, info: dict[str, Any]) -> int | None: - duplex = info.get("duplex") - if isinstance(duplex, dict): - epoch = self._coerce_epoch(duplex.get("epoch")) - if epoch is not None: - return epoch - meta = info.get("meta") - if isinstance(meta, dict): - epoch = self._coerce_epoch(meta.get("epoch")) - if epoch is not None: - return epoch - return self._coerce_epoch(info.get("epoch")) - - def _stream_turn_id(self, info: dict[str, Any]) -> int | None: - duplex = info.get("duplex") - if isinstance(duplex, dict): - turn_id = self._coerce_turn_id(duplex.get("model_turn_id")) - if turn_id is not None: - return turn_id - turn_id = self._coerce_turn_id(duplex.get("turn_id")) - if turn_id is not None: - return turn_id - meta = info.get("meta") - if isinstance(meta, dict): - turn_id = self._coerce_turn_id(meta.get("turn_id")) - if turn_id is not None: - return turn_id - return self._coerce_turn_id(info.get("turn_id")) - - def _remember_talker_request_key(self, info: dict[str, Any], key: str) -> None: - aliases = { - self._coerce_request_key(info.get("request_id")), - self._coerce_request_key(info.get("_omni_req_id")), - } - duplex = info.get("duplex") - if isinstance(duplex, dict): - aliases.add(self._coerce_request_key(duplex.get("request_id"))) - request_keys = getattr(self, "_talker_request_keys", None) - if request_keys is None: - request_keys = {} - self._talker_request_keys = request_keys - for alias in aliases: - if alias and alias != key: - request_keys[alias] = key - - def _empty_audio_chunk(self) -> torch.Tensor: - return torch.zeros((0,), dtype=torch.float32) - - @staticmethod - def _extract_tts_handoff(info: dict[str, Any]) -> tuple[Any, Any]: - return get_tts_handoff(info) - - def _native_duplex_input_ends_turn(self, info: dict[str, Any]) -> bool: - if info.get("native_duplex") is not True: - return False - meta = info.get("meta") - if not isinstance(meta, dict): - meta = {} - if bool(info.get("turn_end") or info.get("end_of_turn") or meta.get("turn_end") or meta.get("end_of_turn")): - return True - turn_eos_id = self._coerce_epoch(meta.get("turn_eos_token_id")) - if turn_eos_id is None: - return False - tts_token_ids, _ = self._extract_tts_handoff(info) - if isinstance(tts_token_ids, torch.Tensor): - return bool((tts_token_ids == turn_eos_id).any().item()) - if isinstance(tts_token_ids, np.ndarray): - return bool(np.any(tts_token_ids == turn_eos_id)) - if isinstance(tts_token_ids, (list, tuple)): - return turn_eos_id in tts_token_ids - return False - - def _t2w_pre_lookahead(self) -> int: - flow = getattr(self.audio_tokenizer, "flow", None) - try: - return int(getattr(flow, "pre_lookahead_len", 3) or 3) - except (TypeError, ValueError): - return 3 - - def _token2wav_lock(self) -> threading.RLock: - lock = getattr(self, "_token2wav_state_lock", None) - if lock is None: - lock = threading.RLock() - self._token2wav_state_lock = lock - return lock - - def _begin_turn_vocoder_cache( - self, - prompt_wav_path: str | None, - *, - state: _TalkerTurnState | None = None, - ) -> None: - """Restore a fresh per-turn clone of the ref-audio vocoder caches.""" - import torchaudio - - with self._token2wav_lock(): - cache_key = prompt_wav_path or "" - base = self._t2w_base_caches.get(cache_key) - if base is None: - if prompt_wav_path is None: - base = (None, {}) - else: - _orig_save = torchaudio.save - prev_dtype = torch.get_default_dtype() - torch.set_default_dtype(torch.float32) - try: - torchaudio.save = _soundfile_patched_save(_orig_save) - stream_cache, hift_cache_dict = self.audio_tokenizer.set_stream_cache(prompt_wav_path) - finally: - torch.set_default_dtype(prev_dtype) - torchaudio.save = _orig_save - base = ( - _torch_clone_recursive(stream_cache), - _torch_clone_recursive(hift_cache_dict), - ) - self._t2w_base_caches[cache_key] = base - stream_cache = _torch_clone_recursive(base[0]) - hift_cache_dict = _torch_clone_recursive(base[1]) - if state is None: - self.audio_tokenizer.stream_cache = stream_cache - self.audio_tokenizer.hift_cache_dict = hift_cache_dict - else: - state.stream_cache = stream_cache - state.hift_cache_dict = hift_cache_dict - state.vocoder_initialized = True - self.audio_tokenizer.stream_cache = None - self.audio_tokenizer.hift_cache_dict = {} - - def _t2w_stream_window(self, token_list: list[int], prompt_wav_path: str | None, *, last_chunk: bool): - import torchaudio - - _orig_save = torchaudio.save - prev_dtype = torch.get_default_dtype() - autocast_context, _ = self._token2wav_autocast_context() - torch.set_default_dtype(torch.float32) - try: - torchaudio.save = _soundfile_patched_save(_orig_save) - with autocast_context: - wav_np = self.audio_tokenizer.stream( - token_list, - prompt_wav_path, - last_chunk=bool(last_chunk), - return_waveform=True, - ) - finally: - torch.set_default_dtype(prev_dtype) - torchaudio.save = _orig_save - return torch.as_tensor(np.asarray(wav_np).reshape(-1), dtype=torch.float32).cpu().contiguous() - - def _run_vocoder_window( - self, - state: _TalkerTurnState, - token_list: list[int], - *, - last_chunk: bool, - ) -> torch.Tensor: - with self._token2wav_lock(): - if not state.vocoder_initialized: - self._begin_turn_vocoder_cache(state.prompt_wav_path, state=state) - self.audio_tokenizer.stream_cache = _torch_clone_recursive(state.stream_cache) - self.audio_tokenizer.hift_cache_dict = _torch_clone_recursive(state.hift_cache_dict) - try: - waveform = self._t2w_stream_window( - token_list, - state.prompt_wav_path, - last_chunk=last_chunk, - ) - state.stream_cache = _torch_clone_recursive(self.audio_tokenizer.stream_cache) - state.hift_cache_dict = _torch_clone_recursive(self.audio_tokenizer.hift_cache_dict) - return waveform - finally: - self.audio_tokenizer.stream_cache = None - self.audio_tokenizer.hift_cache_dict = {} - - def _native_duplex_vocode_tokens( - self, - state: _TalkerTurnState, - new_tokens: torch.Tensor, - *, - turn_end: bool, - force_flush: bool, - chunk_size: int, - ) -> list[torch.Tensor]: - """Run Token2wav with the same buffering policy as official duplex.""" - pre_lookahead = self._t2w_pre_lookahead() - token_list = new_tokens.reshape(-1).detach().cpu().tolist() - state.token2wav_buffer.extend(int(t) for t in token_list) - pieces: list[torch.Tensor] = [] - - if force_flush: - while len(state.token2wav_buffer) >= pre_lookahead + 5: - chunk_to_process = min(chunk_size + pre_lookahead, len(state.token2wav_buffer)) - window = state.token2wav_buffer[:chunk_to_process] - pieces.append(self._run_vocoder_window(state, window, last_chunk=False)) - state.token2wav_buffer = state.token2wav_buffer[min(chunk_size, chunk_to_process - pre_lookahead) :] - else: - while len(state.token2wav_buffer) >= chunk_size + pre_lookahead: - window = state.token2wav_buffer[: chunk_size + pre_lookahead] - pieces.append(self._run_vocoder_window(state, window, last_chunk=False)) - state.token2wav_buffer = state.token2wav_buffer[chunk_size:] - - if turn_end and state.token2wav_buffer: - pieces.append( - self._run_vocoder_window( - state, - list(state.token2wav_buffer), - last_chunk=True, - ) - ) - state.token2wav_buffer = [] - - return pieces - - def _close_turn_state( - self, - key: str, - *, - expected_epoch: int | None = None, - expected_turn_id: int | None = None, - ) -> bool: - state = self._talker_turn_states.get(key) - if ( - state is not None - and expected_epoch is not None - and state.epoch is not None - and state.epoch != expected_epoch - ): - return False - if ( - state is not None - and expected_turn_id is not None - and state.turn_id is not None - and state.turn_id != expected_turn_id - ): - return False - state = self._talker_turn_states.pop(key, None) - self._talker_consumed_tokens.pop(key, None) - request_keys = getattr(self, "_talker_request_keys", None) - if isinstance(request_keys, dict): - request_keys.pop(key, None) - for alias, mapped_key in list(request_keys.items()): - if mapped_key == key: - request_keys.pop(alias, None) - if state is None: - return True - if self.audio_tokenizer is not None: - with self._token2wav_lock(): - self.audio_tokenizer.stream_cache = None - self.audio_tokenizer.hift_cache_dict = {} - temp_path = state.temp_prompt_wav_path - if temp_path and not self._is_cached_ref_audio_prompt_wav(temp_path): - with self._token2wav_lock(): - self._t2w_base_caches.pop(temp_path, None) - try: - os.unlink(temp_path) - except OSError: - pass - return True - - @staticmethod - def _stream_identity_relation( - state: _TalkerTurnState | None, - *, - epoch: int | None, - turn_id: int | None, - ) -> str: - if state is None: - return "current" - if epoch is not None and state.epoch is not None: - if epoch < state.epoch: - return "stale" - if epoch > state.epoch: - return "newer" - if turn_id is not None and state.turn_id is not None: - if turn_id < state.turn_id: - return "stale" - if turn_id > state.turn_id: - return "newer" - return "current" - - def _warmup_duplex_vocoder(self) -> None: - """Pre-compile the two token2wav stream() modes used per turn. - - The first stream() call of each mode (mid-turn 25+pre_lookahead - window, and the variable-size last_chunk tail flush) costs ~20s of - one-time compilation; without this warmup the first spoken turn of - the first session stalls for both. Mirrors the official demo's - precompile step. - """ - if getattr(self, "_t2w_warmed", False): - return - self._t2w_warmed = True - try: - self._lazy_init_tts() - if self.audio_tokenizer is None: - return - prompt_wav_path, _ = self._resolve_prompt_wav_path(None, None) - if prompt_wav_path is None: - return - t0 = time.perf_counter() - chunk_size = self._tts_runtime_config().streaming_generator_chunk - pre_lookahead = self._t2w_pre_lookahead() - self._begin_turn_vocoder_cache(prompt_wav_path) - self._t2w_stream_window( - [_T2W_SILENCE_TOKEN] * (chunk_size + pre_lookahead), - prompt_wav_path, - last_chunk=False, - ) - self._begin_turn_vocoder_cache(prompt_wav_path) - self._t2w_stream_window( - [_T2W_SILENCE_TOKEN] * (chunk_size + pre_lookahead), - prompt_wav_path, - last_chunk=True, - ) - self.audio_tokenizer.stream_cache = None - self.audio_tokenizer.hift_cache_dict = {} - logger.info("4.5 Talker duplex vocoder warmup done in %.1fs", time.perf_counter() - t0) - except Exception: - logger.exception("4.5 Talker duplex vocoder warmup failed") - - def _create_native_duplex_stream_gen(self, info: dict[str, Any]): - """Per-segment generator over a persistent per-turn talker stream. - - Mirrors the official duplex talker: each unit calls - ``MiniCPMTTS.generate_chunk`` while carrying ``past_key_values`` and - ``text_start_pos`` across the spoken turn, then feeds the generated - audio tokens through the same per-turn Token2wav buffer/cache. - """ - key = self._stream_request_key(info) - self._remember_talker_request_key(info, key) - stream_epoch = self._stream_epoch(info) - stream_turn_id = self._stream_turn_id(info) - meta_info = info.get("meta") if isinstance(info.get("meta"), dict) else {} - codes_info = info.get("codes") if isinstance(info.get("codes"), dict) else {} - tts_token_ids, tts_hidden_states = self._extract_tts_handoff(info) - - self._lazy_init_tts() - if getattr(self, "tts_obj", None) is None or self.audio_tokenizer is None: - logger.warning("4.5 Talker duplex streaming: TTS runtime unavailable") - yield self._empty_audio_chunk(), True - return - - if isinstance(tts_token_ids, torch.Tensor): - ids_list = tts_token_ids.reshape(-1).tolist() - elif isinstance(tts_token_ids, list): - ids_list = [int(t) for t in tts_token_ids] - else: - ids_list = [] - - state = self._talker_turn_states.get(key) - identity_relation = self._stream_identity_relation( - state, - epoch=stream_epoch, - turn_id=stream_turn_id, - ) - if identity_relation == "stale": - logger.info( - "4.5 Talker duplex drop stale handoff: key=%s state_epoch=%s state_turn_id=%s " - "handoff_epoch=%s handoff_turn_id=%s", - key, - getattr(state, "epoch", None), - getattr(state, "turn_id", None), - stream_epoch, - stream_turn_id, - ) - yield self._empty_audio_chunk(), True - return - if identity_relation == "newer": - self._close_turn_state(key) - state = None - consumed = self._talker_consumed_tokens.get(key, 0) - if consumed > len(ids_list): - consumed = 0 - pending_ids = ids_list[consumed:] - - turn_eos_raw = meta_info.get("turn_eos_token_id") - try: - turn_eos_id = int(turn_eos_raw) if turn_eos_raw is not None else None - except (TypeError, ValueError): - turn_eos_id = None - explicit_turn_end = bool( - info.get("end_of_turn") or info.get("turn_end") or meta_info.get("end_of_turn") or meta_info.get("turn_end") - ) - turn_end = explicit_turn_end or (turn_eos_id is not None and turn_eos_id in pending_ids) - terminal_only_new_turn = ( - state is None - and turn_end - and turn_eos_id is not None - and bool(pending_ids) - and all(token_id == turn_eos_id for token_id in pending_ids) - ) - if terminal_only_new_turn: - # A model-owned empty turn can be [speak, turn_eos]. There is no - # spoken state to flush, so conditioning TTS on turn_eos alone - # would synthesize an unrelated audio-only response. - yield self._empty_audio_chunk(), True - return - if state is None and not pending_ids: - # No turn open and nothing new to speak: nothing to synthesize. - yield self._empty_audio_chunk(), True - return - - tts = self.tts_obj - if not hasattr(tts.model.config, "rope_theta"): - tts.model.config.rope_theta = 10000.0 - if not callable(getattr(tts, "generate_chunk", None)): - logger.warning("4.5 Talker duplex streaming: MiniCPMTTS.generate_chunk unavailable") - yield self._empty_audio_chunk(), True - return - sampling_params = self._build_tts_sampling_params() - if sampling_params is None: - logger.warning("4.5 Talker duplex streaming: sampling params unavailable") - yield self._empty_audio_chunk(), True - return - chunk_size = self._tts_runtime_config().streaming_generator_chunk - if chunk_size <= 0: - raise ValueError("MiniCPM-o 4.5 TTS streaming generator chunk must be positive") - - if state is None: - ref_audio = codes_info.get("ref", info.get("ref_audio")) - ref_audio_sr = meta_info.get("ref_audio_sr", info.get("ref_audio_sr")) - prompt_wav_path, temp_prompt_wav_path = self._resolve_prompt_wav_path(ref_audio, ref_audio_sr) - if prompt_wav_path is None: - logger.warning("4.5 Talker duplex streaming: no ref_audio prompt; skipping audio synthesis") - yield self._empty_audio_chunk(), True - return - state = _TalkerTurnState( - prompt_wav_path, - temp_prompt_wav_path, - epoch=stream_epoch, - turn_id=stream_turn_id, - ) - self._begin_turn_vocoder_cache(prompt_wav_path, state=state) - self._talker_turn_states[key] = state - - _queue_native_duplex_segment_text( - state, - meta_info.get("native_duplex_segment_text", ""), - ) - - if pending_ids: - pending_hidden = ( - tts_hidden_states[consumed:] - if isinstance(tts_hidden_states, list) - else torch.as_tensor(tts_hidden_states)[consumed:] - ) - cond_ids, cond_hidden = self._normalize_tts_handoff_tensors(pending_ids, pending_hidden) - condition = self._build_tts_condition_embeds(cond_ids, cond_hidden).unsqueeze(0) - else: - emb_dim = int(tts.emb_text.weight.shape[1]) - condition = tts.emb_text.weight.new_zeros((1, 0, emb_dim)) - audio_bos = tts.emb_text( - torch.tensor( - [tts.audio_bos_token_id], - dtype=torch.long, - device=tts.emb_text.weight.device, - ) - ).unsqueeze(0) - condition = torch.cat([condition, audio_bos], dim=1) - max_token_per_chunk = chunk_size + 1 - min_token_per_chunk = 0 if turn_end else max_token_per_chunk - force_flush = False - if state.text_start_pos == 0: - min_token_per_chunk = 0 - force_flush = True - eos_token = torch.tensor( - [tts.config.num_audio_tokens - 1], - dtype=torch.long, - device=tts.emb_text.weight.device, - ) - temperature = torch.tensor( - [float(sampling_params.temperature)], - dtype=torch.float, - device=tts.emb_text.weight.device, - ) - new_tokens, past_key_values = tts.generate_chunk( - inputs_embeds=condition, - temperature=temperature, - repetition_penalty=sampling_params.repetition_penalty, - eos_token=eos_token, - force_no_stop=False, - max_new_token=max_token_per_chunk, - min_new_tokens=min_token_per_chunk, - past_key_values=state.past_key_values, - logits_processors=None, - text_start_pos=state.text_start_pos, - ) - if turn_end: - state.past_key_values = None - state.text_start_pos = 0 - else: - state.past_key_values = past_key_values - state.text_start_pos += int(condition.shape[1]) + int(new_tokens.shape[1]) - waveforms = self._native_duplex_vocode_tokens( - state, - new_tokens, - turn_end=turn_end, - force_flush=force_flush, - chunk_size=chunk_size, - ) - self._talker_consumed_tokens[key] = len(ids_list) - unit_waveform = _native_duplex_unit_waveform(waveforms, turn_end=turn_end) - if unit_waveform is not None: - self._ar_last_emitted_text = _drain_native_duplex_emitted_text( - state, - has_audio=True, - ) - yield unit_waveform, False - if turn_end: - self._close_turn_state(key) - self._ar_last_emitted_text = "" - yield self._empty_audio_chunk(), True - return - self._ar_last_emitted_text = "" - yield self._empty_audio_chunk(), True - - def _create_stream_gen(self, info: dict[str, Any]): - """Yield waveform chunks from MiniCPM-o remote-code TTS streaming. - - This is the real vLLM streaming path: each yielded tensor is returned - through one scheduler step. The older streaming probe still concatenates - chunks inside generate_speech(), so it cannot improve API TTFA. - """ - if info.get("native_duplex") is True: - yield from self._create_native_duplex_stream_gen(info) - return - tts_token_ids, tts_hidden_states = self._extract_tts_handoff(info) - codes_info = info.get("codes") - meta_info = info.get("meta") - if not isinstance(codes_info, dict): - codes_info = {} - if not isinstance(meta_info, dict): - meta_info = {} - - ref_audio = codes_info.get("ref", info.get("ref_audio")) - ref_audio_sr = meta_info.get("ref_audio_sr", info.get("ref_audio_sr")) - - if tts_token_ids is None or tts_hidden_states is None: - logger.warning("4.5 Talker streaming: missing tts_token_ids or tts_hidden_states") - yield self._empty_audio_chunk(), True - return - tts_token_ids, tts_hidden_states = self._normalize_tts_handoff_tensors( - tts_token_ids, - tts_hidden_states, - ) - - self._lazy_init_tts() - if not hasattr(self, "tts_obj") or self.tts_obj is None: - logger.warning("4.5 Talker streaming: tts_obj not initialized") - yield self._empty_audio_chunk(), True - return - if self.audio_tokenizer is None: - logger.warning("4.5 Talker streaming: audio_tokenizer not initialized") - yield self._empty_audio_chunk(), True - return - - generator_cls = getattr(self, "_tts_streaming_generator_cls", None) - if generator_cls is None or self._tts_gen_logits is None: - logger.warning("4.5 Talker streaming: remote-code TTSStreamingGenerator unavailable") - waveform = self.generate_speech( - tts_token_ids, - tts_hidden_states, - ref_audio=ref_audio, - ref_audio_sr=ref_audio_sr, - ) - if waveform is None: - yield self._empty_audio_chunk(), True - else: - yield torch.as_tensor(waveform, dtype=torch.float32).reshape(-1).cpu().contiguous(), True - return - - tts = self.tts_obj - if not hasattr(tts.model.config, "rope_theta"): - tts.model.config.rope_theta = 10000.0 - - tts_embeds = self._build_tts_condition_embeds(tts_token_ids, tts_hidden_states) - num_text = int(tts_token_ids.shape[-1]) if tts_token_ids.ndim > 0 else 0 - min_new_token, max_new_token = self._max_tts_tokens_for_text(num_text) - sampling_params = self._build_tts_sampling_params() - if sampling_params is None: - logger.warning("4.5 Talker streaming: sampling params unavailable") - yield self._empty_audio_chunk(), True - return - - logits_warpers, logits_processors = self._tts_gen_logits( - num_code=tts.config.num_audio_tokens, - repetition_penalty=sampling_params.repetition_penalty, - top_p=sampling_params.top_p, - top_k=sampling_params.top_k, - ) - eos_token = torch.tensor([tts.config.num_audio_tokens - 1], dtype=torch.long, device=tts.emb_text.weight.device) - chunk_size = self._tts_runtime_config().streaming_generator_chunk - if chunk_size <= 0: - raise ValueError("MiniCPM-o 4.5 TTS streaming generator chunk must be positive") - - tts_streaming_generator = generator_cls( - model=tts, - temperature=sampling_params.temperature, - eos_token=eos_token, - chunk_size=chunk_size, - logits_processors=logits_processors, - logits_warpers=logits_warpers, - ) - - prompt_wav_path, temp_prompt_wav_path = self._resolve_prompt_wav_path(ref_audio, ref_audio_sr) - stream_cache = hift_cache_dict = None - import torchaudio - - _orig_save = torchaudio.save - - def _patched_save(uri, src, sample_rate, **kw): - kw.pop("backend", None) - if hasattr(uri, "write"): - sf.write(uri, src.cpu().numpy().T, sample_rate, format="WAV") - return - return _orig_save(uri, src, sample_rate, backend="soundfile", **kw) - - yielded_any = False - try: - torchaudio.save = _patched_save - prev_dtype = torch.get_default_dtype() - torch.set_default_dtype(torch.float32) - try: - stream_cache, hift_cache_dict = self.audio_tokenizer.set_stream_cache(prompt_wav_path) - finally: - torch.set_default_dtype(prev_dtype) - torchaudio.save = _orig_save - self.audio_tokenizer.stream_cache = stream_cache - self.audio_tokenizer.hift_cache_dict = hift_cache_dict - token_iter = tts_streaming_generator.generate_with_buffer( - condition=tts_embeds.unsqueeze(0), - text_finished=True, - max_new_token=max_new_token, - ) - while True: - try: - audio_token_chunk, is_last = next(token_iter) - except StopIteration: - break - if audio_token_chunk is None: - break - - token_list = audio_token_chunk.reshape(-1).detach().cpu().tolist() - if not token_list: - if is_last: - yield self._empty_audio_chunk(), True - yielded_any = True - break - continue - - autocast_context, _ = self._token2wav_autocast_context() - torchaudio.save = _patched_save - prev_dtype = torch.get_default_dtype() - torch.set_default_dtype(torch.float32) - try: - with autocast_context: - wav_np = self.audio_tokenizer.stream( - token_list, - prompt_wav_path, - last_chunk=bool(is_last), - return_waveform=True, - ) - finally: - torch.set_default_dtype(prev_dtype) - torchaudio.save = _orig_save - chunk = torch.as_tensor(np.asarray(wav_np).reshape(-1), dtype=torch.float32).cpu().contiguous() - yielded_any = True - yield chunk, bool(is_last) - if is_last: - break - finally: - torchaudio.save = _orig_save - self.audio_tokenizer.stream_cache = None - self.audio_tokenizer.hift_cache_dict = {} - if temp_prompt_wav_path and not self._is_cached_ref_audio_prompt_wav(temp_prompt_wav_path): - try: - os.unlink(temp_prompt_wav_path) - except OSError: - pass - - if not yielded_any: - yield self._empty_audio_chunk(), True - - def _move_tts_modules_to_device(self) -> torch.dtype: - device = current_omni_platform.get_torch_device() - target_dtype = torch.bfloat16 if current_omni_platform.is_npu() else self._target_tts_dtype() - if target_dtype is torch.float32: - self.tts_obj = self.tts_obj.to(device) - logger.info("Moved MiniCPM-o 4.5 TTS object to %s dtype=%s", device, target_dtype) - return target_dtype - - for module_name in ( - "emb_text", - "model", - "projector_spk", - "projector_semantic", - "emb_code", - "head_code", - ): - module = getattr(self.tts_obj, module_name, None) - if module is not None: - module.to(device=device, dtype=target_dtype) - logger.info("Moved MiniCPM-o 4.5 TTS AR modules to %s dtype=%s", device, target_dtype) - return target_dtype - - def generate_speech( - self, - tts_token_ids: torch.Tensor, - tts_hidden_states: torch.Tensor, - ref_audio=None, - ref_audio_sr: int | None = None, - ) -> torch.Tensor | np.ndarray | None: - """Run full 4.5 TTS pipeline using original MiniCPMTTS.generate.""" - self._lazy_init_tts() - if not hasattr(self, "tts_obj") or self.tts_obj is None: - logger.warning("generate_speech: tts_obj not initialized") - return None - - tts = self.tts_obj - device = tts.emb_text.weight.device - dtype = tts.emb_text.weight.dtype - - llm_embeds = tts.emb_text(tts_token_ids.to(device)) - hidden_embeds = tts.projector_semantic(tts_hidden_states.to(device=device, dtype=dtype)) - if getattr(tts.config, "normalize_projected_hidden", False): + text_embeds = self.emb_text(token_ids) + hidden_embeds = self.projector_semantic(hidden) + if self._normalize: hidden_embeds = F.normalize(hidden_embeds, p=2, dim=-1) - tts_embeds = llm_embeds + hidden_embeds + text_eos = self.emb_text(torch.tensor([self._text_eos_id], device=device, dtype=torch.long)) + audio_bos = self.emb_text(torch.tensor([self._tts_bos_id], device=device, dtype=torch.long)) + return torch.cat([text_embeds + hidden_embeds, text_eos, audio_bos], dim=0) - text_eos = tts.emb_text(torch.tensor([tts.config.text_eos_token_id], device=device, dtype=torch.long)) - audio_bos = tts.emb_text(torch.tensor([tts.audio_bos_token_id], device=device, dtype=torch.long)) - spk_embeds = torch.zeros(0, tts.config.hidden_size, device=device, dtype=tts_embeds.dtype) + def preprocess( + self, + input_ids: torch.Tensor, + input_embeds: torch.Tensor | None, + **info_dict: Any, + ) -> tuple[torch.Tensor, torch.Tensor, dict[str, Any]]: + """Build request-local prefill/decode embeddings for the vLLM runner.""" + del input_embeds + span_len = int(input_ids.shape[0]) + is_prefill = bool(info_dict.get("_omni_is_prefill", False)) + state = info_dict.get("audio_state") + first_call = not isinstance(state, dict) - inputs_embeds = torch.cat([spk_embeds, tts_embeds, text_eos, audio_bos], dim=0).unsqueeze(0) - - # Scale max_new_token with input text length. A fixed 2048-token floor - # can turn an EOS miss on a very short response into ~82s of audio and - # ~18s E2E latency. Keep a conservative short-text floor while bounding - # the tail. - num_text = int(tts_token_ids.shape[-1]) if tts_token_ids.ndim > 0 else 0 - min_new_token, max_new_token = self._max_tts_tokens_for_text(num_text) - - eos_token = torch.tensor([tts.config.num_audio_tokens - 1], dtype=torch.long, device=device) - sampling_params = self._build_tts_sampling_params() - generate_kwargs = { - "inputs_embeds": inputs_embeds, - "eos_token": eos_token, - "max_new_token": max_new_token, - "min_new_token": min_new_token, - "show_tqdm": False, - } - if sampling_params is not None: - generate_kwargs["sampling_params"] = sampling_params - - if self.audio_tokenizer is None: - logger.warning("No audio_tokenizer") - return None - - prompt_wav_path, temp_prompt_wav_path = self._resolve_prompt_wav_path(ref_audio, ref_audio_sr) - - try: - outputs = tts.generate(**generate_kwargs) - generated_tokens = outputs.new_ids.squeeze(-1) - - import torchaudio - - _orig_save = torchaudio.save - - def _patched_save(uri, src, sample_rate, **kw): - kw.pop("backend", None) - if hasattr(uri, "write"): - sf.write(uri, src.cpu().numpy().T, sample_rate, format="WAV") - return - return _orig_save(uri, src, sample_rate, backend="soundfile", **kw) - - torchaudio.save = _patched_save - prev_dtype = torch.get_default_dtype() - torch.set_default_dtype(torch.float32) - try: - autocast_context, token2wav_autocast = self._token2wav_autocast_context() - with autocast_context: - num_tokens = int(generated_tokens.shape[-1]) - - # For long outputs, the one-shot vocoder path - # (Token2wav.__call__ -> flow.inference) runs full O(N^2) self- - # attention over all audio tokens and OOMs on a 24GB card once - # N exceeds a few thousand (e.g. 4964 tokens needs ~3GiB for a - # single attention matmul). Switch to the chunked / streaming - # vocoder (set_stream_cache + stream) which truncates the flow - # attention caches to prompt_len + 100 steps on every chunk, - # keeping peak memory bounded regardless of total length. - STREAM_THRESHOLD = self._tts_runtime_config().streaming_vocoder_threshold # ~100s @ 25Hz - CHUNK_SIZE = self._tts_runtime_config().streaming_vocoder_chunk # ~2s per chunk - MIN_TAIL = 6 # must exceed flow.pre_lookahead_len (typically 3) - - if num_tokens <= STREAM_THRESHOLD: - if self._should_use_direct_token2wav() and token2wav_autocast == "off": - try: - waveform, _ = self._run_token2wav_direct(generated_tokens, prompt_wav_path) - except Exception as exc: - logger.warning( - "MiniCPM-o 4.5 direct Token2wav path failed; falling back to WAV path: %s", - exc, - exc_info=True, - ) - token_list = generated_tokens.squeeze(0).tolist() - self._reset_token2wav_cache_if_needed(prompt_wav_path) - wav_bytes = self.audio_tokenizer(token_list, prompt_wav_path) - waveform, _ = sf.read(io.BytesIO(wav_bytes)) - waveform = waveform.astype(np.float32) - else: - token_list = generated_tokens.squeeze(0).tolist() - self._reset_token2wav_cache_if_needed(prompt_wav_path) - wav_bytes = self.audio_tokenizer(token_list, prompt_wav_path) - waveform, _ = sf.read(io.BytesIO(wav_bytes)) - waveform = waveform.astype(np.float32) - else: - token_list = generated_tokens.squeeze(0).tolist() - # Build chunk boundaries, merging a too-small tail into the - # previous chunk so every chunk satisfies MIN_TAIL. - boundaries = [] - i = 0 - while i < num_tokens: - end = min(i + CHUNK_SIZE, num_tokens) - if 0 < num_tokens - end < MIN_TAIL: - end = num_tokens - boundaries.append((i, end)) - i = end - - logger.info( - "generate_speech: streaming vocoder, %d tokens -> %d chunks (chunk=%d)", - num_tokens, - len(boundaries), - CHUNK_SIZE, - ) - - stream_cache, hift_cache_dict = self.audio_tokenizer.set_stream_cache(prompt_wav_path) - self.audio_tokenizer.stream_cache = stream_cache - self.audio_tokenizer.hift_cache_dict = hift_cache_dict - - try: - pieces = [] - for idx, (s, e) in enumerate(boundaries): - is_last = idx == len(boundaries) - 1 - wav_np = self.audio_tokenizer.stream( - token_list[s:e], - prompt_wav_path, - last_chunk=is_last, - return_waveform=True, - ) - pieces.append(np.asarray(wav_np).reshape(-1)) - waveform = np.concatenate(pieces, axis=0).astype(np.float32) - finally: - # Free per-request streaming state so the next request starts clean - self.audio_tokenizer.stream_cache = None - self.audio_tokenizer.hift_cache_dict = {} - finally: - torch.set_default_dtype(prev_dtype) - torchaudio.save = _orig_save - - return waveform - finally: - if temp_prompt_wav_path and not self._is_cached_ref_audio_prompt_wav(temp_prompt_wav_path): - try: - os.unlink(temp_prompt_wav_path) - except OSError: - pass - - def _generate_tokens(self, inputs_embeds: torch.Tensor, max_new_token: int = 2048) -> torch.Tensor | None: - """Autoregressive generation of audio tokens using the TTS LlamaModel.""" - device = inputs_embeds.device - eos_token = self._num_audio_tokens - 1 - condition_length = inputs_embeds.shape[1] - num_vq = len(self.emb_code) - - new_tokens = torch.zeros(1, max_new_token, num_vq, device=device, dtype=torch.long) - past_key_values = None - finished = False - - for t in range(max_new_token): - if t == 0: - emb = inputs_embeds - position_ids = torch.arange(condition_length, device=device).unsqueeze(0) - else: - code_emb = [self.emb_code[q](new_tokens[:, t - 1 : t, q]) for q in range(num_vq)] - emb = torch.stack(code_emb, -1).sum(-1) - position_ids = torch.tensor([[condition_length + t - 1]], device=device) - - outputs = self.tts_model( - inputs_embeds=emb, - position_ids=position_ids, - past_key_values=past_key_values, - use_cache=True, + if is_prefill or first_call: + token_ids, hidden_states = get_tts_handoff(info_dict) + # Cross-process stage transport serializes CPU tensors as lists. + # Normalize both local tensor handoffs and transported payloads + # before validating/building the Talker condition. + if isinstance(token_ids, (list, tuple)): + token_ids = torch.as_tensor(token_ids, dtype=torch.long) + if isinstance(hidden_states, (list, tuple)): + hidden_states = torch.as_tensor(hidden_states, dtype=torch.float32) + if not isinstance(token_ids, torch.Tensor) or not isinstance(hidden_states, torch.Tensor): + available = sorted(key for key in info_dict if not key.startswith("_")) + raise ValueError( + "MiniCPM-o Talker requires tensor tts_token_ids and " + "tts_hidden_states conditioning; " + f"received token_ids={type(token_ids).__name__}, " + f"hidden_states={type(hidden_states).__name__}, " + f"available_keys={available}" + ) + if token_ids.numel() == 0 or hidden_states.numel() == 0: + raise ValueError("MiniCPM-o Talker conditioning must not be empty") + full_embeds = self._build_condition_embeddings(token_ids, hidden_states) + offset = int(info_dict.get("_omni_num_computed_tokens", 0)) + if offset == 0 and span_len > full_embeds.shape[0]: + # Async-chunk prewarms Stage 1 with placeholder token IDs. Two + # Thinker handoffs can arrive before the first one executes, + # so the scheduler prompt may already include the newer + # placeholder prefix while additional_information still + # carries the preceding condition. Materialize that prefix as + # the embeddings of its actual zero token IDs and keep the + # complete Talker condition tail-aligned (audio BOS remains + # the final prefill token). + prefix_len = span_len - full_embeds.shape[0] + placeholder_ids = torch.zeros( + prefix_len, + dtype=torch.long, + device=self.emb_text.weight.device, + ) + full_embeds = torch.cat([self.emb_text(placeholder_ids), full_embeds], dim=0) + embeds = full_embeds[offset : offset + span_len] + if embeds.shape[0] != span_len: + raise ValueError( + "MiniCPM-o Talker prefill span exceeds condition: " + f"request_id={info_dict.get('request_id')} offset={offset} " + f"span={span_len} condition={full_embeds.shape[0]} " + f"tts_ids={token_ids.shape[0]} tts_hidden={hidden_states.shape[0]} " + f"prompt_len={info_dict.get('_omni_prompt_len')}" + ) + max_tokens = _max_audio_tokens(int(token_ids.numel())) + state = { + "step": 0, + "max_tokens": max_tokens, + "finished": False, + } + request_id = str(info_dict.get("request_id", "0")) + request_states = getattr(self, "_request_audio_states", None) + if request_states is None: + request_states = {} + self._request_audio_states = request_states + request_states[request_id] = state + empty_codes = torch.empty(0, dtype=torch.long, device=embeds.device) + return ( + input_ids, + embeds, + { + "audio_state": state, + "audio_codes": { + "current": empty_codes, + "accumulated": empty_codes, + }, + }, ) - hidden = outputs.last_hidden_state - past_key_values = outputs.past_key_values - logits = torch.stack([self.head_code[q](hidden[:, -1]) for q in range(num_vq)], dim=-1) - logits = logits.float() / 0.8 + current = (info_dict.get("audio_codes", {}) or {}).get("current") + if not isinstance(current, torch.Tensor) or current.numel() != 1: + raise RuntimeError("MiniCPM-o Talker decode is missing the previous request-local audio code") + code = current.to(device=self.emb_code[0].weight.device, dtype=torch.long).reshape(1) + embeds = self.emb_code[0](code) + return input_ids, embeds, {} - if t < 50: - logits[:, eos_token, :] = -float("inf") + def _request_generator(self, request_id: str, device: torch.device) -> torch.Generator: + generator = self._request_generators.get(request_id) + if generator is None: + generator = torch.Generator(device=device) + generator.manual_seed(self._codec_seed) + self._request_generators[request_id] = generator + return generator - probs = F.softmax(logits, dim=1) - idx = torch.multinomial(probs.view(-1, probs.shape[1]), 1).view(1, num_vq) - new_tokens[:, t] = idx + def _sample_audio_code( + self, + hidden_state: torch.Tensor, + history: torch.Tensor, + request_id: str, + step: int, + ) -> torch.Tensor: + logits = self.head_code[0](hidden_state).float() / self._codec_temperature + eos_id = self._num_audio_tokens - 1 + logits = _apply_repetition_penalty( + logits, + history, + penalty=self._codec_repetition_penalty, + window_size=_REPETITION_WINDOW, + ) + if step < self._codec_min_tokens: + logits[..., eos_id] = float("-inf") + logits = _apply_top_k_top_p( + logits, + top_k=self._codec_top_k, + top_p=self._codec_top_p, + min_tokens_to_keep=3, + ) + probabilities = torch.softmax(logits, dim=-1) + return torch.multinomial( + probabilities, + num_samples=1, + generator=self._request_generator(request_id, probabilities.device), + ).reshape(()) - if (idx == eos_token).any(): - finished = True + def make_omni_output( + self, + model_outputs: torch.Tensor | OmniOutput, + **kwargs: Any, + ) -> OmniOutput: + if isinstance(model_outputs, OmniOutput): + return model_outputs + hidden = model_outputs + infos = kwargs.get("model_intermediate_buffer") or [] + spans = kwargs.get("request_token_spans") + if spans is None or len(spans) != len(infos): + raise RuntimeError("MiniCPM-o continuous Talker requires one request_token_span per request") + sample_eligible = kwargs.get("request_sample_eligible") + if sample_eligible is None: + sample_eligible = [True] * len(infos) + if len(sample_eligible) != len(infos): + raise RuntimeError( + f"MiniCPM-o continuous Talker received {len(sample_eligible)} sampling flags for {len(infos)} requests" + ) + + stop_rows: list[torch.Tensor] = [] + codec_deltas: list[torch.Tensor] = [] + terminal_flags: list[torch.Tensor] = [] + empty_delta = hidden.new_empty((0, 1), dtype=torch.long) + for index, info in enumerate(infos): + if not isinstance(info, dict): + stop_rows.append(hidden.new_tensor([0.0, float("-inf")])) + codec_deltas.append(empty_delta) + terminal_flags.append(torch.tensor(False, dtype=torch.bool)) + continue + start, end = spans[index] + end = min(int(end), int(hidden.shape[0])) + if int(start) >= end: + stop_rows.append(hidden.new_tensor([0.0, float("-inf")])) + codec_deltas.append(empty_delta) + terminal_flags.append(torch.tensor(False, dtype=torch.bool)) + continue + request_id = str(info.get("request_id", index)) + request_states = getattr(self, "_request_audio_states", None) + if request_states is None: + request_states = {} + self._request_audio_states = request_states + state = request_states.get(request_id) + if not isinstance(state, dict): + state = dict(info.get("audio_state", {}) or {}) + request_states[request_id] = state + if state.get("finished"): + stop_rows.append(hidden.new_tensor([float("-inf"), 0.0])) + codec_deltas.append(empty_delta) + terminal_flags.append(torch.tensor(False, dtype=torch.bool)) + continue + if not sample_eligible[index]: + # vLLM computes a logit row for incomplete chunked prefills but + # discards its sampled token. Advancing codec/RNG state here + # would make output depend on prefill chunking and compaction. + stop_rows.append(hidden.new_tensor([0.0, float("-inf")])) + codec_deltas.append(empty_delta) + terminal_flags.append(torch.tensor(False, dtype=torch.bool)) + continue + codes = state.get("codes") + if not isinstance(codes, torch.Tensor): + codes = (info.get("audio_codes", {}) or {}).get("accumulated") + if not isinstance(codes, torch.Tensor): + codes = torch.empty(0, dtype=torch.long, device=hidden.device) + else: + codes = codes.to(device=hidden.device, dtype=torch.long).reshape(-1) + step = int(state.get("step", 0)) + sampled = self._sample_audio_code(hidden[end - 1 : end], codes, request_id, step) + is_eos = int(sampled.item()) == self._num_audio_tokens - 1 + state["step"] = int(state.get("step", 0)) + 1 + reached_limit = int(state["step"]) >= int(state.get("max_tokens", 2048)) + finished = is_eos or reached_limit + state["finished"] = finished + if not is_eos: + codes = torch.cat([codes[-(_REPETITION_WINDOW - 1) :], sampled.reshape(1)]) + delta = sampled.reshape(1, 1) + else: + delta = empty_delta + state["codes"] = codes + info["audio_state"] = state + info["audio_codes"] = { + "current": sampled.reshape(1), + "accumulated": codes, + } + codec_deltas.append(delta) + terminal_flags.append(torch.tensor(finished, dtype=torch.bool)) + stop_rows.append(hidden.new_tensor([float("-inf"), 0.0] if finished else [0.0, float("-inf")])) + + self._batch_stop_logits = torch.stack(stop_rows, dim=0) if stop_rows else hidden.new_empty((0, 2)) + # Lists are deliberate: the runner routes element i to request i, + # preserving compaction alignment while emitting only this step's code. + multimodal_outputs: dict[str, Any] = { + "codes": {"audio": codec_deltas}, + "meta": {"finished": terminal_flags}, + } + return OmniOutput( + text_hidden_states=hidden, + multimodal_outputs=multimodal_outputs, + ) + + def on_requests_finished(self, finished_req_ids: set[str] | list[str]) -> None: + self._deferred_cleanup_ids.update(str(req_id) for req_id in finished_req_ids) + + def _flush_deferred_cleanup(self) -> None: + request_audio_states = getattr(self, "_request_audio_states", {}) + for request_id in self._deferred_cleanup_ids: + self._request_generators.pop(request_id, None) + request_audio_states.pop(request_id, None) + self._deferred_cleanup_ids.clear() + + def _dummy_hidden_states( + self, + input_ids: torch.Tensor | None, + positions: torch.Tensor | None, + inputs_embeds: torch.Tensor | None, + ) -> torch.Tensor: + """Shape-correct zero tensor for vllm KV cache profiling. + + vllm's gpu_model_runner._dummy_run takes forward()'s return value as + ``hidden_states`` and does ``hidden_states[logit_indices_device]``; + returning None on the dummy path crashes with + ``TypeError: 'NoneType' object is not subscriptable``. + """ + for ref in (input_ids, positions, inputs_embeds): + if isinstance(ref, torch.Tensor): + num_tokens = int(ref.shape[0]) if ref.ndim >= 1 else 1 + device = ref.device break - - return new_tokens[:, : t + 1 if finished else t, :] + else: + num_tokens = 1 + device = current_omni_platform.get_torch_device() + hidden_size = int(getattr(self, "_hidden_size", 768) or 768) + return torch.zeros((num_tokens, hidden_size), device=device, dtype=torch.bfloat16) def forward( self, @@ -1617,163 +494,77 @@ class MiniCPMO45OmniTTSForConditionalGeneration(nn.Module, SupportsPP): positions=None, intermediate_tensors=None, inputs_embeds=None, - additional_information=None, **kwargs, ): - if additional_information is None: - additional_information = {} - if not additional_information: - # Profile/dummy run: use it to pre-compile the per-turn vocoder - # stream modes so the first real spoken turn does not stall. - self._warmup_duplex_vocoder() - - tts_token_ids, tts_hidden_states = self._extract_tts_handoff(additional_information) - tts_text = additional_information.get("llm_output_text", [""]) - if isinstance(tts_text, list): - tts_text = tts_text[0] if tts_text else "" - codes_info = additional_information.get("codes") - meta_info = additional_information.get("meta") - if not isinstance(codes_info, dict): - codes_info = {} - if not isinstance(meta_info, dict): - meta_info = {} - ref_audio = codes_info.get("ref") - if ref_audio is None: - ref_audio = additional_information.get("ref_audio") - ref_audio_sr = meta_info.get("ref_audio_sr") - if ref_audio_sr is None: - ref_audio_sr = additional_information.get("ref_audio_sr") - - if tts_token_ids is None or tts_hidden_states is None: - logger.warning("4.5 Talker: missing tts_token_ids or tts_hidden_states") - self._ar_last_chunk_flags = [True] - self._ar_turn_end_flags = [False] - return None, None - tts_token_ids, tts_hidden_states = self._normalize_tts_handoff_tensors( - tts_token_ids, - tts_hidden_states, + self._flush_deferred_cleanup() + if input_ids is None and inputs_embeds is None: + return self._dummy_hidden_states(input_ids, positions, inputs_embeds) + return self.tts_model( + input_ids=input_ids, + positions=positions, + intermediate_tensors=intermediate_tensors, + inputs_embeds=inputs_embeds, ) - if self._should_stream_output(additional_information): - request_key = self._stream_request_key(additional_information) - input_ends_turn = self._native_duplex_input_ends_turn(additional_information) - if request_key not in self._stream_gens: - self._stream_gens[request_key] = self._create_stream_gen(additional_information) - generator = self._stream_gens[request_key] - try: - waveform_chunk, is_last = next(generator) - except StopIteration: - self._stream_gens.pop(request_key, None) - waveform_chunk = self._empty_audio_chunk() - is_last = True - if is_last: - self._stream_gens.pop(request_key, None) - self._ar_last_chunk_flags = [bool(is_last)] - # A TTS generator also ends at ordinary chunk boundaries. Export - # turn_end only on the terminal output for a condition that - # actually contains the model's <|turn_eos|> decision. - self._ar_turn_end_flags = [bool(is_last and input_ends_turn)] - return None, waveform_chunk.reshape(-1).contiguous() - - self._ar_last_chunk_flags = [True] - self._ar_turn_end_flags = [False] - waveform = self.generate_speech( - tts_token_ids, - tts_hidden_states, - ref_audio=ref_audio, - ref_audio_sr=ref_audio_sr, - ) - if waveform is not None: - waveform_tensor = torch.as_tensor(waveform, dtype=torch.float32).detach() - if waveform_tensor.device.type != "cpu": - waveform_tensor = waveform_tensor.cpu() - return waveform_tensor.reshape(-1).contiguous(), None - return None, None - def compute_logits(self, hidden_states, *args, **kwargs): - device = hidden_states.device if isinstance(hidden_states, torch.Tensor) else torch.device("cuda") - if isinstance(hidden_states, torch.Tensor): - if hidden_states.ndim == 1: - num_rows = 1 - else: - num_rows = max(1, int(hidden_states.shape[0])) - else: - num_rows = 1 - eos_id = self._scheduler_eos_token_id() - vocab_size = max(int(getattr(self.config, "vocab_size", eos_id + 1) or (eos_id + 1)), eos_id + 1, 3) - safe_id = 1 if eos_id != 1 else 0 - logits = torch.full((num_rows, vocab_size), -1.0e9, dtype=torch.float32, device=device) - flags = self._ar_last_chunk_flags - default_is_last = bool(flags[-1]) if flags else True - for row in range(num_rows): - is_last = bool(flags[row]) if row < len(flags) else default_is_last - if is_last: - logits[row, eos_id] = 1.0e6 - else: - logits[row, safe_id] = 1.0e6 + if not isinstance(hidden_states, torch.Tensor): + return None + if self._batch_stop_logits is None: + return torch.zeros( + hidden_states.shape[0], + 2, + device=hidden_states.device, + dtype=torch.float32, + ) + logits = self._batch_stop_logits + self._batch_stop_logits = None return logits def sample(self, logits, sampling_metadata): - if logits is None or logits.numel() == 0: - return None - sampled = torch.argmax(logits, dim=-1).to(torch.int32) - return SamplerOutput(sampled_token_ids=sampled.unsqueeze(-1), logprobs_tensors=None) - - def on_requests_finished(self, finished_req_ids: set[str] | list[str]) -> None: - for req_id in finished_req_ids: - request_key = str(req_id) - mapped_key = getattr(self, "_talker_request_keys", {}).pop(request_key, None) - keys = {request_key} - if mapped_key: - keys.add(mapped_key) - for key in list(self._stream_gens): - if key in keys: - gen = self._stream_gens.pop(key, None) - if gen is not None: - try: - gen.close() - except Exception: - logger.exception("MiniCPM-o 4.5 failed to close stream gen for request %s", req_id) - for key in list(self._talker_turn_states): - if key in keys: - self._close_turn_state(key) - for key in list(self._talker_consumed_tokens): - if key in keys: - self._talker_consumed_tokens.pop(key, None) + return Sampler()(logits, sampling_metadata) def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]): - loaded = set() - tts_weights = {} - for k, v in weights: - if k.startswith("tts."): - local_name = k.replace("tts.", "", 1) - tts_weights[local_name] = v - loaded.add(f"tts_obj.{local_name}") + return self._load_native_weights(weights) - if tts_weights and self._tts_config is not None: - self._lazy_init_tts() - if hasattr(self, "tts_obj") and self.tts_obj is not None: - missing, unexpected = self.tts_obj.load_state_dict(tts_weights, strict=False) - if missing: - logger.warning("TTS missing keys (%d): %s", len(missing), missing[:5]) - if unexpected: - logger.warning("TTS unexpected keys (%d): %s", len(unexpected), unexpected[:5]) - tts_dtype = self._move_tts_modules_to_device() - if ( - not current_omni_platform.is_npu() - and self.audio_tokenizer is not None - and hasattr(self.audio_tokenizer, "to") - ): - self.audio_tokenizer.to("cuda") - self.emb_text = self.tts_obj.emb_text - self.projector_semantic = self.tts_obj.projector_semantic - logger.info( - "Loaded %d TTS weights, moved AR modules to %s dtype=%s", - len(tts_weights), - current_omni_platform.get_torch_device(), - tts_dtype, - ) + def _load_native_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: + loaded: set[str] = set() + backbone_weights: list[tuple[str, torch.Tensor]] = [] + direct_params = dict(self.named_parameters()) + head_g = head_v = None + for name, tensor in weights: + if not name.startswith("tts."): + continue + stripped = name[len("tts.") :] + if stripped.startswith("model."): + backbone_weights.append((stripped[len("model.") :], tensor)) + continue + if stripped == "head_code.0.parametrizations.weight.original0": + head_g = tensor + continue + if stripped == "head_code.0.parametrizations.weight.original1": + head_v = tensor + continue + target = stripped + parameter = direct_params.get(target) + if parameter is None: + continue + parameter.data.copy_(tensor.to(device=parameter.device, dtype=parameter.dtype)) + loaded.add(target) + + for name in self.tts_model.load_weights(backbone_weights): + loaded.add(f"tts_model.{name}") + + if head_g is None or head_v is None: + raise ValueError("MiniCPM-o checkpoint is missing weight-norm Talker head parameters") + restored = _restore_weight_norm_weight(head_g, head_v) + self.head_code[0].weight.data.copy_( + restored.to( + device=self.head_code[0].weight.device, + dtype=self.head_code[0].weight.dtype, + ) + ) + loaded.add("head_code.0.weight") return loaded def get_input_embeddings(self, input_ids, multimodal_embeddings=None, **kwargs): diff --git a/vllm_omni/model_executor/models/minicpmo_4_5/pipeline.py b/vllm_omni/model_executor/models/minicpmo_4_5/pipeline.py index 57d8c4614..31d02c7cc 100644 --- a/vllm_omni/model_executor/models/minicpmo_4_5/pipeline.py +++ b/vllm_omni/model_executor/models/minicpmo_4_5/pipeline.py @@ -3,12 +3,11 @@ """MiniCPM-o 4.5 pipeline topology (frozen). Stage 0: Thinker — multimodal understanding + text generation. -Stage 1: Talker — MiniCPMTTS + Token2Wav, emits the final audio waveform. +Stage 1: Talker — MiniCPMTTS, emits codec tokens. +Stage 2: Code2Wav — codec tokens to the final audio waveform. -The thinker -> talker bridge passes the hidden states + token ids extracted -from the thinker output through ``minicpmo_4_5_omni.llm2tts``; the talker -runs MiniCPMTTS and the on-device Token2wav vocoder in the same process and -returns the waveform directly as the pipeline's final audio output. +The thinker -> talker bridge uses ``llm2tts``. The talker -> Code2Wav bridge +streams request-routed codec chunks. """ from vllm_omni.config.stage_config import ( @@ -56,28 +55,24 @@ MINICPMO_4_5_PIPELINE = PipelineConfig( StagePipelineConfig( stage_id=1, model_stage="tts", - # Stage 1 shares the top-level wrapper class - # (``MiniCPMO45OmniForConditionalGeneration``) inherited from - # ``model_arch`` above. The wrapper dispatches on ``model_stage`` - # and, for ``"tts"``, instantiates the standalone TTS submodule - # (``MiniCPMO45OmniTTSForConditionalGeneration``) internally. - # Routing through the wrapper is required so that the runner-side - # ``runtime_additional_information`` payload reaches the talker - # (the standalone TTS class only reads ``additional_information``, - # so wiring stage 1 directly to it would always trigger the dummy - # path) and so the resulting waveform is packaged as - # ``OmniOutput.multimodal_outputs["model_outputs"]`` instead of - # being returned as a bare tuple that the AR runner would mistake - # for hidden states. ``hf_config_name="tts_config"`` keeps KV - # cache / mrope sizing scoped to the talker sub-config. execution_type=StageExecutionType.LLM_AR, input_sources=(0,), + hf_config_name="tts_config", + engine_output_type="latent", + custom_process_input_func=f"{_PROC}.llm2tts", + async_chunk_process_next_stage_input_func=f"{_PROC}.tts2code2wav_async_chunk", + sampling_constraints={"detokenize": False}, + ), + StagePipelineConfig( + stage_id=2, + model_stage="code2wav", + execution_type=StageExecutionType.LLM_GENERATION, + input_sources=(1,), final_output=True, final_output_type="audio", - hf_config_name="tts_config", engine_output_type="audio", - custom_process_input_func=f"{_PROC}.llm2tts", - sampling_constraints={"detokenize": False}, + model_arch="MiniCPMO45Code2Wav", + sampling_constraints={"detokenize": True}, ), ), ) diff --git a/vllm_omni/model_executor/models/registry.py b/vllm_omni/model_executor/models/registry.py index 6749d5cfc..6f0302499 100644 --- a/vllm_omni/model_executor/models/registry.py +++ b/vllm_omni/model_executor/models/registry.py @@ -360,6 +360,11 @@ _OMNI_MODELS = { "minicpmo_4_5_omni_tts", "MiniCPMO45OmniTTSForConditionalGeneration", ), + "MiniCPMO45Code2Wav": ( + "minicpmo_4_5", + "minicpmo_4_5_code2wav", + "MiniCPMO45Code2Wav", + ), "AuraQwen3VLForConditionalGeneration": ( "aura_omni", "qwen3_vl", diff --git a/vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py b/vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py index 561c6ac8f..7d319de72 100644 --- a/vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py +++ b/vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py @@ -1,18 +1,15 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Stage input processor for MiniCPM-o 4.5: Thinker (LLM) -> Talker (TTS). - -This bridge converts thinker hidden states and token ids into the talker -prompt payload. The talker runs Token2Wav internally; MiniCPM-o 4.5 does not -have an independent code2wav pipeline stage. -""" +"""MiniCPM-o 4.5 Thinker-to-Talker and Talker-to-Code2Wav bridges.""" import logging -from collections.abc import Callable, Mapping +from collections.abc import Callable, Mapping, Sequence +from typing import Any import torch from vllm.inputs import TextPrompt +from vllm_omni.data_entry_keys import CodesStruct, MetaStruct, OmniPayloadStruct from vllm_omni.experimental.fullduplex.engine.intermediate import ( build_duplex_intermediate_buffer, set_ref_audio, @@ -21,6 +18,21 @@ from vllm_omni.experimental.fullduplex.engine.intermediate import ( from vllm_omni.inputs.data import OmniTokensPrompt logger = logging.getLogger(__name__) +_MINICPMO45_ASYNC_STATE = "_minicpmo45_async_codec_state" +_MINICPMO45_STREAM_RECORD = "_minicpmo45_async_stream_record" +_MINICPMO45_SILENCE_CODE = 4218 + + +class _MiniCPMO45MetaStruct(MetaStruct): + """Model-owned metadata for the split Talker-to-Code2Wav bridge.""" + + ref_audio_sr: int | None = None + native_duplex_segment_text: str | None = None + duplex_turn_id: int | None = None + duplex_epoch: int | None = None + segment_end: bool | None = None + turn_end: bool | None = None + tts_is_last_chunk: bool | None = None def _extract_first_audio_ref(multi_modal_data): @@ -111,6 +123,224 @@ def _coerce_int(value): return None +def _codec_config(transfer_manager: Any) -> tuple[int, int]: + connector = getattr(transfer_manager, "connector", None) + raw_config = getattr(connector, "config", {}) or {} + config = raw_config.get("extra", raw_config) if isinstance(raw_config, dict) else {} + config = config if isinstance(config, dict) else {} + chunk_frames = int(config.get("codec_chunk_frames", 25)) + left_context_frames = int(config.get("codec_left_context_frames", 3)) + if chunk_frames <= 0 or left_context_frames < 0: + raise ValueError( + "Invalid MiniCPM-o codec chunk config: " + f"codec_chunk_frames={chunk_frames}, " + f"codec_left_context_frames={left_context_frames}" + ) + return chunk_frames, left_context_frames + + +def _codec_scalars(value: Any) -> list[int]: + """Normalize one request-routed codec delta to CPU scalar token IDs.""" + if value is None: + return [] + if isinstance(value, torch.Tensor): + if value.numel() == 0: + return [] + return value.detach().to(device="cpu", dtype=torch.long).reshape(-1).tolist() + if isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)): + scalars: list[int] = [] + for item in value: + scalars.extend(_codec_scalars(item)) + return scalars + if isinstance(value, (int, bool)): + return [int(value)] + raise TypeError(f"Unsupported MiniCPM-o codec delta type: {type(value).__name__}") + + +def _extract_codec_delta(pooling_output: Any, request_id: str) -> list[int]: + if pooling_output is None: + return [] + if isinstance(pooling_output, Mapping): + meta = pooling_output.get("meta") + routed_id = meta.get("request_id") if isinstance(meta, Mapping) else None + routed_id = routed_id or pooling_output.get("request_id") + if routed_id is not None and str(routed_id) != request_id: + return [] + codes = pooling_output.get("codes") + audio = codes.get("audio") if isinstance(codes, Mapping) else None + return _codec_scalars(audio) + if isinstance(pooling_output, Sequence) and not isinstance( + pooling_output, + (str, bytes, bytearray), + ): + delta: list[int] = [] + for item in pooling_output: + values = _extract_codec_delta(item, request_id) if isinstance(item, Mapping) else _codec_scalars(item) + delta.extend(values) + return delta + return _codec_scalars(pooling_output) + + +def _drop_codec_state(transfer_manager: Any, request_id: str) -> None: + request_payload = getattr(transfer_manager, "request_payload", None) + if isinstance(request_payload, dict): + container = request_payload.get(request_id) + if isinstance(container, dict): + container.pop(_MINICPMO45_ASYNC_STATE, None) + if not container: + request_payload.pop(request_id, None) + else: + request_payload.pop(request_id, None) + code_accumulators = getattr(transfer_manager, "code_prompt_token_ids", None) + if hasattr(code_accumulators, "pop"): + code_accumulators.pop(request_id, None) + + +def _is_aborted(request: Any) -> bool: + status_name = getattr(getattr(request, "status", None), "name", "") + return any(marker in status_name for marker in ("ABORT", "CANCEL", "IGNORED", "ERROR")) + + +def tts2code2wav_async_chunk( + transfer_manager: Any, + multimodal_output: Any, + request: Any, + is_finished: bool = False, +) -> OmniPayloadStruct | None: + """Stream request-owned MiniCPM-o codec windows to Code2Wav.""" + external_id = getattr(request, "external_req_id", None) + internal_id = getattr(request, "request_id", None) + request_id = str(external_id if external_id is not None else internal_id) + internal_id = str(internal_id if internal_id is not None else request_id) + + request_payload = getattr(transfer_manager, "request_payload", None) + if request_payload is None: + request_payload = {} + transfer_manager.request_payload = request_payload + container = request_payload.get(request_id) + if not isinstance(container, dict): + container = {} + request_payload[request_id] = container + + record = container.get(_MINICPMO45_STREAM_RECORD) + if not isinstance(record, dict): + record = { + "internal_id": internal_id, + "cache_epoch": 0, + "retired_internal_ids": set(), + } + container[_MINICPMO45_STREAM_RECORD] = record + elif internal_id in record["retired_internal_ids"]: + return None + elif record["internal_id"] != internal_id: + record["retired_internal_ids"].add(record["internal_id"]) + record["internal_id"] = internal_id + record["cache_epoch"] = int(record["cache_epoch"]) + 1 + _drop_codec_state(transfer_manager, request_id) + + if _is_aborted(request): + record["retired_internal_ids"].add(internal_id) + _drop_codec_state(transfer_manager, request_id) + return None + + state = container.get(_MINICPMO45_ASYNC_STATE) + if not isinstance(state, dict): + request_info = getattr(request, "additional_information", None) + if not isinstance(request_info, Mapping): + request_info = {} + codes_info = request_info.get("codes") + if not isinstance(codes_info, Mapping): + codes_info = {} + meta_info = request_info.get("meta") + if not isinstance(meta_info, Mapping): + meta_info = {} + duplex_info = request_info.get("duplex") + if not isinstance(duplex_info, Mapping): + duplex_info = {} + state = { + "internal_id": internal_id, + "pending": [], + "left_context": [], + "codec_end": 0, + "ref_audio": codes_info.get("ref"), + "ref_audio_sr": meta_info.get("ref_audio_sr"), + "segment_text": meta_info.get("native_duplex_segment_text"), + "segment_end": bool(meta_info.get("segment_end", False)), + "turn_end": bool(meta_info.get("turn_end", False)), + "duplex_turn_id": duplex_info.get("model_turn_id", duplex_info.get("turn_id")), + "duplex_epoch": duplex_info.get("epoch"), + } + container[_MINICPMO45_ASYNC_STATE] = state + + pending = state["pending"] + pending.extend(_extract_codec_delta(multimodal_output, request_id)) + request_finished = getattr(request, "is_finished", None) + finished = bool(is_finished or (callable(request_finished) and request_finished())) + chunk_frames, left_context_frames = _codec_config(transfer_manager) + if not finished and len(pending) < chunk_frames: + return None + + new_token_count = len(pending) if finished else chunk_frames + new_codes = pending[:new_token_count] + del pending[:new_token_count] + codec_start = int(state["codec_end"]) + codec_end = codec_start + new_token_count + + if new_token_count: + if codec_start == 0: + context = [_MINICPMO45_SILENCE_CODE] * left_context_frames + else: + context = list(state["left_context"]) + output_codes = [*context, *new_codes] + history = [*state["left_context"], *new_codes] + state["left_context"] = history[-left_context_frames:] if left_context_frames else [] + elif finished and codec_start > 0 and state["left_context"]: + context = list(state["left_context"]) + output_codes = context + else: + context = [] + output_codes = [] + state["codec_end"] = codec_end + + last_chunk = bool(finished and not pending) + if last_chunk: + record["retired_internal_ids"].add(internal_id) + _drop_codec_state(transfer_manager, request_id) + + chunk_seq = int(getattr(transfer_manager, "put_req_chunk", {}).get(request_id, 0)) + finished_tensor = torch.tensor(last_chunk, dtype=torch.bool) + ref_audio = state.get("ref_audio") if codec_start == 0 else None + return OmniPayloadStruct( + codes=CodesStruct( + audio=torch.tensor(output_codes, dtype=torch.long), + ref=torch.as_tensor(ref_audio, dtype=torch.float32).reshape(-1) if ref_audio is not None else None, + ), + meta=_MiniCPMO45MetaStruct( + request_id=request_id, + chunk_seq=chunk_seq, + cache_epoch=int(record["cache_epoch"]), + code_flat_numel=len(output_codes), + codec_chunk_frames=new_token_count, + codec_left_context_frames=len(context), + left_context_size=len(context), + last_chunk=last_chunk, + stream_finished=finished_tensor, + finished=finished_tensor, + req_id=[request_id], + ref_audio_sr=_coerce_int(state.get("ref_audio_sr")), + native_duplex_segment_text=( + str(state["segment_text"]) if isinstance(state.get("segment_text"), str) else None + ), + duplex_turn_id=_coerce_int(state.get("duplex_turn_id")), + duplex_epoch=_coerce_int(state.get("duplex_epoch")), + segment_end=bool(state.get("segment_end", False)), + turn_end=bool(state.get("turn_end", False)), + tts_is_last_chunk=last_chunk, + ), + request_id=request_id, + ) + + def _special_token_ids_from_mm_output(mm_output): if not isinstance(mm_output, Mapping): return {} @@ -344,19 +574,7 @@ def llm2tts( requires_multimodal_data: bool = False, _streaming_context=None, ): - """Convert thinker stage output to talker stage input for MiniCPMO Omni. - - Extracts from thinker output: - - Full hidden states (prompt + generated) for speaker embedding extraction - - Prompt token IDs (for finding spk_bos/spk_eos positions) - - Generated token IDs (for decoding TTS text) - - The talker model will: - 1. Find <|spk_bos|>/<|spk_eos|> positions in prompt_token_ids - 2. Extract speaker embedding from hidden states at those positions - 3. Decode generated text and extract TTS content - 4. Run ConditionalChatTTS pipeline - """ + """Build Talker conditioning for ordinary and full-duplex streaming.""" if not source_outputs: raise ValueError("source_outputs cannot be empty") @@ -375,7 +593,14 @@ def llm2tts( for llm_output in llm_outputs: output = llm_output.outputs[0] - mm_output = output.multimodal_output if isinstance(output.multimodal_output, Mapping) else {} + request_mm_output = getattr(llm_output, "multimodal_output", None) + completion_mm_output = getattr(output, "multimodal_output", None) + if isinstance(request_mm_output, Mapping): + mm_output = request_mm_output + elif isinstance(completion_mm_output, Mapping): + mm_output = completion_mm_output + else: + mm_output = {} special_token_ids = _special_token_ids_from_mm_output(mm_output) prompt_token_ids = ( _coerce_token_id_list(mm_output.get("duplex_prompt_token_ids")) @@ -445,6 +670,11 @@ def llm2tts( for idx_t in range(search_start, len(full_token_ids)): if full_token_ids[idx_t] == tts_bos_id: tts_bos_idx = idx_t + 1 + if tts_bos_idx is None and not is_native_duplex_handoff and llm_output_ids: + # Audio routing is a model-stage concern, not an OpenAI serving + # default. Plain chat templates do not include <|tts_bos|>; in + # that case condition the Talker on the generated assistant span. + tts_bos_idx = prompt_token_ids_len tts_eos_idx = None if tts_bos_idx is not None: @@ -591,13 +821,21 @@ def llm2tts( continue set_tts_handoff(model_intermediate_buffer, handoff_ids, handoff_hidden) if native_turn_end_handoff: + model_intermediate_buffer.setdefault("meta", {})["turn_end"] = True _reset_native_tts_handoff(_streaming_context) - scheduler_prompt_token_ids = _build_tts_scheduler_prompt_token_ids( - tts_token_ids_slice, - llm_output_ids, - prompt_token_ids, - ) + if handoff_ids is not None and handoff_hidden is not None: + condition_length = max(len(handoff_ids), len(handoff_hidden)) + 2 + scheduler_prompt_token_ids = [0] * condition_length + handoff_meta = model_intermediate_buffer.setdefault("meta", {}) + handoff_meta["replace_streaming_prompt"] = True + handoff_meta["next_stage_prompt_len"] = condition_length + else: + scheduler_prompt_token_ids = _build_tts_scheduler_prompt_token_ids( + tts_token_ids_slice, + llm_output_ids, + prompt_token_ids, + ) tts_inputs.append( OmniTokensPrompt( prompt_token_ids=scheduler_prompt_token_ids, diff --git a/vllm_omni/platforms/npu/worker/npu_ar_model_runner.py b/vllm_omni/platforms/npu/worker/npu_ar_model_runner.py index 0817d7bbf..904974829 100644 --- a/vllm_omni/platforms/npu/worker/npu_ar_model_runner.py +++ b/vllm_omni/platforms/npu/worker/npu_ar_model_runner.py @@ -43,6 +43,7 @@ from vllm_ascend.worker.model_runner_v1 import graph_capture from vllm_omni.data_entry_keys import flatten_payload from vllm_omni.distributed.omni_connectors.kv_transfer_manager import OmniKVTransferManager +from vllm_omni.distributed.omni_connectors.utils.config import stage_sends_async_output from vllm_omni.outputs import OmniModelRunnerOutput from vllm_omni.platforms.npu.worker.npu_model_runner import OmniNPUModelRunner from vllm_omni.utils.mm_outputs import build_mm_cpu, partition_payload_list, to_payload_element @@ -1186,7 +1187,7 @@ class NPUARModelRunner(OmniNPUModelRunner, OmniConnectorModelRunnerMixin): pooler_output.append(flatten_payload(payload)) pooler_output = pooler_output or [] - if self._async_chunk: + if self._async_chunk and stage_sends_async_output(self.model_config): pooler_inter, pooler_client = partition_payload_list(pooler_output) else: # Non-async-chunk ships the full payload to the next stage via @@ -1203,7 +1204,9 @@ class NPUARModelRunner(OmniNPUModelRunner, OmniConnectorModelRunnerMixin): self.accumulate_full_payload_output(rid, pooler_inter[i], req_state) inter_stage_outputs = self._build_multimodal_outputs(pooler_inter) - multimodal_outputs = self._build_multimodal_outputs(pooler_client) + multimodal_outputs = ( + inter_stage_outputs if pooler_client is pooler_inter else self._build_multimodal_outputs(pooler_client) + ) model_runner_output = OmniModelRunnerOutput( req_ids=req_ids_output_copy, req_id_to_index=req_id_to_index_output_copy, diff --git a/vllm_omni/utils/mm_outputs.py b/vllm_omni/utils/mm_outputs.py index 1dc4e37f7..af8f9ac14 100644 --- a/vllm_omni/utils/mm_outputs.py +++ b/vllm_omni/utils/mm_outputs.py @@ -32,6 +32,7 @@ _CLIENT_MM_META_KEYS: frozenset[str] = frozenset( "duplex_epoch", "duplex_turn_id", "llm_output_text_utf8", + "segment_end", "tts_is_last_chunk", "turn_end", } diff --git a/vllm_omni/worker/gpu_ar_model_runner.py b/vllm_omni/worker/gpu_ar_model_runner.py index 3e1ea6ddb..ca7c2a21a 100644 --- a/vllm_omni/worker/gpu_ar_model_runner.py +++ b/vllm_omni/worker/gpu_ar_model_runner.py @@ -42,6 +42,10 @@ from vllm.v1.worker.utils import is_residual_scattered_for_sp from vllm_omni.data_entry_keys import flatten_payload from vllm_omni.distributed.omni_connectors.kv_transfer_manager import OmniKVTransferManager +from vllm_omni.distributed.omni_connectors.utils.config import ( + get_stage_connector_role, + stage_sends_async_output, +) from vllm_omni.outputs import OmniModelRunnerOutput from vllm_omni.utils.mm_outputs import build_mm_cpu, partition_payload_list, to_payload_element from vllm_omni.worker.gpu_model_runner import OmniGPUModelRunner @@ -321,7 +325,10 @@ class GPUARModelRunner(OmniGPUModelRunner, OmniConnectorModelRunnerMixin): "DyninOmniForConditionalGeneration", "IndexTTS2TalkerForConditionalGeneration", } - if getattr(self.model_config, "model_arch", None) in _OMNI_CONNECTOR_INIT_ARCHS: + if ( + getattr(self.model_config, "model_arch", None) in _OMNI_CONNECTOR_INIT_ARCHS + or get_stage_connector_role(self.model_config) is not None + ): self.init_omni_connectors( model_config=self.model_config, kv_transfer_manager=self.kv_transfer_manager, @@ -1841,14 +1848,11 @@ class GPUARModelRunner(OmniGPUModelRunner, OmniConnectorModelRunnerMixin): pooler_output.append(flatten_payload(payload)) pooler_output = pooler_output or [] - if self._async_chunk: + if self._async_chunk and stage_sends_async_output(self.model_config): pooler_inter, pooler_client = partition_payload_list(pooler_output) else: - # Non-async-chunk still ships the full payload to the next stage (via - # accumulate_full_payload_output and the inter_stage_outputs field); only - # client mm keys are split out when async_chunk is enabled. #4527 set this - # to (None, pooler_output), which skipped accumulation and starved the - # downstream stage (300s connector-input timeout / empty audio). (PR #4792) + # Connector-less stages expose the same payload through the + # orchestrator bridge; non-async stages preserve legacy behavior. pooler_inter, pooler_client = pooler_output, pooler_output if pooler_inter and self._should_accumulate_full_payload_output(): @@ -1860,7 +1864,9 @@ class GPUARModelRunner(OmniGPUModelRunner, OmniConnectorModelRunnerMixin): with record_function_or_nullcontext("omni_output_builder:build_multimodal_outputs"): inter_stage_outputs = self._build_multimodal_outputs(pooler_inter) - multimodal_outputs = self._build_multimodal_outputs(pooler_client) + multimodal_outputs = ( + inter_stage_outputs if pooler_client is pooler_inter else self._build_multimodal_outputs(pooler_client) + ) with record_function_or_nullcontext("gpu_model_runner: ModelRunnerOutput"): routed_experts_lists = None diff --git a/vllm_omni/worker/gpu_generation_model_runner.py b/vllm_omni/worker/gpu_generation_model_runner.py index 6208f73e9..2c81643dd 100644 --- a/vllm_omni/worker/gpu_generation_model_runner.py +++ b/vllm_omni/worker/gpu_generation_model_runner.py @@ -36,6 +36,7 @@ from vllm.v1.worker.gpu_model_runner import ( from vllm.v1.worker.ubatch_utils import maybe_create_ubatch_slices from vllm.v1.worker.utils import sanity_check_mm_encoder_outputs +from vllm_omni.distributed.omni_connectors.utils.config import get_stage_connector_role from vllm_omni.outputs import OmniModelRunnerOutput from vllm_omni.utils.mm_outputs import partition_payload_list from vllm_omni.worker.gpu_ar_model_runner import ExecuteModelState, _ensure_tensor_values @@ -68,7 +69,10 @@ class GPUGenerationModelRunner(OmniGPUModelRunner, OmniConnectorModelRunnerMixin "DyninOmniForConditionalGeneration", "IndexTTS2S2MelDecoder", } - if getattr(self.model_config, "model_arch", None) in _OMNI_CONNECTOR_INIT_ARCHS: + if ( + getattr(self.model_config, "model_arch", None) in _OMNI_CONNECTOR_INIT_ARCHS + or get_stage_connector_role(self.model_config) is not None + ): self.init_omni_connectors( model_config=self.model_config, ) @@ -317,6 +321,8 @@ class GPUGenerationModelRunner(OmniGPUModelRunner, OmniConnectorModelRunnerMixin ) # [Omni] Pass token counts per request for code2wav output slicing model_kwargs["seq_token_counts"] = tokens + if getattr(self.model, "requires_request_ids", False): + model_kwargs["request_ids"] = list(req_ids) # Set cudagraph mode to none if calc_kv_scales is true. # KV scales calculation involves dynamic operations that are incompatible diff --git a/vllm_omni/worker/gpu_model_runner.py b/vllm_omni/worker/gpu_model_runner.py index 0e14784ca..cb52a2b67 100644 --- a/vllm_omni/worker/gpu_model_runner.py +++ b/vllm_omni/worker/gpu_model_runner.py @@ -1361,6 +1361,14 @@ class OmniGPUModelRunner(GPUModelRunner): if nstp is not None and len(nstp) == len(self.input_batch.req_ids): try: model_kwargs_extra["request_token_spans"] = self._compute_request_token_spans(nstp) + if getattr(self.model, "requires_request_sample_eligibility", False): + model_kwargs_extra["request_sample_eligible"] = [ + bool( + (req := self.requests.get(req_id)) is not None + and int(req.num_computed_tokens) + int(nstp[req_index]) >= int(req.num_tokens) + ) + for req_index, req_id in enumerate(self.input_batch.req_ids) + ] except Exception as e: # Visible on purpose: the fallback is the equal rows-per-request # split, which can re-introduce the cross-request corruption this @@ -1482,14 +1490,17 @@ class OmniGPUModelRunner(GPUModelRunner): if isinstance(model_buffer, dict) and model_buffer: self._update_intermediate_buffer(new_req.req_id, model_buffer) payload_info = getattr(new_req, "additional_information", None) - if isinstance(payload_info, dict): - self._update_intermediate_buffer(new_req.req_id, payload_info) + decoded_info = deserialize_additional_information(payload_info) + if decoded_info: + self._update_intermediate_buffer(new_req.req_id, decoded_info) if hasattr(scheduler_output.scheduled_cached_reqs, "additional_information"): cached_infos = getattr(scheduler_output.scheduled_cached_reqs, "additional_information", {}) if isinstance(cached_infos, dict): for req_id, req_infos in cached_infos.items(): - self._update_intermediate_buffer(req_id, req_infos) + decoded_info = deserialize_additional_information(req_infos) + if decoded_info: + self._update_intermediate_buffer(req_id, decoded_info) def _maybe_attach_mimo_audio_req_infos( self, @@ -1661,9 +1672,8 @@ class OmniGPUModelRunner(GPUModelRunner): # (e.g., code2wav) so runtime_additional_information can be refreshed # from scheduler cached infos on every step. if hasattr(self.model, "has_preprocess") or hasattr(self.model, "enable_update_additional_information"): - if self.vllm_config.model_config.async_chunk: - self._update_additional_information(scheduler_output) - else: + self._update_additional_information(scheduler_output) + if not self.vllm_config.model_config.async_chunk: # In full-payload (non-async-chunk) mode, connector-delivered # stage payloads must override any earlier engine-level # additional_information written by the legacy @@ -1672,7 +1682,11 @@ class OmniGPUModelRunner(GPUModelRunner): self._sync_local_stage_payloads() if hasattr(self.model, "has_preprocess") and self.model.has_preprocess: - preprocess_device = input_ids.device if input_ids is not None else inputs_embeds.device + # Multimodal wrappers forward embeddings and therefore intentionally + # set ``input_ids`` to None. Model-specific preprocess hooks still + # need the scheduled token ids to build or replace those embeddings. + preprocess_input_ids = input_ids if input_ids is not None else self.input_ids.gpu[:num_input_tokens] + preprocess_device = preprocess_input_ids.device self._maybe_run_batch_preprocess(self.input_batch.req_ids, preprocess_device) # Overlay custom prompt_embeds per request for the prompt portion; @@ -1690,21 +1704,27 @@ class OmniGPUModelRunner(GPUModelRunner): req_ids_b = [item[0] for item in decode_batch_items] start_offsets_b = [item[1] for item in decode_batch_items] req_infos_b = [item[2] for item in decode_batch_items] - ids_b = torch.stack([input_ids[offset : offset + 1].reshape(-1)[0] for offset in start_offsets_b]) + ids_b = torch.stack( + [preprocess_input_ids[offset : offset + 1].reshape(-1)[0] for offset in start_offsets_b] + ) req_input_ids, req_embeds, last_talker_hidden, text_step, updates = batch_decode_preprocess( input_ids=ids_b, req_infos=req_infos_b, ) if inputs_embeds is None: inputs_embeds = torch.empty( - (input_ids.shape[0], req_embeds.shape[-1]), + (preprocess_input_ids.shape[0], req_embeds.shape[-1]), device=req_embeds.device, dtype=req_embeds.dtype, ) offsets_t = torch.tensor(start_offsets_b, device=req_embeds.device, dtype=torch.long) inputs_embeds.index_copy_(0, offsets_t, req_embeds) - input_ids.index_copy_(0, offsets_t, req_input_ids.reshape(-1).to(dtype=input_ids.dtype)) + preprocess_input_ids.index_copy_( + 0, + offsets_t, + req_input_ids.reshape(-1).to(dtype=preprocess_input_ids.dtype), + ) dst = slice(len(decode_req_ids), len(decode_req_ids) + len(req_ids_b)) self.talker_mtp_input_ids.gpu[dst].copy_(req_input_ids.reshape(-1)) @@ -1751,7 +1771,9 @@ class OmniGPUModelRunner(GPUModelRunner): embed_slice = inputs_embeds[s:e] if inputs_embeds is not None else None req_input_ids, req_embeds, update_dict = self.model.preprocess( - input_ids=preprocess_input_ids[s:e], input_embeds=embed_slice, **req_infos + input_ids=preprocess_input_ids[s:e], + input_embeds=embed_slice, + **req_infos, ) if inputs_embeds is None: inputs_embeds = torch.empty( From c56d0da1bbd4382c838d807811b95a204d093710 Mon Sep 17 00:00:00 2001 From: Canlin Guo Date: Fri, 24 Jul 2026 19:08:17 +0800 Subject: [PATCH 082/240] [Perf] Refactor MOSS-TTS-Local v1.5 talker and local structure for CUDA Graph (#5197) Signed-off-by: gcanlin Signed-off-by: Canlin Guo Signed-off-by: boatman <109857087+sphinxkkkbc@users.noreply.github.com> Co-authored-by: boatman <109857087+sphinxkkkbc@users.noreply.github.com> --- vllm_omni/deploy/moss_tts_local.yaml | 7 +- .../moss_tts/modeling_moss_tts_codec.py | 8 +- .../moss_tts/modeling_moss_tts_talker.py | 227 ++++++++---------- .../stage_input_processors/moss_tts.py | 5 - 4 files changed, 105 insertions(+), 142 deletions(-) diff --git a/vllm_omni/deploy/moss_tts_local.yaml b/vllm_omni/deploy/moss_tts_local.yaml index ab66df300..deffe4a17 100644 --- a/vllm_omni/deploy/moss_tts_local.yaml +++ b/vllm_omni/deploy/moss_tts_local.yaml @@ -24,7 +24,7 @@ stages: gpu_memory_utilization: 0.60 enable_prefix_caching: false async_scheduling: true - max_num_batched_tokens: 4096 + max_num_batched_tokens: 65536 max_model_len: 4096 devices: "0" output_connectors: @@ -40,9 +40,12 @@ stages: - stage_id: 1 max_num_seqs: 8 gpu_memory_utilization: 0.15 - enforce_eager: true + enforce_eager: false enable_prefix_caching: false async_scheduling: true + compilation_config: + cudagraph_capture_sizes: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15] + cudagraph_num_of_warmups: 1 max_num_batched_tokens: 65536 max_model_len: 65536 devices: "0" diff --git a/vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_codec.py b/vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_codec.py index 8f1a72e9b..d0f9ea1bc 100644 --- a/vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_codec.py +++ b/vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_codec.py @@ -307,11 +307,15 @@ class MossTTSCodecDecoder(nn.Module): "MossTTS codec requires seq_token_counts; otherwise concatenated " "codec tokens cannot be split per request." ) - if sum(token_counts) != int(ids_flat.shape[0]): + real_token_count = sum(token_counts) + input_token_count = int(ids_flat.shape[0]) + if real_token_count > input_token_count: raise RuntimeError( "MossTTS codec seq_token_counts mismatch: " - f"counts={token_counts}, sum={sum(token_counts)}, input_tokens={int(ids_flat.shape[0])}." + f"counts={token_counts}, sum={real_token_count}, input_tokens={input_token_count}." ) + if real_token_count < input_token_count: + ids_flat = ids_flat[:real_token_count] num_req = len(token_counts) if len(info_list) < num_req: diff --git a/vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_talker.py b/vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_talker.py index 9758c922e..59313ecdf 100644 --- a/vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_talker.py +++ b/vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_talker.py @@ -1321,7 +1321,14 @@ class MossTTSLocalTalkerForGeneration(nn.Module): # additive-fusion gathers (avoids an n_vq-iteration Python loop). self._stacked_audio_emb_w: torch.Tensor | None = None self.mtp_hidden_size = hidden_size - self.talker_mtp_accepts_req_infos = True + self.talker_mtp_graph_safe = True + self.talker_mtp_output_key = ("audio_codes", "current") + self.use_async_omni_output = False + self.eager_omni_postprocess_before_async_output = True + self.omni_pooler_payload_include_hidden = False + self.postprocess_uses_multimodal_outputs = False + self.postprocess_uses_req_infos = False + self._batch_should_continue: torch.Tensor | None = None self.gpu_resident_buffer_keys: set[tuple[str, str]] = { ("audio_codes", "current"), @@ -1373,6 +1380,38 @@ class MossTTSLocalTalkerForGeneration(nn.Module): logits = hidden_states.new_full((num_rows, self.text_vocab_size), float("-inf")) states = self._batch_state or [] + batch_continue_mask = self._batch_should_continue + if isinstance(batch_continue_mask, torch.Tensor) and batch_continue_mask.numel() > 0: + should_by_req = batch_continue_mask.reshape(-1) + if should_by_req.device != logits.device or should_by_req.dtype != torch.bool: + should_by_req = should_by_req.to(device=logits.device, dtype=torch.bool) + if int(should_by_req.numel()) == num_rows: + should_continue = should_by_req + else: + row_req_indices: list[int] = [] + for req_idx, (_state, row_start, row_end) in enumerate( + _iter_state_row_spans( + states, + getattr(self, "_batch_state_spans", None), + num_rows, + ) + ): + if req_idx >= int(should_by_req.shape[0]): + continue + row_req_indices.extend([req_idx] * max(0, row_end - row_start)) + if row_req_indices: + req_t = torch.tensor(row_req_indices, device=logits.device, dtype=torch.long) + should_continue = should_by_req.index_select(0, req_t) + else: + should_continue = torch.ones((num_rows,), device=logits.device, dtype=torch.bool) + if int(should_continue.numel()) != num_rows: + should_continue = torch.ones((num_rows,), device=logits.device, dtype=torch.bool) + zeros = logits.new_zeros((num_rows,)) + neg_inf = logits.new_full((num_rows,), float("-inf")) + logits[:, self.audio_assistant_slot_token_id] = torch.where(should_continue, zeros, neg_inf) + logits[:, self.im_end_token_id] = torch.where(should_continue, neg_inf, zeros) + return logits + for state, row_start, row_end in _iter_state_row_spans( states, getattr(self, "_batch_state_spans", None), @@ -1441,7 +1480,6 @@ class MossTTSLocalTalkerForGeneration(nn.Module): ref_codes = (info_dict.get("codes", {}) or {}).get("ref") ref_offset = int(info_dict.get("ref_offset", 0)) - last_ref_row = None if isinstance(ref_codes, torch.Tensor) and ref_codes.numel() > 0: if ref_codes.dim() == 1 and ref_codes.numel() % self.n_vq == 0: ref_codes = ref_codes.view(-1, self.n_vq) @@ -1451,23 +1489,9 @@ class MossTTSLocalTalkerForGeneration(nn.Module): if chunk.numel() > 0 and chunk.shape[0] == span_len: codes = chunk.to(device=device, dtype=torch.long) embeds = embeds + self._audio_embed(codes) - last_ref_row = codes[-1].detach() - - current_codes = ( - last_ref_row - if last_ref_row is not None - else torch.full((self.n_vq,), self.audio_pad_token_id, dtype=torch.long, device=device) - ) - - max_new_frames = info_dict.get("max_new_frames", [-1])[0] info_update: dict[str, Any] = { - "audio_state": { - "is_stopping": False, - "step": 0, - "max_new_frames": max_new_frames, - }, - "audio_codes": {"current": current_codes, "emit": False}, + "audio_state": {"is_stopping": False}, "ref_offset": ref_offset + span_len, } return input_ids, embeds, info_update @@ -1483,7 +1507,11 @@ class MossTTSLocalTalkerForGeneration(nn.Module): mtp_hidden = last_hidden.to(device=device, dtype=text_embed.dtype).reshape(1, -1) else: mtp_hidden = torch.zeros((1, self.hidden_size), device=device, dtype=text_embed.dtype) - return input_ids, text_embed, {"mtp_inputs": (mtp_hidden, torch.zeros_like(mtp_hidden))} + state = info_dict.get("audio_state", {}) or {} + active = isinstance(state, dict) and not bool(state.get("is_stopping")) + mtp_control = torch.zeros_like(mtp_hidden) + mtp_control[:, :1] = 1.0 if active else 0.0 + return input_ids, text_embed, {"mtp_inputs": (mtp_hidden, mtp_control)} def postprocess(self, hidden_states: torch.Tensor, **_: Any) -> dict[str, Any]: if hidden_states.numel() == 0: @@ -1502,21 +1530,15 @@ class MossTTSLocalTalkerForGeneration(nn.Module): top_k: int | None = None, top_p: float | None = None, generator: torch.Generator | None = None, - req_infos: list[dict[str, Any]] | None = None, **_: Any, - ) -> tuple[torch.Tensor, None]: + ) -> tuple[torch.Tensor, torch.Tensor]: bsz = int(input_embeds.shape[0]) - dev = input_embeds.device - input_embeds_out = input_embeds.reshape(bsz, -1).clone() - req_infos = req_infos or [{} for _ in range(bsz)] - - def write_update(info: dict[str, Any], update: dict[str, Any]) -> None: - for key, value in update.items(): - if isinstance(value, dict): - target = info.setdefault(key, {}) - target.update(value) - else: - info[key] = value + input_embeds_out = input_embeds.reshape(bsz, -1) + last_talker_hidden = last_talker_hidden.reshape(bsz, -1).to( + device=input_embeds.device, + dtype=input_embeds.dtype, + ) + active_mask = text_step.reshape(bsz, -1)[:, :1] > 0 # Match MOSS-TTS Local v1.5's model-card/SGLang defaults. The generic # stage SamplingParams are for the backbone token loop and should not @@ -1526,89 +1548,32 @@ class MossTTSLocalTalkerForGeneration(nn.Module): audio_top_p = 0.8 do_sample_val = True if do_sample is None else bool(do_sample) - for i in range(bsz): - info = req_infos[i] if i < len(req_infos) and isinstance(req_infos[i], dict) else {} - state = dict(info.get("audio_state", {}) or {}) - step = int(state.get("step", 0)) - max_new_frames = int(state.get("max_new_frames", -1)) - acc = (info.get("audio_codes", {}) or {}).get("accumulated") + should_continue_t, new_codes = self.local_transformer.generate_frame( + last_talker_hidden, + self.audio_lm_heads, + self.audio_embeddings, + self.local_text_lm_head, + n_vq=self.n_vq, + do_sample=do_sample_val, + temperature=audio_temperature, + top_k=audio_top_k, + top_p=audio_top_p, + repetition_penalty=1.0, + history_per_codebook=None, + generator=generator, + ) + new_codes = new_codes.to(device=input_embeds.device, dtype=torch.long) + emit_mask = active_mask & should_continue_t.reshape(bsz, 1) + audio_embed = self._audio_embed(new_codes).to(dtype=input_embeds_out.dtype) + input_embeds_out = input_embeds_out + audio_embed * emit_mask.to(dtype=input_embeds_out.dtype) - if state.get("is_stopping"): - write_update( - info, - { - "audio_state": state, - "audio_codes": { - "current": input_embeds.new_empty((0, self.n_vq), dtype=torch.long), - "emit": False, - }, - }, - ) - continue - - rep_window = 50 - if isinstance(acc, torch.Tensor) and acc.numel() > 0: - tail = acc[-rep_window:].long().cpu().tolist() - hist_per_cb = [[row[cb] for row in tail] for cb in range(self.n_vq)] - else: - hist_per_cb = [[] for _ in range(self.n_vq)] - - should_continue_t, new_codes_b = self.local_transformer.generate_frame( - last_talker_hidden[i : i + 1], - self.audio_lm_heads, - self.audio_embeddings, - self.local_text_lm_head, - n_vq=self.n_vq, - do_sample=do_sample_val, - temperature=audio_temperature, - top_k=audio_top_k, - top_p=audio_top_p, - repetition_penalty=1.0, - history_per_codebook=hist_per_cb, - generator=generator, - ) - should_continue = bool(should_continue_t[0].item()) - new_codes = new_codes_b.squeeze(0).to(device=dev, dtype=torch.long) - - if not should_continue or (0 <= max_new_frames <= step): - state["is_stopping"] = True - state["step"] = step + 1 - write_update( - info, - { - "audio_state": state, - "audio_codes": { - "current": input_embeds.new_empty((0, self.n_vq), dtype=torch.long), - "accumulated": acc, - "emit": False, - }, - }, - ) - continue - - if isinstance(acc, torch.Tensor) and acc.numel() > 0: - updated_acc = torch.cat([acc.to(dev), new_codes.unsqueeze(0)], dim=0) - else: - updated_acc = new_codes.unsqueeze(0) - - state["step"] = step + 1 - current = new_codes.unsqueeze(0) - input_embeds_out[i : i + 1] = input_embeds_out[i : i + 1] + self._audio_embed(current).to( - dtype=input_embeds_out.dtype - ) - write_update( - info, - { - "audio_state": state, - "audio_codes": { - "current": current, - "accumulated": updated_acc, - "emit": True, - }, - }, - ) - - return input_embeds_out, None + keep = emit_mask.reshape(-1) + output_codes = torch.where( + keep.unsqueeze(1), + new_codes, + torch.full_like(new_codes, self.audio_pad_token_id), + ) + return input_embeds_out, output_codes # ------------------------------------------------------------------ # Package runner-generated audio frames @@ -1622,6 +1587,7 @@ class MossTTSLocalTalkerForGeneration(nn.Module): if isinstance(model_outputs, OmniOutput): self._batch_state = None self._batch_state_spans = None + self._batch_should_continue = None return model_outputs hidden = model_outputs # (S, H) @@ -1631,46 +1597,41 @@ class MossTTSLocalTalkerForGeneration(nn.Module): for info in info_dicts: if isinstance(info, dict) and not isinstance(info.get("audio_state"), dict): - info["audio_state"] = {"is_stopping": False, "step": 0, "max_new_frames": -1} + info["audio_state"] = {"is_stopping": False} self._batch_state = [(info["audio_state"] if isinstance(info, dict) else {}) for info in info_dicts] self._batch_state_spans = kwargs.get("request_token_spans") - + should_values: list[torch.Tensor] = [] + have_mtp_output = False per_req_codes: list[torch.Tensor] = [] have_codes = False for info in info_dicts: - current = (info.get("audio_codes", {}) or {}).get("current") if isinstance(info, dict) else None - emit = bool((info.get("audio_codes", {}) or {}).get("emit")) if isinstance(info, dict) else False - if emit and isinstance(current, torch.Tensor) and current.numel() > 0: + audio_codes = (info.get("audio_codes", {}) or {}) if isinstance(info, dict) else {} + current = audio_codes.pop("current", None) + if isinstance(current, torch.Tensor) and current.numel() > 0: codes = current.to(device=hidden.device, dtype=torch.long) if codes.dim() == 1: codes = codes.unsqueeze(0) - per_req_codes.append(codes) - have_codes = True + should_continue = codes.ne(self.audio_pad_token_id).any(dim=-1) + should_values.append(should_continue.reshape(-1)[:1]) + have_mtp_output = True + emitted_codes = codes[should_continue] + per_req_codes.append(emitted_codes) + have_codes = have_codes or emitted_codes.numel() > 0 else: + should_values.append(torch.ones((1,), device=hidden.device, dtype=torch.bool)) per_req_codes.append(hidden.new_empty((0, self.n_vq), dtype=torch.long)) + self._batch_should_continue = torch.cat(should_values, dim=0) if have_mtp_output and should_values else None if not have_codes: return OmniOutput(text_hidden_states=hidden, multimodal_outputs={}) # Local-v1.5 emits the current raw code frame (no delay pattern); the # pipeline routes those rows through ``talker2codec_raw_async_chunk``. - per_req_ref_codes: list[torch.Tensor] = [] - for info in info_dicts: - ref_codes = (info.get("codes", {}) or {}).get("ref") if isinstance(info, dict) else None - if isinstance(ref_codes, torch.Tensor) and ref_codes.numel() > 0: - ref_codes = ref_codes.detach() - if ref_codes.dim() == 1 and ref_codes.numel() % self.n_vq == 0: - ref_codes = ref_codes.view(-1, self.n_vq) - if ref_codes.dim() == 2 and ref_codes.shape[1] == self.n_vq: - per_req_ref_codes.append(ref_codes.to(device=hidden.device, dtype=torch.long)) - continue - per_req_ref_codes.append(hidden.new_empty((0, self.n_vq), dtype=torch.long)) - return OmniOutput( text_hidden_states=hidden, multimodal_outputs={ - "codes": {"audio": per_req_codes, "ref": per_req_ref_codes}, + "codes": {"audio": per_req_codes}, }, ) diff --git a/vllm_omni/model_executor/stage_input_processors/moss_tts.py b/vllm_omni/model_executor/stage_input_processors/moss_tts.py index 3dfe2a603..2a7da4a4d 100644 --- a/vllm_omni/model_executor/stage_input_processors/moss_tts.py +++ b/vllm_omni/model_executor/stage_input_processors/moss_tts.py @@ -291,17 +291,12 @@ def talker2codec_raw_async_chunk( transfer_manager.code_prompt_token_ids.pop(req_id, None) transfer_manager.request_payload.pop(req_id, None) return OmniPayloadStruct( - # A non-empty sentinel is required so Stage-1 is scheduled. - # ``code_flat_numel=0`` tells the codec this is a control-only - # finish packet, not an audio code. - codes=CodesStruct(audio=torch.tensor([0], dtype=torch.long)), meta=MetaStruct( req_id=[req_id], left_context_size=0, codec_streaming=True, codec_chunk_frames=0, codec_left_context_frames=0, - code_flat_numel=0, stream_finished=torch.tensor(True, dtype=torch.bool), finished=torch.tensor(True, dtype=torch.bool), ), From 784c27f4930127eb37266f0782b84da60772479d Mon Sep 17 00:00:00 2001 From: Viacheslav Klimkov Date: Fri, 24 Jul 2026 13:31:47 +0200 Subject: [PATCH 083/240] [Bugfix][Core] Fix async streaming segment-stop accounting (#5183) Signed-off-by: Viacheslav Klimkov --- .../sched/test_omni_ar_scheduler_streaming.py | 65 +++++++++++++++++++ vllm_omni/core/sched/omni_ar_scheduler.py | 9 ++- 2 files changed, 72 insertions(+), 2 deletions(-) diff --git a/tests/core/sched/test_omni_ar_scheduler_streaming.py b/tests/core/sched/test_omni_ar_scheduler_streaming.py index 32284eeac..e2ad98f56 100644 --- a/tests/core/sched/test_omni_ar_scheduler_streaming.py +++ b/tests/core/sched/test_omni_ar_scheduler_streaming.py @@ -1,8 +1,12 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + """Unit tests for Omni AR streaming-session async placeholder handling.""" from __future__ import annotations from types import SimpleNamespace +from unittest.mock import MagicMock import pytest @@ -11,6 +15,8 @@ import pytest # isort: off import vllm_omni # noqa: F401 - import for side effects (patch vLLM) from vllm.sampling_params import SamplingParams +from vllm.v1.core.sched.output import SchedulerOutput +from vllm.v1.outputs import ModelRunnerOutput from vllm.v1.request import Request, RequestStatus, StreamingUpdate from vllm_omni.core.sched.omni_ar_scheduler import OmniARScheduler @@ -51,6 +57,65 @@ def _make_update(prompt_token_ids: list[int] | None = None) -> StreamingUpdate: ) +def _run_resumable_segment_stop(session: Request) -> None: + sched = MagicMock() + sched.requests = {session.request_id: session} + sched.perf_metrics = None + sched.structured_output_manager.should_advance.return_value = False + sched._update_request_with_output.return_value = ([42], True) + sched._handle_stopped_request.return_value = False + sched.chunk_transfer_adapter = None + sched.running = [session] + sched.waiting_for_transfer_free = set() + sched.transfer_triggered_requests = set() + sched.active_kv_transfers = set() + sched.pending_stop_after_extraction = set() + sched.connector = None + sched.kv_cache_manager.take_events.return_value = None + sched.finished_req_ids_dict = {} + sched.make_stats.return_value = None + + scheduler_output = MagicMock(spec=SchedulerOutput) + scheduler_output.num_scheduled_tokens = {session.request_id: 1} + scheduler_output.scheduled_spec_decode_tokens = {} + scheduler_output.num_invalid_spec_tokens = 0 + + model_runner_output = MagicMock(spec=ModelRunnerOutput) + model_runner_output.sampled_token_ids = [[42]] + model_runner_output.logprobs = None + model_runner_output.prompt_logprobs_dict = {} + model_runner_output.pooler_output = None + model_runner_output.num_nans_in_logits = None + model_runner_output.kv_connector_output = None + model_runner_output.cudagraph_stats = None + model_runner_output.req_id_to_index = {session.request_id: 0} + model_runner_output.routed_experts = None + + OmniARScheduler.update_from_output(sched, scheduler_output, model_runner_output) + + +@pytest.mark.parametrize("outstanding_async_tokens", [0, 1, 2]) +def test_resumable_segment_stop_reconciles_async_placeholders( + outstanding_async_tokens: int, +) -> None: + """A segment stop discards and rolls back only in-flight async tokens.""" + session = _make_request() + session.status = RequestStatus.RUNNING + session.resumable = True + session.append_output_token_ids([7, 8]) + session.num_computed_tokens = session.num_tokens + outstanding_async_tokens + session.num_output_placeholders = outstanding_async_tokens + session.spec_token_ids = [-1] * outstanding_async_tokens + + _run_resumable_segment_stop(session) + + assert session.async_tokens_to_discard == outstanding_async_tokens + assert session.num_computed_tokens == session.num_tokens + assert session.num_output_placeholders == 0 + assert session.spec_token_ids == [] + assert session._output_token_ids == [] + + def test_stage0_streaming_update_discards_outstanding_async_placeholder_token() -> None: sched = _make_scheduler(stage_id=0) session = _make_request() diff --git a/vllm_omni/core/sched/omni_ar_scheduler.py b/vllm_omni/core/sched/omni_ar_scheduler.py index 93654f27f..a9702eeb3 100644 --- a/vllm_omni/core/sched/omni_ar_scheduler.py +++ b/vllm_omni/core/sched/omni_ar_scheduler.py @@ -438,8 +438,13 @@ class OmniARScheduler(OmniSchedulerMixin, VLLMScheduler): # Downstream async-chunk stages receive real payloads from the # connector. This update only resumes polling for the next segment. self.chunk_transfer_adapter.segment_finished_requests.discard(request.request_id) - request.async_tokens_to_discard = 1 - request.num_output_placeholders = 0 + outstanding_async_tokens = request.num_output_placeholders + if outstanding_async_tokens > 0: + # Discard only outputs that are already in flight and + # roll back their optimistic computed-token accounting. + request.async_tokens_to_discard = outstanding_async_tokens + request.num_computed_tokens -= outstanding_async_tokens + request.num_output_placeholders = 0 request.spec_token_ids = [] request._output_token_ids.clear() if finished: From 536daea1074159d2ac91aa4dd53e8f1c1510c627 Mon Sep 17 00:00:00 2001 From: akshatvishu Date: Fri, 24 Jul 2026 17:11:56 +0530 Subject: [PATCH 084/240] =?UTF-8?q?[BugFix]=20Fix=20Helios=20Cholesky=20po?= =?UTF-8?q?sitive-definite=20crashes=20and=20CPU=20LAPACK=E2=80=A6=20(#492?= =?UTF-8?q?1)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: akshatvishu Co-authored-by: Hongsheng Liu --- vllm_omni/diffusion/models/helios/pipeline_helios.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/vllm_omni/diffusion/models/helios/pipeline_helios.py b/vllm_omni/diffusion/models/helios/pipeline_helios.py index 655ef8821..90ee0ab3f 100644 --- a/vllm_omni/diffusion/models/helios/pipeline_helios.py +++ b/vllm_omni/diffusion/models/helios/pipeline_helios.py @@ -1561,11 +1561,12 @@ class HeliosPipeline( device = generator.device if generator is not None else self.device - cov = torch.eye(block_size) * (1 + gamma) - torch.ones(block_size, block_size) * gamma - cov += torch.eye(block_size) * 1e-8 - cov = cov.float() # Upcast to fp32 for numerical stability — cholesky is unreliable in fp16/bf16. - - L = torch.linalg.cholesky(cov).to(device) + # Allocate directly on the execution device in float32 to use the device solver + # and avoid fp16/bf16 Cholesky on the covariance matrix. + eye = torch.eye(block_size, device=device, dtype=torch.float32) + cov = eye * (1 + gamma) - torch.ones(block_size, block_size, device=device, dtype=torch.float32) * gamma + cov += eye * 1e-8 + L = torch.linalg.cholesky(cov) block_number = batch_size * channel * num_frames * (height // ph) * (width // pw) z = torch.randn(block_number, block_size, generator=generator, device=device) noise = z @ L.T From a5d9d57adc8ff323dcbbe097fb470967ef4e8c60 Mon Sep 17 00:00:00 2001 From: wangyu <53896905+yenuo26@users.noreply.github.com> Date: Fri, 24 Jul 2026 22:28:43 +0800 Subject: [PATCH 085/240] [Refactor][CI] Unify skip-ci targeting and mirror_hardwares pipeline presets (#5254) Signed-off-by: wangyu <410167048@qq.com> Signed-off-by: wangyu <53896905+yenuo26@users.noreply.github.com> Signed-off-by: [Your Name] [Your Email] Co-authored-by: Weiming Liao --- .buildkite/amd/scripts/bootstrap-amd-omni.sh | 108 +- .buildkite/common/ci_mirror_hardwares.yml | 239 +++ .buildkite/common/scripts/resolve_skip_ci.sh | 23 + .buildkite/common/scripts/skip_ci.py | 617 ++++++++ .buildkite/common/scripts/upload_pipeline.py | 449 ++++++ .buildkite/cuda/bootstrap-upload-steps.yml | 55 + .buildkite/cuda/pipeline.yml | 83 +- .buildkite/cuda/rebase-pipeline.yml | 6 +- .buildkite/cuda/scripts/upload_pipeline.py | 547 ------- .buildkite/cuda/test-merge.yml | 434 +----- .buildkite/cuda/test-nightly.yml | 1341 +---------------- .buildkite/cuda/test-ready.yml | 492 +----- .buildkite/cuda/test-weekly.yml | 347 +---- .../intel/scripts/bootstrap-intel-omni.sh | 41 +- .buildkite/npu/bootstrap-upload-steps.yml | 181 +++ .buildkite/npu/pipeline-npu.yml | 198 +-- .buildkite/npu/test-npu-nightly.yml | 20 +- .buildkite/npu/test-npu-ready.yml | 10 +- .claude/skills/add-diffusion-model/SKILL.md | 2 +- .claude/skills/add-tts-model/SKILL.md | 2 +- .claude/skills/vllm-omni-test/SKILL.md | 4 +- .../vllm-omni-test/references/test-routing.md | 2 +- docs/contributing/README.md | 8 +- docs/contributing/ci/.nav.yaml | 8 +- docs/contributing/ci/ci_settings.md | 344 +++++ docs/contributing/ci/failures.md | 2 +- .../l4_functionality_tests.inc.md | 2 +- .../test_examples/l4_performance_tests.inc.md | 2 +- ...{test_guide.md => test_execution_guide.md} | 6 +- docs/contributing/ci/test_system_overview.md | 210 +++ .../{CI_5levels.md => test_writing_guide.md} | 535 +++---- docs/contributing/ci/tests_markers.md | 207 --- docs/contributing/ci/tests_style.md | 466 ------ .../model/adding_diffusion_model.md | 2 +- docs/contributing/model/adding_omni_model.md | 2 +- docs/contributing/model/adding_tts_model.md | 2 +- tests/buildkite/test_skip_ci.py | 370 +++++ tests/buildkite/test_upload_pipeline.py | 141 ++ 38 files changed, 3018 insertions(+), 4490 deletions(-) create mode 100644 .buildkite/common/ci_mirror_hardwares.yml create mode 100644 .buildkite/common/scripts/resolve_skip_ci.sh create mode 100644 .buildkite/common/scripts/skip_ci.py create mode 100644 .buildkite/common/scripts/upload_pipeline.py create mode 100644 .buildkite/cuda/bootstrap-upload-steps.yml delete mode 100644 .buildkite/cuda/scripts/upload_pipeline.py create mode 100644 .buildkite/npu/bootstrap-upload-steps.yml create mode 100644 docs/contributing/ci/ci_settings.md rename docs/contributing/ci/{test_guide.md => test_execution_guide.md} (97%) create mode 100644 docs/contributing/ci/test_system_overview.md rename docs/contributing/ci/{CI_5levels.md => test_writing_guide.md} (70%) delete mode 100644 docs/contributing/ci/tests_markers.md delete mode 100644 docs/contributing/ci/tests_style.md create mode 100644 tests/buildkite/test_skip_ci.py create mode 100644 tests/buildkite/test_upload_pipeline.py diff --git a/.buildkite/amd/scripts/bootstrap-amd-omni.sh b/.buildkite/amd/scripts/bootstrap-amd-omni.sh index ee858068a..32097cb8c 100755 --- a/.buildkite/amd/scripts/bootstrap-amd-omni.sh +++ b/.buildkite/amd/scripts/bootstrap-amd-omni.sh @@ -6,6 +6,8 @@ set -euo pipefail +source .buildkite/common/scripts/resolve_skip_ci.sh + if [[ -z "${RUN_ALL:-}" ]]; then RUN_ALL=0 fi @@ -22,10 +24,6 @@ if [[ -z "${AMD_MIRROR_HW:-}" ]]; then AMD_MIRROR_HW="amdproduction" fi -if [[ -z "${DOCS_ONLY_DISABLE:-}" ]]; then - DOCS_ONLY_DISABLE=0 -fi - fail_fast() { DISABLE_LABEL="ci-no-fail-fast" # If BUILDKITE_PULL_REQUEST != "false", then we check the PR labels using curl and jq @@ -56,88 +54,6 @@ check_run_all_label() { fi } -is_docs_only_change() { - local file_path - local has_any=0 - - while IFS= read -r file_path; do - [[ -z "${file_path}" ]] && continue - has_any=1 - - if [[ "${file_path}" == docs/* ]]; then - continue - fi - if [[ "${file_path}" == *.md ]]; then - continue - fi - if [[ "${file_path}" == "mkdocs.yml" ]]; then - continue - fi - return 1 - done - - [[ "${has_any}" -eq 1 ]] -} - -resolve_skip_ci() { - local is_pr_build=0 - local files - local base_branch base_ref - - if [[ "${BUILDKITE_PULL_REQUEST:-false}" != "false" && -n "${BUILDKITE_PULL_REQUEST:-}" ]]; then - is_pr_build=1 - fi - - if [[ "${is_pr_build}" -eq 1 ]]; then - base_branch="${BUILDKITE_PULL_REQUEST_BASE_BRANCH:-main}" - if ! git rev-parse --verify "origin/${base_branch}" >/dev/null 2>&1; then - echo "resolve_skip_ci: origin/${base_branch} not found locally; trying fetch" >&2 - git fetch --depth=200 origin "${base_branch}" >/dev/null 2>&1 || true - fi - - base_ref="" - if git rev-parse --verify "origin/${base_branch}" >/dev/null 2>&1; then - base_ref="origin/${base_branch}" - elif git rev-parse --verify "${base_branch}" >/dev/null 2>&1; then - base_ref="${base_branch}" - else - echo "resolve_skip_ci: cannot resolve PR base ${base_branch}; skip-ci=0" >&2 - echo -n 0 - return 0 - fi - - if ! files="$(git diff --name-only "${base_ref}...${BUILDKITE_COMMIT}" 2>/dev/null)"; then - echo "resolve_skip_ci: failed to compute PR changed files; skip-ci=0" >&2 - echo -n 0 - return 0 - fi - elif [[ "${BUILDKITE_BRANCH:-}" == "main" ]]; then - if ! git rev-parse --verify "${BUILDKITE_COMMIT}^" >/dev/null 2>&1; then - echo "resolve_skip_ci: commit has no parent on main; skip-ci=0" >&2 - echo -n 0 - return 0 - fi - if ! files="$(git diff --name-only "${BUILDKITE_COMMIT}^..${BUILDKITE_COMMIT}" 2>/dev/null)"; then - echo "resolve_skip_ci: failed to compute main changed files; skip-ci=0" >&2 - echo -n 0 - return 0 - fi - else - echo "resolve_skip_ci: not PR/main build; skip-ci=0" >&2 - echo -n 0 - return 0 - fi - - if is_docs_only_change <<< "${files}"; then - echo "resolve_skip_ci: docs-only change detected; skip-ci=1" >&2 - echo -n 1 - return 0 - fi - - echo "resolve_skip_ci: non-doc changes detected; skip-ci=0" >&2 - echo -n 0 -} - if [[ -z "${COV_ENABLED:-}" ]]; then COV_ENABLED=0 fi @@ -234,23 +150,13 @@ if [[ $BUILDKITE_BRANCH == "main" ]]; then file_diff=$(get_diff_main) fi -# ---------------------------------------------------------------------- -# Early exit start: skip pipeline if conditions are met -# ---------------------------------------------------------------------- - -# Match CUDA Buildkite skip-ci behavior for docs/markdown-only changes. -if [[ "${DOCS_ONLY_DISABLE}" != "1" ]]; then - SKIP_CI="$(resolve_skip_ci)" - if [[ "${SKIP_CI}" == "1" ]]; then - echo "[docs-only] Docs/markdown-only changes detected. Exiting before pipeline upload." - exit 0 - fi +# Early exit: unified skip-ci (docs / skip marks) and CI-yaml-only level targeting. +if [[ $BUILDKITE_BRANCH == "main" ]]; then + gate_bootstrap_ci amd l3 +else + gate_bootstrap_ci amd l2 fi -# ---------------------------------------------------------------------- -# Early exit end -# ---------------------------------------------------------------------- - patterns=( "docker/Dockerfile" "CMakeLists.txt" diff --git a/.buildkite/common/ci_mirror_hardwares.yml b/.buildkite/common/ci_mirror_hardwares.yml new file mode 100644 index 000000000..b7d40c75d --- /dev/null +++ b/.buildkite/common/ci_mirror_hardwares.yml @@ -0,0 +1,239 @@ +# Uploader-only hardware presets for CUDA / NPU test YAML. +# Referenced as ``mirror_hardwares: l4_1`` on a step; ``upload_pipeline.py`` expands to +# ``agents`` (+ optional top-level ``image`` for NPU) + ``plugins``, then strips +# ``mirror_hardwares`` before Buildkite upload. +# +# CUDA naming: {chip}_{count} +# l4_1, l4_4, h100_1 … h100_4 +# NPU naming: {queue}_{resource_class} +# a2b3_npu_1 — ascend-a2b3, resource_class npu-1 (A2/B3 CI image) +# a2b3_npu_4 — ascend-a2b3, resource_class npu-4 +# a3_npu_2 — ascend-a3, resource_class npu-2 (A3 CI image tag …:$BUILDKITE_COMMIT-a3) + +x-npu-k8s-plugins: &npu_k8s_plugins + - kubernetes: + podSpecPatch: + imagePullSecrets: + - name: swr-secret + + +mirror_hardwares: + l4_1: + agents: + queue: gpu_1_queue + plugins: + - docker#v5.2.0: + image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT + always-pull: true + propagate-environment: true + shm-size: "8gb" + environment: + - HF_HOME=/fsx/hf_cache + - HF_TOKEN + volumes: + - /fsx/hf_cache:/fsx/hf_cache + + l4_4: + agents: + queue: gpu_4_queue + plugins: + - docker#v5.2.0: + image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT + always-pull: true + propagate-environment: true + shm-size: "8gb" + environment: + - HF_HOME=/fsx/hf_cache + - HF_TOKEN + volumes: + - /fsx/hf_cache:/fsx/hf_cache + + h100_1: + agents: + queue: mithril-h100-pool + plugins: + - kubernetes: + podSpec: + containers: + - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT + resources: + limits: + nvidia.com/gpu: 1 + volumeMounts: + - name: devshm + mountPath: /dev/shm + - name: hf-cache + mountPath: /root/.cache/huggingface + env: + - name: HF_HOME + value: /root/.cache/huggingface + - name: HF_TOKEN + valueFrom: + secretKeyRef: + name: hf-token-secret + key: token + nodeSelector: + node.kubernetes.io/instance-type: gpu-h100-sxm + volumes: + - name: devshm + emptyDir: + medium: Memory + - name: hf-cache + hostPath: + path: /mnt/hf-cache + type: DirectoryOrCreate + + h100_2: + agents: + queue: mithril-h100-pool + plugins: + - kubernetes: + podSpec: + containers: + - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT + resources: + limits: + nvidia.com/gpu: 2 + volumeMounts: + - name: devshm + mountPath: /dev/shm + - name: hf-cache + mountPath: /root/.cache/huggingface + env: + - name: HF_HOME + value: /root/.cache/huggingface + - name: HF_TOKEN + valueFrom: + secretKeyRef: + name: hf-token-secret + key: token + nodeSelector: + node.kubernetes.io/instance-type: gpu-h100-sxm + volumes: + - name: devshm + emptyDir: + medium: Memory + - name: hf-cache + hostPath: + path: /mnt/hf-cache + type: DirectoryOrCreate + + h100_3: + agents: + queue: mithril-h100-pool + plugins: + - kubernetes: + podSpec: + containers: + - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT + resources: + limits: + nvidia.com/gpu: 3 + volumeMounts: + - name: devshm + mountPath: /dev/shm + - name: hf-cache + mountPath: /root/.cache/huggingface + env: + - name: HF_HOME + value: /root/.cache/huggingface + - name: HF_TOKEN + valueFrom: + secretKeyRef: + name: hf-token-secret + key: token + nodeSelector: + node.kubernetes.io/instance-type: gpu-h100-sxm + volumes: + - name: devshm + emptyDir: + medium: Memory + - name: hf-cache + hostPath: + path: /mnt/hf-cache + type: DirectoryOrCreate + + h100_4: + agents: + queue: mithril-h100-pool + plugins: + - kubernetes: + podSpec: + containers: + - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT + resources: + limits: + nvidia.com/gpu: 4 + volumeMounts: + - name: devshm + mountPath: /dev/shm + - name: hf-cache + mountPath: /root/.cache/huggingface + env: + - name: HF_HOME + value: /root/.cache/huggingface + - name: HF_TOKEN + valueFrom: + secretKeyRef: + name: hf-token-secret + key: token + nodeSelector: + node.kubernetes.io/instance-type: gpu-h100-sxm + volumes: + - name: devshm + emptyDir: + medium: Memory + - name: hf-cache + hostPath: + path: /mnt/hf-cache + type: DirectoryOrCreate + + a2b3_npu_1: + agents: + queue: "ascend-a2b3" + resource_class: "npu-1" + image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}" + plugins: *npu_k8s_plugins + + a2b3_npu_4: + agents: + queue: "ascend-a2b3" + resource_class: "npu-4" + image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}" + plugins: *npu_k8s_plugins + + + a2b3_npu_8: + agents: + queue: "ascend-a2b3" + resource_class: "npu-8" + image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}" + plugins: *npu_k8s_plugins + + a3_npu_2: + agents: + queue: "ascend-a3" + resource_class: "npu-2" + image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}-a3" + plugins: *npu_k8s_plugins + + a3_npu_4: + agents: + queue: "ascend-a3" + resource_class: "npu-4" + image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}-a3" + plugins: *npu_k8s_plugins + + a3_npu_8: + agents: + queue: "ascend-a3" + resource_class: "npu-8" + image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}-a3" + plugins: *npu_k8s_plugins + + a3_npu_16: + agents: + queue: "ascend-a3" + resource_class: "npu-16" + image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}-a3" + plugins: *npu_k8s_plugins diff --git a/.buildkite/common/scripts/resolve_skip_ci.sh b/.buildkite/common/scripts/resolve_skip_ci.sh new file mode 100644 index 000000000..666573f66 --- /dev/null +++ b/.buildkite/common/scripts/resolve_skip_ci.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# Shared Buildkite skip-ci helpers for AMD / Intel bootstrap scripts. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SKIP_CI_PY="${SCRIPT_DIR}/skip_ci.py" + +# Resolve git diff once. Exit 0 when this bootstrap should stop uploading. +# Decision details are logged by skip_ci.py; this helper only annotates skip-all. +# Usage: gate_bootstrap_ci +gate_bootstrap_ci() { + local platform="$1" + local level="$2" + local kind + if kind="$(python3 "${SKIP_CI_PY}" gate "${platform}" "${level}")"; then + if [[ "${kind}" == "skip-all" ]]; then + buildkite-agent annotate \ + ":memo: CI skipped — docs or pytest skip-mark changes only" \ + --style "info" 2>/dev/null || true + fi + exit 0 + fi +} diff --git a/.buildkite/common/scripts/skip_ci.py b/.buildkite/common/scripts/skip_ci.py new file mode 100644 index 000000000..b452b5adf --- /dev/null +++ b/.buildkite/common/scripts/skip_ci.py @@ -0,0 +1,617 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +"""Shared Buildkite skip-ci and CI-YAML level targeting logic. + +``resolve_ci_decision(changed_files, ...)`` buckets explicit file lists (tests, dry-run). +``resolve_ci_context_from_git()`` resolves git diff once and returns decision + changed files +for Buildkite bootstrap / upload scripts. + +CLI (for bootstrap scripts; exit code is the signal): + python3 skip_ci.py gate + Exit 0 when bootstrap should stop (skip-all, or that L2/L3 target is off). + Prints ``skip-all`` or ``skip-l23`` on stdout; logs the decision once. + python3 skip_ci.py check-skip-all + Exit 0 when only docs/skip-mark changes → bootstrap skips entire CI upload. + python3 skip_ci.py check-skip-l2-l3 + Exit 0 when that L2/L3 pipeline should be skipped (CI config YAML-only diffs). + python3 skip_ci.py print-annotate + Print decision message for Buildkite annotation. +""" + +from __future__ import annotations + +import argparse +import os +import re +import subprocess +import sys +from dataclasses import dataclass, field, replace +from pathlib import Path + +LOG = "[skip-ci]" +ROOT = Path(__file__).resolve().parent.parent.parent.parent + +PLATFORMS = ("cuda", "amd", "intel", "npu") +L23_LEVELS = frozenset({"l2", "l3"}) + +L2_YAML_FILES: dict[str, str] = { + ".buildkite/cuda/test-ready.yml": "cuda", + ".buildkite/npu/test-npu-ready.yml": "npu", + ".buildkite/amd/test-amd-ready.yml": "amd", + ".buildkite/intel/pipeline-intel.yml": "intel", +} + +L3_YAML_FILES: dict[str, str] = { + ".buildkite/cuda/test-merge.yml": "cuda", + ".buildkite/amd/test-amd-merge.yml": "amd", +} + +L45_YAML_FILES: dict[str, str] = { + ".buildkite/cuda/test-nightly.yml": "cuda", + ".buildkite/cuda/test-weekly.yml": "cuda", + ".buildkite/npu/test-npu-nightly.yml": "npu", +} + + +def _empty_platform_buckets() -> dict[str, list[str]]: + return {platform: [] for platform in PLATFORMS} + + +@dataclass(frozen=True) +class CiDevice: + """Per-platform / per-level skip flags for yaml-gated bootstrap (``skip_l2_l3``).""" + + skip_cuda_l2: bool = False + skip_cuda_l3: bool = False + skip_amd_l2: bool = False + skip_amd_l3: bool = False + skip_intel_l2: bool = False + skip_intel_l3: bool = False + skip_npu_l2: bool = False + skip_npu_l3: bool = False + + def is_skip(self, platform: str, level: str) -> bool: + return bool(getattr(self, f"skip_{platform}_{level}", True)) + + +@dataclass +class DiffBuckets: + """Changed files grouped by doc / skip-mark / CI yaml / other.""" + + docs: list[str] = field(default_factory=list) + skip_tests: list[str] = field(default_factory=list) + other: list[str] = field(default_factory=list) + l2: dict[str, list[str]] = field(default_factory=_empty_platform_buckets) + l3: dict[str, list[str]] = field(default_factory=_empty_platform_buckets) + l45: dict[str, list[str]] = field(default_factory=_empty_platform_buckets) + + @property + def has_doc_changes(self) -> bool: + return bool(self.docs) + + @property + def has_skip_test_changes(self) -> bool: + return bool(self.skip_tests) + + @property + def has_other_changes(self) -> bool: + """Product code or paths outside skip-ci scope.""" + return bool(self.other) + + @property + def has_l23_yaml_changes(self) -> bool: + """At least one whitelisted L2/L3 CI yaml file changed.""" + return any(self.l2[platform] or self.l3[platform] for platform in PLATFORMS) + + @property + def has_l45_yaml_changes(self) -> bool: + """At least one whitelisted L4/L5 (nightly/weekly) CI yaml file changed.""" + return any(self.l45[platform] for platform in PLATFORMS) + + @property + def has_skip_ci_scope_changes(self) -> bool: + """At least one doc / skip-mark / whitelisted CI yaml change.""" + return ( + self.has_doc_changes or self.has_skip_test_changes or self.has_l23_yaml_changes or self.has_l45_yaml_changes + ) + + def l2_platforms(self) -> frozenset[str]: + return frozenset(platform for platform in PLATFORMS if self.l2[platform]) + + def l3_platforms(self) -> frozenset[str]: + return frozenset(platform for platform in PLATFORMS if self.l3[platform]) + + +@dataclass(frozen=True) +class SkipL2L3Basis: + skip_all_l2: bool + skip_all_l3: bool + + @classmethod + def from_buckets(cls, buckets: DiffBuckets) -> SkipL2L3Basis | None: + """L4/L5 yaml → skip both L2 and L3 (L2/L3 buckets may rescue). + + Docs / skip-mark changes do not block this path; they are ignored for + L2/L3 targeting when CI YAML is also present. + """ + if buckets.has_other_changes: + return None + if not buckets.has_l45_yaml_changes: + return None + return cls( + skip_all_l2=not buckets.l2_platforms(), + skip_all_l3=not buckets.l3_platforms(), + ) + + +@dataclass(frozen=True) +class SkipL3Basis: + enable_l2_platforms: frozenset[str] + + @classmethod + def from_buckets(cls, buckets: DiffBuckets) -> SkipL3Basis | None: + """L2 yaml changed → skip L3; keep L2 on touched platforms.""" + platforms = buckets.l2_platforms() + if not platforms: + return None + return cls(enable_l2_platforms=platforms) + + +@dataclass(frozen=True) +class SkipL2Basis: + enable_l3_platforms: frozenset[str] + + @classmethod + def from_buckets(cls, buckets: DiffBuckets) -> SkipL2Basis | None: + """L3 yaml changed → skip L2; keep L3 on touched platforms.""" + platforms = buckets.l3_platforms() + if not platforms: + return None + return cls(enable_l3_platforms=platforms) + + +@dataclass(frozen=True) +class CiDecision: + """Final CI skip decision for one diff.""" + + skip_all: bool = False + skip_l2_l3: bool = False + device: CiDevice = field(default_factory=CiDevice) + message: str = "" + + @classmethod + def skip_all_basis(cls, buckets: DiffBuckets, *, diff_range: str | None) -> bool: + """Docs / pytest skip-mark only → suppress entire default CI.""" + if buckets.has_other_changes: + return False + if (buckets.has_l23_yaml_changes or buckets.has_l45_yaml_changes) and ( + buckets.has_doc_changes or buckets.has_skip_test_changes + ): + return False + if not buckets.has_doc_changes and not buckets.has_skip_test_changes: + return False + if diff_range is None and not buckets.has_doc_changes: + return False + return True + + @classmethod + def from_yaml_basis( + cls, + l2_l3_basis: SkipL2L3Basis | None, + l3_basis: SkipL3Basis | None, + l2_basis: SkipL2Basis | None, + ) -> CiDecision: + """Build a yaml-gated decision by merging L2/L3 skip bases. + + Bases are applied as a union of enables: L2 yaml keeps L2 on touched + platforms, L3 yaml keeps L3 on touched platforms. Cross-platform mixes + (e.g. CUDA ready + AMD merge) must preserve both enables. The resulting + per-platform / per-level matrix is stored on ``device`` as ``skip_*``. + """ + run = {platform: {"l2": True, "l3": True} for platform in PLATFORMS} + + if l2_l3_basis is not None: + if l2_l3_basis.skip_all_l2: + for platform in PLATFORMS: + run[platform]["l2"] = False + if l2_l3_basis.skip_all_l3: + for platform in PLATFORMS: + run[platform]["l3"] = False + + if l3_basis is not None: + for platform in PLATFORMS: + run[platform]["l3"] = False + run[platform]["l2"] = platform in l3_basis.enable_l2_platforms + + if l2_basis is not None: + for platform in PLATFORMS: + if platform in l2_basis.enable_l3_platforms: + run[platform]["l3"] = True + if l3_basis is None or platform not in l3_basis.enable_l2_platforms: + run[platform]["l2"] = False + elif l3_basis is None: + # L3-yaml-only: disable platforms that did not change L3. + run[platform]["l2"] = False + run[platform]["l3"] = False + # else: leave L2 enables from SkipL3Basis; keep this platform's L3 off. + + return cls( + skip_l2_l3=True, + device=cls._device_from_run(run), + ) + + @staticmethod + def _device_from_run(run: dict[str, dict[str, bool]]) -> CiDevice: + return CiDevice( + **{f"skip_{platform}_{level}": not run[platform][level] for platform in PLATFORMS for level in ("l2", "l3")} + ) + + def is_run(self, platform: str, level: str) -> bool: + if self.skip_all: + return False + if not self.skip_l2_l3: + return True + if level not in L23_LEVELS: + return True + return not self.device.is_skip(platform, level) + + +@dataclass(frozen=True) +class CiContext: + """Skip-ci decision plus git diff inputs used to compute it.""" + + decision: CiDecision + changed_files: list[str] | None + diff_range: str | None + + +def _log(message: str) -> None: + print(f"{LOG}: {message}", file=sys.stderr) + + +def _git(*args: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + ["git", *args], + check=False, + capture_output=True, + text=True, + ) + + +# --- Diff helpers --- + + +def _resolve_diff_range() -> str | None: + """Return a git diff range for PR or main builds, or None when unavailable.""" + is_pr = os.environ.get("BUILDKITE_PULL_REQUEST", "false") != "false" and os.environ.get( + "BUILDKITE_PULL_REQUEST", "" + ) + commit = os.environ.get("BUILDKITE_COMMIT", "") + + if is_pr: + base_branch = os.environ.get("BUILDKITE_PULL_REQUEST_BASE_BRANCH", "main") + base_ref = f"origin/{base_branch}" + if _git("rev-parse", "--verify", base_ref).returncode != 0: + _log(f"origin/{base_branch} not found locally; trying fetch") + _git("fetch", "--depth=200", "origin", base_branch) + if _git("rev-parse", "--verify", base_ref).returncode != 0: + if _git("rev-parse", "--verify", base_branch).returncode == 0: + base_ref = base_branch + else: + _log(f"cannot resolve PR base {base_branch}; using safe defaults") + return None + return f"{base_ref}...{commit}" + + if os.environ.get("BUILDKITE_BRANCH", "") == "main": + if _git("rev-parse", "--verify", f"{commit}^").returncode != 0: + _log("main commit has no parent; using safe defaults") + return None + return f"{commit}^..{commit}" + + _log("not PR/main build; using safe defaults") + return None + + +def _changed_files_for_diff_range(diff_range: str | None) -> list[str] | None: + """Return changed file paths for *diff_range*, or None when unavailable.""" + if diff_range is None: + return None + + result = _git("diff", "--name-only", diff_range) + if result.returncode != 0: + _log(f"git diff failed ({diff_range}); using safe defaults") + return None + + files = [line for line in result.stdout.splitlines() if line.strip()] + _log(f"{len(files)} changed file(s)") + return files + + +def resolve_ci_context_from_git() -> CiContext: + """Resolve git diff once and return skip-ci decision plus changed files.""" + diff_range = _resolve_diff_range() + changed_files = _changed_files_for_diff_range(diff_range) + decision = resolve_ci_decision(changed_files, diff_range=diff_range) + return CiContext(decision=decision, changed_files=changed_files, diff_range=diff_range) + + +# --- Doc / skip-mark helpers --- + + +_SKIP_MARK_RE = re.compile(r"pytest\.mark\.skip(?:if)?\b|pytest\.skip\s*\(") +_PYTESTMARK_SKIP_RE = re.compile(r"pytest\.mark\.skip\b") +_PYTEST_MARK_ONLY_RE = re.compile(r"pytest\.mark\.\w+") + + +def _diff_only_contains_skip_mark_changes(diff_text: str) -> bool: + """Return True when a unified diff only *adds* pytest skip marks. + + Deleting a skip mark re-enables the test and must not suppress CI. Editing an + existing skip (``-`` old / ``+`` new) still qualifies when at least one skip + mark is added. + """ + + def paren_balance(line: str) -> int: + return line.count("(") - line.count(")") + + def is_blank_or_comment(line: str) -> bool: + stripped = line.strip() + return not stripped or stripped.startswith("#") + + def is_skip_mark_line(line: str) -> bool: + return _SKIP_MARK_RE.search(line.strip()) is not None + + def is_pytestmark_adjacent_line(line: str) -> bool: + stripped = line.strip().rstrip(",") + if not stripped or stripped in {"[", "]"} or stripped.startswith("#"): + return True + if stripped.startswith("pytestmark"): + return True + return _PYTEST_MARK_ONLY_RE.search(stripped) is not None + + pending_depth = 0 + saw_change = False + has_added_skip_mark = False + for raw_line in diff_text.splitlines(): + if raw_line.startswith("@@"): + pending_depth = 0 + continue + if not (raw_line.startswith("+") or raw_line.startswith("-")): + continue + if raw_line.startswith("+++") or raw_line.startswith("---"): + continue + + saw_change = True + is_add = raw_line.startswith("+") + content = raw_line[1:] + + if pending_depth > 0: + pending_depth += paren_balance(content) + if pending_depth < 0: + pending_depth = 0 + continue + + if is_blank_or_comment(content): + continue + + if is_skip_mark_line(content): + if is_add: + has_added_skip_mark = True + pending_depth = max(0, paren_balance(content)) + continue + + if is_pytestmark_adjacent_line(content): + continue + + return False + + return saw_change and has_added_skip_mark + + +def _is_skip_testcase_change(file_path: str, *, diff_range: str) -> bool: + """Return True when a test file diff is skip-mark-only (including new-file cases).""" + diff_result = _git("diff", diff_range, "--", file_path) + if diff_result.returncode != 0: + _log(f"skip-mark-only: git diff failed for {file_path}") + return False + + diff_text = diff_result.stdout + if not diff_text.strip(): + _log(f"skip-mark-only: empty diff for {file_path}") + return False + + if _diff_only_contains_skip_mark_changes(diff_text): + return True + + if "new file mode" not in diff_text and "--- /dev/null" not in diff_text: + _log(f"skip-mark-only: non-skip changes in {file_path}") + return False + + commit = os.environ.get("BUILDKITE_COMMIT", "") + if not commit: + return False + + content_result = _git("show", f"{commit}:{file_path}") + if content_result.returncode != 0: + _log(f"skip-mark-only: cannot read {file_path} at {commit}") + return False + + content = content_result.stdout + match = re.search(r"^pytestmark\s*=\s*\[(.*?)^\s*\]", content, re.MULTILINE | re.DOTALL) + if match is not None and _PYTESTMARK_SKIP_RE.search(match.group(1)) is not None: + _log(f"skip-mark-only: new test file with module-level pytest.mark.skip: {file_path}") + return True + if re.search(r"^pytestmark\s*=\s*pytest\.mark\.skip\b", content, re.MULTILINE): + _log(f"skip-mark-only: new test file with module-level pytest.mark.skip: {file_path}") + return True + + _log(f"skip-mark-only: new test file without module-level pytest.mark.skip: {file_path}") + return False + + +def _classify_changed_files_into_buckets(changed_files: list[str], *, diff_range: str | None) -> DiffBuckets: + """Classify each changed file path into a ``DiffBuckets`` category.""" + buckets = DiffBuckets() + for file_path in changed_files: + if not file_path: + continue + if file_path.startswith("docs/") or file_path.endswith(".md") or file_path == "mkdocs.yml": + buckets.docs.append(file_path) + continue + path = Path(file_path) + if path.suffix == ".py" and path.parts and path.parts[0] == "tests": + if diff_range is not None and _is_skip_testcase_change(file_path, diff_range=diff_range): + buckets.skip_tests.append(file_path) + else: + buckets.other.append(file_path) + continue + if file_path in L2_YAML_FILES: + platform = L2_YAML_FILES[file_path] + buckets.l2[platform].append(file_path) + elif file_path in L3_YAML_FILES: + platform = L3_YAML_FILES[file_path] + buckets.l3[platform].append(file_path) + elif file_path in L45_YAML_FILES: + platform = L45_YAML_FILES[file_path] + buckets.l45[platform].append(file_path) + else: + buckets.other.append(file_path) + return buckets + + +# --- Decision --- + + +def _finish(decision: CiDecision, message: str) -> CiDecision: + finished = replace(decision, message=message) + _log(message) + return finished + + +def _format_yaml_gated_message(buckets: DiffBuckets, decision: CiDecision) -> str: + """Human-readable summary of which CI YAML changed and which L2/L3 targets run.""" + changed_parts: list[str] = [] + for level_name, by_platform in (("L2", buckets.l2), ("L3", buckets.l3), ("L4/L5", buckets.l45)): + files = [path for platform in PLATFORMS for path in by_platform[platform]] + if files: + changed_parts.append(f"{level_name}=[{', '.join(files)}]") + + run_targets = [ + f"{platform}/{level}" for platform in PLATFORMS for level in ("l2", "l3") if decision.is_run(platform, level) + ] + skip_targets = [ + f"{platform}/{level}" + for platform in PLATFORMS + for level in ("l2", "l3") + if not decision.is_run(platform, level) + ] + + changed = "; ".join(changed_parts) if changed_parts else "none" + run = ", ".join(run_targets) if run_targets else "none" + skip = ", ".join(skip_targets) if skip_targets else "none" + if buckets.has_doc_changes or buckets.has_skip_test_changes: + return ( + "CI config YAML + docs/skip-mark changed — follow CI YAML gating; " + f"changed: {changed}; run: {run}; skip: {skip}" + ) + return f"only CI config YAML changed — changed: {changed}; run: {run}; skip: {skip}" + + +def resolve_ci_decision( + changed_files: list[str] | None = None, + *, + diff_range: str | None = None, +) -> CiDecision: + if changed_files is None: + return _finish(CiDecision(), "could not resolve changed files; run normal CI") + + buckets = _classify_changed_files_into_buckets(changed_files, diff_range=diff_range) + if buckets.has_other_changes or not buckets.has_skip_ci_scope_changes: + return _finish(CiDecision(), "product or non-whitelisted changes present; run normal CI") + + if CiDecision.skip_all_basis(buckets, diff_range=diff_range): + return _finish( + CiDecision(skip_all=True), + "CI skipped — docs or pytest skip-mark changes only", + ) + + # Docs / skip-mark mixed with whitelisted CI YAML → same yaml-gated path + # as CI-YAML-only (docs/skip-mark do not widen to normal CI). + if not buckets.has_l23_yaml_changes and not buckets.has_l45_yaml_changes: + if buckets.has_skip_test_changes and diff_range is None: + message = ( + "only pytest skip-mark test changes detected, but no git diff range " + "is available to verify them (skip-all requires diff_range); run normal CI" + ) + elif buckets.has_skip_test_changes: + message = "pytest skip-mark test changes did not qualify for skip-all and no ci yaml changed; run normal CI" + else: + message = "doc/skip-mark changes did not qualify for skip-all; run normal CI" + return _finish(CiDecision(), message) + + decision = CiDecision.from_yaml_basis( + SkipL2L3Basis.from_buckets(buckets), + SkipL3Basis.from_buckets(buckets), + SkipL2Basis.from_buckets(buckets), + ) + return _finish(decision, _format_yaml_gated_message(buckets, decision)) + + +def main() -> int: + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + subparsers = parser.add_subparsers(dest="command", required=True) + + gate_parser = subparsers.add_parser( + "gate", + help="exit 0 when bootstrap should stop (skip-all or PLATFORM/LEVEL disabled); logs once", + ) + gate_parser.add_argument("platform", help="cuda, amd, intel, or npu") + gate_parser.add_argument("level", choices=("l2", "l3"), help="test level") + + subparsers.add_parser( + "check-skip-all", + help="exit 0 when docs/skip-mark-only diff should skip entire CI upload", + ) + + skip_parser = subparsers.add_parser( + "check-skip-l2-l3", + help="exit 0 when PLATFORM l2/l3 pipeline should be skipped (CI config YAML-only diffs)", + ) + skip_parser.add_argument("platform", help="cuda, amd, intel, or npu") + skip_parser.add_argument("level", choices=("l2", "l3"), help="test level") + + subparsers.add_parser( + "print-annotate", + help="print human-readable skip reason for Buildkite annotation", + ) + + args = parser.parse_args() + decision = resolve_ci_context_from_git().decision + + if args.command == "gate": + if decision.skip_all: + print("skip-all") + return 0 + if not decision.is_run(args.platform, args.level): + print("skip-l23") + return 0 + return 1 + + if args.command == "check-skip-all": + return 0 if decision.skip_all else 1 + + if args.command == "check-skip-l2-l3": + return 0 if not decision.is_run(args.platform, args.level) else 1 + + if args.command == "print-annotate": + print(decision.message or "CI will run normally") + return 0 + + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.buildkite/common/scripts/upload_pipeline.py b/.buildkite/common/scripts/upload_pipeline.py new file mode 100644 index 000000000..b27f78c3a --- /dev/null +++ b/.buildkite/common/scripts/upload_pipeline.py @@ -0,0 +1,449 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +"""Render and optionally upload Buildkite pipeline YAML with diff-aware logic. + +Bootstrap mode (``bootstrap-upload-steps.yml``): + - Hook uploads the entry YAML (``pipeline.yml`` / ``pipeline-npu.yml``) with one step that runs + ``upload_pipeline.py --upload /bootstrap-upload-steps.yml``. + - Injects ``if`` by step ``key`` from skip-ci and uploads child steps (image build, L2–L5 upload). + - Detect docs-only, pytest skip-mark-only, or combined skip-ci from git diff. + - When only CI level YAML changes, enable **L2/L3** upload steps for affected levels only. + +Test pipeline mode (e.g. test-merge.yml): + - Drop steps whose ``source_file_dependencies`` do not match changed files. + - Expand uploader-only ``mirror_hardwares: l4_1`` into ``agents`` (+ optional ``image`` + for NPU) + ``plugins`` (see ci_mirror_hardwares.yml). + +Usage: + python3 upload_pipeline.py [--upload] [--all | --e2e] + +Requires PyYAML (``pip install pyyaml``); installs it automatically when missing. +""" + +from __future__ import annotations + +import argparse +import copy +import subprocess +import sys +from functools import lru_cache +from pathlib import Path +from typing import Any + +try: + import yaml +except ModuleNotFoundError: + subprocess.run( + [sys.executable, "-m", "pip", "install", "-q", "pyyaml"], + check=True, + ) + import yaml + +from skip_ci import ( + ROOT, + resolve_ci_context_from_git, +) + +# --- Constants --- + +LOG = "upload_pipeline" +BOOTSTRAP_STEPS_FILENAME = "bootstrap-upload-steps.yml" +BOOTSTRAP_IMAGE_BUILD_KEYS = frozenset({"image-build", "image-build-a2", "image-build-a3"}) +BOOTSTRAP_UPLOAD_IF_KEYS = { + "upload-ready-pipeline": "ready", + "upload-merge-pipeline": "merge", + "upload-nightly-pipeline": "nightly", + "upload-weekly-pipeline": "weekly", +} +E2E_GROUP_MARKER = "E2E Test" +CI_MIRROR_HARDWARES_PATH = ROOT / ".buildkite/common/ci_mirror_hardwares.yml" + +CUDA_NIGHTLY_ONLY = ( + '(build.pull_request.labels includes "nightly-test") || (build.branch == "main" && build.env("NIGHTLY") == "1")' +) +NPU_NIGHTLY_ONLY = ( + '(build.branch == "main" && build.env("NIGHTLY") == "1") || ' + '(build.branch != "main" && (' + 'build.pull_request.labels includes "nightly-test" || ' + 'build.pull_request.labels includes "omni-test" || ' + 'build.pull_request.labels includes "tts-test" || ' + 'build.pull_request.labels includes "diffusion-x2iat-test" || ' + 'build.pull_request.labels includes "diffusion-x2v-test"' + "))" +) + + +# --- Logging --- + + +def _log(message: str) -> None: + print(f"{LOG}: {message}", file=sys.stderr) + + +# --- Bootstrap pipeline (bootstrap-upload-steps.yml) --- + + +def _get_bootstrap_platform(path: Path) -> str: + parts = path.as_posix().split("/") + return "npu" if "npu" in parts else "cuda" + + +def _load_bootstrap_steps(path: Path) -> str: + if path.name != BOOTSTRAP_STEPS_FILENAME: + raise ValueError(f"expected {BOOTSTRAP_STEPS_FILENAME}, got {path.name}") + return path.read_text(encoding="utf-8") + + +def _format_bootstrap_if(expr: str) -> str: + """Return a Buildkite ``if`` string. Buildkite rejects YAML bool ``if`` values.""" + if expr in ("true", "false"): + return expr + return f"({expr})" + + +def _compute_bootstrap_if_exprs(*, decision, platform: str) -> dict[str, str]: + disabled = "false" + nightly_main = 'build.branch == "main" && build.env("NIGHTLY") == "1"' + nightly_only = NPU_NIGHTLY_ONLY if platform == "npu" else CUDA_NIGHTLY_ONLY + + if platform == "npu": + ready_pr = ( + 'build.branch != "main" && (' + 'build.pull_request.labels includes "npu-test" || ' + 'build.pull_request.labels includes "ready"' + ")" + ) + nightly_label_if = nightly_only + weekly_label_if = disabled + merge_base = disabled + else: + ready_pr = 'build.branch != "main" && build.pull_request.labels includes "ready"' + merge_main = 'build.branch == "main" && build.env("NIGHTLY") != "1" && build.env("WEEKLY") != "1"' + merge_pr = 'build.branch != "main" && build.pull_request.labels includes "merge-test"' + nightly_label_if = ( + '(build.branch == "main" && build.env("NIGHTLY") == "1") || ' + '(build.branch != "main" && (' + 'build.pull_request.labels includes "nightly-test" || ' + 'build.pull_request.labels includes "omni-test" || ' + 'build.pull_request.labels includes "tts-test" || ' + 'build.pull_request.labels includes "diffusion-x2iat-test" || ' + 'build.pull_request.labels includes "diffusion-x2v-test"' + "))" + ) + weekly_label_if = ( + '(build.branch == "main" && build.env("WEEKLY") == "1") || ' + '(build.branch != "main" && build.pull_request.labels includes "weekly-test")' + ) + merge_base = f"({nightly_main}) || (({merge_main}) || ({merge_pr}))" + + ready_base = f"({nightly_main}) || ({ready_pr})" + weekly_main = 'build.branch == "main" && build.env("WEEKLY") == "1"' + + if decision.skip_all: + # Docs / skip-mark only: no PR-label escape hatch. Main scheduled + # NIGHTLY=1 still runs L4 + L2/L3 (--e2e); WEEKLY=1 still runs L5. + image_expr = f"({nightly_main}) || ({weekly_main})" if platform == "cuda" else nightly_main + ready_expr = nightly_main + merge_expr = nightly_main if platform == "cuda" else disabled + nightly_expr = nightly_main + weekly_expr = weekly_main if platform == "cuda" else disabled + elif decision.skip_l2_l3: + l2_enabled = decision.is_run("npu", "l2") if platform == "npu" else decision.is_run("cuda", "l2") + l3_enabled = platform == "cuda" and decision.is_run("cuda", "l3") + + ready_expr = ready_base if l2_enabled else disabled + merge_expr = merge_base if l3_enabled else disabled + nightly_expr = nightly_label_if + weekly_expr = weekly_label_if if platform == "cuda" else disabled + + image_parts = [f"({nightly_label_if})"] + if platform == "cuda": + image_parts.append(f"({weekly_label_if})") + if l2_enabled: + image_parts.insert(0, f"({ready_base})") + if l3_enabled: + image_parts.insert(1 if l2_enabled else 0, f"({merge_base})") + image_expr = " || ".join(image_parts) + else: + image_expr = "true" + ready_expr = ready_base + merge_expr = merge_base if platform == "cuda" else disabled + nightly_expr = nightly_label_if + weekly_expr = weekly_label_if if platform == "cuda" else disabled + + return { + "image": _format_bootstrap_if(image_expr), + "ready": _format_bootstrap_if(ready_expr), + "merge": _format_bootstrap_if(merge_expr), + "nightly": _format_bootstrap_if(nightly_expr), + "weekly": _format_bootstrap_if(weekly_expr), + } + + +def _apply_bootstrap_if(steps: list[Any], if_exprs: dict[str, str]) -> list[Any]: + """Inject ``if`` by step key; drop steps that are unconditionally disabled.""" + kept: list[Any] = [] + for step in steps: + if not isinstance(step, dict): + kept.append(step) + continue + nested = step.get("steps") + if isinstance(nested, list): + nested_kept = _apply_bootstrap_if(nested, if_exprs) + if nested_kept: + kept.append({**step, "steps": nested_kept}) + continue + key = step.get("key") + if key in BOOTSTRAP_IMAGE_BUILD_KEYS: + step["if"] = if_exprs["image"] + elif key in BOOTSTRAP_UPLOAD_IF_KEYS: + step["if"] = if_exprs[BOOTSTRAP_UPLOAD_IF_KEYS[key]] + if_expr = step.get("if") + if if_expr == "false": + _log(f"omit disabled bootstrap step {key!r}") + continue + if if_expr == "true": + # Unconditional step: omit ``if`` (Buildkite requires string ``if``, not YAML bool). + step.pop("if", None) + kept.append(step) + return kept + + +def _render_bootstrap_pipeline( + steps_yaml: str, + *, + decision, + path: Path, +) -> str: + """Load bootstrap steps YAML and inject ``if`` expressions from skip-ci decision.""" + doc = yaml.safe_load(steps_yaml) + if not isinstance(doc, dict): + raise ValueError(f"invalid bootstrap steps YAML for {path}") + steps = doc.get("steps") + if not isinstance(steps, list): + raise ValueError(f"bootstrap steps YAML must contain steps: list in {path}") + + platform = _get_bootstrap_platform(path) + if_exprs = _compute_bootstrap_if_exprs(decision=decision, platform=platform) + doc["steps"] = _apply_bootstrap_if(steps, if_exprs) + return yaml.safe_dump(doc, sort_keys=False) + + +# --- Test pipeline (test-ready.yml, test-merge.yml) --- + + +@lru_cache(maxsize=1) +def _load_mirror_hardwares() -> dict[str, dict[str, Any]]: + if not CI_MIRROR_HARDWARES_PATH.is_file(): + raise FileNotFoundError(f"missing CI mirror_hardwares registry: {CI_MIRROR_HARDWARES_PATH}") + doc = yaml.safe_load(CI_MIRROR_HARDWARES_PATH.read_text(encoding="utf-8")) + if not isinstance(doc, dict): + raise ValueError(f"invalid CI mirror_hardwares registry: {CI_MIRROR_HARDWARES_PATH}") + presets = doc.get("mirror_hardwares") + if not isinstance(presets, dict): + raise ValueError(f"mirror_hardwares must be a mapping in {CI_MIRROR_HARDWARES_PATH}") + return presets + + +def _expand_mirror_hardwares(step: dict[str, Any]) -> dict[str, Any]: + """Replace uploader-only ``mirror_hardwares`` with preset fields from ci_mirror_hardwares.yml.""" + hardware = step.get("mirror_hardwares") + if hardware is None: + return step + + if not isinstance(hardware, str) or not hardware.strip(): + raise ValueError( + f"mirror_hardwares must be a non-empty string in step {_get_step_label(step)!r}", + ) + + preset = _load_mirror_hardwares().get(hardware) + if preset is None: + known = ", ".join(sorted(_load_mirror_hardwares())) + raise ValueError( + f"unknown mirror_hardwares {hardware!r} in step {_get_step_label(step)!r}; known: {known}", + ) + + if step.get("agents") is not None or step.get("plugins") is not None or step.get("image") is not None: + raise ValueError( + f"step {_get_step_label(step)!r} sets mirror_hardwares together with agents/plugins/image; " + "use mirror_hardwares only", + ) + + expanded = copy.deepcopy(preset) + return {key: value for key, value in step.items() if key != "mirror_hardwares"} | expanded + + +def _match_source_file(changed_files: list[str], prefixes: list[str]) -> bool: + for path in changed_files: + for prefix in prefixes: + normalized = prefix.rstrip("/") + if path == normalized or path.startswith(f"{normalized}/"): + return True + return False + + +def _get_step_label(step: dict[str, Any]) -> str: + return str(step.get("group") or step.get("label") or "") + + +def _process_test_steps( + steps: list[Any], + changed_files: list[str] | None, +) -> list[Any]: + """Drop steps by ``source_file_dependencies`` when *changed_files* is set; always strip that field.""" + processed: list[Any] = [] + for step in steps: + if not isinstance(step, dict): + processed.append(step) + continue + + deps = step.get("source_file_dependencies") + if deps is not None and not isinstance(deps, list): + raise ValueError( + f"source_file_dependencies must be a list in step {_get_step_label(step)!r}", + ) + if changed_files is not None and deps is not None and not _match_source_file(changed_files, deps): + _log(f"skip {_get_step_label(step)!r} (no changes under {deps})") + continue + + nested = step.get("steps") + if nested is not None: + kept_nested = _process_test_steps(nested, changed_files) + if changed_files is not None and not kept_nested: + _log(f"omit empty group {_get_step_label(step)!r}") + continue + new_step = {key: value for key, value in step.items() if key != "source_file_dependencies"} + new_step["steps"] = kept_nested + processed.append(new_step) + continue + + if deps is not None: + processed.append( + _expand_mirror_hardwares( + {key: value for key, value in step.items() if key != "source_file_dependencies"}, + ), + ) + else: + processed.append(_expand_mirror_hardwares(step)) + + return processed + + +def _select_e2e_group_steps(steps: list[Any]) -> list[Any]: + """Keep only top-level groups whose name contains ``E2E_GROUP_MARKER``.""" + selected = [ + step + for step in steps + if isinstance(step, dict) and isinstance(step.get("group"), str) and E2E_GROUP_MARKER in step["group"] + ] + if not selected: + _log(f"no group matching {E2E_GROUP_MARKER!r} found") + else: + _log(f"keep {len(selected)} group(s) matching {E2E_GROUP_MARKER!r}") + return selected + + +def _render_test_pipeline( + doc: dict[str, Any], + changed_files: list[str] | None, + *, + e2e_only: bool = False, +) -> dict[str, Any]: + """Filter steps by PR diff and strip uploader-only ``source_file_dependencies`` metadata.""" + steps = doc.get("steps") + if not isinstance(steps, list): + return doc + if e2e_only: + steps = _select_e2e_group_steps(steps) + steps = _process_test_steps(steps, changed_files) + return {**doc, "steps": steps} + + +# --- Entry (read file → bootstrap or test render → YAML string) --- + + +def _render_pipeline( + path: Path, + *, + force_all: bool = False, + e2e_only: bool = False, +) -> str: + if path.name == BOOTSTRAP_STEPS_FILENAME: + ctx = resolve_ci_context_from_git() + continuation = _load_bootstrap_steps(path) + return _render_bootstrap_pipeline( + continuation, + decision=ctx.decision, + path=path, + ) + + text = path.read_text(encoding="utf-8") + ctx = resolve_ci_context_from_git() + if force_all or e2e_only: + changed_files = None + else: + changed_files = ctx.changed_files + + doc = yaml.safe_load(text) + if not isinstance(doc, dict): + raise ValueError(f"invalid pipeline YAML: {path}") + + doc = _render_test_pipeline(doc, changed_files, e2e_only=e2e_only) + return yaml.safe_dump(doc, sort_keys=False) + + +def _upload_to_buildkite(content: str) -> None: + subprocess.run( + ["buildkite-agent", "pipeline", "upload"], + input=content, + text=True, + check=True, + ) + + +# --- CLI --- + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "pipeline", + nargs="?", + default=".buildkite/cuda/bootstrap-upload-steps.yml", + help="Pipeline YAML path (default: .buildkite/cuda/bootstrap-upload-steps.yml)", + ) + parser.add_argument( + "--upload", + action="store_true", + help="Pipe rendered YAML to buildkite-agent pipeline upload", + ) + mode = parser.add_mutually_exclusive_group() + mode.add_argument( + "--all", + action="store_true", + help="Keep all steps (disable diff-aware skipping)", + ) + mode.add_argument( + "--e2e", + action="store_true", + help="Keep only the E2E Test group", + ) + args = parser.parse_args() + + path = Path(args.pipeline) + if not path.is_absolute(): + path = ROOT / path + if not path.is_file(): + _log(f"missing pipeline file: {path}") + return 1 + + rendered = _render_pipeline(path, force_all=args.all, e2e_only=args.e2e) + if args.upload: + _upload_to_buildkite(rendered) + else: + sys.stdout.write(rendered) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.buildkite/cuda/bootstrap-upload-steps.yml b/.buildkite/cuda/bootstrap-upload-steps.yml new file mode 100644 index 000000000..f2467bf42 --- /dev/null +++ b/.buildkite/cuda/bootstrap-upload-steps.yml @@ -0,0 +1,55 @@ +# Uploaded by upload_pipeline.py --upload (not the Buildkite hook). +# ``if`` is injected by step ``key`` from skip-ci at upload time. +steps: + - label: ":docker: Build image" + key: image-build + commands: + - "bash .buildkite/cuda/scripts/build-ci-image.sh" + agents: + queue: "cpu_queue_premerge" + + # L2 Test — PR ready label (diff-aware) or main+NIGHTLY=1 (--e2e, full E2E group) + - label: "Upload Ready Pipeline" + depends_on: image-build + key: upload-ready-pipeline + commands: + - | + EXTRA="" + if [ "${NIGHTLY:-}" = "1" ] && [ "${BUILDKITE_BRANCH}" = "main" ]; then + EXTRA="--e2e" + fi + python3 .buildkite/common/scripts/upload_pipeline.py --upload $$EXTRA .buildkite/cuda/test-ready.yml + agents: + queue: "cpu_queue_premerge" + + # L3 Test — PR merge-test label (diff-aware) or main+NIGHTLY=1 (--e2e, full E2E group) + - label: "Upload Merge Pipeline" + depends_on: image-build + key: upload-merge-pipeline + commands: + - | + EXTRA="" + if [ "${NIGHTLY:-}" = "1" ] && [ "${BUILDKITE_BRANCH}" = "main" ]; then + EXTRA="--e2e" + fi + python3 .buildkite/common/scripts/upload_pipeline.py --upload $$EXTRA .buildkite/cuda/test-merge.yml + agents: + queue: "cpu_queue_premerge" + + # L4 Test — main+NIGHTLY=1 (scheduled), or PR with specific label (e.g. add label then Rebuild) + - label: "Upload Nightly Pipeline" + depends_on: image-build + key: upload-nightly-pipeline + commands: + - python3 .buildkite/common/scripts/upload_pipeline.py --upload .buildkite/cuda/test-nightly.yml + agents: + queue: "cpu_queue_premerge" + + # L5 Test — main+WEEKLY=1 (scheduled), or PR with weekly-test label + - label: "Upload Weekly Pipeline" + depends_on: image-build + key: upload-weekly-pipeline + commands: + - python3 .buildkite/common/scripts/upload_pipeline.py --upload .buildkite/cuda/test-weekly.yml + agents: + queue: "cpu_queue_premerge" diff --git a/.buildkite/cuda/pipeline.yml b/.buildkite/cuda/pipeline.yml index e9441a7b5..aeec9d1aa 100644 --- a/.buildkite/cuda/pipeline.yml +++ b/.buildkite/cuda/pipeline.yml @@ -1,84 +1,9 @@ -# Document 1: Buildkite loads only this block on first parse. The next step resolves docs-only, -# pytest skip-mark-only, or combined skip-ci from git diff, then uploads document 2. When skip-ci -# applies, image-build still runs for the nightly exception in upload_pipeline.py (nightly-test -# label or main NIGHTLY=1); otherwise upload-nightly can be skipped together with test-ready/test-merge. -# Weekly uploads also depend on image-build. - -# Document 2: appended after `---`; same file, read by upload_pipeline.py (not evaluated as a second pipeline by Buildkite). Child uploads include test-ready.yml, test-merge.yml, test-nightly.yml, and test-weekly.yml. +# Hook uploads this entry YAML; upload_pipeline.py --upload reads bootstrap-upload-steps.yml +# and injects ``if`` by step ``key`` from skip-ci. steps: - - label: ":github: Resolve skip-ci (docs / skip marks) & upload pipeline" + - label: ":github: Resolve skip-ci & upload pipeline" key: upload-ci-pipeline commands: - - "python3 .buildkite/cuda/scripts/upload_pipeline.py --upload .buildkite/cuda/pipeline.yml" - agents: - queue: "cpu_queue_premerge" - ---- -steps: - - label: ":docker: Build image" - key: image-build - if: __IMAGE_BUILD_IF__ - commands: - - "bash .buildkite/cuda/scripts/build-ci-image.sh" - agents: - queue: "cpu_queue_premerge" - - # L2 Test — PR ready label (diff-aware) or main+NIGHTLY=1 (--e2e, full E2E group) - - label: "Upload Ready Pipeline" - depends_on: image-build - key: upload-ready-pipeline - if: __UPLOAD_READY_IF__ - commands: - - | - EXTRA="" - if [ "${NIGHTLY:-}" = "1" ] && [ "${BUILDKITE_BRANCH}" = "main" ]; then - EXTRA="--e2e" - fi - python3 .buildkite/cuda/scripts/upload_pipeline.py --upload $$EXTRA .buildkite/cuda/test-ready.yml - agents: - queue: "cpu_queue_premerge" - - # L3 Test — PR merge-test label (diff-aware) or main+NIGHTLY=1 (--e2e, full E2E group) - - label: "Upload Merge Pipeline" - depends_on: image-build - key: upload-merge-pipeline - if: __UPLOAD_MERGE_IF__ - commands: - - | - EXTRA="" - if [ "${NIGHTLY:-}" = "1" ] && [ "${BUILDKITE_BRANCH}" = "main" ]; then - EXTRA="--e2e" - fi - python3 .buildkite/cuda/scripts/upload_pipeline.py --upload $$EXTRA .buildkite/cuda/test-merge.yml - agents: - queue: "cpu_queue_premerge" - - # L4 Test — main+NIGHTLY=1 (scheduled), or PR with specific label (e.g. add label then Rebuild) - - label: "Upload Nightly Pipeline" - depends_on: image-build - key: upload-nightly-pipeline - if: >- - (build.branch == "main" && build.env("NIGHTLY") == "1") || - (build.branch != "main" && ( - build.pull_request.labels includes "nightly-test" || - build.pull_request.labels includes "omni-test" || - build.pull_request.labels includes "tts-test" || - build.pull_request.labels includes "diffusion-x2iat-test" || - build.pull_request.labels includes "diffusion-x2v-test" - )) - commands: - - buildkite-agent pipeline upload .buildkite/cuda/test-nightly.yml - agents: - queue: "cpu_queue_premerge" - - # L5 Test — main+WEEKLY=1 (scheduled), or PR with weekly-test label - - label: "Upload Weekly Pipeline" - depends_on: image-build - key: upload-weekly-pipeline - if: >- - (build.branch == "main" && build.env("WEEKLY") == "1") || - (build.branch != "main" && build.pull_request.labels includes "weekly-test") - commands: - - buildkite-agent pipeline upload .buildkite/cuda/test-weekly.yml + - "python3 .buildkite/common/scripts/upload_pipeline.py --upload .buildkite/cuda/bootstrap-upload-steps.yml" agents: queue: "cpu_queue_premerge" diff --git a/.buildkite/cuda/rebase-pipeline.yml b/.buildkite/cuda/rebase-pipeline.yml index 0cb21edcd..391079c4a 100644 --- a/.buildkite/cuda/rebase-pipeline.yml +++ b/.buildkite/cuda/rebase-pipeline.yml @@ -11,7 +11,7 @@ steps: depends_on: image-build key: upload-ready-pipeline commands: - - python3 .buildkite/cuda/scripts/upload_pipeline.py --upload --all .buildkite/cuda/test-ready.yml + - python3 .buildkite/common/scripts/upload_pipeline.py --upload --all .buildkite/cuda/test-ready.yml agents: queue: "cpu_queue_premerge" @@ -20,7 +20,7 @@ steps: depends_on: image-build key: upload-merge-pipeline commands: - - python3 .buildkite/cuda/scripts/upload_pipeline.py --upload --all .buildkite/cuda/test-merge.yml + - python3 .buildkite/common/scripts/upload_pipeline.py --upload --all .buildkite/cuda/test-merge.yml agents: queue: "cpu_queue_premerge" @@ -30,6 +30,6 @@ steps: key: upload-nightly-pipeline if: build.env("NIGHTLY") == "1" commands: - - python3 .buildkite/cuda/scripts/upload_pipeline.py --upload --all .buildkite/cuda/test-nightly.yml + - python3 .buildkite/common/scripts/upload_pipeline.py --upload --all .buildkite/cuda/test-nightly.yml agents: queue: "cpu_queue_premerge" diff --git a/.buildkite/cuda/scripts/upload_pipeline.py b/.buildkite/cuda/scripts/upload_pipeline.py deleted file mode 100644 index dc233af7a..000000000 --- a/.buildkite/cuda/scripts/upload_pipeline.py +++ /dev/null @@ -1,547 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-License-Identifier: Apache-2.0 -"""Render and optionally upload Buildkite pipeline YAML with diff-aware logic. - -Bootstrap mode (pipeline.yml with __IMAGE_BUILD_IF__ placeholders): - - Detect docs-only, pytest skip-mark-only, or combined docs+skip-mark PR/main changes and - substitute skip-ci ``if`` expressions. - -Test pipeline mode (e.g. test-merge.yml): - - Drop steps or groups whose ``source_file_dependencies`` do not match changed files. - - ``source_file_dependencies`` may be set on a leaf step or on a group step. - - Always strip ``source_file_dependencies`` from uploaded YAML (Buildkite does not - recognize this uploader-only key). - -Usage: - python3 upload_pipeline.py [--upload] [--all | --e2e] - - # Bootstrap (replaces upload_pipeline_with_skip_ci.sh): - python3 upload_pipeline.py --upload .buildkite/cuda/pipeline.yml - - # Test pipeline (replaces upload_test_pipeline_with_diff_skip.py): - python3 upload_pipeline.py --upload .buildkite/cuda/test-merge.yml - - # Full suite (rebase pipeline): keep all steps, still strip the uploader-only key: - python3 upload_pipeline.py --upload --all .buildkite/cuda/test-merge.yml - - # Nightly L2 E2E only: keep the E2E Test group, ignore source_file_dependencies: - python3 upload_pipeline.py --upload --e2e .buildkite/cuda/test-merge.yml -""" - -from __future__ import annotations - -import argparse -import os -import re -import subprocess -import sys -from pathlib import Path -from typing import Any - -try: - import yaml -except ImportError: - subprocess.check_call( - [sys.executable, "-m", "pip", "install", "-q", "pyyaml"], - ) - import yaml - -LOG = "upload_pipeline" -ROOT = Path(__file__).resolve().parent.parent.parent.parent -DOC_SEP = "\n---\n" -BOOTSTRAP_MARKER = "__IMAGE_BUILD_IF__" -E2E_GROUP_MARKER = "E2E Test" - - -def _log(message: str) -> None: - print(f"{LOG}: {message}", file=sys.stderr) - - -def _git(*args: str) -> subprocess.CompletedProcess[str]: - return subprocess.run( - ["git", *args], - check=False, - capture_output=True, - text=True, - ) - - -def resolve_diff_range() -> str | None: - """Return a git diff range for PR or main builds, or None when unavailable.""" - is_pr = os.environ.get("BUILDKITE_PULL_REQUEST", "false") != "false" and os.environ.get( - "BUILDKITE_PULL_REQUEST", "" - ) - commit = os.environ.get("BUILDKITE_COMMIT", "") - - if is_pr: - base_branch = os.environ.get("BUILDKITE_PULL_REQUEST_BASE_BRANCH", "main") - base_ref = f"origin/{base_branch}" - if _git("rev-parse", "--verify", base_ref).returncode != 0: - _log(f"origin/{base_branch} not found locally; trying fetch") - _git("fetch", "--depth=200", "origin", base_branch) - if _git("rev-parse", "--verify", base_ref).returncode != 0: - if _git("rev-parse", "--verify", base_branch).returncode == 0: - base_ref = base_branch - else: - _log(f"cannot resolve PR base {base_branch}; using safe defaults") - return None - return f"{base_ref}...{commit}" - - if os.environ.get("BUILDKITE_BRANCH", "") == "main": - if _git("rev-parse", "--verify", f"{commit}^").returncode != 0: - _log("main commit has no parent; using safe defaults") - return None - return f"{commit}^..{commit}" - - _log("not PR/main build; using safe defaults") - return None - - -def resolve_changed_files() -> list[str] | None: - """Return changed file paths, or None when diff cannot be resolved.""" - diff_range = resolve_diff_range() - if diff_range is None: - return None - - result = _git("diff", "--name-only", diff_range) - if result.returncode != 0: - _log(f"git diff failed ({diff_range}); using safe defaults") - return None - - files = [line for line in result.stdout.splitlines() if line.strip()] - _log(f"{len(files)} changed file(s)") - return files - - -def _is_doc_file(file_path: str) -> bool: - if not file_path: - return False - if file_path.startswith("docs/"): - return True - if file_path.endswith(".md"): - return True - return file_path == "mkdocs.yml" - - -def is_docs_only_change(changed_files: list[str]) -> bool: - has_any = False - for file_path in changed_files: - if not file_path: - continue - has_any = True - if not _is_doc_file(file_path): - return False - return has_any - - -def _is_test_python_file(file_path: str) -> bool: - path = Path(file_path) - return path.suffix == ".py" and bool(path.parts) and path.parts[0] == "tests" - - -_SKIP_MARK_RE = re.compile(r"pytest\.mark\.skip(?:if)?\b|pytest\.skip\s*\(") -_PYTESTMARK_SKIP_RE = re.compile(r"pytest\.mark\.skip\b") -_PYTEST_MARK_ONLY_RE = re.compile(r"pytest\.mark\.\w+") - - -def _paren_balance(line: str) -> int: - return line.count("(") - line.count(")") - - -def _is_skip_mark_related_content(line: str) -> bool: - stripped = line.strip() - if not stripped: - return True - if stripped.startswith("#"): - return True - return _SKIP_MARK_RE.search(stripped) is not None - - -def _is_pytestmark_adjacent_content(line: str) -> bool: - """Allow pytestmark list refactors that only add skip/skipif alongside existing marks.""" - stripped = line.strip().rstrip(",") - if not stripped: - return True - if stripped in {"[", "]"}: - return True - if stripped.startswith("#"): - return True - if stripped.startswith("pytestmark"): - return True - return _PYTEST_MARK_ONLY_RE.search(stripped) is not None - - -def diff_only_contains_skip_mark_changes(diff_text: str) -> bool: - """True when diff only edits skip marks or reformats pytestmark to add them.""" - pending_depth = 0 - saw_change = False - has_skip_mark_edit = False - for raw_line in diff_text.splitlines(): - if raw_line.startswith("@@"): - pending_depth = 0 - continue - if not (raw_line.startswith("+") or raw_line.startswith("-")): - continue - if raw_line.startswith("+++") or raw_line.startswith("---"): - continue - - saw_change = True - content = raw_line[1:] - - if pending_depth > 0: - pending_depth += _paren_balance(content) - if pending_depth < 0: - pending_depth = 0 - continue - - if _is_skip_mark_related_content(content): - has_skip_mark_edit = True - pending_depth = max(0, _paren_balance(content)) - continue - - if _is_pytestmark_adjacent_content(content): - continue - - return False - - return saw_change and has_skip_mark_edit - - -def _git_file_diff(file_path: str, diff_range: str) -> str | None: - result = _git("diff", diff_range, "--", file_path) - if result.returncode != 0: - _log(f"skip-mark-only: git diff failed for {file_path}") - return None - return result.stdout - - -def _is_new_file_diff(diff_text: str) -> bool: - return "new file mode" in diff_text or "--- /dev/null" in diff_text - - -def _file_has_module_level_pytest_skip(content: str) -> bool: - """True when ``pytestmark`` applies unconditional ``pytest.mark.skip`` to the module.""" - match = re.search(r"^pytestmark\s*=\s*\[(.*?)^\s*\]", content, re.MULTILINE | re.DOTALL) - if match is not None: - return _PYTESTMARK_SKIP_RE.search(match.group(1)) is not None - match = re.search(r"^pytestmark\s*=\s*pytest\.mark\.skip\b", content, re.MULTILINE) - return match is not None - - -def _file_content_at_commit(file_path: str, commit: str) -> str | None: - result = _git("show", f"{commit}:{file_path}") - if result.returncode != 0: - _log(f"skip-mark-only: cannot read {file_path} at {commit}") - return None - return result.stdout - - -def _qualifies_as_skip_mark_test_change(file_path: str, diff_range: str) -> bool: - diff_text = _git_file_diff(file_path, diff_range) - if diff_text is None: - return False - if not diff_text.strip(): - _log(f"skip-mark-only: empty diff for {file_path}") - return False - - if diff_only_contains_skip_mark_changes(diff_text): - return True - - if not _is_new_file_diff(diff_text): - _log(f"skip-mark-only: non-skip changes in {file_path}") - return False - - commit = os.environ.get("BUILDKITE_COMMIT", "") - if not commit: - return False - content = _file_content_at_commit(file_path, commit) - if content is None: - return False - if _file_has_module_level_pytest_skip(content): - _log(f"skip-mark-only: new test file with module-level pytest.mark.skip: {file_path}") - return True - - _log(f"skip-mark-only: new test file without module-level pytest.mark.skip: {file_path}") - return False - - -def is_skip_mark_only_change(changed_files: list[str], *, diff_range: str) -> bool: - """True when every changed file is a tests/ module with only skip-mark edits.""" - has_any = False - for file_path in changed_files: - if not file_path: - continue - has_any = True - if not _is_test_python_file(file_path): - return False - if not _qualifies_as_skip_mark_test_change(file_path, diff_range): - return False - return has_any - - -def is_docs_or_skip_mark_only_change(changed_files: list[str], *, diff_range: str) -> bool: - """True when each changed file is a doc path or a qualifying skip-mark test change.""" - has_any = False - for file_path in changed_files: - if not file_path: - continue - has_any = True - if _is_doc_file(file_path): - continue - if _is_test_python_file(file_path) and _qualifies_as_skip_mark_test_change(file_path, diff_range): - continue - _log(f"docs/skip-mark-only: rejecting {file_path}") - return False - return has_any - - -def resolve_skip_ci(changed_files: list[str] | None, *, diff_range: str | None = None) -> bool: - if changed_files is None: - _log("skip-ci=0 (could not resolve changed files)") - return False - - if diff_range is not None and is_docs_or_skip_mark_only_change(changed_files, diff_range=diff_range): - if is_docs_only_change(changed_files): - _log("docs-only change detected; skip-ci=1") - elif is_skip_mark_only_change(changed_files, diff_range=diff_range): - _log("pytest skip-mark-only change detected; skip-ci=1") - else: - _log("docs + pytest skip-mark-only change detected; skip-ci=1") - return True - - if diff_range is None and is_docs_only_change(changed_files): - _log("docs-only change detected; skip-ci=1") - return True - - _log("non-doc/non-skip-mark changes detected; skip-ci=0") - return False - - -def render_bootstrap_pipeline(text: str, *, skip_ci: bool) -> str: - if DOC_SEP in text: - _, continuation = text.split(DOC_SEP, 1) - else: - continuation = text - - nightly_only = ( - '(build.pull_request.labels includes "nightly-test") || (build.branch == "main" && build.env("NIGHTLY") == "1")' - ) - nightly_main = 'build.branch == "main" && build.env("NIGHTLY") == "1"' - ready_pr = 'build.branch != "main" && build.pull_request.labels includes "ready"' - merge_main = 'build.branch == "main" && build.env("NIGHTLY") != "1" && build.env("WEEKLY") != "1"' - merge_pr = 'build.branch != "main" && build.pull_request.labels includes "merge-test"' - if skip_ci: - image_if = f"'{nightly_only}'" - ready_if = f"'({nightly_main})'" - merge_if = f"'({nightly_main})'" - else: - image_if = "'true'" - ready_if = f"'(({nightly_main}) || ({ready_pr}))'" - merge_if = f"'(({nightly_main}) || (({merge_main}) || ({merge_pr})))'" - - return ( - continuation.replace("__IMAGE_BUILD_IF__", image_if) - .replace("__UPLOAD_READY_IF__", ready_if) - .replace("__UPLOAD_MERGE_IF__", merge_if) - ) - - -def _matches_dependencies(changed_files: list[str], prefixes: list[str]) -> bool: - for path in changed_files: - for prefix in prefixes: - normalized = prefix.rstrip("/") - if path == normalized or path.startswith(f"{normalized}/"): - return True - return False - - -def _strip_source_file_dependencies(step: dict[str, Any]) -> dict[str, Any]: - return {key: value for key, value in step.items() if key != "source_file_dependencies"} - - -def _step_label(step: dict[str, Any]) -> str: - return str(step.get("group") or step.get("label") or "") - - -def _filter_steps(steps: list[Any], changed_files: list[str]) -> list[Any]: - filtered: list[Any] = [] - for step in steps: - if not isinstance(step, dict): - filtered.append(step) - continue - - deps = step.get("source_file_dependencies") - if deps is not None and not isinstance(deps, list): - raise ValueError( - f"source_file_dependencies must be a list in step {_step_label(step)!r}", - ) - if deps is not None and not _matches_dependencies(changed_files, deps): - _log( - f"skip {_step_label(step)!r} (no changes under {deps})", - ) - continue - - nested = step.get("steps") - if nested is not None: - kept_nested = _filter_steps(nested, changed_files) - if not kept_nested: - _log(f"omit empty group {_step_label(step)!r}") - continue - new_step = _strip_source_file_dependencies(step) - new_step["steps"] = kept_nested - filtered.append(new_step) - continue - - if deps is not None: - filtered.append(_strip_source_file_dependencies(step)) - else: - filtered.append(step) - - return filtered - - -def _is_e2e_group(step: dict[str, Any]) -> bool: - group = step.get("group") - return isinstance(group, str) and E2E_GROUP_MARKER in group - - -def _select_e2e_group_steps(steps: list[Any]) -> list[Any]: - selected = [step for step in steps if isinstance(step, dict) and _is_e2e_group(step)] - if not selected: - _log(f"no group matching {E2E_GROUP_MARKER!r} found") - else: - _log(f"keep {len(selected)} group(s) matching {E2E_GROUP_MARKER!r}") - return selected - - -def _strip_uploader_metadata_from_steps(steps: list[Any]) -> list[Any]: - """Remove uploader-only keys while keeping all steps (no diff filtering).""" - stripped: list[Any] = [] - for step in steps: - if not isinstance(step, dict): - stripped.append(step) - continue - - deps = step.get("source_file_dependencies") - if deps is not None and not isinstance(deps, list): - raise ValueError( - f"source_file_dependencies must be a list in step {_step_label(step)!r}", - ) - - nested = step.get("steps") - new_step = _strip_source_file_dependencies(step) - if nested is not None: - new_step["steps"] = _strip_uploader_metadata_from_steps(nested) - stripped.append(new_step) - - return stripped - - -def render_test_pipeline( - doc: dict[str, Any], - changed_files: list[str] | None, - *, - e2e_only: bool = False, -) -> dict[str, Any]: - steps = doc.get("steps") - if not isinstance(steps, list): - return doc - if e2e_only: - steps = _select_e2e_group_steps(steps) - steps = _strip_uploader_metadata_from_steps(steps) - elif changed_files is not None: - steps = _filter_steps(steps, changed_files) - else: - steps = _strip_uploader_metadata_from_steps(steps) - return {**doc, "steps": steps} - - -def resolve_pipeline_path(arg: str) -> Path: - path = Path(arg) - if path.is_absolute(): - return path - return ROOT / path - - -def render_pipeline( - path: Path, - *, - force_all: bool = False, - e2e_only: bool = False, -) -> str: - text = path.read_text(encoding="utf-8") - diff_range = resolve_diff_range() - changed_files = resolve_changed_files() - - # ``--all`` forces the keep-all-steps path (no diff-aware skipping) while still - # stripping ``source_file_dependencies``. Used by the rebase pipeline so main builds - # run the full e2e suite (see .buildkite/cuda/rebase-pipeline.yml). - if force_all or e2e_only: - changed_files = None - - if BOOTSTRAP_MARKER in text: - rendered = render_bootstrap_pipeline( - text, - skip_ci=resolve_skip_ci(changed_files, diff_range=diff_range), - ) - return rendered - - doc = yaml.safe_load(text) - if not isinstance(doc, dict): - raise ValueError(f"invalid pipeline YAML: {path}") - - doc = render_test_pipeline(doc, changed_files, e2e_only=e2e_only) - - return yaml.safe_dump(doc, sort_keys=False) - - -def upload_to_buildkite(content: str) -> None: - subprocess.run( - ["buildkite-agent", "pipeline", "upload"], - input=content, - text=True, - check=True, - ) - - -def main() -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument( - "pipeline", - nargs="?", - default=".buildkite/cuda/pipeline.yml", - help="Pipeline YAML path (default: .buildkite/cuda/pipeline.yml)", - ) - parser.add_argument( - "--upload", - action="store_true", - help="Pipe rendered YAML to buildkite-agent pipeline upload", - ) - mode = parser.add_mutually_exclusive_group() - mode.add_argument( - "--all", - action="store_true", - help="Keep all steps (disable diff-aware skipping); still strips source_file_dependencies", - ) - mode.add_argument( - "--e2e", - action="store_true", - help="Keep only the E2E Test group; disable diff-aware skipping for those steps", - ) - args = parser.parse_args() - - path = resolve_pipeline_path(args.pipeline) - if not path.is_file(): - _log(f"missing pipeline file: {path}") - return 1 - - rendered = render_pipeline(path, force_all=args.all, e2e_only=args.e2e) - if args.upload: - upload_to_buildkite(rendered) - else: - sys.stdout.write(rendered) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/.buildkite/cuda/test-merge.yml b/.buildkite/cuda/test-merge.yml index 7a78eda4a..46ed531f3 100644 --- a/.buildkite/cuda/test-merge.yml +++ b/.buildkite/cuda/test-merge.yml @@ -11,34 +11,12 @@ steps: - label: "Simple · Diffusion & Model Executor Test" commands: - "timeout 20m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml" - agents: - queue: "gpu_1_queue" - plugins: - - docker#v5.2.0: - image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - always-pull: true - propagate-environment: true - environment: - - "HF_HOME=/fsx/hf_cache" - - "HF_TOKEN" - volumes: - - "/fsx/hf_cache:/fsx/hf_cache" + mirror_hardwares: l4_1 - label: "Simple · Other Test" commands: - "timeout 20m pytest -sv -m 'core_model and cpu' --ignore=tests/diffusion --ignore=tests/model_executor --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml" - agents: - queue: "gpu_1_queue" - plugins: - - docker#v5.2.0: - image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - always-pull: true - propagate-environment: true - environment: - - "HF_HOME=/fsx/hf_cache" - - "HF_TOKEN" - volumes: - - "/fsx/hf_cache:/fsx/hf_cache" + mirror_hardwares: l4_1 - group: ":card_index_dividers: Diffusion Test" depends_on: upload-merge-pipeline @@ -47,35 +25,13 @@ steps: timeout_in_minutes: 30 commands: - pytest -sv tests/diffusion/lora/ -m 'advanced_model and cuda' --run-level "advanced_model" - agents: - queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU - plugins: - - docker#v5.2.0: - image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - always-pull: true - propagate-environment: true - environment: - - "HF_HOME=/fsx/hf_cache" - - "HF_TOKEN" - volumes: - - "/fsx/hf_cache:/fsx/hf_cache" + mirror_hardwares: l4_1 - label: "Diffusion · Offloader Test" timeout_in_minutes: 40 commands: - timeout 40m pytest -s -v tests/diffusion/offloader -m "advanced_model and cuda" - agents: - queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU - plugins: - - docker#v5.2.0: - image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - always-pull: true - propagate-environment: true - environment: - - "HF_HOME=/fsx/hf_cache" - - "HF_TOKEN" - volumes: - - "/fsx/hf_cache:/fsx/hf_cache" + mirror_hardwares: l4_1 - label: "Distributed Test" depends_on: upload-merge-pipeline @@ -87,39 +43,7 @@ steps: export VLLM_IMAGE_FETCH_TIMEOUT=60 pytest -s -v tests/distributed -m 'advanced_model and cuda and H100' --run-level 'advanced_model' " - agents: - queue: "mithril-h100-pool" - plugins: - - kubernetes: - podSpec: - containers: - - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - resources: - limits: - nvidia.com/gpu: 1 - volumeMounts: - - name: devshm - mountPath: /dev/shm - - name: hf-cache - mountPath: /root/.cache/huggingface - env: - - name: HF_HOME - value: /root/.cache/huggingface - - name: HF_TOKEN - valueFrom: - secretKeyRef: - name: hf-token-secret - key: token - nodeSelector: - node.kubernetes.io/instance-type: gpu-h100-sxm - volumes: - - name: devshm - emptyDir: - medium: Memory - - name: hf-cache - hostPath: - path: /mnt/hf-cache - type: DirectoryOrCreate + mirror_hardwares: h100_1 - group: ":card_index_dividers: Entrypoints Test" depends_on: upload-merge-pipeline @@ -128,56 +52,13 @@ steps: timeout_in_minutes: 60 commands: - pytest -s -v tests/entrypoints/ -m "advanced_model and cuda and H100" --run-level "advanced_model" - agents: - queue: "mithril-h100-pool" - plugins: - - kubernetes: - podSpec: - containers: - - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - resources: - limits: - nvidia.com/gpu: 2 - volumeMounts: - - name: devshm - mountPath: /dev/shm - - name: hf-cache - mountPath: /root/.cache/huggingface - env: - - name: HF_HOME - value: /root/.cache/huggingface - - name: HF_TOKEN - valueFrom: - secretKeyRef: - name: hf-token-secret - key: token - nodeSelector: - node.kubernetes.io/instance-type: gpu-h100-sxm - volumes: - - name: devshm - emptyDir: - medium: Memory - - name: hf-cache - hostPath: - path: /mnt/hf-cache - type: DirectoryOrCreate + mirror_hardwares: h100_2 - label: "Entrypoint Test with L4" timeout_in_minutes: 30 commands: - pytest -s -v tests/entrypoints/ -m "advanced_model and cuda and L4" --run-level "advanced_model" - agents: - queue: "gpu_1_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU - plugins: - - docker#v5.2.0: - image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - always-pull: true - propagate-environment: true - environment: - - "HF_HOME=/fsx/hf_cache" - - "HF_TOKEN" - volumes: - - "/fsx/hf_cache:/fsx/hf_cache" + mirror_hardwares: l4_1 - group: ":card_index_dividers: E2E Test" depends_on: upload-merge-pipeline @@ -196,18 +77,7 @@ steps: export VLLM_ALLOW_LONG_MAX_MODEL_LEN=1 pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py tests/e2e/offline_inference/test_qwen3_tts_customvoice.py -m 'advanced_model and cuda' --run-level 'advanced_model' " - agents: - queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU - plugins: - - docker#v5.2.0: - image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - always-pull: true - propagate-environment: true - shm-size: "8gb" - environment: - - "HF_HOME=/fsx/hf_cache" - volumes: - - "/fsx/hf_cache:/fsx/hf_cache" + mirror_hardwares: l4_1 - label: "TTS · Qwen3-TTS Base Test" source_file_dependencies: @@ -223,18 +93,7 @@ steps: export VLLM_ALLOW_LONG_MAX_MODEL_LEN=1 pytest -s -v tests/e2e/online_serving/test_qwen3_tts_base.py tests/e2e/offline_inference/test_qwen3_tts_base.py -m 'advanced_model and cuda' --run-level 'advanced_model' " - agents: - queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU - plugins: - - docker#v5.2.0: - image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - always-pull: true - propagate-environment: true - shm-size: "8gb" - environment: - - "HF_HOME=/fsx/hf_cache" - volumes: - - "/fsx/hf_cache:/fsx/hf_cache" + mirror_hardwares: l4_1 - label: "TTS · Higgs-Audio-V3 Test" source_file_dependencies: @@ -250,19 +109,7 @@ steps: export VLLM_MOE_USE_DEEP_GEMM=0 pytest -s -v tests/e2e/online_serving/test_higgs_audio_v3.py -m 'advanced_model' --run-level 'advanced_model' " - agents: - queue: "gpu_1_queue" - plugins: - - docker#v5.2.0: - image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - always-pull: true - propagate-environment: true - shm-size: "8gb" - environment: - - "HF_HOME=/fsx/hf_cache" - - "HF_TOKEN" - volumes: - - "/fsx/hf_cache:/fsx/hf_cache" + mirror_hardwares: l4_1 - label: "Omni · Qwen3-Omni Test" source_file_dependencies: @@ -274,39 +121,7 @@ steps: timeout_in_minutes: 50 commands: - pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py tests/e2e/online_serving/test_qwen3_omni.py -m 'advanced_model and cuda' --run-level 'advanced_model' - agents: - queue: "mithril-h100-pool" - plugins: - - kubernetes: - podSpec: - containers: - - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - resources: - limits: - nvidia.com/gpu: 2 - volumeMounts: - - name: devshm - mountPath: /dev/shm - - name: hf-cache - mountPath: /root/.cache/huggingface - env: - - name: HF_HOME - value: /root/.cache/huggingface - - name: HF_TOKEN - valueFrom: - secretKeyRef: - name: hf-token-secret - key: token - nodeSelector: - node.kubernetes.io/instance-type: gpu-h100-sxm - volumes: - - name: devshm - emptyDir: - medium: Memory - - name: hf-cache - hostPath: - path: /mnt/hf-cache - type: DirectoryOrCreate + mirror_hardwares: h100_2 - label: "TTS · VoxCPM2 Test" source_file_dependencies: @@ -322,19 +137,7 @@ steps: export VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -s -v tests/e2e/offline_inference/test_voxcpm2_tts.py tests/e2e/online_serving/test_voxcpm2_tts.py -m 'advanced_model and cuda' --run-level 'advanced_model' " - agents: - queue: "gpu_1_queue" - plugins: - - docker#v5.2.0: - image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - always-pull: true - propagate-environment: true - shm-size: "8gb" - environment: - - "HF_HOME=/fsx/hf_cache" - - "HF_TOKEN" - volumes: - - "/fsx/hf_cache:/fsx/hf_cache" + mirror_hardwares: l4_1 - label: "TTS · SoulX-Singer Test" source_file_dependencies: @@ -350,19 +153,7 @@ steps: export VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -s -v tests/e2e/offline_inference/test_soulxsinger.py -k 'svc or precomputed' -m 'advanced_model and cuda' --run-level 'advanced_model' " - agents: - queue: "gpu_1_queue" - plugins: - - docker#v5.2.0: - image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - always-pull: true - propagate-environment: true - shm-size: "8gb" - environment: - - "HF_HOME=/fsx/hf_cache" - - "HF_TOKEN" - volumes: - - "/fsx/hf_cache:/fsx/hf_cache" + mirror_hardwares: l4_1 - label: "Diffusion · Qwen Image Edit Test" source_file_dependencies: @@ -371,40 +162,7 @@ steps: timeout_in_minutes: 20 commands: - pytest -s -v tests/e2e/online_serving/test_qwen_image_edit.py -m 'advanced_model and cuda' --run-level 'advanced_model' - agents: - queue: "mithril-h100-pool" - plugins: - - kubernetes: - podSpec: - containers: - - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - resources: - limits: - nvidia.com/gpu: 1 - volumeMounts: - - name: devshm - mountPath: /dev/shm - - name: hf-cache - mountPath: /root/.cache/huggingface - env: - - name: HF_HOME - value: /root/.cache/huggingface - - name: HF_TOKEN - valueFrom: - secretKeyRef: - name: hf-token-secret - key: token - nodeSelector: - node.kubernetes.io/instance-type: gpu-h100-sxm - volumes: - - name: devshm - emptyDir: - medium: Memory - - name: hf-cache - hostPath: - path: /mnt/hf-cache - type: DirectoryOrCreate - + mirror_hardwares: h100_1 - label: "Diffusion · Bagel Test" source_file_dependencies: - tests/e2e/offline_inference/test_bagel.py @@ -416,40 +174,7 @@ steps: timeout_in_minutes: 20 commands: - pytest -s -v tests/e2e/offline_inference/test_bagel.py tests/e2e/online_serving/test_bagel.py -m 'advanced_model and cuda' --run-level 'advanced_model' - agents: - queue: "mithril-h100-pool" - plugins: - - kubernetes: - podSpec: - containers: - - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - resources: - limits: - nvidia.com/gpu: 1 - volumeMounts: - - name: devshm - mountPath: /dev/shm - - name: hf-cache - mountPath: /root/.cache/huggingface - env: - - name: HF_HOME - value: /root/.cache/huggingface - - name: HF_TOKEN - valueFrom: - secretKeyRef: - name: hf-token-secret - key: token - nodeSelector: - node.kubernetes.io/instance-type: gpu-h100-sxm - volumes: - - name: devshm - emptyDir: - medium: Memory - - name: hf-cache - hostPath: - path: /mnt/hf-cache - type: DirectoryOrCreate - + mirror_hardwares: h100_1 - label: "Diffusion · Qwen Image Test" source_file_dependencies: @@ -461,39 +186,7 @@ steps: export VLLM_IMAGE_FETCH_TIMEOUT=60 pytest -s -v tests/e2e/online_serving/test_qwen_image.py -m 'advanced_model and cuda' --run-level 'advanced_model' ' - agents: - queue: "mithril-h100-pool" - plugins: - - kubernetes: - podSpec: - containers: - - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - resources: - limits: - nvidia.com/gpu: 1 - volumeMounts: - - name: devshm - mountPath: /dev/shm - - name: hf-cache - mountPath: /root/.cache/huggingface - env: - - name: HF_HOME - value: /root/.cache/huggingface - - name: HF_TOKEN - valueFrom: - secretKeyRef: - name: hf-token-secret - key: token - nodeSelector: - node.kubernetes.io/instance-type: gpu-h100-sxm - volumes: - - name: devshm - emptyDir: - medium: Memory - - name: hf-cache - hostPath: - path: /mnt/hf-cache - type: DirectoryOrCreate + mirror_hardwares: h100_1 - label: "Diffusion · Qwen Image Layered Test" source_file_dependencies: @@ -505,39 +198,7 @@ steps: export VLLM_IMAGE_FETCH_TIMEOUT=60 pytest -s -v tests/e2e/online_serving/test_qwen_image_layered.py -m 'advanced_model and cuda' --run-level 'advanced_model' " - agents: - queue: "mithril-h100-pool" - plugins: - - kubernetes: - podSpec: - containers: - - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - resources: - limits: - nvidia.com/gpu: 1 - volumeMounts: - - name: devshm - mountPath: /dev/shm - - name: hf-cache - mountPath: /root/.cache/huggingface - env: - - name: HF_HOME - value: /root/.cache/huggingface - - name: HF_TOKEN - valueFrom: - secretKeyRef: - name: hf-token-secret - key: token - nodeSelector: - node.kubernetes.io/instance-type: gpu-h100-sxm - volumes: - - name: devshm - emptyDir: - medium: Memory - - name: hf-cache - hostPath: - path: /mnt/hf-cache - type: DirectoryOrCreate + mirror_hardwares: h100_1 - label: "Diffusion · Wan22 Test" source_file_dependencies: @@ -550,39 +211,7 @@ steps: export VLLM_IMAGE_FETCH_TIMEOUT=60 pytest -s -v tests/e2e/offline_inference/test_wan22_t2v.py tests/e2e/online_serving/test_wan22_t2v.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model' " - agents: - queue: "mithril-h100-pool" - plugins: - - kubernetes: - podSpec: - containers: - - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - resources: - limits: - nvidia.com/gpu: 1 - volumeMounts: - - name: devshm - mountPath: /dev/shm - - name: hf-cache - mountPath: /root/.cache/huggingface - env: - - name: HF_HOME - value: /root/.cache/huggingface - - name: HF_TOKEN - valueFrom: - secretKeyRef: - name: hf-token-secret - key: token - nodeSelector: - node.kubernetes.io/instance-type: gpu-h100-sxm - volumes: - - name: devshm - emptyDir: - medium: Memory - - name: hf-cache - hostPath: - path: /mnt/hf-cache - type: DirectoryOrCreate + mirror_hardwares: h100_1 - label: "Diffusion · Z Image Test" source_file_dependencies: @@ -592,19 +221,7 @@ steps: timeout_in_minutes: 30 commands: - pytest -s -v tests/e2e/offline_inference/test_z_image.py tests/e2e/online_serving/test_z_image.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model' - agents: - queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU - plugins: - - docker#v5.2.0: - image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - always-pull: true - propagate-environment: true - environment: - - "HF_HOME=/fsx/hf_cache" - - "HF_TOKEN" - volumes: - - "/fsx/hf_cache:/fsx/hf_cache" - + mirror_hardwares: l4_1 - label: "Diffusion · FLUX.2 Klein Test" source_file_dependencies: - tests/e2e/offline_inference/test_flux2_klein.py @@ -612,15 +229,4 @@ steps: timeout_in_minutes: 30 commands: - pytest -s -v tests/e2e/offline_inference/test_flux2_klein.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model' - agents: - queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU - plugins: - - docker#v5.2.0: - image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - always-pull: true - propagate-environment: true - environment: - - "HF_HOME=/fsx/hf_cache" - - "HF_TOKEN" - volumes: - - "/fsx/hf_cache:/fsx/hf_cache" + mirror_hardwares: l4_1 diff --git a/.buildkite/cuda/test-nightly.yml b/.buildkite/cuda/test-nightly.yml index 9568f60c4..2aed12819 100644 --- a/.buildkite/cuda/test-nightly.yml +++ b/.buildkite/cuda/test-nightly.yml @@ -11,100 +11,25 @@ steps: depends_on: upload-nightly-pipeline if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" || build.pull_request.labels includes "omni-test" steps: + - label: ":full_moon: Omni · Function Test with H100" timeout_in_minutes: 120 commands: - pytest -sv tests/e2e --run-level "full_model" -m "full_model and H100 and omni" --ignore=tests/e2e/accuracy --ignore=tests/e2e/online_serving/test_qwen3_omni_multi_replicas.py - agents: - queue: "mithril-h100-pool" - plugins: - - kubernetes: - podSpec: - containers: - - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - resources: - limits: - nvidia.com/gpu: 2 - volumeMounts: - - name: devshm - mountPath: /dev/shm - - name: hf-cache - mountPath: /root/.cache/huggingface - env: - - name: HF_HOME - value: /root/.cache/huggingface - - name: HF_TOKEN - valueFrom: - secretKeyRef: - name: hf-token-secret - key: token - nodeSelector: - node.kubernetes.io/instance-type: gpu-h100-sxm - volumes: - - name: devshm - emptyDir: - medium: Memory - - name: hf-cache - hostPath: - path: /mnt/hf-cache - type: DirectoryOrCreate + mirror_hardwares: h100_2 - label: ":full_moon: Omni · Doc Test with L4" timeout_in_minutes: 90 commands: - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1" - pytest -s -v tests/examples/ -m "full_model and omni and L4" --run-level "full_model" - agents: - queue: "gpu_4_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU - plugins: - - docker#v5.2.0: - image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - always-pull: true - propagate-environment: true - shm-size: "8gb" - environment: - - "HF_HOME=/fsx/hf_cache" - - "HF_TOKEN" - volumes: - - "/fsx/hf_cache:/fsx/hf_cache" + mirror_hardwares: l4_4 - label: ":full_moon: Omni · Doc Test with H100" timeout_in_minutes: 90 commands: - pytest -s -v tests/examples/ -m "full_model and omni and H100" --run-level "full_model" - agents: - queue: "mithril-h100-pool" - plugins: - - kubernetes: - podSpec: - containers: - - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - resources: - limits: - nvidia.com/gpu: 2 - volumeMounts: - - name: devshm - mountPath: /dev/shm - - name: hf-cache - mountPath: /root/.cache/huggingface - env: - - name: HF_HOME - value: /root/.cache/huggingface - - name: HF_TOKEN - valueFrom: - secretKeyRef: - name: hf-token-secret - key: token - nodeSelector: - node.kubernetes.io/instance-type: gpu-h100-sxm - volumes: - - name: devshm - emptyDir: - medium: Memory - - name: hf-cache - hostPath: - path: /mnt/hf-cache - type: DirectoryOrCreate + mirror_hardwares: h100_2 - label: ":full_moon: Omni · Accuracy Test" timeout_in_minutes: 180 @@ -117,39 +42,7 @@ steps: EXIT=$$? buildkite-agent artifact upload "tests/e2e/accuracy/qwen3_omni/results/qwen_omni_acc/*.json" exit $$EXIT - agents: - queue: "mithril-h100-pool" - plugins: - - kubernetes: - podSpec: - containers: - - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - resources: - limits: - nvidia.com/gpu: 2 - volumeMounts: - - name: devshm - mountPath: /dev/shm - - name: hf-cache - mountPath: /root/.cache/huggingface - env: - - name: HF_HOME - value: /root/.cache/huggingface - - name: HF_TOKEN - valueFrom: - secretKeyRef: - name: hf-token-secret - key: token - nodeSelector: - node.kubernetes.io/instance-type: gpu-h100-sxm - volumes: - - name: devshm - emptyDir: - medium: Memory - - name: hf-cache - hostPath: - path: /mnt/hf-cache - type: DirectoryOrCreate + mirror_hardwares: h100_2 - label: ":full_moon: Omni · Perf Test · No Async Chunk" key: nightly-omni-performance-no-async-chunk @@ -162,39 +55,7 @@ steps: EXIT=$$? buildkite-agent artifact upload "tests/dfx/perf/results/*.json" exit $$EXIT - agents: - queue: "mithril-h100-pool" - plugins: - - kubernetes: - podSpec: - containers: - - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - resources: - limits: - nvidia.com/gpu: 2 - volumeMounts: - - name: devshm - mountPath: /dev/shm - - name: hf-cache - mountPath: /root/.cache/huggingface - env: - - name: HF_HOME - value: /root/.cache/huggingface - - name: HF_TOKEN - valueFrom: - secretKeyRef: - name: hf-token-secret - key: token - nodeSelector: - node.kubernetes.io/instance-type: gpu-h100-sxm - volumes: - - name: devshm - emptyDir: - medium: Memory - - name: hf-cache - hostPath: - path: /mnt/hf-cache - type: DirectoryOrCreate + mirror_hardwares: h100_2 - label: ":full_moon: Omni · Perf Test · Async Chunk" key: nightly-omni-performance-async-chunk @@ -207,39 +68,7 @@ steps: EXIT=$$? buildkite-agent artifact upload "tests/dfx/perf/results/*.json" exit $$EXIT - agents: - queue: "mithril-h100-pool" - plugins: - - kubernetes: - podSpec: - containers: - - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - resources: - limits: - nvidia.com/gpu: 2 - volumeMounts: - - name: devshm - mountPath: /dev/shm - - name: hf-cache - mountPath: /root/.cache/huggingface - env: - - name: HF_HOME - value: /root/.cache/huggingface - - name: HF_TOKEN - valueFrom: - secretKeyRef: - name: hf-token-secret - key: token - nodeSelector: - node.kubernetes.io/instance-type: gpu-h100-sxm - volumes: - - name: devshm - emptyDir: - medium: Memory - - name: hf-cache - hostPath: - path: /mnt/hf-cache - type: DirectoryOrCreate + mirror_hardwares: h100_2 - label: ":full_moon: Omni · Perf Test · vLLM Text" key: nightly-omni-performance-vllm-text @@ -252,39 +81,7 @@ steps: EXIT=$$? buildkite-agent artifact upload "tests/dfx/perf/results/*.json" exit $$EXIT - agents: - queue: "mithril-h100-pool" - plugins: - - kubernetes: - podSpec: - containers: - - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - resources: - limits: - nvidia.com/gpu: 2 - volumeMounts: - - name: devshm - mountPath: /dev/shm - - name: hf-cache - mountPath: /root/.cache/huggingface - env: - - name: HF_HOME - value: /root/.cache/huggingface - - name: HF_TOKEN - valueFrom: - secretKeyRef: - name: hf-token-secret - key: token - nodeSelector: - node.kubernetes.io/instance-type: gpu-h100-sxm - volumes: - - name: devshm - emptyDir: - medium: Memory - - name: hf-cache - hostPath: - path: /mnt/hf-cache - type: DirectoryOrCreate + mirror_hardwares: h100_2 - label: ":full_moon: Omni · Perf Test · Multi-Replica" key: nightly-omni-performance-multi-replicas @@ -297,84 +94,20 @@ steps: EXIT=$$? buildkite-agent artifact upload "tests/dfx/perf/results/*.json" exit $$EXIT - agents: - queue: "mithril-h100-pool" - plugins: - - kubernetes: - podSpec: - containers: - - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - resources: - limits: - nvidia.com/gpu: 3 - volumeMounts: - - name: devshm - mountPath: /dev/shm - - name: hf-cache - mountPath: /root/.cache/huggingface - env: - - name: HF_HOME - value: /root/.cache/huggingface - - name: HF_TOKEN - valueFrom: - secretKeyRef: - name: hf-token-secret - key: token - nodeSelector: - node.kubernetes.io/instance-type: gpu-h100-sxm - volumes: - - name: devshm - emptyDir: - medium: Memory - - name: hf-cache - hostPath: - path: /mnt/hf-cache - type: DirectoryOrCreate - + mirror_hardwares: h100_3 - label: ":full_moon: Omni · Multi-Replica Startup Test with 4x H100" timeout_in_minutes: 45 commands: - pytest -s -v tests/e2e/online_serving/test_qwen3_omni_multi_replicas.py -m "full_model" --run-level "core_model" - agents: - queue: "mithril-h100-pool" - plugins: - - kubernetes: - podSpec: - containers: - - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - resources: - limits: - nvidia.com/gpu: 4 - volumeMounts: - - name: devshm - mountPath: /dev/shm - - name: hf-cache - mountPath: /root/.cache/huggingface - env: - - name: HF_HOME - value: /root/.cache/huggingface - - name: HF_TOKEN - valueFrom: - secretKeyRef: - name: hf-token-secret - key: token - nodeSelector: - node.kubernetes.io/instance-type: gpu-h100-sxm - volumes: - - name: devshm - emptyDir: - medium: Memory - - name: hf-cache - hostPath: - path: /mnt/hf-cache - type: DirectoryOrCreate + mirror_hardwares: h100_4 - group: ":card_index_dividers: TTS Model Test" key: nightly-tts-test-group depends_on: upload-nightly-pipeline if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" || build.pull_request.labels includes "tts-test" steps: + - label: ":full_moon: TTS · Function Test with L4" timeout_in_minutes: 120 commands: @@ -382,58 +115,7 @@ steps: - export VLLM_USE_DEEP_GEMM="0" - export VLLM_MOE_USE_DEEP_GEMM="0" - pytest -s -v tests/e2e/ -m "full_model and L4 and tts" --run-level "full_model" --ignore=tests/e2e/accuracy --ignore=tests/e2e/online_serving/test_voxcpm2_tts_expansion.py - agents: - queue: "gpu_1_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU - plugins: - - docker#v5.2.0: - image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - always-pull: true - propagate-environment: true - shm-size: "8gb" - environment: - - "HF_HOME=/fsx/hf_cache" - - "HF_TOKEN" - volumes: - - "/fsx/hf_cache:/fsx/hf_cache" - - # - label: ":full_moon: TTS · Function Test with H100" - # timeout_in_minutes: 90 - # commands: - # - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1" - # - pytest -s -v tests/e2e/ -m "full_model and H100 and tts" --run-level "full_model" --ignore=tests/e2e/accuracy - # agents: - # queue: "mithril-h100-pool" - # plugins: - # - kubernetes: - # podSpec: - # containers: - # - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - # resources: - # limits: - # nvidia.com/gpu: 1 - # volumeMounts: - # - name: devshm - # mountPath: /dev/shm - # - name: hf-cache - # mountPath: /root/.cache/huggingface - # env: - # - name: HF_HOME - # value: /root/.cache/huggingface - # - name: HF_TOKEN - # valueFrom: - # secretKeyRef: - # name: hf-token-secret - # key: token - # nodeSelector: - # node.kubernetes.io/instance-type: gpu-h100-sxm - # volumes: - # - name: devshm - # emptyDir: - # medium: Memory - # - name: hf-cache - # hostPath: - # path: /mnt/hf-cache - # type: DirectoryOrCreate + mirror_hardwares: l4_1 - label: ":full_moon: TTS · Accuracy Test · VoxCPM2" timeout_in_minutes: 30 @@ -445,19 +127,7 @@ steps: export VLLM_ALLOW_LONG_MAX_MODEL_LEN=1 pytest -s -v tests/e2e/online_serving/test_voxcpm2_tts_expansion.py -m 'full_model and cuda and tts' --run-level 'full_model' " - agents: - queue: "gpu_1_queue" - plugins: - - docker#v5.2.0: - image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - always-pull: true - propagate-environment: true - shm-size: "8gb" - environment: - - "HF_HOME=/fsx/hf_cache" - - "HF_TOKEN" - volumes: - - "/fsx/hf_cache:/fsx/hf_cache" + mirror_hardwares: l4_1 - label: ":full_moon: TTS · Perf Test" key: nightly-tts-performance @@ -471,39 +141,7 @@ steps: EXIT=$$? buildkite-agent artifact upload "tests/dfx/perf/results/*.json" exit $$EXIT - agents: - queue: "mithril-h100-pool" - plugins: - - kubernetes: - podSpec: - containers: - - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - resources: - limits: - nvidia.com/gpu: 2 - volumeMounts: - - name: devshm - mountPath: /dev/shm - - name: hf-cache - mountPath: /root/.cache/huggingface - env: - - name: HF_HOME - value: /root/.cache/huggingface - - name: HF_TOKEN - valueFrom: - secretKeyRef: - name: hf-token-secret - key: token - nodeSelector: - node.kubernetes.io/instance-type: gpu-h100-sxm - volumes: - - name: devshm - emptyDir: - medium: Memory - - name: hf-cache - hostPath: - path: /mnt/hf-cache - type: DirectoryOrCreate + mirror_hardwares: h100_2 - label: ":full_moon: TTS · VoxCPM2 · Perf Test" key: nightly-tts-performance-voxcpm2 @@ -517,39 +155,7 @@ steps: EXIT=$$? buildkite-agent artifact upload "tests/dfx/perf/results/*.json" exit $$EXIT - agents: - queue: "mithril-h100-pool" - plugins: - - kubernetes: - podSpec: - containers: - - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - resources: - limits: - nvidia.com/gpu: 1 - volumeMounts: - - name: devshm - mountPath: /dev/shm - - name: hf-cache - mountPath: /root/.cache/huggingface - env: - - name: HF_HOME - value: /root/.cache/huggingface - - name: HF_TOKEN - valueFrom: - secretKeyRef: - name: hf-token-secret - key: token - nodeSelector: - node.kubernetes.io/instance-type: gpu-h100-sxm - volumes: - - name: devshm - emptyDir: - medium: Memory - - name: hf-cache - hostPath: - path: /mnt/hf-cache - type: DirectoryOrCreate + mirror_hardwares: h100_1 - label: ":full_moon: TTS · Higgs-Audio-V3 · Perf Test" key: nightly-tts-performance-higgs-audio-v3 @@ -564,39 +170,7 @@ steps: EXIT=$$? buildkite-agent artifact upload "tests/dfx/perf/results/*.json" exit $$EXIT - agents: - queue: "mithril-h100-pool" - plugins: - - kubernetes: - podSpec: - containers: - - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - resources: - limits: - nvidia.com/gpu: 1 - volumeMounts: - - name: devshm - mountPath: /dev/shm - - name: hf-cache - mountPath: /root/.cache/huggingface - env: - - name: HF_HOME - value: /root/.cache/huggingface - - name: HF_TOKEN - valueFrom: - secretKeyRef: - name: hf-token-secret - key: token - nodeSelector: - node.kubernetes.io/instance-type: gpu-h100-sxm - volumes: - - name: devshm - emptyDir: - medium: Memory - - name: hf-cache - hostPath: - path: /mnt/hf-cache - type: DirectoryOrCreate + mirror_hardwares: h100_1 - group: ":card_index_dividers: Tiny Model Tests [Multi-GPU]" key: nightly-tiny-model-test-group @@ -605,6 +179,7 @@ steps: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" steps: + - label: ":full_moon: Tiny Model Tests · Diffusion (Multi-GPU accelerations)" timeout_in_minutes: 30 # NOTE: Single-GPU (core_model) tiny model tests run on the ready label, @@ -616,21 +191,8 @@ steps: # the full weights or not. commands: - timeout 25m pytest -sv tests/model_tests/diffusion -m 'full_model and cuda' --run-level "core_model" - agents: - queue: "gpu_4_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU - plugins: - - docker#v5.2.0: - image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - always-pull: true - propagate-environment: true - shm-size: "8gb" - environment: - - "HF_HOME=/fsx/hf_cache" - - "HF_TOKEN" - volumes: - - "/fsx/hf_cache:/fsx/hf_cache" + mirror_hardwares: l4_4 - # Diffusion X2I suite: x2i / x2a / x2t and related non-video paths; x2v is only in "Diffusion X2V Model Test" below. - group: ":card_index_dividers: Diffusion X2I(&A&T) Model Test" key: nightly-diffusion-x2iat-group depends_on: upload-nightly-pipeline @@ -641,6 +203,7 @@ steps: steps: # Use explicit file shards so single-card cases do not reserve 2+ GPUs, # and Bagel feature coverage is not duplicated via a broad tests/e2e sweep. + - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with H100 · Single-GPU" timeout_in_minutes: 120 commands: @@ -658,39 +221,7 @@ steps: tests/e2e/offline_inference/test_krea2_expansion.py -m "full_model and diffusion and H100 and not distributed_cuda" --run-level "full_model" - agents: - queue: "mithril-h100-pool" - plugins: - - kubernetes: - podSpec: - containers: - - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - resources: - limits: - nvidia.com/gpu: 1 - volumeMounts: - - name: devshm - mountPath: /dev/shm - - name: hf-cache - mountPath: /root/.cache/huggingface - env: - - name: HF_HOME - value: /root/.cache/huggingface - - name: HF_TOKEN - valueFrom: - secretKeyRef: - name: hf-token-secret - key: token - nodeSelector: - node.kubernetes.io/instance-type: gpu-h100-sxm - volumes: - - name: devshm - emptyDir: - medium: Memory - - name: hf-cache - hostPath: - path: /mnt/hf-cache - type: DirectoryOrCreate + mirror_hardwares: h100_1 - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with H100 · Multi-GPU Qwen-Image" timeout_in_minutes: 120 @@ -702,39 +233,7 @@ steps: tests/e2e/offline_inference/test_krea2_expansion.py -m "full_model and diffusion and H100 and distributed_cuda" --run-level "full_model" - agents: - queue: "mithril-h100-pool" - plugins: - - kubernetes: - podSpec: - containers: - - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - resources: - limits: - nvidia.com/gpu: 2 - volumeMounts: - - name: devshm - mountPath: /dev/shm - - name: hf-cache - mountPath: /root/.cache/huggingface - env: - - name: HF_HOME - value: /root/.cache/huggingface - - name: HF_TOKEN - valueFrom: - secretKeyRef: - name: hf-token-secret - key: token - nodeSelector: - node.kubernetes.io/instance-type: gpu-h100-sxm - volumes: - - name: devshm - emptyDir: - medium: Memory - - name: hf-cache - hostPath: - path: /mnt/hf-cache - type: DirectoryOrCreate + mirror_hardwares: h100_2 - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with H100 · Multi-GPU Layered/LongCat/FLUX" timeout_in_minutes: 90 @@ -744,39 +243,7 @@ steps: tests/e2e/online_serving/test_qwen_image_layered_expansion.py -m "full_model and diffusion and H100 and distributed_cuda" --run-level "full_model" - agents: - queue: "mithril-h100-pool" - plugins: - - kubernetes: - podSpec: - containers: - - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - resources: - limits: - nvidia.com/gpu: 2 - volumeMounts: - - name: devshm - mountPath: /dev/shm - - name: hf-cache - mountPath: /root/.cache/huggingface - env: - - name: HF_HOME - value: /root/.cache/huggingface - - name: HF_TOKEN - valueFrom: - secretKeyRef: - name: hf-token-secret - key: token - nodeSelector: - node.kubernetes.io/instance-type: gpu-h100-sxm - volumes: - - name: devshm - emptyDir: - medium: Memory - - name: hf-cache - hostPath: - path: /mnt/hf-cache - type: DirectoryOrCreate + mirror_hardwares: h100_2 - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with H100 · Multi-GPU Bagel" timeout_in_minutes: 120 @@ -786,39 +253,7 @@ steps: tests/e2e/online_serving/test_bagel_expansion.py -m "full_model and diffusion and H100 and distributed_cuda" --run-level "full_model" - agents: - queue: "mithril-h100-pool" - plugins: - - kubernetes: - podSpec: - containers: - - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - resources: - limits: - nvidia.com/gpu: 2 - volumeMounts: - - name: devshm - mountPath: /dev/shm - - name: hf-cache - mountPath: /root/.cache/huggingface - env: - - name: HF_HOME - value: /root/.cache/huggingface - - name: HF_TOKEN - valueFrom: - secretKeyRef: - name: hf-token-secret - key: token - nodeSelector: - node.kubernetes.io/instance-type: gpu-h100-sxm - volumes: - - name: devshm - emptyDir: - medium: Memory - - name: hf-cache - hostPath: - path: /mnt/hf-cache - type: DirectoryOrCreate + mirror_hardwares: h100_2 - label: ":full_moon: Diffusion X2I(&A&T) · GLM-Image Function Test with H100" timeout_in_minutes: 120 @@ -829,137 +264,27 @@ steps: tests/e2e/offline_inference/test_glm_image_autoround_w4a16_expansion.py -m "full_model and diffusion and H100" --run-level "full_model" - agents: - queue: "mithril-h100-pool" - plugins: - - kubernetes: - podSpec: - containers: - - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - resources: - limits: - nvidia.com/gpu: 2 - volumeMounts: - - name: devshm - mountPath: /dev/shm - - name: hf-cache - mountPath: /root/.cache/huggingface - env: - - name: HF_HOME - value: /root/.cache/huggingface - - name: HF_TOKEN - valueFrom: - secretKeyRef: - name: hf-token-secret - key: token - nodeSelector: - node.kubernetes.io/instance-type: gpu-h100-sxm - volumes: - - name: devshm - emptyDir: - medium: Memory - - name: hf-cache - hostPath: - path: /mnt/hf-cache - type: DirectoryOrCreate - + mirror_hardwares: h100_2 - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with L4" timeout_in_minutes: 120 commands: - pytest -sv tests/e2e/ -k "not test_wan and not test_bagel_expansion and not hunyuan" -m "full_model and diffusion and L4" --run-level "full_model" --ignore=tests/e2e/accuracy - agents: - queue: "gpu_4_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU - plugins: - - docker#v5.2.0: - image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - always-pull: true - propagate-environment: true - shm-size: "8gb" - environment: - - "HF_HOME=/fsx/hf_cache" - - "HF_TOKEN" - volumes: - - "/fsx/hf_cache:/fsx/hf_cache" + mirror_hardwares: l4_4 - label: ":full_moon: Diffusion X2I(&A&T) · Doc Test" timeout_in_minutes: 60 commands: - pytest -s -v tests/examples/*/test_text_to_image.py -m "full_model and example and H100" --run-level "full_model" - agents: - queue: "mithril-h100-pool" - plugins: - - kubernetes: - podSpec: - containers: - - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - resources: - limits: - nvidia.com/gpu: 2 - volumeMounts: - - name: devshm - mountPath: /dev/shm - - name: hf-cache - mountPath: /root/.cache/huggingface - env: - - name: HF_HOME - value: /root/.cache/huggingface - - name: HF_TOKEN - valueFrom: - secretKeyRef: - name: hf-token-secret - key: token - nodeSelector: - node.kubernetes.io/instance-type: gpu-h100-sxm - volumes: - - name: devshm - emptyDir: - medium: Memory - - name: hf-cache - hostPath: - path: /mnt/hf-cache - type: DirectoryOrCreate + mirror_hardwares: h100_2 - label: ":full_moon: Diffusion X2I(&A&T) · Accuracy Test" timeout_in_minutes: 180 commands: + - export VLLM_HTTP_TIMEOUT_KEEP_ALIVE=120 - pytest -s -v tests/e2e/accuracy/test_qwen_image*.py --run-level full_model - pytest -s -v tests/e2e/accuracy/test_diffusers_backend_similarity.py -k '2i_matches_diffusers' -m full_model --run-level full_model - agents: - queue: "mithril-h100-pool" - plugins: - - kubernetes: - podSpec: - containers: - - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - resources: - limits: - nvidia.com/gpu: 1 - volumeMounts: - - name: devshm - mountPath: /dev/shm - - name: hf-cache - mountPath: /root/.cache/huggingface - env: - - name: HF_HOME - value: /root/.cache/huggingface - - name: VLLM_HTTP_TIMEOUT_KEEP_ALIVE - value: "120" - - name: HF_TOKEN - valueFrom: - secretKeyRef: - name: hf-token-secret - key: token - nodeSelector: - node.kubernetes.io/instance-type: gpu-h100-sxm - volumes: - - name: devshm - emptyDir: - medium: Memory - - name: hf-cache - hostPath: - path: /mnt/hf-cache - type: DirectoryOrCreate + mirror_hardwares: h100_1 - label: ":full_moon: HunyuanImage3-DIT · Accuracy Test" timeout_in_minutes: 180 @@ -967,39 +292,7 @@ steps: - export HUNYUAN_IMAGE3_MODEL="tencent/HunyuanImage-3.0-Instruct" - export HUNYUAN_IMAGE3_DEVICES="0,1,2,3" - pytest -s -v tests/e2e/accuracy/test_hunyuan_image3_pixel_accuracy.py -m full_model --run-level full_model - agents: - queue: "mithril-h100-pool" - plugins: - - kubernetes: - podSpec: - containers: - - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - resources: - limits: - nvidia.com/gpu: 4 - volumeMounts: - - name: devshm - mountPath: /dev/shm - - name: hf-cache - mountPath: /root/.cache/huggingface - env: - - name: HF_HOME - value: /root/.cache/huggingface - - name: HF_TOKEN - valueFrom: - secretKeyRef: - name: hf-token-secret - key: token - nodeSelector: - node.kubernetes.io/instance-type: gpu-h100-sxm - volumes: - - name: devshm - emptyDir: - medium: Memory - - name: hf-cache - hostPath: - path: /mnt/hf-cache - type: DirectoryOrCreate + mirror_hardwares: h100_4 - label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image" key: nightly-diffusion-x2iat-performance-qwen-image @@ -1015,39 +308,7 @@ steps: buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json" buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log" exit $$EXIT - agents: - queue: "mithril-h100-pool" - plugins: - - kubernetes: - podSpec: - containers: - - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - resources: - limits: - nvidia.com/gpu: 4 - volumeMounts: - - name: devshm - mountPath: /dev/shm - - name: hf-cache - mountPath: /root/.cache/huggingface - env: - - name: HF_HOME - value: /root/.cache/huggingface - - name: HF_TOKEN - valueFrom: - secretKeyRef: - name: hf-token-secret - key: token - nodeSelector: - node.kubernetes.io/instance-type: gpu-h100-sxm - volumes: - - name: devshm - emptyDir: - medium: Memory - - name: hf-cache - hostPath: - path: /mnt/hf-cache - type: DirectoryOrCreate + mirror_hardwares: h100_4 - label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image-Edit" key: nightly-diffusion-x2iat-performance-qwen-image-edit @@ -1063,39 +324,7 @@ steps: buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json" buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log" exit $$EXIT - agents: - queue: "mithril-h100-pool" - plugins: - - kubernetes: - podSpec: - containers: - - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - resources: - limits: - nvidia.com/gpu: 4 - volumeMounts: - - name: devshm - mountPath: /dev/shm - - name: hf-cache - mountPath: /root/.cache/huggingface - env: - - name: HF_HOME - value: /root/.cache/huggingface - - name: HF_TOKEN - valueFrom: - secretKeyRef: - name: hf-token-secret - key: token - nodeSelector: - node.kubernetes.io/instance-type: gpu-h100-sxm - volumes: - - name: devshm - emptyDir: - medium: Memory - - name: hf-cache - hostPath: - path: /mnt/hf-cache - type: DirectoryOrCreate + mirror_hardwares: h100_4 - label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image-Edit-2511" key: nightly-diffusion-x2iat-performance-qwen-image-edit-2511 @@ -1111,39 +340,7 @@ steps: buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json" buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log" exit $$EXIT - agents: - queue: "mithril-h100-pool" - plugins: - - kubernetes: - podSpec: - containers: - - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - resources: - limits: - nvidia.com/gpu: 4 - volumeMounts: - - name: devshm - mountPath: /dev/shm - - name: hf-cache - mountPath: /root/.cache/huggingface - env: - - name: HF_HOME - value: /root/.cache/huggingface - - name: HF_TOKEN - valueFrom: - secretKeyRef: - name: hf-token-secret - key: token - nodeSelector: - node.kubernetes.io/instance-type: gpu-h100-sxm - volumes: - - name: devshm - emptyDir: - medium: Memory - - name: hf-cache - hostPath: - path: /mnt/hf-cache - type: DirectoryOrCreate + mirror_hardwares: h100_4 - label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image-Layered" key: nightly-diffusion-x2iat-performance-qwen-image-layered @@ -1159,39 +356,7 @@ steps: buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json" buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log" exit $$EXIT - agents: - queue: "mithril-h100-pool" - plugins: - - kubernetes: - podSpec: - containers: - - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - resources: - limits: - nvidia.com/gpu: 1 - volumeMounts: - - name: devshm - mountPath: /dev/shm - - name: hf-cache - mountPath: /root/.cache/huggingface - env: - - name: HF_HOME - value: /root/.cache/huggingface - - name: HF_TOKEN - valueFrom: - secretKeyRef: - name: hf-token-secret - key: token - nodeSelector: - node.kubernetes.io/instance-type: gpu-h100-sxm - volumes: - - name: devshm - emptyDir: - medium: Memory - - name: hf-cache - hostPath: - path: /mnt/hf-cache - type: DirectoryOrCreate + mirror_hardwares: h100_1 - label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · BAGEL" key: nightly-diffusion-x2iat-performance-bagel @@ -1207,41 +372,8 @@ steps: buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json" buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log" exit $$EXIT - agents: - queue: "mithril-h100-pool" - plugins: - - kubernetes: - podSpec: - containers: - - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - resources: - limits: - nvidia.com/gpu: 1 - volumeMounts: - - name: devshm - mountPath: /dev/shm - - name: hf-cache - mountPath: /root/.cache/huggingface - env: - - name: HF_HOME - value: /root/.cache/huggingface - - name: HF_TOKEN - valueFrom: - secretKeyRef: - name: hf-token-secret - key: token - nodeSelector: - node.kubernetes.io/instance-type: gpu-h100-sxm - volumes: - - name: devshm - emptyDir: - medium: Memory - - name: hf-cache - hostPath: - path: /mnt/hf-cache - type: DirectoryOrCreate + mirror_hardwares: h100_1 - # Diffusion x2v only (Wan, HunyuanVideo, …). x2i/x2a/x2t live in the X2I group above, not here. - group: ":card_index_dividers: Diffusion X2V Model Test" key: nightly-diffusion-x2v-group depends_on: upload-nightly-pipeline @@ -1250,235 +382,44 @@ steps: build.pull_request.labels includes "nightly-test" || build.pull_request.labels includes "diffusion-x2v-test" steps: + - label: ":full_moon: Diffusion X2V · Wan2.2 T2V Function Test" timeout_in_minutes: 90 commands: - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k "t2v" -m "full_model and cuda" --run-level "full_model" - agents: - queue: "mithril-h100-pool" - plugins: - - kubernetes: - podSpec: - containers: - - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - resources: - limits: - nvidia.com/gpu: 2 - volumeMounts: - - name: devshm - mountPath: /dev/shm - - name: hf-cache - mountPath: /root/.cache/huggingface - env: - - name: HF_HOME - value: /root/.cache/huggingface - - name: HF_TOKEN - valueFrom: - secretKeyRef: - name: hf-token-secret - key: token - nodeSelector: - node.kubernetes.io/instance-type: gpu-h100-sxm - volumes: - - name: devshm - emptyDir: - medium: Memory - - name: hf-cache - hostPath: - path: /mnt/hf-cache - type: DirectoryOrCreate + mirror_hardwares: h100_2 - label: ":full_moon: Diffusion X2V · Wan2.2 I2V/TI2V Function Test" timeout_in_minutes: 90 commands: - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k "i2v" -m "full_model and cuda" --run-level "full_model" - agents: - queue: "mithril-h100-pool" - plugins: - - kubernetes: - podSpec: - containers: - - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - resources: - limits: - nvidia.com/gpu: 2 - volumeMounts: - - name: devshm - mountPath: /dev/shm - - name: hf-cache - mountPath: /root/.cache/huggingface - env: - - name: HF_HOME - value: /root/.cache/huggingface - - name: HF_TOKEN - valueFrom: - secretKeyRef: - name: hf-token-secret - key: token - nodeSelector: - node.kubernetes.io/instance-type: gpu-h100-sxm - volumes: - - name: devshm - emptyDir: - medium: Memory - - name: hf-cache - hostPath: - path: /mnt/hf-cache - type: DirectoryOrCreate + mirror_hardwares: h100_2 - label: ":full_moon: Diffusion X2V · LingBot Video MoE Function Test" timeout_in_minutes: 90 commands: - pytest -s -v tests/e2e/online_serving/test_lingbot_video_moe.py -m "full_model and cuda" --run-level "full_model" - agents: - queue: "mithril-h100-pool" - plugins: - - kubernetes: - podSpec: - containers: - - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - resources: - limits: - nvidia.com/gpu: 1 - volumeMounts: - - name: devshm - mountPath: /dev/shm - - name: hf-cache - mountPath: /root/.cache/huggingface - env: - - name: HF_HOME - value: /root/.cache/huggingface - - name: HF_TOKEN - valueFrom: - secretKeyRef: - name: hf-token-secret - key: token - nodeSelector: - node.kubernetes.io/instance-type: gpu-h100-sxm - volumes: - - name: devshm - emptyDir: - medium: Memory - - name: hf-cache - hostPath: - path: /mnt/hf-cache - type: DirectoryOrCreate + mirror_hardwares: h100_1 - label: ":full_moon: Diffusion X2V · Doc Test" timeout_in_minutes: 60 commands: - pytest -s -v tests/examples/offline_inference/test_image_to_video.py -m "full_model and example and H100" --run-level "full_model" - agents: - queue: "mithril-h100-pool" - plugins: - - kubernetes: - podSpec: - containers: - - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - resources: - limits: - nvidia.com/gpu: 1 - volumeMounts: - - name: devshm - mountPath: /dev/shm - - name: hf-cache - mountPath: /root/.cache/huggingface - env: - - name: HF_HOME - value: /root/.cache/huggingface - - name: HF_TOKEN - valueFrom: - secretKeyRef: - name: hf-token-secret - key: token - nodeSelector: - node.kubernetes.io/instance-type: gpu-h100-sxm - volumes: - - name: devshm - emptyDir: - medium: Memory - - name: hf-cache - hostPath: - path: /mnt/hf-cache - type: DirectoryOrCreate + mirror_hardwares: h100_1 - label: ":full_moon: Diffusion X2V · Other Function Test" timeout_in_minutes: 90 commands: - pytest -s -v tests/e2e/offline_inference/test_wan22_autoround_w4a16_expansion.py -m "full_model and cuda" --run-level "full_model" - pytest -s -v tests/e2e/online_serving/test_wan_2_1_vace_expansion.py tests/e2e/online_serving/test_ltx2_expansion.py -m "full_model and cuda" --run-level "full_model" - agents: - queue: "mithril-h100-pool" - plugins: - - kubernetes: - podSpec: - containers: - - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - resources: - limits: - nvidia.com/gpu: 2 - volumeMounts: - - name: devshm - mountPath: /dev/shm - - name: hf-cache - mountPath: /root/.cache/huggingface - env: - - name: HF_HOME - value: /root/.cache/huggingface - - name: HF_TOKEN - valueFrom: - secretKeyRef: - name: hf-token-secret - key: token - nodeSelector: - node.kubernetes.io/instance-type: gpu-h100-sxm - volumes: - - name: devshm - emptyDir: - medium: Memory - - name: hf-cache - hostPath: - path: /mnt/hf-cache - type: DirectoryOrCreate + mirror_hardwares: h100_2 - label: ":full_moon: Diffusion X2V · Wan2.2 Accuracy Test" timeout_in_minutes: 180 commands: - pytest -s -v tests/e2e/accuracy/wan22_i2v/test_wan22_i2v_video_similarity.py -m full_model --run-level full_model - pytest -s -v tests/e2e/accuracy/test_diffusers_backend_similarity.py -k '2v_matches_diffusers' -m full_model --run-level full_model - agents: - queue: "mithril-h100-pool" - plugins: - - kubernetes: - podSpec: - containers: - - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - resources: - limits: - nvidia.com/gpu: 2 - volumeMounts: - - name: devshm - mountPath: /dev/shm - - name: hf-cache - mountPath: /root/.cache/huggingface - env: - - name: HF_HOME - value: /root/.cache/huggingface - - name: HF_TOKEN - valueFrom: - secretKeyRef: - name: hf-token-secret - key: token - nodeSelector: - node.kubernetes.io/instance-type: gpu-h100-sxm - volumes: - - name: devshm - emptyDir: - medium: Memory - - name: hf-cache - hostPath: - path: /mnt/hf-cache - type: DirectoryOrCreate + mirror_hardwares: h100_2 - label: ":full_moon: Diffusion X2V · Other Accuracy Test" timeout_in_minutes: 180 @@ -1487,39 +428,7 @@ steps: - pytest -s -v tests/e2e/accuracy/hunyuanvideo15_i2v/test_hunyuanvideo15_i2v_video_similarity.py -m full_model --run-level full_model - pytest -s -v tests/e2e/accuracy/test_ltx2_3_video_similarity.py -m full_model --run-level full_model - pytest -s -v tests/e2e/accuracy/test_video_streaming_output_similarity.py -m full_model --run-level full_model - agents: - queue: "mithril-h100-pool" - plugins: - - kubernetes: - podSpec: - containers: - - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - resources: - limits: - nvidia.com/gpu: 2 - volumeMounts: - - name: devshm - mountPath: /dev/shm - - name: hf-cache - mountPath: /root/.cache/huggingface - env: - - name: HF_HOME - value: /root/.cache/huggingface - - name: HF_TOKEN - valueFrom: - secretKeyRef: - name: hf-token-secret - key: token - nodeSelector: - node.kubernetes.io/instance-type: gpu-h100-sxm - volumes: - - name: devshm - emptyDir: - medium: Memory - - name: hf-cache - hostPath: - path: /mnt/hf-cache - type: DirectoryOrCreate + mirror_hardwares: h100_2 - label: ":full_moon: Diffusion X2V · Perf Test" key: nightly-diffusion-x2v-performance @@ -1537,114 +446,29 @@ steps: buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log" if [ $$EXIT1 -ne 0 ]; then exit $$EXIT1; fi exit $$EXIT2 - agents: - queue: "mithril-h100-pool" - plugins: - - kubernetes: - podSpec: - containers: - - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - resources: - limits: - nvidia.com/gpu: 2 - volumeMounts: - - name: devshm - mountPath: /dev/shm - - name: hf-cache - mountPath: /root/.cache/huggingface - env: - - name: HF_HOME - value: /root/.cache/huggingface - - name: HF_TOKEN - valueFrom: - secretKeyRef: - name: hf-token-secret - key: token - nodeSelector: - node.kubernetes.io/instance-type: gpu-h100-sxm - volumes: - - name: devshm - emptyDir: - medium: Memory - - name: hf-cache - hostPath: - path: /mnt/hf-cache - type: DirectoryOrCreate + mirror_hardwares: h100_2 - group: ":card_index_dividers: Diffusion Test" depends_on: upload-nightly-pipeline if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" || build.pull_request.labels includes "diffusion-x2iat-test" steps: + - label: ":full_moon: Diffusion · Offloader Test" timeout_in_minutes: 40 commands: - timeout 40m pytest -s -v tests/diffusion/offloader -m "full_model and cuda" - agents: - queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU - plugins: - - docker#v5.2.0: - image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - always-pull: true - propagate-environment: true - environment: - - "HF_HOME=/fsx/hf_cache" - - "HF_TOKEN" - volumes: - - "/fsx/hf_cache:/fsx/hf_cache" + mirror_hardwares: l4_1 - label: ":full_moon: Diffusion · Quantization Test with H100" commands: - timeout 60m pytest -sv tests/diffusion/quantization -m 'full_model and cuda and H100' --run-level "full_model" - agents: - queue: "mithril-h100-pool" - plugins: - - kubernetes: - podSpec: - containers: - - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - resources: - limits: - nvidia.com/gpu: 1 - volumeMounts: - - name: devshm - mountPath: /dev/shm - - name: hf-cache - mountPath: /root/.cache/huggingface - env: - - name: HF_HOME - value: /root/.cache/huggingface - - name: HF_TOKEN - valueFrom: - secretKeyRef: - name: hf-token-secret - key: token - nodeSelector: - node.kubernetes.io/instance-type: gpu-h100-sxm - volumes: - - name: devshm - emptyDir: - medium: Memory - - name: hf-cache - hostPath: - path: /mnt/hf-cache - type: DirectoryOrCreate + mirror_hardwares: h100_1 - label: ":full_moon: Diffusion · Quantization Test with L4" timeout_in_minutes: 60 commands: - pytest -sv tests/diffusion/quantization -m 'full_model and cuda and L4' --run-level "full_model" - agents: - queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU - plugins: - - docker#v5.2.0: - image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - always-pull: true - propagate-environment: true - environment: - - "HF_HOME=/fsx/hf_cache" - - "HF_TOKEN" - volumes: - - "/fsx/hf_cache:/fsx/hf_cache" + mirror_hardwares: l4_1 - label: ":full_moon: Diffusion · Wan VAE Spatial Shard Correctness Test" timeout_in_minutes: 60 @@ -1653,39 +477,7 @@ steps: pytest -sv tests/diffusion/distributed/test_wan_spatial_shard.py -m "full_model and diffusion and H100 and distributed_cuda" --run-level "full_model" - agents: - queue: "mithril-h100-pool" - plugins: - - kubernetes: - podSpec: - containers: - - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - resources: - limits: - nvidia.com/gpu: 2 - volumeMounts: - - name: devshm - mountPath: /dev/shm - - name: hf-cache - mountPath: /root/.cache/huggingface - env: - - name: HF_HOME - value: /root/.cache/huggingface - - name: HF_TOKEN - valueFrom: - secretKeyRef: - name: hf-token-secret - key: token - nodeSelector: - node.kubernetes.io/instance-type: gpu-h100-sxm - volumes: - - name: devshm - emptyDir: - medium: Memory - - name: hf-cache - hostPath: - path: /mnt/hf-cache - type: DirectoryOrCreate + mirror_hardwares: h100_2 - label: ":bar_chart: Testcase Statistics" key: nightly-testcase-statistics @@ -1695,41 +487,10 @@ steps: commands: - python tools/nightly/buildkite_testcase_statistics.py -o tests/dfx/perf/results/buildkite_testcase_statistics.html - buildkite-agent artifact upload "tests/dfx/perf/results/*.html" - agents: - queue: "mithril-h100-pool" - plugins: - - kubernetes: - podSpec: - containers: - - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - resources: - limits: - nvidia.com/gpu: 1 - volumeMounts: - - name: devshm - mountPath: /dev/shm - - name: hf-cache - mountPath: /root/.cache/huggingface - env: - - name: HF_HOME - value: /root/.cache/huggingface - - name: HF_TOKEN - valueFrom: - secretKeyRef: - name: hf-token-secret - key: token - nodeSelector: - node.kubernetes.io/instance-type: gpu-h100-sxm - volumes: - - name: devshm - emptyDir: - medium: Memory - - name: hf-cache - hostPath: - path: /mnt/hf-cache - type: DirectoryOrCreate + mirror_hardwares: h100_1 - # No need to run this step for PRs with label nightly-test + # NOTE: ":email: Nightly Collection & Email" is deprecated and will be removed soon. + # Do not add new depends_on entries or extend this step. - label: ":email: Nightly Collection & Email" key: nightly-perf-distribution depends_on: diff --git a/.buildkite/cuda/test-ready.yml b/.buildkite/cuda/test-ready.yml index 75e834be9..d622aecbe 100644 --- a/.buildkite/cuda/test-ready.yml +++ b/.buildkite/cuda/test-ready.yml @@ -8,104 +8,41 @@ steps: - group: ":card_index_dividers: Simple Test" depends_on: upload-ready-pipeline steps: + - label: "Simple · Diffusion & Model Executor Test" commands: - "timeout 20m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml" - agents: - queue: "gpu_1_queue" - plugins: - - docker#v5.2.0: - image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - always-pull: true - propagate-environment: true - environment: - - "HF_HOME=/fsx/hf_cache" - - "HF_TOKEN" - volumes: - - "/fsx/hf_cache:/fsx/hf_cache" + mirror_hardwares: l4_1 - label: "Simple · Other Test" commands: - "timeout 20m pytest -sv -m 'core_model and cpu' --ignore=tests/diffusion --ignore=tests/model_executor --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml" - agents: - queue: "gpu_1_queue" - plugins: - - docker#v5.2.0: - image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - always-pull: true - propagate-environment: true - environment: - - "HF_HOME=/fsx/hf_cache" - - "HF_TOKEN" - volumes: - - "/fsx/hf_cache:/fsx/hf_cache" + mirror_hardwares: l4_1 - label: "Custom Pipeline Test" depends_on: upload-ready-pipeline commands: - timeout 20m pytest -s -v tests/e2e/offline_inference/custom_pipeline/ -m "core_model" - agents: - queue: "gpu_1_queue" - plugins: - - docker#v5.2.0: - image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - always-pull: true - propagate-environment: true - environment: - - "HF_HOME=/fsx/hf_cache" - volumes: - - "/fsx/hf_cache:/fsx/hf_cache" + mirror_hardwares: l4_1 - group: ":card_index_dividers: Diffusion Test" depends_on: upload-ready-pipeline steps: + - label: "Diffusion · Offloader Test" commands: - timeout 40m pytest -s -v tests/diffusion/offloader -m "core_model and cuda" --run-level "core_model" - agents: - queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU - plugins: - - docker#v5.2.0: - image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - always-pull: true - propagate-environment: true - environment: - - "HF_HOME=/fsx/hf_cache" - - "HF_TOKEN" - volumes: - - "/fsx/hf_cache:/fsx/hf_cache" + mirror_hardwares: l4_1 - label: "Diffusion · Cache Test" commands: - timeout 15m pytest -sv tests/diffusion/cache -m 'core_model and cuda' --run-level "core_model" - agents: - queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU - plugins: - - docker#v5.2.0: - image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - always-pull: true - propagate-environment: true - environment: - - "HF_HOME=/fsx/hf_cache" - - "HF_TOKEN" - volumes: - - "/fsx/hf_cache:/fsx/hf_cache" + mirror_hardwares: l4_1 - label: "Diffusion · Batch Test" commands: - timeout 15m pytest -sv tests/diffusion/batching -m 'core_model and cuda' --run-level "core_model" - agents: - queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU - plugins: - - docker#v5.2.0: - image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - always-pull: true - propagate-environment: true - environment: - - "HF_HOME=/fsx/hf_cache" - - "HF_TOKEN" - volumes: - - "/fsx/hf_cache:/fsx/hf_cache" + mirror_hardwares: l4_1 - label: "Diffusion · Distributed Test" source_file_dependencies: @@ -116,73 +53,26 @@ steps: - vllm_omni/diffusion/models/ commands: - timeout 20m pytest -sv tests/diffusion/distributed -m 'core_model and cuda' --run-level "core_model" - agents: - queue: "gpu_4_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU - plugins: - - docker#v5.2.0: - image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - always-pull: true - propagate-environment: true - shm-size: "8gb" - environment: - - "HF_HOME=/fsx/hf_cache" - - "HF_TOKEN" - volumes: - - "/fsx/hf_cache:/fsx/hf_cache" + mirror_hardwares: l4_4 - label: "Diffusion · Model Test" commands: - timeout 15m pytest -sv tests/diffusion/models/ -m 'core_model and cuda' --run-level "core_model" - agents: - queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU - plugins: - - docker#v5.2.0: - image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - always-pull: true - propagate-environment: true - environment: - - "HF_HOME=/fsx/hf_cache" - - "HF_TOKEN" - volumes: - - "/fsx/hf_cache:/fsx/hf_cache" + mirror_hardwares: l4_1 - label: "Diffusion · Other Test" commands: - timeout 15m pytest -sv tests/diffusion/*.py -m 'core_model and cuda' --run-level "core_model" - agents: - queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU - plugins: - - docker#v5.2.0: - image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - always-pull: true - propagate-environment: true - environment: - - "HF_HOME=/fsx/hf_cache" - - "HF_TOKEN" - volumes: - - "/fsx/hf_cache:/fsx/hf_cache" - + mirror_hardwares: l4_1 - group: ":card_index_dividers: Tiny Model Tests [Single GPU]" depends_on: upload-ready-pipeline steps: + - label: "Tiny Model Tests · Diffusion (base)" commands: - timeout 15m pytest -sv tests/model_tests/diffusion -m 'core_model and cuda' --run-level "core_model" - agents: - # Ready label only runs the cases that don't need parallelism for tiny - # model tests, which is usually just a smoke test for TTI and I2I - queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU - plugins: - - docker#v5.2.0: - image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - always-pull: true - propagate-environment: true - environment: - - "HF_HOME=/fsx/hf_cache" - - "HF_TOKEN" - volumes: - - "/fsx/hf_cache:/fsx/hf_cache" + mirror_hardwares: l4_1 - label: "Engine&Model Executor Test" depends_on: upload-ready-pipeline @@ -191,20 +81,7 @@ steps: timeout 15m bash -c " pytest -sv tests/engine/ tests/model_executor/ -m 'core_model and cuda' --run-level 'core_model' " - agents: - queue: "gpu_1_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU - plugins: - - docker#v5.2.0: - image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - always-pull: true - propagate-environment: true - shm-size: "8gb" - environment: - - "HF_HOME=/fsx/hf_cache" - - "HF_TOKEN" - volumes: - - "/fsx/hf_cache:/fsx/hf_cache" - + mirror_hardwares: l4_1 - label: "Distributed Test" depends_on: upload-ready-pipeline @@ -213,19 +90,7 @@ steps: timeout 20m bash -c " pytest -sv tests/distributed/ -m 'core_model and cuda and L4' --run-level 'core_model' " - agents: - queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU - plugins: - - docker#v5.2.0: - image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - always-pull: true - propagate-environment: true - shm-size: "8gb" - environment: - - "HF_HOME=/fsx/hf_cache" - - "HF_TOKEN" - volumes: - - "/fsx/hf_cache:/fsx/hf_cache" + mirror_hardwares: l4_1 - label: "Entrypoints Test" depends_on: upload-ready-pipeline @@ -235,23 +100,12 @@ steps: export VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -sv tests/entrypoints/ -m 'core_model and cuda' --run-level 'core_model' " - agents: - queue: "gpu_1_queue" - plugins: - - docker#v5.2.0: - image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - always-pull: true - propagate-environment: true - shm-size: "8gb" - environment: - - "HF_HOME=/fsx/hf_cache" - - "HF_TOKEN" - volumes: - - "/fsx/hf_cache:/fsx/hf_cache" + mirror_hardwares: l4_1 - group: ":card_index_dividers: E2E Test" depends_on: upload-ready-pipeline steps: + - label: "RLHF · VeRL-Omni E2E Test" source_file_dependencies: - tests/e2e/offline_inference/rlhf_test/test_verl_omni_e2e.py @@ -266,19 +120,7 @@ steps: timeout 20m bash -c " pytest -s -v tests/e2e/offline_inference/rlhf_test/test_verl_omni_e2e.py " - agents: - queue: "gpu_1_queue" - plugins: - - docker#v5.2.0: - image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - always-pull: true - propagate-environment: true - shm-size: "8gb" - environment: - - "HF_HOME=/fsx/hf_cache" - - "HF_TOKEN" - volumes: - - "/fsx/hf_cache:/fsx/hf_cache" + mirror_hardwares: l4_1 - label: "Omni · Qwen3-Omni Test" source_file_dependencies: @@ -291,34 +133,7 @@ steps: timeout 20m bash -c " pytest -s -v tests/e2e/online_serving/test_qwen3_omni.py -m 'core_model' --run-level 'core_model' " - agents: - queue: "mithril-h100-pool" - plugins: - - kubernetes: - podSpec: - containers: - - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - resources: - limits: - nvidia.com/gpu: 2 - volumeMounts: - - name: devshm - mountPath: /dev/shm - - name: hf-cache - mountPath: /root/.cache/huggingface - env: - - name: HF_HOME - value: /root/.cache/huggingface - nodeSelector: - node.kubernetes.io/instance-type: gpu-h100-sxm - volumes: - - name: devshm - emptyDir: - medium: Memory - - name: hf-cache - hostPath: - path: /mnt/hf-cache - type: DirectoryOrCreate + mirror_hardwares: h100_2 - label: "TTS · VoxCPM2 Test" source_file_dependencies: @@ -333,19 +148,7 @@ steps: export VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -s -v tests/e2e/online_serving/test_voxcpm2_tts.py -m 'core_model' --run-level 'core_model' " - agents: - queue: "gpu_1_queue" - plugins: - - docker#v5.2.0: - image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - always-pull: true - propagate-environment: true - shm-size: "8gb" - environment: - - "HF_HOME=/fsx/hf_cache" - - "HF_TOKEN" - volumes: - - "/fsx/hf_cache:/fsx/hf_cache" + mirror_hardwares: l4_1 - label: "TTS · Qwen3-TTS CustomVoice Test" source_file_dependencies: @@ -359,19 +162,7 @@ steps: export VLLM_ALLOW_LONG_MAX_MODEL_LEN=1 pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py -m 'core_model' --run-level 'core_model' " - agents: - queue: "gpu_1_queue" - plugins: - - docker#v5.2.0: - image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - always-pull: true - propagate-environment: true - shm-size: "8gb" - environment: - - "HF_HOME=/fsx/hf_cache" - - "HF_TOKEN" - volumes: - - "/fsx/hf_cache:/fsx/hf_cache" + mirror_hardwares: l4_1 - label: "TTS · Qwen3-TTS Base Test" source_file_dependencies: @@ -387,19 +178,7 @@ steps: export VLLM_OMNI_USE_V2_RUNNER=1 pytest -s -v tests/e2e/online_serving/test_qwen3_tts_base.py -m 'core_model' --run-level 'core_model' " - agents: - queue: "gpu_1_queue" - plugins: - - docker#v5.2.0: - image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - always-pull: true - propagate-environment: true - shm-size: "8gb" - environment: - - "HF_HOME=/fsx/hf_cache" - - "HF_TOKEN" - volumes: - - "/fsx/hf_cache:/fsx/hf_cache" + mirror_hardwares: l4_1 - label: "TTS · Higgs-Audio-V3 Test" source_file_dependencies: @@ -415,19 +194,7 @@ steps: export VLLM_MOE_USE_DEEP_GEMM=0 pytest -s -v tests/e2e/online_serving/test_higgs_audio_v3.py -m 'core_model' --run-level 'core_model' " - agents: - queue: "gpu_1_queue" - plugins: - - docker#v5.2.0: - image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - always-pull: true - propagate-environment: true - shm-size: "8gb" - environment: - - "HF_HOME=/fsx/hf_cache" - - "HF_TOKEN" - volumes: - - "/fsx/hf_cache:/fsx/hf_cache" + mirror_hardwares: l4_1 - label: "Diffusion · Qwen Image Edit Test" source_file_dependencies: @@ -438,34 +205,7 @@ steps: timeout 20m bash -c " pytest -s -v tests/e2e/online_serving/test_qwen_image_edit.py -m 'core_model' --run-level 'core_model' " - agents: - queue: "mithril-h100-pool" - plugins: - - kubernetes: - podSpec: - containers: - - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - resources: - limits: - nvidia.com/gpu: 1 - volumeMounts: - - name: devshm - mountPath: /dev/shm - - name: hf-cache - mountPath: /root/.cache/huggingface - env: - - name: HF_HOME - value: /root/.cache/huggingface - nodeSelector: - node.kubernetes.io/instance-type: gpu-h100-sxm - volumes: - - name: devshm - emptyDir: - medium: Memory - - name: hf-cache - hostPath: - path: /mnt/hf-cache - type: DirectoryOrCreate + mirror_hardwares: h100_1 - label: "Diffusion · Bagel Test" source_file_dependencies: @@ -480,39 +220,7 @@ steps: export VLLM_IMAGE_FETCH_TIMEOUT=60 pytest -s -v tests/e2e/online_serving/test_bagel.py -m 'core_model' --run-level 'core_model' " - agents: - queue: "mithril-h100-pool" - plugins: - - kubernetes: - podSpec: - containers: - - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - resources: - limits: - nvidia.com/gpu: 1 - volumeMounts: - - name: devshm - mountPath: /dev/shm - - name: hf-cache - mountPath: /root/.cache/huggingface - env: - - name: HF_HOME - value: /root/.cache/huggingface - - name: HF_TOKEN - valueFrom: - secretKeyRef: - name: hf-token-secret - key: token - nodeSelector: - node.kubernetes.io/instance-type: gpu-h100-sxm - volumes: - - name: devshm - emptyDir: - medium: Memory - - name: hf-cache - hostPath: - path: /mnt/hf-cache - type: DirectoryOrCreate + mirror_hardwares: h100_1 - label: "Diffusion · Wan22 Test" source_file_dependencies: @@ -524,39 +232,7 @@ steps: export VLLM_IMAGE_FETCH_TIMEOUT=60 pytest -s -v tests/e2e/online_serving/test_wan22_t2v.py -m 'core_model' --run-level 'core_model' " - agents: - queue: "mithril-h100-pool" - plugins: - - kubernetes: - podSpec: - containers: - - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - resources: - limits: - nvidia.com/gpu: 1 - volumeMounts: - - name: devshm - mountPath: /dev/shm - - name: hf-cache - mountPath: /root/.cache/huggingface - env: - - name: HF_HOME - value: /root/.cache/huggingface - - name: HF_TOKEN - valueFrom: - secretKeyRef: - name: hf-token-secret - key: token - nodeSelector: - node.kubernetes.io/instance-type: gpu-h100-sxm - volumes: - - name: devshm - emptyDir: - medium: Memory - - name: hf-cache - hostPath: - path: /mnt/hf-cache - type: DirectoryOrCreate + mirror_hardwares: h100_1 - label: "Diffusion · Cosmos3 Test" source_file_dependencies: @@ -568,40 +244,7 @@ steps: export VLLM_IMAGE_FETCH_TIMEOUT=60 pytest -s -v tests/e2e/online_serving/test_cosmos3.py -m 'core_model' --run-level 'core_model' " - agents: - queue: "mithril-h100-pool" - plugins: - - kubernetes: - podSpec: - containers: - - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - resources: - limits: - nvidia.com/gpu: 1 - volumeMounts: - - name: devshm - mountPath: /dev/shm - - name: hf-cache - mountPath: /root/.cache/huggingface - env: - - name: HF_HOME - value: /root/.cache/huggingface - - name: HF_TOKEN - valueFrom: - secretKeyRef: - name: hf-token-secret - key: token - nodeSelector: - node.kubernetes.io/instance-type: gpu-h100-sxm - volumes: - - name: devshm - emptyDir: - medium: Memory - - name: hf-cache - hostPath: - path: /mnt/hf-cache - type: DirectoryOrCreate - + mirror_hardwares: h100_1 - label: "Diffusion · Qwen Image Test" source_file_dependencies: @@ -613,39 +256,7 @@ steps: export VLLM_IMAGE_FETCH_TIMEOUT=60 pytest -s -v tests/e2e/online_serving/test_qwen_image.py -m 'core_model' --run-level 'core_model' " - agents: - queue: "mithril-h100-pool" - plugins: - - kubernetes: - podSpec: - containers: - - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - resources: - limits: - nvidia.com/gpu: 1 - volumeMounts: - - name: devshm - mountPath: /dev/shm - - name: hf-cache - mountPath: /root/.cache/huggingface - env: - - name: HF_HOME - value: /root/.cache/huggingface - - name: HF_TOKEN - valueFrom: - secretKeyRef: - name: hf-token-secret - key: token - nodeSelector: - node.kubernetes.io/instance-type: gpu-h100-sxm - volumes: - - name: devshm - emptyDir: - medium: Memory - - name: hf-cache - hostPath: - path: /mnt/hf-cache - type: DirectoryOrCreate + mirror_hardwares: h100_1 - label: "Diffusion · Qwen Image Layered Test" source_file_dependencies: @@ -657,39 +268,7 @@ steps: export VLLM_IMAGE_FETCH_TIMEOUT=60 pytest -s -v tests/e2e/online_serving/test_qwen_image_layered.py -m 'core_model' --run-level 'core_model' " - agents: - queue: "mithril-h100-pool" - plugins: - - kubernetes: - podSpec: - containers: - - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - resources: - limits: - nvidia.com/gpu: 1 - volumeMounts: - - name: devshm - mountPath: /dev/shm - - name: hf-cache - mountPath: /root/.cache/huggingface - env: - - name: HF_HOME - value: /root/.cache/huggingface - - name: HF_TOKEN - valueFrom: - secretKeyRef: - name: hf-token-secret - key: token - nodeSelector: - node.kubernetes.io/instance-type: gpu-h100-sxm - volumes: - - name: devshm - emptyDir: - medium: Memory - - name: hf-cache - hostPath: - path: /mnt/hf-cache - type: DirectoryOrCreate + mirror_hardwares: h100_1 - label: "Diffusion · GGUF Plugin Test" soft_fail: true @@ -699,15 +278,4 @@ steps: commands: - pip install "vllm-gguf-plugin>=0.0.3" - timeout 30m pytest -sv tests/plugin/test_gguf_diffusion.py - agents: - queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU - plugins: - - docker#v5.2.0: - image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - always-pull: true - propagate-environment: true - environment: - - "HF_HOME=/fsx/hf_cache" - - "HF_TOKEN" - volumes: - - "/fsx/hf_cache:/fsx/hf_cache" + mirror_hardwares: l4_1 diff --git a/.buildkite/cuda/test-weekly.yml b/.buildkite/cuda/test-weekly.yml index d71bdae0a..cc7f99647 100644 --- a/.buildkite/cuda/test-weekly.yml +++ b/.buildkite/cuda/test-weekly.yml @@ -5,45 +5,14 @@ env: HF_HUB_ETAG_TIMEOUT: 60 steps: + - label: "Reliability Test - qwen3-omni" timeout_in_minutes: 180 depends_on: upload-weekly-pipeline if: build.env("WEEKLY") == "1" || build.pull_request.labels includes "weekly-test" commands: - pytest -s -v tests/dfx/reliability/test_reliability_qwen3_omni.py -m "slow" - agents: - queue: "mithril-h100-pool" - plugins: - - kubernetes: - podSpec: - containers: - - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - resources: - limits: - nvidia.com/gpu: 2 - volumeMounts: - - name: devshm - mountPath: /dev/shm - - name: hf-cache - mountPath: /root/.cache/huggingface - env: - - name: HF_HOME - value: /root/.cache/huggingface - - name: HF_TOKEN - valueFrom: - secretKeyRef: - name: hf-token-secret - key: token - nodeSelector: - node.kubernetes.io/instance-type: gpu-h100-sxm - volumes: - - name: devshm - emptyDir: - medium: Memory - - name: hf-cache - hostPath: - path: /mnt/hf-cache - type: DirectoryOrCreate + mirror_hardwares: h100_2 - label: "Reliability Test - wan22" timeout_in_minutes: 180 @@ -51,39 +20,7 @@ steps: if: build.env("WEEKLY") == "1" || build.pull_request.labels includes "weekly-test" commands: - pytest -s -v tests/dfx/reliability/test_reliability_wan22.py -m "slow" - agents: - queue: "mithril-h100-pool" - plugins: - - kubernetes: - podSpec: - containers: - - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - resources: - limits: - nvidia.com/gpu: 2 - volumeMounts: - - name: devshm - mountPath: /dev/shm - - name: hf-cache - mountPath: /root/.cache/huggingface - env: - - name: HF_HOME - value: /root/.cache/huggingface - - name: HF_TOKEN - valueFrom: - secretKeyRef: - name: hf-token-secret - key: token - nodeSelector: - node.kubernetes.io/instance-type: gpu-h100-sxm - volumes: - - name: devshm - emptyDir: - medium: Memory - - name: hf-cache - hostPath: - path: /mnt/hf-cache - type: DirectoryOrCreate + mirror_hardwares: h100_2 - label: "Reliability Test - hunyuan-image" timeout_in_minutes: 180 @@ -91,39 +28,7 @@ steps: if: build.env("WEEKLY") == "1" || build.pull_request.labels includes "weekly-test" commands: - pytest -s -v tests/dfx/reliability/test_reliability_hunyuan_image.py -m "slow" - agents: - queue: "mithril-h100-pool" - plugins: - - kubernetes: - podSpec: - containers: - - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - resources: - limits: - nvidia.com/gpu: 4 - volumeMounts: - - name: devshm - mountPath: /dev/shm - - name: hf-cache - mountPath: /root/.cache/huggingface - env: - - name: HF_HOME - value: /root/.cache/huggingface - - name: HF_TOKEN - valueFrom: - secretKeyRef: - name: hf-token-secret - key: token - nodeSelector: - node.kubernetes.io/instance-type: gpu-h100-sxm - volumes: - - name: devshm - emptyDir: - medium: Memory - - name: hf-cache - hostPath: - path: /mnt/hf-cache - type: DirectoryOrCreate + mirror_hardwares: h100_4 - label: "Reliability Test - voxcpm2" timeout_in_minutes: 180 @@ -131,86 +36,30 @@ steps: if: build.env("WEEKLY") == "1" || build.pull_request.labels includes "weekly-test" commands: - pytest -s -v tests/dfx/reliability/test_reliability_voxcpm2.py -m "slow" - agents: - queue: "gpu_1_queue" - plugins: - - docker#v5.2.0: - image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - always-pull: true - propagate-environment: true - shm-size: "8gb" - environment: - - "HF_HOME=/fsx/hf_cache" - - "HF_TOKEN" - volumes: - - "/fsx/hf_cache:/fsx/hf_cache" + mirror_hardwares: l4_1 - # Invalid HTTP / WS payloads vs live Omni servers (Issue #3649 tracks known skips). - group: "Reliability Test - Invalid parameters Test" depends_on: upload-weekly-pipeline if: build.env("WEEKLY") == "1" || build.pull_request.labels includes "weekly-test" steps: + - label: "Invalid parameters Test · H100" timeout_in_minutes: 120 commands: - pytest -s -v tests/dfx/reliability/invalid_param_test/ -m "slow and H100" - agents: - queue: "mithril-h100-pool" - plugins: - - kubernetes: - podSpec: - containers: - - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - resources: - limits: - nvidia.com/gpu: 2 - volumeMounts: - - name: devshm - mountPath: /dev/shm - - name: hf-cache - mountPath: /root/.cache/huggingface - env: - - name: HF_HOME - value: /root/.cache/huggingface - - name: HF_TOKEN - valueFrom: - secretKeyRef: - name: hf-token-secret - key: token - nodeSelector: - node.kubernetes.io/instance-type: gpu-h100-sxm - volumes: - - name: devshm - emptyDir: - medium: Memory - - name: hf-cache - hostPath: - path: /mnt/hf-cache - type: DirectoryOrCreate + mirror_hardwares: h100_2 - # Stable Audio Open, Qwen3-TTS speech/batch/voices paths marked ``L4``. - label: "Invalid parameters Test · L4" timeout_in_minutes: 120 commands: - pytest -s -v tests/dfx/reliability/invalid_param_test/ -m "slow and L4" - agents: - queue: "gpu_1_queue" - plugins: - - docker#v5.2.0: - image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - always-pull: true - propagate-environment: true - shm-size: "8gb" - environment: - - "HF_HOME=/fsx/hf_cache" - - "HF_TOKEN" - volumes: - - "/fsx/hf_cache:/fsx/hf_cache" + mirror_hardwares: l4_1 - group: ":card_index_dividers: E2E Tests" depends_on: upload-weekly-pipeline if: build.env("WEEKLY") == "1" || build.pull_request.labels includes "weekly-test" steps: + - label: "Omni · L4" timeout_in_minutes: 90 commands: @@ -222,19 +71,7 @@ steps: --ignore=tests/e2e/offline_inference/test_qwen2_5_omni_expansion.py -m "slow and L4 and omni" --run-level "full_model" - agents: - queue: "gpu_4_queue" - plugins: - - docker#v5.2.0: - image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - always-pull: true - propagate-environment: true - shm-size: "8gb" - environment: - - "HF_HOME=/fsx/hf_cache" - - "HF_TOKEN" - volumes: - - "/fsx/hf_cache:/fsx/hf_cache" + mirror_hardwares: l4_4 - label: "Omni · H100" timeout_in_minutes: 90 @@ -243,39 +80,7 @@ steps: pytest -sv tests/e2e/ -m "slow and H100 and omni" --run-level "full_model" - agents: - queue: "mithril-h100-pool" - plugins: - - kubernetes: - podSpec: - containers: - - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - resources: - limits: - nvidia.com/gpu: 2 - volumeMounts: - - name: devshm - mountPath: /dev/shm - - name: hf-cache - mountPath: /root/.cache/huggingface - env: - - name: HF_HOME - value: /root/.cache/huggingface - - name: HF_TOKEN - valueFrom: - secretKeyRef: - name: hf-token-secret - key: token - nodeSelector: - node.kubernetes.io/instance-type: gpu-h100-sxm - volumes: - - name: devshm - emptyDir: - medium: Memory - - name: hf-cache - hostPath: - path: /mnt/hf-cache - type: DirectoryOrCreate + mirror_hardwares: h100_2 - label: "TTS · L4" timeout_in_minutes: 90 @@ -289,19 +94,7 @@ steps: pytest -s -v "$$f" -m "slow and L4 and tts" --run-level "full_model" || failed=1 done exit $$failed - agents: - queue: "gpu_1_queue" - plugins: - - docker#v5.2.0: - image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - always-pull: true - propagate-environment: true - shm-size: "8gb" - environment: - - "HF_HOME=/fsx/hf_cache" - - "HF_TOKEN" - volumes: - - "/fsx/hf_cache:/fsx/hf_cache" + mirror_hardwares: l4_1 - label: "TTS · H100" timeout_in_minutes: 90 @@ -313,39 +106,7 @@ steps: pytest -sv tests/e2e/ -m "slow and H100 and tts" --run-level "full_model" - agents: - queue: "mithril-h100-pool" - plugins: - - kubernetes: - podSpec: - containers: - - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - resources: - limits: - nvidia.com/gpu: 1 - volumeMounts: - - name: devshm - mountPath: /dev/shm - - name: hf-cache - mountPath: /root/.cache/huggingface - env: - - name: HF_HOME - value: /root/.cache/huggingface - - name: HF_TOKEN - valueFrom: - secretKeyRef: - name: hf-token-secret - key: token - nodeSelector: - node.kubernetes.io/instance-type: gpu-h100-sxm - volumes: - - name: devshm - emptyDir: - medium: Memory - - name: hf-cache - hostPath: - path: /mnt/hf-cache - type: DirectoryOrCreate + mirror_hardwares: h100_1 - label: "Diffusion · L4" timeout_in_minutes: 60 @@ -354,19 +115,7 @@ steps: pytest -sv tests/e2e/ -m "slow and diffusion and L4" --run-level "full_model" - agents: - queue: "gpu_4_queue" - plugins: - - docker#v5.2.0: - image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - always-pull: true - propagate-environment: true - shm-size: "8gb" - environment: - - "HF_HOME=/fsx/hf_cache" - - "HF_TOKEN" - volumes: - - "/fsx/hf_cache:/fsx/hf_cache" + mirror_hardwares: l4_4 - label: "Diffusion · H100 · Single-GPU" timeout_in_minutes: 120 @@ -375,39 +124,7 @@ steps: pytest -sv tests/e2e/ -m "slow and diffusion and H100 and not distributed_cuda" --run-level "full_model" - agents: - queue: "mithril-h100-pool" - plugins: - - kubernetes: - podSpec: - containers: - - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - resources: - limits: - nvidia.com/gpu: 1 - volumeMounts: - - name: devshm - mountPath: /dev/shm - - name: hf-cache - mountPath: /root/.cache/huggingface - env: - - name: HF_HOME - value: /root/.cache/huggingface - - name: HF_TOKEN - valueFrom: - secretKeyRef: - name: hf-token-secret - key: token - nodeSelector: - node.kubernetes.io/instance-type: gpu-h100-sxm - volumes: - - name: devshm - emptyDir: - medium: Memory - - name: hf-cache - hostPath: - path: /mnt/hf-cache - type: DirectoryOrCreate + mirror_hardwares: h100_1 - label: "Diffusion · H100 · Multi-GPU" timeout_in_minutes: 90 @@ -416,36 +133,4 @@ steps: pytest -sv tests/e2e/ -m "slow and diffusion and H100 and distributed_cuda" --run-level "full_model" - agents: - queue: "mithril-h100-pool" - plugins: - - kubernetes: - podSpec: - containers: - - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - resources: - limits: - nvidia.com/gpu: 2 - volumeMounts: - - name: devshm - mountPath: /dev/shm - - name: hf-cache - mountPath: /root/.cache/huggingface - env: - - name: HF_HOME - value: /root/.cache/huggingface - - name: HF_TOKEN - valueFrom: - secretKeyRef: - name: hf-token-secret - key: token - nodeSelector: - node.kubernetes.io/instance-type: gpu-h100-sxm - volumes: - - name: devshm - emptyDir: - medium: Memory - - name: hf-cache - hostPath: - path: /mnt/hf-cache - type: DirectoryOrCreate + mirror_hardwares: h100_2 diff --git a/.buildkite/intel/scripts/bootstrap-intel-omni.sh b/.buildkite/intel/scripts/bootstrap-intel-omni.sh index e51252bd9..a5d3f00a6 100755 --- a/.buildkite/intel/scripts/bootstrap-intel-omni.sh +++ b/.buildkite/intel/scripts/bootstrap-intel-omni.sh @@ -1,51 +1,16 @@ #!/bin/bash # vllm-omni Intel bootstrap -# Uses static pipeline-intel.yaml for Intel XPU tests -# Based on: bootstrap-amd-omni.sh +# Uses static pipeline-intel.yml for Intel XPU tests set -euo pipefail -if [[ -z "${DOCS_ONLY_DISABLE:-}" ]]; then - DOCS_ONLY_DISABLE=0 -fi +source .buildkite/common/scripts/resolve_skip_ci.sh upload_pipeline() { echo "--- 🛠 Preparing Intel pipeline" buildkite-agent pipeline upload .buildkite/intel/pipeline-intel.yml } -get_diff() { - $(git add .) - echo $(git diff --name-only --diff-filter=ACMDR $(git merge-base origin/main HEAD)) -} - -get_diff_main() { - $(git add .) - echo $(git diff --name-only --diff-filter=ACMDR HEAD~1) -} - -file_diff=$(get_diff) -if [[ $BUILDKITE_BRANCH == "main" ]]; then - file_diff=$(get_diff_main) -fi - -# ---------------------------------------------------------------------- -# Early exit: skip pipeline if all changed files are under docs/ -# ---------------------------------------------------------------------- -if [[ "${DOCS_ONLY_DISABLE}" != "1" ]] && [[ -n "${file_diff:-}" ]]; then - docs_only=1 - while IFS= read -r f; do - [[ -z "$f" ]] && continue - if [[ "$f" != docs/* ]]; then - docs_only=0 - break - fi - done < <(printf '%s\n' "$file_diff" | tr ' ' '\n' | tr -d '\r') - - if [[ "$docs_only" -eq 1 ]]; then - buildkite-agent annotate ":memo: CI skipped — docs only" --style "info" - exit 0 - fi -fi +gate_bootstrap_ci intel l2 upload_pipeline diff --git a/.buildkite/npu/bootstrap-upload-steps.yml b/.buildkite/npu/bootstrap-upload-steps.yml new file mode 100644 index 000000000..64b97f895 --- /dev/null +++ b/.buildkite/npu/bootstrap-upload-steps.yml @@ -0,0 +1,181 @@ +# Uploaded by upload_pipeline.py --upload (not the Buildkite hook). +# ``if`` is injected by step ``key`` from skip-ci at upload time. +steps: + - label: ":buildkit: Build and Push NPU Test Image - A2" + key: image-build-a2 + agents: + queue: "ascend-a2b3" + resource_class: "npu-1" + plugins: + - kubernetes: + metadata: + annotations: + vault.hashicorp.com/agent-init-first: "true" + vault.hashicorp.com/agent-inject: "true" + vault.hashicorp.com/agent-inject-perms-ca.pem: "0400" + vault.hashicorp.com/agent-inject-perms-cert.pem: "0400" + vault.hashicorp.com/agent-inject-perms-config.json: "0400" + vault.hashicorp.com/agent-inject-perms-key.pem: "0400" + vault.hashicorp.com/agent-inject-secret-ca.pem: internal/data/ascend/buildkitd + vault.hashicorp.com/agent-inject-secret-cert.pem: internal/data/ascend/buildkitd + vault.hashicorp.com/agent-inject-secret-config.json: internal/data/ascend/buildkitd + vault.hashicorp.com/agent-inject-secret-key.pem: internal/data/ascend/buildkitd + vault.hashicorp.com/agent-inject-template-ca.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.RootCA }}\n{{- end }}" + vault.hashicorp.com/agent-inject-template-cert.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.ClientCaCert }}\n{{- end }}" + vault.hashicorp.com/agent-inject-template-config.json: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.dockerConfig }}\n{{- end }}" + vault.hashicorp.com/agent-inject-template-key.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.ClientCaKey }}\n{{- end }}" + vault.hashicorp.com/agent-pre-populate-only: "true" + vault.hashicorp.com/agent-run-as-group: "1000" + vault.hashicorp.com/agent-run-as-user: "1000" + vault.hashicorp.com/agent-service-account-token-volume-name: token-vol + vault.hashicorp.com/role: ascend-gha-runners + vault.hashicorp.com/secret-volume-path: /home/user/.docker/ + vault.hashicorp.com/tls-skip-verify: "true" + podSpecPatch: + volumes: + - name: token-vol + projected: + defaultMode: 420 + sources: + - serviceAccountToken: + audience: api + expirationSeconds: 600 + path: token + env: + VLLM_IMAGE_TAG: "${BUILDKITE_COMMIT}" + IMAGE_NAME: "vllm-omni-ci-npu" + IMAGE_REGISTRY: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry" + BUILDKITD_ADDR: "tcp://buildkitd-service.buildkitd:1234" + command: | + set -ex + + echo "--- Building and pushing NPU Test Image" + echo "Image: $${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG}" + echo "buildkitd address: $${BUILDKITD_ADDR}" + + if ! command -v buildctl &> /dev/null; then + echo "Installing buildctl..." + mkdir -p /tmp/buildkit + BUILDKIT_VERSION="v0.29.0" + wget -q "https://github.com/moby/buildkit/releases/download/$${BUILDKIT_VERSION}/buildkit-$${BUILDKIT_VERSION}.linux-arm64.tar.gz" -O /tmp/buildkit.tar.gz + tar -xzf /tmp/buildkit.tar.gz -C /tmp/buildkit + cp /tmp/buildkit/bin/buildctl /usr/local/bin/ + fi + + export DOCKER_CONFIG=/home/user/.docker + buildctl \ + --addr="$${BUILDKITD_ADDR}" \ + --tlscacert=/home/user/.docker/ca.pem \ + --tlscert=/home/user/.docker/cert.pem \ + --tlskey=/home/user/.docker/key.pem \ + build \ + --frontend dockerfile.v0 \ + --local context=. \ + --local dockerfile=./docker \ + --opt filename=Dockerfile.npu.ci \ + --secret id=dockerconfig,src=/home/user/.docker/config.json \ + --output type=image,name=$${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG},push=true \ + --progress=plain + + echo "--- Image pushed successfully" + echo "$${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG}" + + - label: ":buildkit: Build and Push NPU Test Image - A3" + key: image-build-a3 + agents: + queue: "ascend-a2b3" + resource_class: "npu-1" + plugins: + - kubernetes: + metadata: + annotations: + vault.hashicorp.com/agent-init-first: "true" + vault.hashicorp.com/agent-inject: "true" + vault.hashicorp.com/agent-inject-perms-ca.pem: "0400" + vault.hashicorp.com/agent-inject-perms-cert.pem: "0400" + vault.hashicorp.com/agent-inject-perms-config.json: "0400" + vault.hashicorp.com/agent-inject-perms-key.pem: "0400" + vault.hashicorp.com/agent-inject-secret-ca.pem: internal/data/ascend/buildkitd + vault.hashicorp.com/agent-inject-secret-cert.pem: internal/data/ascend/buildkitd + vault.hashicorp.com/agent-inject-secret-config.json: internal/data/ascend/buildkitd + vault.hashicorp.com/agent-inject-secret-key.pem: internal/data/ascend/buildkitd + vault.hashicorp.com/agent-inject-template-ca.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.RootCA }}\n{{- end }}" + vault.hashicorp.com/agent-inject-template-cert.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.ClientCaCert }}\n{{- end }}" + vault.hashicorp.com/agent-inject-template-config.json: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.dockerConfig }}\n{{- end }}" + vault.hashicorp.com/agent-inject-template-key.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.ClientCaKey }}\n{{- end }}" + vault.hashicorp.com/agent-pre-populate-only: "true" + vault.hashicorp.com/agent-run-as-group: "1000" + vault.hashicorp.com/agent-run-as-user: "1000" + vault.hashicorp.com/agent-service-account-token-volume-name: token-vol + vault.hashicorp.com/role: ascend-gha-runners + vault.hashicorp.com/secret-volume-path: /home/user/.docker/ + vault.hashicorp.com/tls-skip-verify: "true" + podSpecPatch: + volumes: + - name: token-vol + projected: + defaultMode: 420 + sources: + - serviceAccountToken: + audience: api + expirationSeconds: 600 + path: token + env: + VLLM_IMAGE_TAG: "${BUILDKITE_COMMIT}-a3" + IMAGE_NAME: "vllm-omni-ci-npu" + IMAGE_REGISTRY: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry" + BUILDKITD_ADDR: "tcp://buildkitd-service.buildkitd:1234" + command: | + set -ex + + echo "--- Building and pushing NPU Test Image" + echo "Image: $${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG}" + echo "buildkitd address: $${BUILDKITD_ADDR}" + + if ! command -v buildctl &> /dev/null; then + echo "Installing buildctl..." + mkdir -p /tmp/buildkit + BUILDKIT_VERSION="v0.29.0" + wget -q "https://github.com/moby/buildkit/releases/download/$${BUILDKIT_VERSION}/buildkit-$${BUILDKIT_VERSION}.linux-arm64.tar.gz" -O /tmp/buildkit.tar.gz + tar -xzf /tmp/buildkit.tar.gz -C /tmp/buildkit + cp /tmp/buildkit/bin/buildctl /usr/local/bin/ + fi + + export DOCKER_CONFIG=/home/user/.docker + buildctl \ + --addr="$${BUILDKITD_ADDR}" \ + --tlscacert=/home/user/.docker/ca.pem \ + --tlscert=/home/user/.docker/cert.pem \ + --tlskey=/home/user/.docker/key.pem \ + build \ + --frontend dockerfile.v0 \ + --local context=. \ + --local dockerfile=./docker \ + --opt filename=Dockerfile.npu.ci.a3 \ + --secret id=dockerconfig,src=/home/user/.docker/config.json \ + --output type=image,name=$${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG},push=true \ + --progress=plain + + echo "--- Image pushed successfully" + echo "$${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG}" + + - label: "Upload Ready Pipeline" + depends_on: + - image-build-a2 + - image-build-a3 + key: upload-ready-pipeline + agents: + queue: "cpu_queue_premerge" + command: | + python3 .buildkite/common/scripts/upload_pipeline.py --upload .buildkite/npu/test-npu-ready.yml + + # L4 Test — main+NIGHTLY=1 (scheduled), or PR with specific label (e.g. add label then Rebuild) + - label: "Upload Nightly Pipeline" + depends_on: + - image-build-a2 + - image-build-a3 + key: upload-nightly-pipeline + commands: + - python3 .buildkite/common/scripts/upload_pipeline.py --upload .buildkite/npu/test-npu-nightly.yml + agents: + queue: "cpu_queue_premerge" diff --git a/.buildkite/npu/pipeline-npu.yml b/.buildkite/npu/pipeline-npu.yml index fd67711f0..237b65593 100644 --- a/.buildkite/npu/pipeline-npu.yml +++ b/.buildkite/npu/pipeline-npu.yml @@ -1,195 +1,9 @@ +# Hook uploads this entry YAML; upload_pipeline.py --upload reads bootstrap-upload-steps.yml +# and injects ``if`` by step ``key`` from skip-ci. steps: - - label: ":buildkit: Build and Push NPU Test Image - A2" - key: image-build - agents: - queue: "ascend-a2b3" - resource_class: "npu-1" - plugins: - - kubernetes: - metadata: - annotations: - vault.hashicorp.com/agent-init-first: "true" - vault.hashicorp.com/agent-inject: "true" - vault.hashicorp.com/agent-inject-perms-ca.pem: "0400" - vault.hashicorp.com/agent-inject-perms-cert.pem: "0400" - vault.hashicorp.com/agent-inject-perms-config.json: "0400" - vault.hashicorp.com/agent-inject-perms-key.pem: "0400" - vault.hashicorp.com/agent-inject-secret-ca.pem: internal/data/ascend/buildkitd - vault.hashicorp.com/agent-inject-secret-cert.pem: internal/data/ascend/buildkitd - vault.hashicorp.com/agent-inject-secret-config.json: internal/data/ascend/buildkitd - vault.hashicorp.com/agent-inject-secret-key.pem: internal/data/ascend/buildkitd - vault.hashicorp.com/agent-inject-template-ca.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.RootCA }}\n{{- end }}" - vault.hashicorp.com/agent-inject-template-cert.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.ClientCaCert }}\n{{- end }}" - vault.hashicorp.com/agent-inject-template-config.json: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.dockerConfig }}\n{{- end }}" - vault.hashicorp.com/agent-inject-template-key.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.ClientCaKey }}\n{{- end }}" - vault.hashicorp.com/agent-pre-populate-only: "true" - vault.hashicorp.com/agent-run-as-group: "1000" - vault.hashicorp.com/agent-run-as-user: "1000" - vault.hashicorp.com/agent-service-account-token-volume-name: token-vol - vault.hashicorp.com/role: ascend-gha-runners - vault.hashicorp.com/secret-volume-path: /home/user/.docker/ - vault.hashicorp.com/tls-skip-verify: "true" - podSpecPatch: - volumes: - - name: token-vol - projected: - defaultMode: 420 - sources: - - serviceAccountToken: - audience: api - expirationSeconds: 600 - path: token - env: - VLLM_IMAGE_TAG: "${BUILDKITE_COMMIT}" - IMAGE_NAME: "vllm-omni-ci-npu" - IMAGE_REGISTRY: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry" - BUILDKITD_ADDR: "tcp://buildkitd-service.buildkitd:1234" - command: | - set -ex - - echo "--- Building and pushing NPU Test Image" - echo "Image: $${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG}" - echo "buildkitd address: $${BUILDKITD_ADDR}" - - if ! command -v buildctl &> /dev/null; then - echo "Installing buildctl..." - mkdir -p /tmp/buildkit - BUILDKIT_VERSION="v0.29.0" - wget -q "https://github.com/moby/buildkit/releases/download/$${BUILDKIT_VERSION}/buildkit-$${BUILDKIT_VERSION}.linux-arm64.tar.gz" -O /tmp/buildkit.tar.gz - tar -xzf /tmp/buildkit.tar.gz -C /tmp/buildkit - cp /tmp/buildkit/bin/buildctl /usr/local/bin/ - fi - - export DOCKER_CONFIG=/home/user/.docker - buildctl \ - --addr="$${BUILDKITD_ADDR}" \ - --tlscacert=/home/user/.docker/ca.pem \ - --tlscert=/home/user/.docker/cert.pem \ - --tlskey=/home/user/.docker/key.pem \ - build \ - --frontend dockerfile.v0 \ - --local context=. \ - --local dockerfile=./docker \ - --opt filename=Dockerfile.npu.ci \ - --secret id=dockerconfig,src=/home/user/.docker/config.json \ - --output type=image,name=$${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG},push=true \ - --progress=plain - - echo "--- Image pushed successfully" - echo "$${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG}" - - - label: ":buildkit: Build and Push NPU Test Image - A3" - key: image-build-a3 - agents: - queue: "ascend-a2b3" - resource_class: "npu-1" - plugins: - - kubernetes: - metadata: - annotations: - vault.hashicorp.com/agent-init-first: "true" - vault.hashicorp.com/agent-inject: "true" - vault.hashicorp.com/agent-inject-perms-ca.pem: "0400" - vault.hashicorp.com/agent-inject-perms-cert.pem: "0400" - vault.hashicorp.com/agent-inject-perms-config.json: "0400" - vault.hashicorp.com/agent-inject-perms-key.pem: "0400" - vault.hashicorp.com/agent-inject-secret-ca.pem: internal/data/ascend/buildkitd - vault.hashicorp.com/agent-inject-secret-cert.pem: internal/data/ascend/buildkitd - vault.hashicorp.com/agent-inject-secret-config.json: internal/data/ascend/buildkitd - vault.hashicorp.com/agent-inject-secret-key.pem: internal/data/ascend/buildkitd - vault.hashicorp.com/agent-inject-template-ca.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.RootCA }}\n{{- end }}" - vault.hashicorp.com/agent-inject-template-cert.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.ClientCaCert }}\n{{- end }}" - vault.hashicorp.com/agent-inject-template-config.json: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.dockerConfig }}\n{{- end }}" - vault.hashicorp.com/agent-inject-template-key.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.ClientCaKey }}\n{{- end }}" - vault.hashicorp.com/agent-pre-populate-only: "true" - vault.hashicorp.com/agent-run-as-group: "1000" - vault.hashicorp.com/agent-run-as-user: "1000" - vault.hashicorp.com/agent-service-account-token-volume-name: token-vol - vault.hashicorp.com/role: ascend-gha-runners - vault.hashicorp.com/secret-volume-path: /home/user/.docker/ - vault.hashicorp.com/tls-skip-verify: "true" - podSpecPatch: - volumes: - - name: token-vol - projected: - defaultMode: 420 - sources: - - serviceAccountToken: - audience: api - expirationSeconds: 600 - path: token - env: - VLLM_IMAGE_TAG: "${BUILDKITE_COMMIT}-a3" - IMAGE_NAME: "vllm-omni-ci-npu" - IMAGE_REGISTRY: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry" - BUILDKITD_ADDR: "tcp://buildkitd-service.buildkitd:1234" - command: | - set -ex - - echo "--- Building and pushing NPU Test Image" - echo "Image: $${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG}" - echo "buildkitd address: $${BUILDKITD_ADDR}" - - if ! command -v buildctl &> /dev/null; then - echo "Installing buildctl..." - mkdir -p /tmp/buildkit - BUILDKIT_VERSION="v0.29.0" - wget -q "https://github.com/moby/buildkit/releases/download/$${BUILDKIT_VERSION}/buildkit-$${BUILDKIT_VERSION}.linux-arm64.tar.gz" -O /tmp/buildkit.tar.gz - tar -xzf /tmp/buildkit.tar.gz -C /tmp/buildkit - cp /tmp/buildkit/bin/buildctl /usr/local/bin/ - fi - - export DOCKER_CONFIG=/home/user/.docker - buildctl \ - --addr="$${BUILDKITD_ADDR}" \ - --tlscacert=/home/user/.docker/ca.pem \ - --tlscert=/home/user/.docker/cert.pem \ - --tlskey=/home/user/.docker/key.pem \ - build \ - --frontend dockerfile.v0 \ - --local context=. \ - --local dockerfile=./docker \ - --opt filename=Dockerfile.npu.ci.a3 \ - --secret id=dockerconfig,src=/home/user/.docker/config.json \ - --output type=image,name=$${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG},push=true \ - --progress=plain - - echo "--- Image pushed successfully" - echo "$${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG}" - - - label: "Upload Ready Pipeline" - depends_on: - - image-build - - image-build-a3 - key: upload-ready-pipeline - if: >- - build.branch != "main" && ( - build.pull_request.labels includes "npu-test" || - build.pull_request.labels includes "ready" - ) - agents: - queue: "ascend-a2b3" - resource_class: "npu-1" - command: | - buildkite-agent pipeline upload .buildkite/npu/test-npu-ready.yml - - # L4 Test — main+NIGHTLY=1 (scheduled), or PR with specific label (e.g. add label then Rebuild) - - label: "Upload Nightly Pipeline" - depends_on: - - image-build - - image-build-a3 - key: upload-nightly-pipeline - if: >- - (build.branch == "main" && build.env("NIGHTLY") == "1") || - (build.branch != "main" && ( - build.pull_request.labels includes "nightly-test" || - build.pull_request.labels includes "omni-test" || - build.pull_request.labels includes "tts-test" || - build.pull_request.labels includes "diffusion-x2iat-test" || - build.pull_request.labels includes "diffusion-x2v-test" - )) + - label: ":github: Resolve skip-ci & upload pipeline" + key: upload-ci-pipeline commands: - - buildkite-agent pipeline upload .buildkite/npu/test-npu-nightly.yml + - "python3 .buildkite/common/scripts/upload_pipeline.py --upload .buildkite/npu/bootstrap-upload-steps.yml" agents: - queue: "ascend-a2b3" - resource_class: "npu-1" + queue: "cpu_queue_premerge" diff --git a/.buildkite/npu/test-npu-nightly.yml b/.buildkite/npu/test-npu-nightly.yml index 4dbda17f9..4050cca8d 100644 --- a/.buildkite/npu/test-npu-nightly.yml +++ b/.buildkite/npu/test-npu-nightly.yml @@ -8,15 +8,7 @@ steps: build.pull_request.labels includes "diffusion-x2v-test" steps: - label: ":full_moon: Diffusion X2V · Function Test" - agents: - queue: "ascend-a2b3" - resource_class: "npu-4" - image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}" - plugins: - - kubernetes: - podSpecPatch: - imagePullSecrets: - - name: swr-secret + mirror_hardwares: a2b3_npu_4 env: VLLM_WORKER_MULTIPROC_METHOD: spawn HF_TOKEN: "${HF_TOKEN}" @@ -28,15 +20,7 @@ steps: if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" || build.pull_request.labels includes "tts-test" steps: - label: ":full_moon: TTS · Perf Test" - agents: - queue: "ascend-a3" - resource_class: "npu-2" - image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}-a3" - plugins: - - kubernetes: - podSpecPatch: - imagePullSecrets: - - name: swr-secret + mirror_hardwares: a3_npu_2 env: VLLM_WORKER_MULTIPROC_METHOD: spawn VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1" diff --git a/.buildkite/npu/test-npu-ready.yml b/.buildkite/npu/test-npu-ready.yml index e1fe20d60..317517398 100644 --- a/.buildkite/npu/test-npu-ready.yml +++ b/.buildkite/npu/test-npu-ready.yml @@ -4,15 +4,7 @@ steps: depends_on: upload-ready-pipeline steps: - label: ":sound: TTS · Qwen3-TTS CustomVoice Test" - agents: - queue: "ascend-a2b3" - resource_class: "npu-1" - image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}" - plugins: - - kubernetes: - podSpecPatch: - imagePullSecrets: - - name: swr-secret + mirror_hardwares: a2b3_npu_1 env: VLLM_WORKER_MULTIPROC_METHOD: spawn VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1" diff --git a/.claude/skills/add-diffusion-model/SKILL.md b/.claude/skills/add-diffusion-model/SKILL.md index 76c894473..e06af9f21 100644 --- a/.claude/skills/add-diffusion-model/SKILL.md +++ b/.claude/skills/add-diffusion-model/SKILL.md @@ -317,7 +317,7 @@ Required updates: ### Step 8: Add E2E Tests -**Follow the [vllm-omni-test skill](../vllm-omni-test/SKILL.md)** for markers, file naming, Buildkite wiring, and run commands. Also read [l4_functionality_tests.inc.md](https://github.com/vllm-project/vllm-omni/blob/main/docs/contributing/ci/test_examples/l4_functionality_tests.inc.md) and [CI_5levels.md](https://github.com/vllm-project/vllm-omni/blob/main/docs/contributing/ci/CI_5levels.md). +**Follow the [vllm-omni-test skill](../vllm-omni-test/SKILL.md)** for markers, file naming, Buildkite wiring, and run commands. Also read [l4_functionality_tests.inc.md](https://github.com/vllm-project/vllm-omni/blob/main/docs/contributing/ci/test_examples/l4_functionality_tests.inc.md), [test_system_overview.md](https://github.com/vllm-project/vllm-omni/blob/main/docs/contributing/ci/test_system_overview.md), and [test_writing_guide.md](https://github.com/vllm-project/vllm-omni/blob/main/docs/contributing/ci/test_writing_guide.md). Classify the model's **CI priority** first: diff --git a/.claude/skills/add-tts-model/SKILL.md b/.claude/skills/add-tts-model/SKILL.md index 260a7e924..dc423825a 100644 --- a/.claude/skills/add-tts-model/SKILL.md +++ b/.claude/skills/add-tts-model/SKILL.md @@ -307,7 +307,7 @@ def build_voice_clone_prompt(ref_audio_path: str, text: str, codec) -> list: ### Test Case Writing (CI Levels) -**Follow the [vllm-omni-test skill](../vllm-omni-test/SKILL.md)** for markers, file naming (`test_{slug}.py` / `test_{slug}_expansion.py`), Buildkite wiring, and copy-paste run commands. Also read [CI_5levels.md](https://github.com/vllm-project/vllm-omni/blob/main/docs/contributing/ci/CI_5levels.md) and [tests_style.md](https://github.com/vllm-project/vllm-omni/blob/main/docs/contributing/ci/tests_style.md). +**Follow the [vllm-omni-test skill](../vllm-omni-test/SKILL.md)** for markers, file naming (`test_{slug}.py` / `test_{slug}_expansion.py`), Buildkite wiring, and copy-paste run commands. Also read [test_system_overview.md](https://github.com/vllm-project/vllm-omni/blob/main/docs/contributing/ci/test_system_overview.md) and [test_writing_guide.md](https://github.com/vllm-project/vllm-omni/blob/main/docs/contributing/ci/test_writing_guide.md). Classify the model's **CI priority** first (high / medium / low). High-priority TTS models are typically those on the integration hot path or listed in tracking issues such as [#1832](https://github.com/vllm-project/vllm-omni/issues/1832); medium and low tiers cover the long tail. When unsure, ask the reviewer which tier applies. diff --git a/.claude/skills/vllm-omni-test/SKILL.md b/.claude/skills/vllm-omni-test/SKILL.md index db5068a50..d5102b3f2 100644 --- a/.claude/skills/vllm-omni-test/SKILL.md +++ b/.claude/skills/vllm-omni-test/SKILL.md @@ -9,7 +9,7 @@ description: Generate and run tests for vllm-project/vllm-omni with CI-aligned l Use this skill to generate minimal, stable test cases and run them with the correct marker/level strategy for [vllm-project/vllm-omni](https://github.com/vllm-project/vllm-omni). -**Link convention:** Paths such as `.buildkite/` and `docs/contributing/` live at the **vllm-omni repo root**. Markdown links use repo-relative paths from this skill file (e.g. `../../../.buildkite/cuda/test-ready.yml`, `../../../docs/contributing/ci/CI_5levels.md`). +**Link convention:** Paths such as `.buildkite/` and `docs/contributing/` live at the **vllm-omni repo root**. Markdown links use repo-relative paths from this skill file (e.g. `../../../.buildkite/cuda/test-ready.yml`, `../../../docs/contributing/ci/test_system_overview.md`). Default priorities: @@ -559,7 +559,7 @@ L1 tests under `tests/entrypoints/` may keep **minimal** asserts next to the han #### Omni Test Writing Guidance (L1-L4 Layering) -When the goal is a general Omni/multimodal test case, prioritize mapping the test to the correct purpose, directory, and resource assumptions aligned with the layers (see [CI_5levels.md](../../../docs/contributing/ci/CI_5levels.md)): +When the goal is a general Omni/multimodal test case, prioritize mapping the test to the correct purpose, directory, and resource assumptions aligned with the layers (see [test_system_overview.md](../../../docs/contributing/ci/test_system_overview.md) and [test_writing_guide.md](../../../docs/contributing/ci/test_writing_guide.md)): - **L1**: Unit/logic validation on CPU (`core_model and cpu`). Cover input validation, branches, and exception paths (`tests//test_*.py`). **Mock with `mocker` / `monkeypatch` only — not `unittest.mock`.** - **L2**: Basic e2e (online/offline basic scenarios). Prefer dummy/lightweight models to validate the end-to-end request-to-output-structure/streaming chain (typically `tests/e2e/online_serving/` and `tests/e2e/offline_inference/`). diff --git a/.claude/skills/vllm-omni-test/references/test-routing.md b/.claude/skills/vllm-omni-test/references/test-routing.md index 83b9b2007..394046b91 100644 --- a/.claude/skills/vllm-omni-test/references/test-routing.md +++ b/.claude/skills/vllm-omni-test/references/test-routing.md @@ -2,7 +2,7 @@ Use this reference to map testing goals to levels, markers, and runnable commands. -**Repo paths** (`.buildkite/`, `docs/contributing/…`): link with repo-relative paths from this file (e.g. `../../../../.buildkite/cuda/test-ready.yml`, `../../../../docs/contributing/ci/CI_5levels.md`). +**Repo paths** (`.buildkite/`, `docs/contributing/…`): link with repo-relative paths from this file (e.g. `../../../../.buildkite/cuda/test-ready.yml`, `../../../../docs/contributing/ci/test_system_overview.md`). ## Model-centric e2e filename convention (L2–L4) diff --git a/docs/contributing/README.md b/docs/contributing/README.md index 63a62eabf..4c05fe0f2 100644 --- a/docs/contributing/README.md +++ b/docs/contributing/README.md @@ -71,7 +71,7 @@ For additional features and advanced configurations, refer to the: ### Testing vLLM-Omni uses `pytest` to test the codebase. -Please refer to the [test instructions](./ci/test_guide.md) for detailed testing information. +Please refer to the [test instructions](./ci/test_execution_guide.md) for detailed testing information. !!! warning Currently, not all unit tests pass when run on CPU platforms. If you don't have access to a GPU platform to run unit tests locally, rely on the continuous integration system to run the tests for now. @@ -127,11 +127,11 @@ The skill offers two modes: The precheck covers five PR types: Bug Fix, Performance, New Model, Diffusion Model, and General. Each type has a tailored checklist that validates evidence quality (repro steps, A/B benchmarks, registry entries, etc.). See the [precheck-pr skill](https://github.com/vllm-project/vllm-omni/blob/main/.claude/skills/precheck-pr/SKILL.md) for the full checklist. ### Local Test -Please run the L1 and L2 test cases locally first and attach the results before contacting us to add the "ready" label. Please refer to the [test instructions](./ci/test_guide.md) for running the test cases. +Please run the L1 and L2 test cases locally first and attach the results before contacting us to add the "ready" label. Please refer to the [test instructions](./ci/test_execution_guide.md) for running the test cases. ### Automatic skip-ci (docs and pytest skip marks) -On pull requests and `main` pushes, the bootstrap step in [`.buildkite/cuda/pipeline.yml`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/cuda/pipeline.yml) runs [`.buildkite/cuda/scripts/upload_pipeline.py`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/cuda/scripts/upload_pipeline.py) against the git diff. When every changed file qualifies, **L2 (`ready`) and L3 (`merge-test`) pipelines are not uploaded**, so the default GPU CI jobs are skipped. +On pull requests and `main` pushes, the bootstrap step in [`.buildkite/cuda/pipeline.yml`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/cuda/pipeline.yml) runs [`.buildkite/common/scripts/upload_pipeline.py`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/common/scripts/upload_pipeline.py) against the git diff. When every changed file qualifies, **L2 (`ready`) and L3 (`merge-test`) pipelines are not uploaded**, so the default GPU CI jobs are skipped. | Change per file | Examples | | --- | --- | @@ -148,7 +148,7 @@ These PR shapes all trigger skip-ci: Skip-ci does **not** apply when the diff also touches product code (for example `vllm_omni/`), or when test files change assertions, imports, fixtures, or other non-skip logic. If the diff cannot be resolved (non-PR branches outside `main`), CI runs as usual. !!! note - Skipping L2/L3 does **not** disable the Docker image build step. Nightly (L4) upload can still run when the PR has a `nightly-test` label or on scheduled `main` builds with `NIGHTLY=1`. See [CI levels](./ci/CI_5levels.md) for how bootstrap skip-ci relates to diff-gated E2E jobs. + Skipping L2/L3 does **not** disable the Docker image build step. Nightly (L4) upload can still run when the PR has a `nightly-test` label or on scheduled `main` builds with `NIGHTLY=1`. Bootstrap child steps live in `bootstrap-upload-steps.yml`; the hook entry step runs `upload_pipeline.py --upload /bootstrap-upload-steps.yml`, which injects `if` by step `key` from skip-ci before upload. See [CI Settings — Diff-aware CI](./ci/ci_settings.md#diff-aware-ci) and [Test System Overview](./ci/test_system_overview.md). ### Code Quality diff --git a/docs/contributing/ci/.nav.yaml b/docs/contributing/ci/.nav.yaml index 0f187f3a1..867bcc330 100644 --- a/docs/contributing/ci/.nav.yaml +++ b/docs/contributing/ci/.nav.yaml @@ -1,6 +1,6 @@ nav: - - CI_5levels.md + - test_system_overview.md + - ci_settings.md + - test_writing_guide.md - failures.md - - test_guide.md - - test_markers.md - - test_style.md + - test_execution_guide.md diff --git a/docs/contributing/ci/ci_settings.md b/docs/contributing/ci/ci_settings.md new file mode 100644 index 000000000..ff4b8e107 --- /dev/null +++ b/docs/contributing/ci/ci_settings.md @@ -0,0 +1,344 @@ +# CI Settings + +This document describes **where** Buildkite YAML lives in the repo, **how each platform organizes CI**, and **how to add a new job**. It does not document agent queues, GPU types, or container plugin details—those belong in infra / preset files (for example `.buildkite/common/ci_mirror_hardwares.yml`). + +For CI levels (L1–L5) and triggers, see [Test System Overview](./test_system_overview.md). For test authoring, see [Test Writing Guide](./test_writing_guide.md). For running tests locally or replaying CI jobs, see [Test Execution Guide](./test_execution_guide.md). + +## Directory layout + +Canonical layout (prefer these paths for new changes): + +``` +.buildkite/ +├── common/ # Shared across platforms +│ ├── scripts/ +│ │ ├── skip_ci.py # skip-ci decision (docs / skip-mark / CI YAML paths) +│ │ ├── upload_pipeline.py # Bootstrap + test-pipeline uploader (CUDA/NPU) +│ │ └── resolve_skip_ci.sh # Shell helpers for AMD/Intel bootstrap +│ └── ci_mirror_hardwares.yml # CUDA uploader presets (referenced by name only) +├── cuda/ # Primary NVIDIA CUDA CI +│ ├── pipeline.yml # Bootstrap entry (hook upload) +│ ├── bootstrap-upload-steps.yml # Bootstrap child steps (upload_pipeline --upload) +│ ├── test-ready.yml # L2 +│ ├── test-merge.yml # L3 +│ ├── test-nightly.yml # L4 +│ ├── test-weekly.yml # L5 +│ └── rebase-pipeline.yml +├── npu/ +│ ├── pipeline-npu.yml # Bootstrap entry (hook upload) +│ ├── bootstrap-upload-steps.yml # Bootstrap child steps (upload_pipeline --upload) +│ ├── pipeline-npu-a3.yml # A3 variant (when used) +│ ├── test-npu-ready.yml # L2 +│ ├── test-npu-nightly.yml # L4 +│ └── scripts/ +├── amd/ +│ ├── test-amd-ready.yml # L2 job definitions (template input) +│ ├── test-amd-merge.yml # L3 job definitions +│ ├── test-template-amd-omni.j2 # Renders final pipeline.yaml +│ └── scripts/ +│ ├── bootstrap-amd-omni.sh # Entry: skip-ci → Jinja → upload +│ └── run-amd-test.sh # Wraps pytest inside ROCm docker +├── intel/ +│ ├── pipeline-intel.yml # Static Intel XPU pipeline +│ └── scripts/ +│ ├── bootstrap-intel-omni.sh +│ └── run-xpu-test.sh +└── release/ + ├── release-pipeline.yml + └── scripts/ +``` + +**Placement rules** + +| Rule | Detail | +| ---- | ------ | +| Platform code under platform dir | New CUDA jobs go in `.buildkite/cuda/`; do not add new top-level `.buildkite/test-*.yml` files. | +| Shared logic in `common/` | Skip-ci and CUDA upload rendering stay in `.buildkite/common/scripts/`. | +| Bootstrap vs test YAML | **Bootstrap** (`pipeline*.yml`) builds images and uploads **child** test pipelines. **Test** YAML (`test-*.yml`) lists pytest steps only. | +| Register CI YAML in `skip_ci.py` | If you add a new whitelisted test pipeline file, update `L2_YAML_FILES`, `L3_YAML_FILES`, or `L45_YAML_FILES` in `.buildkite/common/scripts/skip_ci.py` so skip-ci paths stay correct. | + +There are still **legacy copies** at `.buildkite/*.yaml` (without the `cuda/` prefix). Treat `.buildkite/cuda/*` as source of truth. + +## Platform comparison + +| Platform | Bootstrap entry | Test job files | Upload mechanism | Job hardware in YAML | +| -------- | ---------------- | -------------- | ---------------- | -------------------- | +| **CUDA** | `cuda/pipeline.yml` | `test-ready.yml`, `test-merge.yml`, `test-nightly.yml`, `test-weekly.yml` | `upload_pipeline.py --upload` (expands uploader-only keys) | `mirror_hardwares: ` (string) | +| **NPU** | `npu/pipeline-npu.yml` | `test-npu-ready.yml`, `test-npu-nightly.yml` | `upload_pipeline.py --upload` | `mirror_hardwares: a2b3_npu_1` / `a2b3_npu_4` / `a3_npu_2` | +| **AMD** | `amd/scripts/bootstrap-amd-omni.sh` | `test-amd-ready.yml`, `test-amd-merge.yml` | Jinja (`test-template-amd-omni.j2`) → `pipeline upload` | `agent_pool` + `mirror_hardwares: [amdproduction]` (array, template filter) | +| **Intel** | `intel/scripts/bootstrap-intel-omni.sh` | `intel/pipeline-intel.yml` (steps inline) | Direct `pipeline upload` | Inline `agents.queue` on each step | + +## Platform configuration style + +=== "CUDA" + + **Bootstrap:** [`cuda/pipeline.yml`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/cuda/pipeline.yml) (hook upload) + [`cuda/bootstrap-upload-steps.yml`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/cuda/bootstrap-upload-steps.yml) (child steps). Document 1 runs [`upload_pipeline.py --upload .buildkite/cuda/bootstrap-upload-steps.yml`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/common/scripts/upload_pipeline.py), which injects `if` by step `key` from skip-ci and uploads image build plus L2–L5 child pipeline upload steps (see [Diff-aware CI — Bootstrap skip](#bootstrap-skip)). + + **Test YAML:** `cuda/test-ready.yml` (L2), `test-merge.yml` (L3), `test-nightly.yml` (L4), `test-weekly.yml` (L5). Each file starts with shared `env:` then `steps:`. + + | CI level | File | Typical trigger | + | -------- | ---- | ----------------- | + | L2 | `cuda/test-ready.yml` | `ready` label | + | L3 | `cuda/test-merge.yml` | `merge-test` label / main merge | + | L4 | `cuda/test-nightly.yml` | `nightly-test` label or `NIGHTLY=1` | + | L5 | `cuda/test-weekly.yml` | `weekly-test` label or `WEEKLY=1` | + + **Upload:** `upload_pipeline.py --upload` expands uploader-only keys before Buildkite upload. + + **Hardware in YAML:** `mirror_hardwares: ` (string)—preset names in [`common/ci_mirror_hardwares.yml`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/common/ci_mirror_hardwares.yml). Do **not** set `agents` / `plugins` on the same step. + + **Conventions** + + - **`depends_on`:** leaf jobs depend on `upload-ready-pipeline`, `upload-merge-pipeline`, etc. + - **`group` / `label`:** `:card_index_dividers:` groups; labels like `Diffusion · Qwen Image Test`. + - **`commands`:** `timeout … pytest …` with markers and `--run-level` for the pipeline level. + - **`source_file_dependencies`:** required on **E2E Test** leaf jobs in L2/L3; see [Step filtering](#step-filtering). + + **Adding a job** + + 1. Pick the level file (ready / merge / nightly / weekly). + 2. Add a step under the right **group** (usually **E2E Test** for model pytest). + 3. Set `label`, `commands`, `mirror_hardwares`, `depends_on: upload--pipeline`. + 4. For L2/L3 E2E, add `source_file_dependencies` (pytest + model + deploy YAML prefixes). + 5. Dry-run: + + ```bash + python3 .buildkite/common/scripts/upload_pipeline.py .buildkite/cuda/test-ready.yml + ``` + +=== "NPU" + + **Bootstrap:** [`npu/pipeline-npu.yml`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/npu/pipeline-npu.yml) + [`npu/bootstrap-upload-steps.yml`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/npu/bootstrap-upload-steps.yml)—same split as CUDA; builds A2/B3 and A3 CI images, then uploads child test pipelines. + + **Test YAML:** `npu/test-npu-ready.yml` (L2), `test-npu-nightly.yml` (L4). + + **Upload:** same as CUDA—`upload_pipeline.py --upload`. + + **Hardware in YAML:** `mirror_hardwares` preset (string), expanded to `agents`, top-level `image`, and `plugins`. Presets: `a2b3_npu_1`, `a2b3_npu_4`, `a3_npu_2` in `common/ci_mirror_hardwares.yml`. + + **Conventions** + + - **`depends_on: upload-ready-pipeline`** (or `upload-nightly-pipeline`) ties jobs to bootstrap upload keys. + - Do not duplicate `agents` / `image` / `plugins` when using `mirror_hardwares`. + + **Adding a job** + + 1. Edit `test-npu-ready.yml` (L2) or `test-npu-nightly.yml` (L4). + 2. Add a step with `mirror_hardwares` (add a new preset in `ci_mirror_hardwares.yml` first if needed). + 3. Set `commands` to your pytest file and markers. + 4. Dry-run: + + ```bash + python3 .buildkite/common/scripts/upload_pipeline.py .buildkite/npu/test-npu-ready.yml + ``` + + 5. Register new pipeline paths in `skip_ci.py` (`L2_YAML_FILES` or `L45_YAML_FILES`) when applicable. + +=== "AMD" + + **Bootstrap:** [`amd/scripts/bootstrap-amd-omni.sh`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/amd/scripts/bootstrap-amd-omni.sh)—skip-ci, diff filtering, Jinja render, then `buildkite-agent pipeline upload`. + + **Test YAML (data):** `amd/test-amd-ready.yml` (L2 / PR), `test-amd-merge.yml` (L3 / main). + + **Rendering:** [`test-template-amd-omni.j2`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/amd/test-template-amd-omni.j2) wraps data steps with `amd-build` image build and `amd_` queues. Do **not** hand-edit generated `pipeline.yaml`. + + **Hardware in YAML:** `agent_pool` (for example `mi325_1`) plus `mirror_hardwares: [amdproduction]` (array—Buildkite template filter, not the CUDA/NPU uploader preset mechanism). + + **Data file fields** + + | Field | Purpose | + | ----- | ------- | + | `agent_pool` | ROCm pool; template maps to `queue: amd_`. | + | `mirror_hardwares` | Which mirror HW runs the step (for example `[amdproduction]`). | + | `commands` | Passed into `run-amd-test.sh` via `TEST_COMMAND`. | + + **Adding a job** + + 1. Edit `test-amd-ready.yml` or `test-amd-merge.yml`. + 2. Copy a neighboring block: `label`, `agent_pool`, `mirror_hardwares`, `commands`, optional `grade`. + 3. Regenerate via bootstrap / Jinja; update `skip_ci.py` if you add a new YAML path. + +=== "Intel" + + **Bootstrap:** [`intel/scripts/bootstrap-intel-omni.sh`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/intel/scripts/bootstrap-intel-omni.sh)—skip-ci, then direct `pipeline upload`. + + **Test YAML:** steps live inline in [`intel/pipeline-intel.yml`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/intel/pipeline-intel.yml); runners under `intel/scripts/` (for example `run-xpu-test.sh`). + + **Upload:** `buildkite-agent pipeline upload` (no `upload_pipeline.py` mirror expansion). + + **Hardware in YAML:** inline `agents.queue` (for example `intel-gpu-omni`) on each step. + + **Adding a job** + + 1. Add a step to `pipeline-intel.yml`, or extend an existing runner script. + 2. Match `agents`, `env`, `timeout_in_minutes`, and `command`/`commands` with sibling steps. + 3. Keep `pipeline-intel.yml` listed in `L2_YAML_FILES` in `skip_ci.py`. + +## Cross-cutting conventions + +### Diff-aware CI {#diff-aware-ci} + +PR diffs drive **two independent skip layers**. Both read changed files from git, but at different pipeline stages and with different granularity: + +| Layer | Script | When | What is skipped | Where you configure | +| ----- | ------ | ---- | --------------- | ------------------- | +| **Bootstrap** | [`skip_ci.py`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/common/scripts/skip_ci.py) + [`upload_pipeline.py`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/common/scripts/upload_pipeline.py) | Before child test pipelines upload (`cuda/pipeline.yml`, `npu/pipeline-npu.yml`, AMD/Intel bootstraps) | Entire default CI, or whole L2/L3 upload for a platform | Whitelists in `skip_ci.py`; bootstrap `if` injected by step `key` in `upload_pipeline.py` | +| **Step filter** | [`upload_pipeline.py`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/common/scripts/upload_pipeline.py) | While uploading CUDA L2/L3 YAML | Individual Buildkite steps inside `test-ready.yml` / `test-merge.yml` | `source_file_dependencies` on each step or group | + +**Changed files** (both layers): + +| Build context | Diff command | +| --- | --- | +| Pull request | `git diff --name-only origin/...` | +| `main` push | `git diff --name-only ^..` | +| Local dry-run / non-PR | Diff unavailable → bootstrap and step filter both keep all steps; uploader-only keys are still stripped | + +Label triggers (`ready`, `merge-test`) are unchanged—diff-aware logic only reduces what runs **after** a pipeline is already scheduled. + +#### Bootstrap skip {#bootstrap-skip} + +[`skip_ci.py`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/common/scripts/skip_ci.py) classifies the git diff into buckets (docs / qualifying skip-mark / whitelisted L2·L3·L4/L5 YAML / other), then picks one decision path. CUDA/NPU apply that decision by injecting Buildkite `if` on `bootstrap-upload-steps.yml` step keys; AMD/Intel call `gate_bootstrap_ci `. + +##### Decision overview + +| Diff shape | Path | Default L2/L3 | +| --- | --- | --- | +| Product code / non-whitelisted paths | normal CI | all on | +| Docs and/or qualifying skip-mark only | `skip_all` | all off (scheduled `main` NIGHTLY/WEEKLY exceptions below) | +| Whitelisted CI YAML only | yaml-gated (`skip_l2_l3`) | per platform/level matrix | +| Docs/skip-mark **+** whitelisted CI YAML | yaml-gated (`skip_l2_l3`) | same as CI-YAML-only (does **not** widen to normal CI) | +| Non-qualifying skip-mark and no CI YAML | normal CI | all on | +| Diff unavailable | normal CI | all on | + +**`skip_all` exceptions (CUDA/NPU bootstrap only):** PR labels (`nightly-test`, `merge-test`, `npu-test`, `weekly-test`, …) do **not** revive jobs. On `main`, scheduled `NIGHTLY=1` still builds the image and uploads L4 plus L2/L3 (with `--e2e`); `WEEKLY=1` still builds the image and uploads L5 (CUDA). + +**Yaml-gated nightly/weekly:** L4/L5 upload steps keep their normal label / `NIGHTLY` / `WEEKLY` conditions (for example PR `nightly-test` still uploads nightly). Only L2/L3 upload steps are matrix-gated. + +##### Whitelisted CI YAML → platform + +Register new files in `L2_YAML_FILES`, `L3_YAML_FILES`, or `L45_YAML_FILES` in `skip_ci.py`. + +| Level | File | Platform | +| --- | --- | --- | +| L2 | `.buildkite/cuda/test-ready.yml` | cuda | +| L2 | `.buildkite/npu/test-npu-ready.yml` | npu | +| L2 | `.buildkite/amd/test-amd-ready.yml` | amd | +| L2 | `.buildkite/intel/pipeline-intel.yml` | intel | +| L3 | `.buildkite/cuda/test-merge.yml` | cuda | +| L3 | `.buildkite/amd/test-amd-merge.yml` | amd | +| L4/L5 | `.buildkite/cuda/test-nightly.yml`, `test-weekly.yml` | cuda | +| L4/L5 | `.buildkite/npu/test-npu-nightly.yml` | npu | + +##### Yaml-gated category branches + +`L2?` / `L3?` / `L45?` = whether any file in that whitelist changed. Untouched platforms/levels end up off. + +| ID | L2? | L3? | L45? | Enabled L2/L3 | +| --- | --- | --- | --- | --- | +| G1 | | | ✓ | none (all L2/L3 off) | +| G2 | ✓ | | | L2 only on platforms whose L2 YAML changed | +| G3 | | ✓ | | L3 only on platforms whose L3 YAML changed | +| G4 | ✓ | ✓ | | union of G2 and G3 (same platform ready+merge → both on) | +| G5 | ✓ | | ✓ | same as G2 (L45 rescued by L2 YAML) | +| G6 | | ✓ | ✓ | same as G3 (L45 rescued by L3 YAML) | +| G7 | ✓ | ✓ | ✓ | same as G4 | + +##### Yaml-gated examples (platform-level) + +Docs/skip-mark mixed with any row below follows the same matrix. + +| Diff (whitelist) | Branch | Enabled L2/L3 | +| --- | --- | --- | +| Only `cuda/test-ready.yml` | G2 | `cuda/l2` | +| Only `amd/test-amd-ready.yml` | G2 | `amd/l2` | +| Only `npu/test-npu-ready.yml` | G2 | `npu/l2` | +| Only `intel/pipeline-intel.yml` | G2 | `intel/l2` | +| Only `cuda/test-merge.yml` | G3 | `cuda/l3` | +| Only `amd/test-amd-merge.yml` | G3 | `amd/l3` | +| CUDA ready + CUDA merge | G4 | `cuda/l2` + `cuda/l3` | +| CUDA ready + AMD merge | G4 | `cuda/l2` + `amd/l3` | +| CUDA ready + AMD ready | G2 | `cuda/l2` + `amd/l2` | +| CUDA merge + AMD merge | G3 | `cuda/l3` + `amd/l3` | +| Only nightly / weekly / npu-nightly YAML | G1 | (none) | +| CUDA ready + nightly | G5 | `cuda/l2` | +| CUDA merge + nightly | G6 | `cuda/l3` | +| CUDA ready + merge + nightly | G7 | `cuda/l2` + `cuda/l3` | + +##### How platforms consume the decision + +| Platform | Mechanism | +| --- | --- | +| **CUDA** | `upload_pipeline.py` injects `if` by step `key` (`image-build`, `upload-ready-pipeline`, `upload-merge-pipeline`, …) from `is_run(cuda, l2/l3)` | +| **NPU** | Same injection; no merge upload step (L3 always off in bootstrap) | +| **AMD / Intel** | `gate_bootstrap_ci ` exits 0 when `skip_all` or that target is off | + +Unit coverage: `tests/buildkite/test_skip_ci.py`. + +#### Step filtering {#step-filtering} + +CUDA **L2** (`.buildkite/cuda/test-ready.yml`) and **L3** (`.buildkite/cuda/test-merge.yml`) only. Bootstrap upload entry: `upload_pipeline.py --upload .buildkite/cuda/bootstrap-upload-steps.yml`. + +**Uploader-only keys** — removed before Buildkite sees the YAML; never used at runtime on agents: + +| Key | Purpose | +| --- | ------- | +| `source_file_dependencies` | List of path **prefixes**. If any changed file equals a prefix or starts with `prefix/`, keep the step (or group); otherwise omit it. | +| `mirror_hardwares` | Expand to `agents` + `plugins` (+ optional `image`) from `ci_mirror_hardwares.yml`. | + +**Policy** + +- **Always uploaded** (no key): groups outside **E2E Test**—Simple Test, Diffusion unit tests, Engine/Model Executor, Distributed, Custom Pipeline, Entrypoints (L2), LoRA / Entrypoints (L3). +- **Diff-gated**: every **E2E Test** leaf job. List the smallest prefix set per step—pytest file(s), model code under `vllm_omni/model_executor/models/` or `vllm_omni/diffusion/models/`, plus `stage_input_processors/` and `vllm_omni/deploy/*.yaml` when applicable. A **group** may define the key instead; the whole group drops if no prefix matches. + +**YAML examples** + +```yaml + - label: "Diffusion · Qwen Image Test" + source_file_dependencies: + - tests/e2e/online_serving/test_qwen_image.py + - vllm_omni/diffusion/models/qwen_image/ + commands: + - pytest -s -v tests/e2e/online_serving/test_qwen_image.py -m 'core_model' ... + mirror_hardwares: h100_1 + + - label: "TTS · Qwen3-TTS CustomVoice Test" + source_file_dependencies: + - tests/e2e/online_serving/test_qwen3_tts_customvoice.py + - vllm_omni/model_executor/models/qwen3_tts/ + - vllm_omni/model_executor/stage_input_processors/qwen3_tts.py + - vllm_omni/deploy/qwen3_tts.yaml + commands: + - pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py ... + mirror_hardwares: l4_4 +``` + +**Local dry-run** + +```bash +python3 .buildkite/common/scripts/upload_pipeline.py .buildkite/cuda/test-ready.yml +python3 .buildkite/common/scripts/upload_pipeline.py .buildkite/cuda/test-merge.yml | grep source_file_dependencies +# (no output expected) +``` + +On PR builds, `upload_pipeline.py` logs `skip '…' (no changes under …)` for omitted steps. + +### Labels and grouping + +- Use **groups** for dashboard readability; keep **E2E Test** as the group name CUDA diff filtering expects for `--e2e` nightly runs on main. +- Prefix labels by model domain: **Omni ·**, **TTS ·**, **Diffusion ·**, **Simple ·**, etc., matching existing steps. + +### Validation checklist + +| Check | Command / location | +| ----- | ------------------ | +| CUDA render | `python3 .buildkite/common/scripts/upload_pipeline.py .buildkite/cuda/test-.yml` | +| No leaked uploader keys | `… \| grep -E 'mirror_hardwares|source_file_dependencies'` → empty | +| Skip-ci / upload unit tests | `pytest tests/buildkite/` | +| Local job replay (CUDA L2+) | `tools/run_ready_jobs.sh`, `tools/run_merge_jobs.sh`, `tools/nightly/run_nightly_jobs.sh` (read YAML from `cuda/`) | + +## Related documentation + +- [Test System Overview](./test_system_overview.md) — L1–L5 scope, triggers, test pyramid +- [Test Writing Guide](./test_writing_guide.md) — markers, directories, L1–L5 examples +- [Test Execution Guide](./test_execution_guide.md) — running CI-aligned jobs locally +- Implementation: `.buildkite/common/scripts/upload_pipeline.py`, `.buildkite/common/scripts/skip_ci.py` diff --git a/docs/contributing/ci/failures.md b/docs/contributing/ci/failures.md index 80699e6a5..9743097cf 100644 --- a/docs/contributing/ci/failures.md +++ b/docs/contributing/ci/failures.md @@ -79,7 +79,7 @@ For resilience behavior under injected process signals/OOM pressure (validation ### Running Tests Locally -For instructions on running tests locally (including specific test files, functions, and markers), see the [Running Tests](https://docs.vllm.ai/projects/vllm-omni/en/latest/contributing/ci/test_guide/#running-tests) section in the Test Guide. +For instructions on running tests locally (including specific test files, functions, and markers), see the [Running Tests](https://docs.vllm.ai/projects/vllm-omni/en/latest/contributing/ci/test_execution_guide/#running-tests) section in the Test Execution guide. ## Step 4: Raise an Issue or Fix It diff --git a/docs/contributing/ci/test_examples/l4_functionality_tests.inc.md b/docs/contributing/ci/test_examples/l4_functionality_tests.inc.md index 688933f13..d390a603a 100644 --- a/docs/contributing/ci/test_examples/l4_functionality_tests.inc.md +++ b/docs/contributing/ci/test_examples/l4_functionality_tests.inc.md @@ -37,7 +37,7 @@ Currently all the features are available in online serving mode. Hence, only nee **Code Style** - Validation: test that the multimodal output files of your model have the correct shapes. `OpenAIClientHandler.send_diffusion_request` should have taken care of this. -- Test marks: always add `full_model` and `diffusion` for L4 nightly `test_*_expansion.py` cases. Add GPU-related marks if needed. Ref: [Markers for Tests](https://docs.vllm.ai/projects/vllm-omni/en/latest/contributing/ci/tests_markers/). +- Test marks: always add `full_model` and `diffusion` for L4 nightly `test_*_expansion.py` cases. Add GPU-related marks if needed. Ref: [Markers for Tests](https://docs.vllm.ai/projects/vllm-omni/en/latest/contributing/ci/test_writing_guide/#markers-for-tests). - To maximize code reuse, you may refer to - `tests/conftest.py` for `omni_server` (running server in subprocess) and `openai_client` fixtures (sending requests and validating output), `generate_synthetic_image` and `assert_XXX_valid` helper. - `tests/helpers/mark.py` for `@hardware_test(...)` and `hardware_marks`. diff --git a/docs/contributing/ci/test_examples/l4_performance_tests.inc.md b/docs/contributing/ci/test_examples/l4_performance_tests.inc.md index 5c59ba499..9bdcd29fe 100644 --- a/docs/contributing/ci/test_examples/l4_performance_tests.inc.md +++ b/docs/contributing/ci/test_examples/l4_performance_tests.inc.md @@ -148,7 +148,7 @@ pytest -sv tests/dfx/perf/scripts/run_diffusion_benchmark.py --assert-baseline \ --test-config-file tests/dfx/perf/tests/test_qwen_image_vllm_omni.json ``` -See also [Markers for Tests](./tests_markers.md) for registered hardware markers (`H100`, `L4`, `cuda`, `distributed_cuda`, …). +See also [Markers for Tests](#markers-for-tests) for registered hardware markers (`H100`, `L4`, `cuda`, `distributed_cuda`, …). **`server_params` Configuration** diff --git a/docs/contributing/ci/test_guide.md b/docs/contributing/ci/test_execution_guide.md similarity index 97% rename from docs/contributing/ci/test_guide.md rename to docs/contributing/ci/test_execution_guide.md index d1179e832..d1c2c560f 100644 --- a/docs/contributing/ci/test_guide.md +++ b/docs/contributing/ci/test_execution_guide.md @@ -1,4 +1,4 @@ -# Test Guide +# Test Execution Guide ## Setting Up the Test Environment ### Creating a Container vLLM-Omni provides an official Docker image for deployment. These images are built upon vLLM Docker images and are available on [Docker Hub](https://hub.docker.com/r/vllm/vllm-omni/tags). The version of vLLM-Omni indicates which vLLM release it is based on. @@ -191,7 +191,7 @@ Failed jobs: 1/2 The latest L5 CI jobs (reliability + invalid-parameter weekly steps) are in [test-weekly.yml](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/cuda/test-weekly.yml). -You can find more information about markers in the documentation: [marker doc](./tests_markers.md) +You can find more information about markers in [Test Writing Guide — Markers for Tests](./test_writing_guide.md#markers-for-tests). ## Adding New Test Cases -Please refer to the [L5 Layering Specification document](./CI_5levels.md). +Please refer to the [Test System Overview](./test_system_overview.md) and [Test Writing Guide](./test_writing_guide.md). diff --git a/docs/contributing/ci/test_system_overview.md b/docs/contributing/ci/test_system_overview.md new file mode 100644 index 000000000..f1d4fb2c2 --- /dev/null +++ b/docs/contributing/ci/test_system_overview.md @@ -0,0 +1,210 @@ +# Multi-Level Automated Testing System Overview + +## Document Overview + +This testing system aims to build a complete, efficient, and well-structured quality assurance framework for the development, integration, and release of model services. It draws on the concept of the test pyramid from modern software engineering, progressively expanding testing activities from basic code logic verification to complex end-to-end (E2E) functionality, performance, accuracy, and even long-term stability validation. + +Through five levels (L1-L5) and common (Common) specifications, the system clarifies the testing objectives, scope, execution frequency, and required resources for different development stages (e.g., each commit, PR merge, daily build, pre-release). This ensures that models meet high standards for functionality, performance, and reliability across various deployment scenarios (online serving and offline inference). + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
LevelScope & FocusModel Coverage StrategyFeature Coverage StrategyInterface Coverage StrategyTagsTime CostTest DirDocFrequencyHardware
CommonContribution Guideline & PR checklist//////PR Checklist//
CI Failure Description//////CI Failures//
L1
(Unit & Logic)
Unit tests for components like entrypoints, models///core_model and cpu<15min/tests/{component_name}/test_xxx + L1 & L2
+ Section 1 L1&L2: Purpose, Test Content, Directory Location, Example +
PR with ready label (also can run locally)CPU
L2
(E2E across models & GPU-required UT)
Online (basic deployment scenarios):
dummy, normal inference function (output format, stream), some instance startup UT
High-priority models + online basic scenarios; request success, non-empty output, format match (no Whisper/accuracy)High-priority features (using random lightweight models)High-priority interfaces (using random lightweight models)core_model and hardware_test(H100, L4, etc.) and omni/tts/diffusion + Model tests:
+ /tests/e2e/online_serving/test_{model_name}.py
+ Feature tests:
+ /tests/{component_name}/test_xxx
+ Interface tests:
+ /tests/entrypoints/test_xxx +
+ L1 & L2
+ L1&L2: Purpose, Test Content, Directory Location, Example +
PR with ready labelGPU
L3
(Important Perf & Integration & Accuracy)
Online & Offline (multiple deployment scenarios):
real model, normal inference function, normal accuracy
High/medium-priority models + key online/offline scenarios; real weights, Whisper/similarity, preset voice gender, basic accuracyMedium-priority features (using random lightweight models)Medium-priority interfaces (using random lightweight models)advanced_model and hardware_test(H100, L4, etc.) and omni/tts/diffusion<30min + Model tests:
+ /tests/e2e/online_serving/test_{model_name}.py
+ /tests/e2e/offline_inference/test_{model_name}.py
+ Feature tests:
+ /tests/{component_name}/test_xxx
+ Interface tests:
+ /tests/entrypoints/test_xxx +
+ L3
+ L3: Purpose, Test Content, Directory Location, Example +
PR Merged (Also run L1&L2 Tests)GPU
L4
(Perf & Integration & Accuracy)
Online: full functional scenarios + performance test + doc test + accuracy testHigh-priority models: function, performance, accuracy, and doc testing
Medium-priority models: function and doc testing
Low-priority features (using real weights)Low-priority interfaces (using real weights)full_model and hardware_test(H100, L4, etc.) and omni/tts/diffusion<3 hour + Model tests:
+ /tests/e2e/online_serving/test_{model_name}_expansion.py
+ Feature tests:
+ /tests/{component_name}/test_xxx
+ Interface tests:
+ /tests/entrypoints/test_xxx
+ Performance:
+ /tests/dfx/perf/tests/test_qwen3_omni_*.json (Omni), test_tts.json (TTS),
+ test_voxcpm2.json, test_higgs_audio_v3.json, and
+ /tests/dfx/perf/tests/test_{diffusion_model}_vllm_omni.json (Diffusion)
+ Doc Test:
+ tests/examples/online_serving/test_{model_name}.py
+ tests/examples/offline_inference/test_{model_name}.py
+ Accuracy Test:
+ /tests/e2e/accuracy/test_{model_name}.py +
+ L4
+ L4: Purpose, Test Content, Directory Location, Example +
NightlyGPU
L5
(Stability & Reliability)
Online: long-term stability test + reliability testLong-term stability and reliability testing for high-priority models
Low-priority models: function and doc testing
/Invalid-parameter validation for high-priority interfacesslow and hardware_test(H100, L4, etc.) and omni/tts/diffusion Depends on reality + Stability:
+ /tests/dfx/stability/tests/test_qwen3_omni.json
+ /tests/dfx/stability/tests/test_wan22.json
+ Reliability:
+ tests/dfx/reliability/test_reliability_{model_key}.py
+ (e.g. test_reliability_qwen3_omni.py, test_reliability_wan22.py, test_reliability_hunyuan_image.py, test_reliability_voxcpm2.py) +
+ L5
+ L5: Purpose, Test Content, Directory Location, Example +
Weekly / Days before ReleaseGPU
+ +For per-level test authoring (directories, markers, examples), see [Test Writing Guide](./test_writing_guide.md). + +## Common Specifications + +Before entering specific testing levels, the project establishes two common specifications aimed at standardizing the development process and quickly locating issues. + +1. ***PR Checklist ([`.github/PULL_REQUEST_TEMPLATE.md`](https://github.com/vllm-project/vllm-omni/blob/main/.github/PULL_REQUEST_TEMPLATE.md))***: This template defines the self-check items that must be completed before submitting a code review (Pull Request). It ensures that each code change meets basic requirements such as code style, dependency updates, and documentation synchronization before entering the automated testing pipeline, serving as the first manual line of defense for quality assurance. +2. ***CI Failure Explanation ([CI Failures](./failures.md))***: This document archives and explains common failure patterns in the Continuous Integration (CI) pipeline, error log interpretation, and preliminary troubleshooting steps. It helps developers and testers quickly diagnose the causes of automated test failures, improving problem-solving efficiency. + +## Notes + +### L2 / L3 diff-aware CI (CUDA) + +On CUDA **L2** and **L3**, **E2E Test** Buildkite jobs may be omitted at pipeline upload when the PR diff does not touch their path prefixes; other groups still always upload. Full two-layer mechanics and YAML examples: [CI Settings — Diff-aware CI](./ci_settings.md#diff-aware-ci) ([step filtering](./ci_settings.md#step-filtering)). + +### Test helper environment variables + +Some shared helpers under `tests/helpers/` honor optional environment variables for local debugging. These are **not** set in CI by default. + +| Variable | Accepted values | Description | +| -------- | --------------- | ----------- | +| `VLLM_OMNI_KEEP_REQUEST_MEDIA` | `1`, `true`, `yes` (case-insensitive) | When enabled, temporary WAV files created by `tests.helpers.media.convert_audio_bytes_to_text` are **not** deleted when the pytest process exits. By default, each call writes a unique file under the system temp directory via `tempfile.mkstemp` and registers `atexit` cleanup. Use this when debugging audio output validation (Whisper transcription, keyword checks, text–audio similarity). The saved path is logged as `audio data is saved: `. | + +Example (Linux / macOS): + +```bash +export VLLM_OMNI_KEEP_REQUEST_MEDIA=1 +pytest -s -v tests/e2e/online_serving/test_qwen3_omni.py -k test_mix_to_text_audio +``` + +Example (Windows PowerShell): + +```powershell +$env:VLLM_OMNI_KEEP_REQUEST_MEDIA = "1" +pytest -s -v tests/e2e/online_serving/test_qwen3_omni.py -k test_mix_to_text_audio +``` + +## Summary + +This multi-level testing system achieves continuous, progressive validation of model service quality by tightly integrating testing activities with the development workflow (commit, review, merge, release). From rapid unit testing to comprehensive end-to-end testing, and further to in-depth performance, stability, and reliability verification, each level has clear objectives, collectively building a robust quality protection net. By following this system, teams can deliver high-quality, highly reliable model services more efficiently. diff --git a/docs/contributing/ci/CI_5levels.md b/docs/contributing/ci/test_writing_guide.md similarity index 70% rename from docs/contributing/ci/CI_5levels.md rename to docs/contributing/ci/test_writing_guide.md index d3c164d9b..131d20b83 100644 --- a/docs/contributing/ci/CI_5levels.md +++ b/docs/contributing/ci/test_writing_guide.md @@ -1,177 +1,9 @@ -# Multi-Level Automated Testing System Documentation +# Test Writing Guide -## Document Overview +This document describes how to author tests for each CI level (L1–L5). For the level matrix, CI triggers, and Buildkite upload behavior, see [Test System Overview](./test_system_overview.md). -This testing system aims to build a complete, efficient, and well-structured quality assurance framework for the development, integration, and release of model services. It draws on the concept of the test pyramid from modern software engineering, progressively expanding testing activities from basic code logic verification to complex end-to-end (E2E) functionality, performance, accuracy, and even long-term stability validation. +## Test Directory Structure -Through five levels (L1-L5) and common (Common) specifications, the system clarifies the testing objectives, scope, execution frequency, and required resources for different development stages (e.g., each commit, PR merge, daily build, pre-release). This ensures that models meet high standards for functionality, performance, and reliability across various deployment scenarios (online serving and offline inference). - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
LevelScope & FocusModel Coverage StrategyFeature Coverage StrategyInterface Coverage StrategyTagsTime CostTest DirDocFrequencyHardware
CommonContribution Guideline & PR checklist//////.github/PULL_REQUEST_TEMPLATE.md Test Style (PR Checklist)//
CI Failure Description////// CI Failures//
L1
(Unit & Logic)
Unit tests for components like entrypoints, models///core_model and cpu<15min/tests/{component_name}/test_xxx - Chapter 1
- Section 1 L1&L2: Purpose, Test Content, Directory Location, Example -
PR with ready label (also can run locally)CPU
L2
(E2E across models & GPU-required UT)
Online (basic deployment scenarios):
dummy, normal inference function (output format, stream), some instance startup UT
High-priority models + online basic scenarios; request success, non-empty output, format match (no Whisper/accuracy)High-priority features (using random lightweight models)High-priority interfaces (using random lightweight models)core_model and hardware_test(H100, L4, etc.) and omni/tts/diffusion - Model tests:
- /tests/e2e/online_serving/test_{model_name}.py
- Feature tests:
- /tests/{component_name}/test_xxx
- Interface tests:
- /tests/entrypoints/test_xxx -
- Chapter 1
- L1&L2: Purpose, Test Content, Directory Location, Example -
PR with ready labelGPU
L3
(Important Perf & Integration & Accuracy)
Online & Offline (multiple deployment scenarios):
real model, normal inference function, normal accuracy
High/medium-priority models + key online/offline scenarios; real weights, Whisper/similarity, preset voice gender, basic accuracyMedium-priority features (using random lightweight models)Medium-priority interfaces (using random lightweight models)advanced_model and hardware_test(H100, L4, etc.) and omni/tts/diffusion<30min - Model tests:
- /tests/e2e/online_serving/test_{model_name}.py
- /tests/e2e/offline_inference/test_{model_name}.py
- Feature tests:
- /tests/{component_name}/test_xxx
- Interface tests:
- /tests/entrypoints/test_xxx -
- Chapter 2
- L3: Purpose, Test Content, Directory Location, Example -
PR Merged (Also run L1&L2 Tests)GPU
L4
(Perf & Integration & Accuracy)
Online: full functional scenarios + performance test + doc test + accuracy testHigh-priority models: function, performance, accuracy, and doc testing
Medium-priority models: function and doc testing
Low-priority features (using real weights)Low-priority interfaces (using real weights)full_model and hardware_test(H100, L4, etc.) and omni/tts/diffusion<3 hour - Model tests:
- /tests/e2e/online_serving/test_{model_name}_expansion.py
- Feature tests:
- /tests/{component_name}/test_xxx
- Interface tests:
- /tests/entrypoints/test_xxx
- Performance:
- /tests/dfx/perf/tests/test_qwen3_omni_*.json (Omni), test_tts.json (TTS),
- test_voxcpm2.json, test_higgs_audio_v3.json, and
- /tests/dfx/perf/tests/test_{diffusion_model}_vllm_omni.json (Diffusion)
- Doc Test:
- tests/examples/online_serving/test_{model_name}.py
- tests/examples/offline_inference/test_{model_name}.py
- Accuracy Test:
- /tests/e2e/accuracy/test_{model_name}.py -
- Chapter 3
- L4: Purpose, Test Content, Directory Location, Example -
NightlyGPU
L5
(Stability & Reliability)
Online: long-term stability test + reliability testLong-term stability and reliability testing for high-priority models
Low-priority models: function and doc testing
/Invalid-parameter validation for high-priority interfacesslow and hardware_test(H100, L4, etc.) and omni/tts/diffusion Depends on reality - Stability:
- /tests/dfx/stability/tests/test_qwen3_omni.json
- /tests/dfx/stability/tests/test_wan22.json
- Reliability:
- tests/dfx/reliability/test_reliability_{model_key}.py
- (e.g. test_reliability_qwen3_omni.py, test_reliability_wan22.py, test_reliability_hunyuan_image.py, test_reliability_voxcpm2.py) -
- Chapter 4
- L5: Purpose, Test Content, Directory Location, Example -
Weekly / Days before ReleaseGPU
- - ---- -
- The folder structure for tests file based on the 5 levels design Legend: `✅` = test exists, `⬜` = suggested to add. ``` vllm_omni/ tests/ @@ -304,134 +136,219 @@ vllm_omni/ tests/ # vllm_omni/deploy/ instead — see docs/configuration/stage_configs.md. ``` +## Markers for Tests -
+By adding markers before test functions, tests can later be executed uniformly by simply declaring the corresponding marker type. +### Current Markers +Defined in `pyproject.toml`: -## Common Specifications +| Marker | Description | +| ------------------ | --------------------------------------------------------- | +| `core_model` | L1&L2 tests (run in each PR) | +| `advanced_model` | L3 tests (run on each merge to main) | +| `full_model` | L4 tests (run nightly) | +| `diffusion` | Diffusion model tests | +| `omni` | Omni multimodal model tests | +| `tts` | TTS model tests | +| `cache` | Cache backend tests | +| `parallel` | Parallelism/distributed tests | +| `cpu` | Tests that run on CPU | +| `gpu` | Tests that run on GPU * | +| `cuda` | Tests that run on CUDA * | +| `rocm` | Tests that run on AMD/ROCm * | +| `xpu` | Tests that run on Intel XPU * | +| `npu` | Tests that run on NPU/Ascend * | +| `H100` | Tests that require H100 GPU * | +| `L4` | Tests that require L4 GPU * | +| `MI325` | Tests that require MI325 GPU (AMD/ROCm) * | +| `A2` | Tests that require A2 NPU * | +| `A3` | Tests that require A3 NPU * | +| `distributed_cuda` | Tests that require multi cards on CUDA platform * | +| `distributed_rocm` | Tests that require multi cards on ROCm platform * | +| `distributed_npu` | Tests that require multi cards on NPU platform * | +| `skipif_cuda` | Skip if the num of CUDA cards is less than the required * | +| `skipif_rocm` | Skip if the num of ROCm cards is less than the required * | +| `skipif_npu` | Skip if the num of NPU cards is less than the required * | +| `slow` | Slow tests (may skip in quick CI) | +| `benchmark` | Benchmark tests (decorator on runner test functions; perf JSON uses `full_model` + type marker instead) | +| `local_model` | Tests requiring local / non-HF-hub model weights | -Before entering specific testing levels, the project establishes two common specifications aimed at standardizing the development process and quickly locating issues. +\* Means those markers are auto-added by `@hardware_test` (parametrization decorator) or `hardware_marks` (only returning the list of marks for flexibility). -1. ***PR Checklist ([Tests Style](../ci/tests_style.md))***: This template defines the self-check items that must be completed before submitting a code review (Pull Request). It ensures that each code change meets basic requirements such as code style, dependency updates, and documentation synchronization before entering the automated testing pipeline, serving as the first manual line of defense for quality assurance. -2. ***CI Failure Explanation ([CI Failures](../ci/failures.md))***: This document archives and explains common failure patterns in the Continuous Integration (CI) pipeline, error log interpretation, and preliminary troubleshooting steps. It helps developers and testers quickly diagnose the causes of automated test failures, improving problem-solving efficiency. +#### Example usage for markers -## Notes +```python +from tests.helpers.mark import hardware_test -### Diff-aware Buildkite uploads (`source_file_dependencies`) - -L2 (`.buildkite/cuda/test-ready.yml`) and L3 (`.buildkite/cuda/test-merge.yml`) pipelines can **skip unrelated GPU jobs at upload time** based on the PR diff. This is implemented by `.buildkite/cuda/scripts/upload_pipeline.py`, which filters steps before calling `buildkite-agent pipeline upload`. - -#### What `source_file_dependencies` is - -- A **uploader-only** YAML key on a Buildkite step or group. **Buildkite does not understand it**; `upload_pipeline.py` always **removes** it from the YAML that is uploaded. -- A list of path **prefixes** (directories or individual files). If **any** changed file in the diff equals a prefix or starts with `prefix/`, the step is kept; otherwise the step (or entire group) is **omitted** from the uploaded pipeline. - -#### When filtering runs - -| Build context | Changed files used for matching | -| --- | --- | -| Pull request | `git diff --name-only origin/...` | -| `main` branch push | `git diff --name-only ^..` | -| Other (e.g. local dry-run, non-PR branch) | Diff cannot be resolved → **no filtering**; all steps are uploaded and `source_file_dependencies` is still stripped | - -Docs-only PRs are handled earlier in bootstrap (`.buildkite/cuda/pipeline.yml`) via skip-ci logic; `source_file_dependencies` applies only to the **uploaded** L2/L3 test pipelines. - -#### Where it is configured - -| Level | Pipeline file | Upload entry | -| --- | --- | --- | -| L2 | `.buildkite/cuda/test-ready.yml` | `upload_pipeline.py --upload .buildkite/cuda/test-ready.yml` (from `cuda/pipeline.yml`) | -| L3 | `.buildkite/cuda/test-merge.yml` | `upload_pipeline.py --upload .buildkite/cuda/test-merge.yml` (from `cuda/pipeline.yml`) | - -Steps **without** `source_file_dependencies` are always uploaded (subject to the usual label conditions: `ready` for L2, `merge-test` for L3). - -#### Current skip policy (L2 / L3) - -To balance CI cost and coverage: - -- **Always run** (no `source_file_dependencies`): baseline groups outside E2E Test—e.g. Simple Test, Diffusion unit tests, Engine/Model Executor, Distributed, Custom Pipeline, Entrypoints (L2), LoRA / Entrypoints (L3). -- **Diff-gated** (`source_file_dependencies` set): **every leaf job under the E2E Test group** in `test-ready.yml` and `test-merge.yml`, regardless of queue (`mithril-h100-pool`, `gpu_1_queue`, or `gpu_4_queue`). Each step lists the smallest set of prefixes that should trigger it—typically: - - pytest file(s) exercised by the job (online and/or offline); - - model code under `vllm_omni/model_executor/models/` or `vllm_omni/diffusion/models/`; - - related `vllm_omni/model_executor/stage_input_processors/` and `vllm_omni/deploy/*.yaml` when applicable. - -Adding a new E2E step: add `source_file_dependencies` on the leaf job with those prefixes. Prefer **per-step** deps rather than a broad group-level list unless every child shares the same paths. - -#### YAML examples - -H100 E2E (kubernetes / `mithril-h100-pool`): - -```yaml - - label: "Diffusion · Qwen Image Test" - source_file_dependencies: - - tests/e2e/online_serving/test_qwen_image.py - - vllm_omni/diffusion/models/qwen_image/ - commands: - - pytest -s -v tests/e2e/online_serving/test_qwen_image.py -m 'core_model' ... - agents: - queue: "mithril-h100-pool" +@pytest.mark.core_model +@pytest.mark.omni +@hardware_test( + res={"cuda": "L4", "rocm": "MI325", "npu": "A2"}, + num_cards=2, +) +@pytest.mark.parametrize("omni_server", test_params, indirect=True) +def test_video_to_audio() + ... ``` -Docker E2E (`gpu_1_queue` / `gpu_4_queue`)—same key, same upload-time filtering: +#### Decorator: `@hardware_test` -```yaml - - label: "TTS · Qwen3-TTS CustomVoice Test" - source_file_dependencies: - - tests/e2e/online_serving/test_qwen3_tts_customvoice.py - - vllm_omni/model_executor/models/qwen3_tts/ - - vllm_omni/model_executor/stage_input_processors/qwen3_tts.py - - vllm_omni/deploy/qwen3_tts.yaml - commands: - - pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py ... - agents: - queue: "gpu_4_queue" +This decorator is intended to make hardware-aware, cross-platform test authoring easier and more robust for CI/CD environments. The `hardware_test` decorator in `vllm-omni/tests/helpers/mark.py` performs the following actions: + +1. **Applies platform and resource markers** + Adds the appropriate pytest markers for each specified hardware platform (e.g., `cuda`, `rocm`, `xpu`, `npu`) and resource type (e.g., `L4`, `H100`, `MI325`, `B60`, `A2`, `A3`). + ``` + @pytest.mark.cuda + @pytest.mark.L4 + ``` +2. **Handles multi-card (distributed) scenarios** + For tests requiring multiple cards, it automatically adds distributed markers such as `distributed_cuda`, `distributed_rocm`, or `distributed_npu`. + ``` + @pytest.mark.distributed_cuda(num_cards=num_cards) + ``` +3. **Supports flexible card requirements** + Accepts `num_cards` as either a single integer for all platforms or as a dictionary with per-platform values. If not specified, defaults to 1 card per platform. + +4. **Integrates resource validation** + On CUDA, adds a skip marker (`skipif_cuda`) if the system does not have the required number of devices. + Support for `skipif_rocm` and `skipif_npu` will be implemented later. + + +5. **Works with pytest filtering** + Allows tests to be filtered and selected at runtime using standard pytest marker expressions (e.g., `-m "distributed_cuda and L4"`). + +##### Example usage for decorator +- Single call for multiple platforms: + ```python + @hardware_test( + res={"cuda": "L4", "rocm": "MI325", "xpu": "B60", "npu": "A2"}, + num_cards={"cuda": 2, "rocm": 2, "xpu": 2, "npu": 2}, + ) + ``` + or + ```python + @hardware_test( + res={"cuda": "L4", "rocm": "MI325", "xpu": "B60", "npu": "A2"}, + num_cards=2, + ) + ``` +- `res` must be a dict; supported resources: CUDA (L4/H100), ROCm (MI325), XPU (B60), MUSA (S5000), NPU (A2/A3) +- `num_cards` can be int (all platforms) or dict (per platform); defaults to 1 when missing +- Distributed markers (`distributed_cuda`, `distributed_rocm`, `distributed_npu`) are auto-added for multi-card cases +- Filtering examples: + - CUDA only: `pytest -m "distributed_cuda and L4"` + - ROCm only: `pytest -m "distributed_rocm and MI325"` + - NPU only: `pytest -m "distributed_npu"` + +#### Function: `hardware_marks` + +`hardware_marks` returns a list of pytest mark objects with the same signature as `@hardware_test`. Use it when you need more flexibility, such as attaching hardware marks to individual `pytest.param` entries rather than an entire test function. + +```python +from tests.helpers.mark import hardware_marks + +MULTI_CARD_MARKS = hardware_marks( + res={"cuda": "H100", "rocm": "MI325", "npu": "A2"}, num_cards=2 +) + +@pytest.mark.parametrize("omni_server", [ + pytest.param(OmniServerParams(...), id="case_001", marks=MULTI_CARD_MARKS), +], indirect=True) +def test_feature(omni_server): + ... ``` -A **group** may also define `source_file_dependencies`; nested steps inherit filtering as a unit—the whole group is dropped if no prefix matches. +#### JSON `mark` field (L4 perf configs) -#### Local dry-run +Perf JSON under `tests/dfx/perf/tests/` can attach pytest marks per **case** (`test_name` block). Parsed by `tests.dfx.conftest.resolve_pytest_marks` and applied to each parametrized run in `run_benchmark.py` / `run_diffusion_benchmark.py`. -```bash -# Render filtered YAML to stdout (no upload) -python3 .buildkite/cuda/scripts/upload_pipeline.py .buildkite/cuda/test-ready.yml +When `mark` is present, it must be an **array** with exactly one ``hardware_marks`` object (same semantics as `hardware_marks()` above), followed by registered pytest marker name strings such as `full_model`, `omni`, `tts`, or `diffusion`. -# Confirm uploader-only keys are stripped -python3 .buildkite/cuda/scripts/upload_pipeline.py .buildkite/cuda/test-merge.yml | grep source_file_dependencies -# (no output expected) +```json +{ + "test_name": "test_cosmos3_t2i_official_demo_2gpu", + "mark": [ + {"hardware_marks": {"res": {"cuda": "H100"}, "num_cards": 2}}, + "full_model", + "diffusion" + ], + "server_type": "vllm-omni", + "server_params": { "...": "..." }, + "benchmark_params": [{ "name": "1024x1024_steps4", "...": "..." }] +} ``` -On a PR build, Buildkite logs from `upload_pipeline.py` include lines such as `skip '…' (no changes under …)` for omitted steps. +- Local bulk load: `pytest -sv tests/dfx/perf/scripts/run_diffusion_benchmark.py -m "full_model and diffusion and H100"` +- Nightly CI perf steps: `--test-config-file tests/dfx/perf/tests/test__vllm_omni.json` (file selects cases; no `-m`) +- Result filenames use **runtime** GPU detection (`get_runtime_resource_label`); `H100` is omitted on the default CI pool -#### Related +See [L4 performance test examples](./test_examples/l4_performance_tests.inc.md) for the full schema. -- Implementation: `.buildkite/cuda/scripts/upload_pipeline.py` -- L2/L3 diff skip does **not** replace label-based triggers (`ready`, `merge-test`); it only reduces which steps appear **after** the pipeline is already scheduled. +### Add Support for a New Platform -### Test helper environment variables +If you want to add support for a new platform (e.g., "tpu" for a new accelerator), follow these steps: -Some shared helpers under `tests/helpers/` honor optional environment variables for local debugging. These are **not** set in CI by default. +1. **Extend the marker list in your pytest config** so that platform/resource markers are defined: + ```toml + # In pyproject.toml or pytest.ini + [tool.pytest.ini_options] + markers = [ + # ... existing markers ... + "tpu: Tests that require TPU device", + "TPU_V3: Tests that require TPU v3 hardware", + "distributed_tpu: Tests that require multiple TPU devices", + ] + ``` +2. **Implement a marker construction function for your platform** in `vllm-omni/tests/helpers/mark.py`: + ```python + # In vllm-omni/tests/helpers/mark.py -| Variable | Accepted values | Description | -| -------- | --------------- | ----------- | -| `VLLM_OMNI_KEEP_REQUEST_MEDIA` | `1`, `true`, `yes` (case-insensitive) | When enabled, temporary WAV files created by `tests.helpers.media.convert_audio_bytes_to_text` are **not** deleted when the pytest process exits. By default, each call writes a unique file under the system temp directory via `tempfile.mkstemp` and registers `atexit` cleanup. Use this when debugging audio output validation (Whisper transcription, keyword checks, text–audio similarity). The saved path is logged as `audio data is saved: `. | + def tpu_marks(*, res: str, num_cards: int): + test_platform = pytest.mark.tpu + if res == "TPU_V3": + test_resource = pytest.mark.TPU_V3 + else: + raise ValueError( + f"Invalid TPU resource type: {res}. Supported: TPU_V3") -Example (Linux / macOS): + if num_cards == 1: + return [test_platform, test_resource] + else: + test_distributed = pytest.mark.distributed_tpu(num_cards=num_cards) + # Optionally: add skipif_tpu when implemented + return [test_platform, test_resource, test_distributed] + ``` +3. **Update `hardware_marks` to recognize your new platform**: + In the relevant place (see the `hardware_marks` implementation), add: + ```python + if platform == "tpu": + marks = tpu_marks(res=resource, num_cards=cards) + ``` + (`hardware_test` calls `hardware_marks` internally, so both will pick up the change.) +4. **(Recommended) Add a test using your new markers**: + ```python + @hardware_test( + res={"tpu": "TPU_V3"}, + num_cards=2, + ) + def test_my_tpu_feature(): + ... + ``` -```bash -export VLLM_OMNI_KEEP_REQUEST_MEDIA=1 -pytest -s -v tests/e2e/online_serving/test_qwen3_omni.py -k test_mix_to_text_audio -``` +**Summary**: +- Add pytest markers for your new platform/resources +- Implement a marker function (`xxx_marks`) +- Plug into `hardware_marks` +- You're done: tests using `@hardware_test` or `hardware_marks` with your platform now automatically get the correct markers, distribution, and isolation! -Example (Windows PowerShell): +See code in `vllm-omni/tests/helpers/mark.py` for existing examples (`cuda_marks`, `rocm_marks`, `npu_marks`). -```powershell -$env:VLLM_OMNI_KEEP_REQUEST_MEDIA = "1" -pytest -s -v tests/e2e/online_serving/test_qwen3_omni.py -k test_mix_to_text_audio -``` +## Test case style -## Chapter 1: L1 & L2 Level Testing - Unit Testing and Basic End-to-End Verification +### L1 & L2 Level Testing - Unit Testing and Basic End-to-End Verification -### 1.1 Testing Purpose +#### 1.1 Testing Purpose L1 and L2 level testing form the foundation of the quality assurance system. L1 level testing focuses on verifying the internal logic correctness of code units (e.g., functions, classes), ensuring each independent component behaves as designed. @@ -439,7 +356,7 @@ L2 level testing builds upon L1 by introducing GPU resources and verifying that -### 1.2 Testing Content and Scope +#### 1.2 Testing Content and Scope - ***L1 (Unit & Logic Testing)***: - - ***Scope***: Tests internal functions and methods of core components such as `entrypoints`, `models`. @@ -454,7 +371,7 @@ L2 level testing builds upon L1 by introducing GPU resources and verifying that - **Omni multimodal** (`assert_omni_response`): L2 asserts successful completion; keyword, transcript, and cross-modal similarity checks run only at L3+. - ***Characteristic***: Requires ***GPU*** resources to perform model computations. -### 1.3 Test Directory and Execution Files +#### 1.3 Test Directory and Execution Files A clear directory structure is key to managing test cases efficiently. @@ -464,10 +381,10 @@ A clear directory structure is key to managing test cases efficiently. - - Online Serving: `/tests/e2e/online_serving/test_{model_name}.py` - Offline Inference: `/tests/e2e/offline_inference/test_{model_name}.py` -### 1.4 Execution Method and Example +#### 1.4 Execution Method and Example - ***Trigger Timing***: **`PR with ready label`**. That is, when a developer adds a "ready for review" or similar label to a PR on platforms like GitHub, L1 and L2 tests are automatically triggered. -- ***Diff-aware step skipping***: On L2, **E2E Test** jobs may be omitted at pipeline upload when the PR diff does not touch their [`source_file_dependencies`](#diff-aware-buildkite-uploads-source_file_dependencies) prefixes (H100 and docker queues alike); non-E2E groups still always upload. See [Diff-aware Buildkite uploads](#diff-aware-buildkite-uploads-source_file_dependencies). +- ***Diff-aware step skipping***: On L2, **E2E Test** jobs may be omitted at pipeline upload when the PR diff does not touch their [`source_file_dependencies`](ci_settings.md#step-filtering) prefixes; non-E2E groups still always upload. See [CI Settings — Diff-aware CI](ci_settings.md#diff-aware-ci). - ***Execution Environment***: L1 uses ***CPU*** environment; L2 requires ***GPU*** environment. - ***Script Example***: @@ -535,7 +452,7 @@ def test_cap_and_align_mel_length_no_mismatch(repeats, code_len, max_mel_frames)
L2 Test Examples -You can refer to Test Examples in Chapter 2 to see example test cases that incorporate both L2 and L3 testing logic. +You can refer to Test Examples in the L3 section to see example test cases that incorporate both L2 and L3 testing logic.
- - ***Run Command***: @@ -543,9 +460,9 @@ You can refer to Test Examples in Chapter 2 to see example test cases that incor `pytest -s -v /tests/e2e/online_serving/test_{model_name}.py` `pytest -s -v -m 'core_model and cpu' --run-level=core_model` -## Chapter 2: L3 Level Testing - Core Integration, Performance, and Accuracy Verification +### L3 Level Testing - Core Integration, Performance, and Accuracy Verification -### 2.1 Testing Purpose +#### 2.1 Testing Purpose L3 level testing executes after code is merged into the main branch. Its core purpose is to verify the integration effect, key performance indicators, and output accuracy of ***real models*** in ***multiple deployment scenarios*** @@ -553,30 +470,30 @@ L3 level testing executes after code is merged into the main branch. Its core pu -### 2.2 Testing Content and Scope +#### 2.2 Testing Content and Scope - ***Deployment Scenarios***: Covers richer `online` and `offline` deployment configurations, which may include different hardware configurations, batch sizes, concurrency levels, etc. - ***Core Verification***: - 1. ***Inference Functionality***: Ensures real models can perform forward computation normally and return results. 2. ***Accuracy Compliance***: Verifies that the model's evaluation metrics (e.g., accuracy) meet the expected baseline, preventing code changes from introducing accuracy issues. 3. ***Important Performance***: Verifies whether performance (e.g., P99 latency, throughput) in core scenarios meets preset thresholds. -- ***L3 response validation (`advanced_model`)***: At `--run-level advanced_model`, `tests/helpers/assertions.py` adds semantic checks on top of L2 payload gates (see Chapter 1): +- ***L3 response validation (`advanced_model`)***: At `--run-level advanced_model`, `tests/helpers/assertions.py` adds semantic checks on top of L2 payload gates (see L1 & L2 above): - **Speech / TTS** (`assert_audio_speech_response`): Whisper transcript of returned audio vs `request_config["input"]` (cosine similarity > 0.9 when input text is set); optional `min_audio_bytes` floor; PCM harmonic-to-noise ratio when `response_format` is `pcm`; **preset voice gender** via `_assert_preset_voice_gender_from_audio` when `voice` matches a known preset in `_PRESET_VOICE_GENDER_MAP` (pitch/F0-based classifier; skipped for unknown voices or `pcm` output). - **Omni multimodal** (`assert_omni_response`): non-empty text/audio outputs; `key_words` in transcript or text; text–audio similarity / containment; preset **`speaker`** gender check (same helper as TTS). - **Diffusion** (`assert_*_diffusion_response`): image/video dimension and frame-count parity with request parameters where configured. - **Accuracy suites**: pixel/video similarity and metric baselines under `/tests/e2e/accuracy/` (separate from inline helper assertions). -### 2.3 Test Directory and Execution Files +#### 2.3 Test Directory and Execution Files - ***Functional Testing***: - - Online Serving: `/tests/e2e/online_serving/test_{model_name}_expansion.py` - Offline Inference: `/tests/e2e/offline_inference/test_{model_name}_expansion.py` - (Note: `_expansion.py` likely means it contains more comprehensive scenario cases compared to L2 tests). -### 2.4 Execution Method and Example +#### 2.4 Execution Method and Example - ***Trigger Timing***: **`PR Merged`**. Automatically triggered after code review is approved and merged into the main branch (typically via `merge-test` label on the PR before merge). -- ***Diff-aware step skipping***: Same [`source_file_dependencies`](#diff-aware-buildkite-uploads-source_file_dependencies) mechanism as L2—**all E2E Test** leaf steps are diff-gated; other L3 groups always upload. See [Diff-aware Buildkite uploads](#diff-aware-buildkite-uploads-source_file_dependencies). +- ***Diff-aware step skipping***: Same [`source_file_dependencies`](ci_settings.md#step-filtering) mechanism as L2—**all E2E Test** leaf steps are diff-gated; other L3 groups always upload. See [CI Settings — Diff-aware CI](ci_settings.md#diff-aware-ci). - ***Execution Environment***: ***GPU*** servers. - ***Script Example***: @@ -692,27 +609,27 @@ L3 level testing executes after code is merged into the main branch. Its core pu **Implicit Validation**: `send_omni_request`, `send_audio_speech_request`, and `send_diffusion_request` call `assert_*_response` helpers with the session `--run-level`. At `core_model`, checks are limited to success plus cheap payload sanity (non-empty audio/media, format/content-type, optional `min_audio_bytes`). At `advanced_model` and `full_model`, deep validation adds Whisper transcripts, keyword/similarity, diffusion dimension checks, PCM HNR, etc. - **Audio output debugging**: Deep validation may transcribe returned audio via `convert_audio_bytes_to_text` (Whisper). If an audio keyword or text–audio similarity assertion fails, set `VLLM_OMNI_KEEP_REQUEST_MEDIA=1` before running pytest to keep the intermediate WAV files for inspection (see [Test helper environment variables](#test-helper-environment-variables)). + **Audio output debugging**: Deep validation may transcribe returned audio via `convert_audio_bytes_to_text` (Whisper). If an audio keyword or text–audio similarity assertion fails, set `VLLM_OMNI_KEEP_REQUEST_MEDIA=1` before running pytest to keep the intermediate WAV files for inspection (see [Test helper environment variables](test_system_overview.md#test-helper-environment-variables)). - ***Run Command (L3 merge)***: `pytest -s -v /tests/e2e/online_serving/test_{model_name}.py -m advanced_model --run-level=advanced_model` - ***Run Command (L4 nightly expansion)***: `pytest -s -v /tests/e2e/online_serving/test_{model_name}_expansion.py -m full_model --run-level=full_model` -## Chapter 3: L4 Level Testing - Full Functionality, Performance, and Documentation Testing +### L4 Level Testing - Full Functionality, Performance, and Documentation Testing -### 3.1 Testing Purpose +#### 3.1 Testing Purpose L4 level testing is a comprehensive quality audit before a version release. It expands upon L3, executing ***full*** functional scenarios, conducting systematic ***performance stress tests***, and simultaneously verifying the correctness of accompanying ***example documentation***. Its purpose is to perform deep validation of the system during off-peak nighttime hours, providing quality trend reports for daytime development and data support for release decisions. -### 3.2 Testing Content and Scope +#### 3.2 Testing Content and Scope - ***Full Functionality Testing***: Executes all test cases defined in `test_{model_name}_expansion.py`, covering all implemented features, positive flows, boundary conditions, and exception handling. - ***Performance Testing***: Uses `tests/dfx/perf/tests/test_qwen3_omni_*.json` (Omni), `test_tts.json` / `test_voxcpm2.json` / `test_higgs_audio_v3.json` (TTS), and diffusion configs `tests/dfx/perf/tests/test_*_vllm_omni.json` (passed to `run_benchmark.py` or `run_diffusion_benchmark.py` via `--test-config-file` in nightly **Perf Test** steps) to drive throughput, latency, and memory benchmarks. Each JSON **case** may declare an optional top-level **`mark`** array: exactly one ``hardware_marks`` object plus pytest marker name strings (`full_model`, `omni` / `tts` / `diffusion`, …). Runners attach those marks to each parametrized `(server, benchmark index)` pair so **local** bulk runs can filter with `-m` (for example `-m "full_model and H100 and diffusion"`). Nightly CI perf jobs select workloads by **`--test-config-file`**, not `-m`. Details are in the Performance Tests example below. - ***Documentation Testing***: Verifies whether the example code provided to users is runnable and its results match the description. -### 3.3 Test Directory and Execution Files +#### 3.3 Test Directory and Execution Files - ***Functional Testing***: Same directories as L3. - ***Performance Test Configuration***: `tests/dfx/perf/tests/test_qwen3_omni_*.json`, `test_tts.json`, `test_voxcpm2.json`, `test_higgs_audio_v3.json`, and diffusion configs `tests/dfx/perf/tests/test_*_vllm_omni.json` (e.g. `test_qwen_image_vllm_omni.json`, `test_cosmos3_vllm_omni.json`). Optional per-case `mark` for local `-m` filtering is documented in Section 3.4 Performance Tests. @@ -720,7 +637,7 @@ L4 level testing is a comprehensive quality audit before a version release. It e - `tests/examples/online_serving/test_{model_name}.py` - `tests/examples/offline_inference/test_{model_name}.py` -### 3.4 Execution Method and Example +#### 3.4 Execution Method and Example - ***Trigger Timing***: **`Nightly`**, automatically executed every night. - ***Execution Environment***: ***GPU*** server clusters to meet the resource demands of performance testing. @@ -740,20 +657,20 @@ L4 level testing is a comprehensive quality audit before a version release. It e - ***Run Command***: (Specific commands would depend on the performance testing tool and configuration defined in `nightly.json`). -## Chapter 4: L5 Level Testing - Stability and Reliability Testing +### L5 Level Testing - Stability and Reliability Testing -### 4.1 Testing Purpose +#### 4.1 Testing Purpose L5 level testing focuses on the performance of model services under ***long-running*** and ***abnormal fault*** scenarios. It aims to uncover deep-seated issues that only manifest under sustained pressure or extreme conditions, such as memory leaks, resource contention, gradual performance degradation, and lack of fault tolerance mechanisms. This is the final, yet crucial, line of defense for ensuring service high availability and production environment robustness. -### 4.2 Testing Content and Scope +#### 4.2 Testing Content and Scope - ***Long-term Stability (Stability) Testing***: Uses JSON under `tests/dfx/stability/tests/` (for example `test_qwen3_omni.json` and `test_wan22.json`) to run the service under moderate load for an extended period (e.g., over 12 hours), monitoring whether metrics like memory/VRAM usage, response time, and throughput degrade over time, and whether the service process remains stable. - ***Reliability Testing***: Uses pytest suites under `tests/dfx/reliability/` to inject controlled faults against a **live** `vllm_omni serve` instance (same **`omni_server` / `omni_server_function`** fixture style as E2E). Current suites emphasize **GPU memory pressure** (CUDA sidecar “memory hog”), **worker / runtime process kill** (`SIGKILL` on `VLLM::Worker` for Qwen3-Omni, `multiprocessing.spawn` for Wan2.2 video workers, or `vLLM-Omni::` for HunyuanImage DiT workers), **large multimodal chat**, **`/v1/videos`**, **`/v1/images/generations`**, or **`/v1/audio/speech`** jobs under OOM, **`/health` → 503** and **fast-fail / non-hanging concurrent** requests after kill, and **OpenAI-style 5xx error contracts** (e.g. text vs text+audio under OOM). **Post-fault recovery** checks exist where enabled (some cases may be `skip` while issues are tracked). See the Reliability `
` block in Section 4.4 for file-level responsibilities and CI markers (`slow`, `hardware_test`, POSIX-only kill). -### 4.3 Test Directory and Execution Files +#### 4.3 Test Directory and Execution Files - ***Stability Test Configuration***: `tests/dfx/stability/tests/test_qwen3_omni.json`, `tests/dfx/stability/tests/test_wan22.json` (one JSON per model / runner family) - ***Reliability Test Suite*** (`tests/dfx/reliability/`): @@ -765,7 +682,7 @@ L5 level testing focuses on the performance of model services under ***long-runn - `conftest.py` — `fault_injector` and `omni_server_after_fault` / `omni_server_after_fault_function` fixtures to run a callable **after** the server is ready. - `README.md` — Short local run commands for this directory. -### 4.4 Execution Method and Example +#### 4.4 Execution Method and Example - ***Trigger Timing***: **`Weekly`** (weekly) or **`Days before Release`** (several days before a major release). Due to long execution times, the frequency is lower. - ***Execution Environment***: ***GPU*** servers, requiring a stable and exclusive testing environment. @@ -799,7 +716,7 @@ When you want to add L5-level stability test cases, add or extend the appropriat } ``` -#### Parameter Explanation +##### Parameter Explanation ***Overview*** @@ -809,16 +726,16 @@ When you want to add L5-level stability test cases, add or extend the appropriat | server_params | Yes | Server-side configuration parameters (model, stage configuration, etc.) | | benchmark_params | Yes | Stability benchmark running parameters (supports multiple configurations) | -#### server_params Configuration +##### server_params Configuration -##### Basic Parameters +###### Basic Parameters | Parameter | Required | Example | Description | | ----------------- | -------- | ---------------------------------- | ----------------------------------- | | model | Yes | "Qwen/Qwen3-Omni-30B-A3B-Instruct" | Model name or path | | stage_config_name | Yes | "qwen3_omni.yaml" | Stage configuration file name | -##### Dynamic Configuration (update/delete) +###### Dynamic Configuration (update/delete) Supports incremental modifications based on the basic configuration: @@ -828,9 +745,9 @@ Supports incremental modifications based on the basic configuration: | delete | Delete specified configuration items | ***Example***: -You can refer to Test Examples in Chapter 3.4 +You can refer to Test Examples in L4 §3.4 -#### benchmark_params Configuration +##### benchmark_params Configuration For stability testing, the key parameters are: @@ -840,18 +757,18 @@ For stability testing, the key parameters are: - `max_concurrency`: Maximum number of concurrent requests. When this is used, `request_rate` is set to `inf` internally. - **num_prompts_per_batch**: Number of prompts sent in each batch. Multiple batches will be executed sequentially within `duration_sec`. -All other optional parameters follow the same rules as the in Chapter 3.4. +All other optional parameters follow the same rules as in L4 §3.4.
Reliability test suite (tests/dfx/reliability) -#### Purpose and relationship to stability +##### Purpose and relationship to stability Reliability tests are **short fault-injection** integration runs (L5 **(b)** in `tests/dfx/reliability/README.md`). They complement **stability** JSON-driven long runs: instead of hours of steady traffic, they **perturb** the server (GPU OOM sidecar, fatal signals on selected processes) and check **failure mode** and **latency bounds** (e.g. chat or `/v1/videos` must not hang under concurrent fault-time load). -#### Directory layout +##### Directory layout | Path | Responsibility | | ---- | -------------- | @@ -863,13 +780,13 @@ Reliability tests are **short fault-injection** integration runs (L5 **(b)** in | `test_reliability_voxcpm2.py` | VoxCPM2: `/v1/audio/speech` under OOM (error contract), **SIGKILL** on `VLLM::` workers and serve/tree targets, health / fast-fail / concurrent speech requests; some OOM cases may be skipped while tracked in issues. | | `README.md` | Minimal run / collect examples. | -#### Parametrization and markers +##### Parametrization and markers - Each test module defines a **`RELIABILITY_SCENARIOS`** list (`test_name`, `server_params`: model, `stage_config_name` or diffusion `server_args`, etc.). **`create_reliability_omni_server_params()`** in `tests/dfx/conftest.py` resolves stage paths (including XPU substitutions where applicable) and builds **`OmniServerParams`** lists consumed by **`@pytest.mark.parametrize(..., indirect=True)`** on `omni_server` or `omni_server_function`. - Cases are tagged **`@pytest.mark.slow`** for weekly / selective CI. GPU-heavy suites use **`@hardware_test(res={"cuda": "H100"}, num_cards=...)`** or **`@hardware_test(res={"cuda": "L4"}, num_cards=1)`** (Qwen3-Omni **2**× H100; Wan2.2 **1**× H100; HunyuanImage DiT **4**× H100; VoxCPM2 **1**× L4). - **Process-kill** tests use **`@pytest.mark.skipif(os.name == "nt", ...)`** because injection uses POSIX **`pgrep` / `kill`**. -#### CI trigger +##### CI trigger Weekly Buildkite (`.buildkite/cuda/test-weekly.yml`) runs one step per model suite (trigger: `WEEKLY=1` or PR label `weekly-test`), for example: @@ -887,7 +804,7 @@ pytest -s -v tests/dfx/reliability/test_reliability_hunyuan_image.py -m "slow" pytest -s -v tests/dfx/reliability/test_reliability_voxcpm2.py -m "slow" ``` -#### Local commands +##### Local commands ```bash pytest --collect-only tests/dfx/reliability @@ -897,7 +814,7 @@ pytest -s -v tests/dfx/reliability/test_reliability_hunyuan_image.py -m slow pytest -s -v tests/dfx/reliability/test_reliability_voxcpm2.py -m slow ``` -#### Adding a new model suite +##### Adding a new model suite 1. Add `test_reliability_.py` under `tests/dfx/reliability/`. 2. Define **`RELIABILITY_SCENARIOS`** and pass them through **`create_reliability_omni_server_params()`** with the correct deploy or e2e stage-config directory (same pattern as existing files). @@ -908,7 +825,3 @@ pytest -s -v tests/dfx/reliability/test_reliability_voxcpm2.py -m slow - - ***Stability***: `pytest -s -v tests/dfx/stability/scripts/test_stability_qwen3_omni.py` or `pytest -s -v tests/dfx/stability/scripts/test_stability_wan22.py` (or add `test_stability_.py` alongside a matching JSON config) - ***Reliability***: `pytest -s -v tests/dfx/reliability/test_reliability_.py -m slow` (current suites: `qwen3_omni`, `wan22`, `hunyuan_image`, `voxcpm2`; add `test_reliability_.py` and a matching step in `.buildkite/cuda/test-weekly.yml` for new models) - -## Summary - -This multi-level testing system achieves continuous, progressive validation of model service quality by tightly integrating testing activities with the development workflow (commit, review, merge, release). From rapid unit testing to comprehensive end-to-end testing, and further to in-depth performance, stability, and reliability verification, each level has clear objectives, collectively building a robust quality protection net. By following this system, teams can deliver high-quality, highly reliable model services more efficiently. diff --git a/docs/contributing/ci/tests_markers.md b/docs/contributing/ci/tests_markers.md deleted file mode 100644 index 0a0a974e6..000000000 --- a/docs/contributing/ci/tests_markers.md +++ /dev/null @@ -1,207 +0,0 @@ -# Markers for Tests - -By adding markers before test functions, tests can later be executed uniformly by simply declaring the corresponding marker type. - -## Current Markers -Defined in `pyproject.toml`: - -| Marker | Description | -| ------------------ | --------------------------------------------------------- | -| `core_model` | L1&L2 tests (run in each PR) | -| `advanced_model` | L3 tests (run on each merge to main) | -| `full_model` | L4 tests (run nightly) | -| `diffusion` | Diffusion model tests | -| `omni` | Omni multimodal model tests | -| `tts` | TTS model tests | -| `cache` | Cache backend tests | -| `parallel` | Parallelism/distributed tests | -| `cpu` | Tests that run on CPU | -| `gpu` | Tests that run on GPU * | -| `cuda` | Tests that run on CUDA * | -| `rocm` | Tests that run on AMD/ROCm * | -| `xpu` | Tests that run on Intel XPU * | -| `npu` | Tests that run on NPU/Ascend * | -| `H100` | Tests that require H100 GPU * | -| `L4` | Tests that require L4 GPU * | -| `MI325` | Tests that require MI325 GPU (AMD/ROCm) * | -| `A2` | Tests that require A2 NPU * | -| `A3` | Tests that require A3 NPU * | -| `distributed_cuda` | Tests that require multi cards on CUDA platform * | -| `distributed_rocm` | Tests that require multi cards on ROCm platform * | -| `distributed_npu` | Tests that require multi cards on NPU platform * | -| `skipif_cuda` | Skip if the num of CUDA cards is less than the required * | -| `skipif_rocm` | Skip if the num of ROCm cards is less than the required * | -| `skipif_npu` | Skip if the num of NPU cards is less than the required * | -| `slow` | Slow tests (may skip in quick CI) | -| `benchmark` | Benchmark tests (decorator on runner test functions; perf JSON uses `full_model` + type marker instead) | -| `local_model` | Tests requiring local / non-HF-hub model weights | - -\* Means those markers are auto-added by `@hardware_test` (parametrization decorator) or `hardware_marks` (only returning the list of marks for flexibility). - -### Example usage for markers - -```python -from tests.helpers.mark import hardware_test - -@pytest.mark.core_model -@pytest.mark.omni -@hardware_test( - res={"cuda": "L4", "rocm": "MI325", "npu": "A2"}, - num_cards=2, -) -@pytest.mark.parametrize("omni_server", test_params, indirect=True) -def test_video_to_audio() - ... -``` - -### Decorator: `@hardware_test` - -This decorator is intended to make hardware-aware, cross-platform test authoring easier and more robust for CI/CD environments. The `hardware_test` decorator in `vllm-omni/tests/helpers/mark.py` performs the following actions: - -1. **Applies platform and resource markers** - Adds the appropriate pytest markers for each specified hardware platform (e.g., `cuda`, `rocm`, `xpu`, `npu`) and resource type (e.g., `L4`, `H100`, `MI325`, `B60`, `A2`, `A3`). - ``` - @pytest.mark.cuda - @pytest.mark.L4 - ``` -2. **Handles multi-card (distributed) scenarios** - For tests requiring multiple cards, it automatically adds distributed markers such as `distributed_cuda`, `distributed_rocm`, or `distributed_npu`. - ``` - @pytest.mark.distributed_cuda(num_cards=num_cards) - ``` -3. **Supports flexible card requirements** - Accepts `num_cards` as either a single integer for all platforms or as a dictionary with per-platform values. If not specified, defaults to 1 card per platform. - -4. **Integrates resource validation** - On CUDA, adds a skip marker (`skipif_cuda`) if the system does not have the required number of devices. - Support for `skipif_rocm` and `skipif_npu` will be implemented later. - - -5. **Works with pytest filtering** - Allows tests to be filtered and selected at runtime using standard pytest marker expressions (e.g., `-m "distributed_cuda and L4"`). - -#### Example usage for decorator -- Single call for multiple platforms: - ```python - @hardware_test( - res={"cuda": "L4", "rocm": "MI325", "xpu": "B60", "npu": "A2"}, - num_cards={"cuda": 2, "rocm": 2, "xpu": 2, "npu": 2}, - ) - ``` - or - ```python - @hardware_test( - res={"cuda": "L4", "rocm": "MI325", "xpu": "B60", "npu": "A2"}, - num_cards=2, - ) - ``` -- `res` must be a dict; supported resources: CUDA (L4/H100), ROCm (MI325), XPU (B60), MUSA (S5000), NPU (A2/A3) -- `num_cards` can be int (all platforms) or dict (per platform); defaults to 1 when missing -- Distributed markers (`distributed_cuda`, `distributed_rocm`, `distributed_npu`) are auto-added for multi-card cases -- Filtering examples: - - CUDA only: `pytest -m "distributed_cuda and L4"` - - ROCm only: `pytest -m "distributed_rocm and MI325"` - - NPU only: `pytest -m "distributed_npu"` - -### Function: `hardware_marks` - -`hardware_marks` returns a list of pytest mark objects with the same signature as `@hardware_test`. Use it when you need more flexibility, such as attaching hardware marks to individual `pytest.param` entries rather than an entire test function. - -```python -from tests.helpers.mark import hardware_marks - -MULTI_CARD_MARKS = hardware_marks( - res={"cuda": "H100", "rocm": "MI325", "npu": "A2"}, num_cards=2 -) - -@pytest.mark.parametrize("omni_server", [ - pytest.param(OmniServerParams(...), id="case_001", marks=MULTI_CARD_MARKS), -], indirect=True) -def test_feature(omni_server): - ... -``` - -### JSON `mark` field (L4 perf configs) - -Perf JSON under `tests/dfx/perf/tests/` can attach pytest marks per **case** (`test_name` block). Parsed by `tests.dfx.conftest.resolve_pytest_marks` and applied to each parametrized run in `run_benchmark.py` / `run_diffusion_benchmark.py`. - -When `mark` is present, it must be an **array** with exactly one ``hardware_marks`` object (same semantics as `hardware_marks()` above), followed by registered pytest marker name strings such as `full_model`, `omni`, `tts`, or `diffusion`. - -```json -{ - "test_name": "test_cosmos3_t2i_official_demo_2gpu", - "mark": [ - {"hardware_marks": {"res": {"cuda": "H100"}, "num_cards": 2}}, - "full_model", - "diffusion" - ], - "server_type": "vllm-omni", - "server_params": { "...": "..." }, - "benchmark_params": [{ "name": "1024x1024_steps4", "...": "..." }] -} -``` - -- Local bulk load: `pytest -sv tests/dfx/perf/scripts/run_diffusion_benchmark.py -m "full_model and diffusion and H100"` -- Nightly CI perf steps: `--test-config-file tests/dfx/perf/tests/test__vllm_omni.json` (file selects cases; no `-m`) -- Result filenames use **runtime** GPU detection (`get_runtime_resource_label`); `H100` is omitted on the default CI pool - -See [L4 performance test examples](./test_examples/l4_performance_tests.inc.md) for the full schema. - -## Add Support for a New Platform - -If you want to add support for a new platform (e.g., "tpu" for a new accelerator), follow these steps: - -1. **Extend the marker list in your pytest config** so that platform/resource markers are defined: - ```toml - # In pyproject.toml or pytest.ini - [tool.pytest.ini_options] - markers = [ - # ... existing markers ... - "tpu: Tests that require TPU device", - "TPU_V3: Tests that require TPU v3 hardware", - "distributed_tpu: Tests that require multiple TPU devices", - ] - ``` -2. **Implement a marker construction function for your platform** in `vllm-omni/tests/helpers/mark.py`: - ```python - # In vllm-omni/tests/helpers/mark.py - - def tpu_marks(*, res: str, num_cards: int): - test_platform = pytest.mark.tpu - if res == "TPU_V3": - test_resource = pytest.mark.TPU_V3 - else: - raise ValueError( - f"Invalid TPU resource type: {res}. Supported: TPU_V3") - - if num_cards == 1: - return [test_platform, test_resource] - else: - test_distributed = pytest.mark.distributed_tpu(num_cards=num_cards) - # Optionally: add skipif_tpu when implemented - return [test_platform, test_resource, test_distributed] - ``` -3. **Update `hardware_marks` to recognize your new platform**: - In the relevant place (see the `hardware_marks` implementation), add: - ```python - if platform == "tpu": - marks = tpu_marks(res=resource, num_cards=cards) - ``` - (`hardware_test` calls `hardware_marks` internally, so both will pick up the change.) -4. **(Recommended) Add a test using your new markers**: - ```python - @hardware_test( - res={"tpu": "TPU_V3"}, - num_cards=2, - ) - def test_my_tpu_feature(): - ... - ``` - -**Summary**: -- Add pytest markers for your new platform/resources -- Implement a marker function (`xxx_marks`) -- Plug into `hardware_marks` -- You're done: tests using `@hardware_test` or `hardware_marks` with your platform now automatically get the correct markers, distribution, and isolation! - -See code in `vllm-omni/tests/helpers/mark.py` for existing examples (`cuda_marks`, `rocm_marks`, `npu_marks`). diff --git a/docs/contributing/ci/tests_style.md b/docs/contributing/ci/tests_style.md deleted file mode 100644 index 420a8821d..000000000 --- a/docs/contributing/ci/tests_style.md +++ /dev/null @@ -1,466 +0,0 @@ -# Test File Structure and Style Guide - -To ensure project maintainability and sustainable development, we encourage contributors to submit test code (unit tests, system tests, or end-to-end tests) alongside their code changes. This document outlines the guidelines for organizing and naming test files. - -## Checklist before submitting your test files - -1. The file is saved in an appropriate place and the file name is clear. -2. The coding style follows the requirements outlined below. -3. All test functions have appropriate pytest markers. -4. For tests that need run in CI, please ensure it labeled as ``@pytest.mark.core_model` the test is configured under the `./buildkite/` folder. - - -## Test Types - -For more details about our [Five Levels Tests design](../ci/CI_5levels.md). - -### Unit Tests and System Tests -For unit tests and system tests, we strongly recommend placing test files in the same directory structure as the source code being tested, using the naming convention `test_*.py`. - -### End-to-End (E2E) Tests for Models -End-to-end tests verify the complete functionality of a system or component. For our project, the E2E tests for different omni models are organized into two subdirectories: - -- **`tests/e2e/offline_inference/`**: Tests for offline inference modes (e.g., Qwen3Omni offline inference) - -- **`tests/e2e/online_serving/`**: Tests for online serving scenarios (e.g., API server tests) - -## Test Directory Structure - -The ideal directory structure mirrors the source code organization. Legend: `✅` = test exists, `⬜` = suggested to add. - -``` -vllm_omni/ tests/ -├── config/ → ├── config/ -│ ├── model.py │ └── test_model.py ⬜ -│ └── lora.py │ └── test_lora.py ⬜ -│ -├── core/ → ├── core/ -│ └── sched/ │ └── sched/ -│ ├── omni_ar_scheduler.py │ ├── test_omni_ar_scheduler.py ⬜ -│ ├── omni_generation_scheduler.py │ ├── test_omni_generation_scheduler.py ⬜ -│ └── output.py │ └── test_output.py ✅ currently in entrypoints/test_omni_new_request_data.py (tests output.OmniNewRequestData) -│ -├── diffusion/ → ├── diffusion/ -│ ├── diffusion_engine.py │ ├── test_diffusion_engine.py ⬜ -│ ├── attention/ │ ├── attention/ -│ │ ├── layer.py │ │ ├── test_attention_sp.py ✅ -│ │ └── backends/ │ │ └── test_flash_attn.py ✅ -│ ├── distributed/ │ ├── distributed/ -│ │ └── ... │ │ ├── test_comm.py ✅ -│ │ │ │ ├── test_cfg_parallel.py ✅ -│ │ │ │ └── test_sp_plan_hooks.py ✅ -│ ├── lora/ │ ├── lora/ -│ │ └── ... │ │ ├── test_base_linear.py ✅ -│ │ │ │ └── test_lora_manager.py ✅ -│ ├── models/ │ ├── models/ -│ │ ├── qwen_image/ │ │ ├── qwen_image/ (e2e coverage) -│ │ ├── ovis_image/ │ │ ├── ovis_image/ -│ │ │ └── ... │ │ │ └── test_ovis_image.py ✅ -│ │ ├── z_image/ │ │ └── z_image/ -│ │ └── ... │ │ └── test_zimage_tp_constraints.py ✅ -│ └── worker/ │ └── worker/ -│ ├── diffusion_worker.py │ └── test_diffusion_worker.py ✅ file at diffusion/test_diffusion_worker.py -│ └── diffusion_model_runner.py │ -│ -├── distributed/ → ├── distributed/ -│ └── omni_connectors/ │ └── omni_connectors/ -│ ├── adapter.py │ ├── test_adapter_and_flow.py ✅ -│ ├── kv_transfer_manager.py │ ├── test_basic_connectors.py ✅ -│ ├── connectors/ │ ├── test_kv_flow.py ✅ -│ └── utils/ │ └── test_omni_connector_configs.py ✅ -│ -├── engine/ → ├── engine/ -│ ├── input_processor.py │ ├── test_input_processor.py ⬜ (no processor.py in source) -│ ├── output_processor.py │ └── test_output_processor.py ⬜ -│ └── arg_utils.py │ └── test_arg_utils.py ⬜ -│ -├── entrypoints/ → ├── entrypoints/ -│ ├── stage_utils.py │ ├── test_stage_utils.py ✅ -│ ├── cli/ │ ├── cli/ (benchmarks/test_serve_cli.py covers CLI serve) -│ │ └── ... │ │ └── test_*.py ⬜ -│ └── openai/ │ └── openai_api/ # maps to entrypoints/openai/ -│ ├── api_server.py │ ├── test_api_server.py ⬜ (e2e indirect coverage) -│ ├── serving_chat.py │ ├── test_serving_chat_sampling_params.py ✅ -│ ├── serving_speech.py │ ├── test_serving_speech.py ✅ -│ └── image_api_utils.py │ └── test_image_server.py ✅ -│ -├── inputs/ → ├── inputs/ -│ ├── data.py │ ├── test_data.py ⬜ -│ ├── parse.py │ ├── test_parse.py ⬜ -│ └── preprocess.py │ └── test_preprocess.py ✅ currently in entrypoints/test_omni_input_preprocessor.py -│ -├── model_executor/ → ├── model_executor/ -│ ├── layers/ │ ├── layers/ -│ │ └── mrope.py │ │ └── test_mrope.py ⬜ -│ ├── model_loader/ │ ├── model_loader/ -│ │ └── weight_utils.py │ │ └── test_weight_utils.py ⬜ -│ ├── models/ │ ├── models/ -│ │ ├── qwen2_5_omni/ │ │ ├── qwen2_5_omni/ -│ │ │ ├── qwen2_5_omni_thinker.py │ │ │ ├── test_audio_length.py ✅ -│ │ │ ├── qwen2_5_omni_talker.py │ │ │ ├── test_qwen2_5_omni_thinker.py ⬜ -│ │ │ └── qwen2_5_omni_token2wav.py │ │ │ ├── test_qwen2_5_omni_talker.py ⬜ -│ │ └── qwen3_omni/ │ │ │ └── test_qwen2_5_omni_token2wav.py ⬜ -│ │ └── ... │ │ └── qwen3_omni/ -│ ├── stage_configs/ │ │ └── test_*.py ⬜ -│ │ └── *.yaml │ └── stage_configs/ (used by e2e, test_*.py can be added) ⬜ -│ └── stage_input_processors/ │ └── stage_input_processors/ -│ └── ... │ └── test_*.py ⬜ -│ -├── sample/ → ├── sample/ -│ └── __init__.py │ └── test_*.py ⬜ -│ -├── utils/ → ├── utils/ -│ └── __init__.py │ └── test_*.py ⬜ (no platform_utils.py currently) -│ -├── worker/ → ├── worker/ -│ ├── gpu_ar_model_runner.py │ ├── test_gpu_ar_model_runner.py ⬜ -│ ├── gpu_ar_worker.py │ ├── test_gpu_ar_worker.py ⬜ -│ ├── gpu_generation_model_runner.py │ ├── test_gpu_generation_model_runner.py ✅ -│ ├── gpu_generation_worker.py │ ├── test_gpu_generation_worker.py ⬜ -│ ├── gpu_model_runner.py │ ├── test_omni_gpu_model_runner.py ✅ -│ └── mixins.py │ └── (npu under platforms/npu/worker/) # not worker/npu/ -│ -├── platforms/ → (no tests/platforms/, e2e and stage_configs provide indirect coverage) -│ ├── cuda/ -│ ├── npu/worker/ # NPU worker here, not vllm_omni/worker/npu/ -│ ├── rocm/ -│ └── xpu/worker/ -│ -├── outputs.py → test_outputs.py ✅ (at tests root) -├── (logger, patch, request, version) → (no corresponding unit test) -│ -└── e2e (tests side only) → ├── e2e/ - ├── online_serving/ ✅ non-empty - │ ├── test_qwen2_5_omni.py - │ ├── test_async_omni.py - │ ├── test_qwen3_omni.py - │ ├── test_qwen3_omni_expansion.py - │ ├── test_mimo_audio.py - │ └── test_images_generations_lora.py - └── offline_inference/ ✅ - ├── test_qwen2_5_omni.py - ├── test_qwen3_omni.py - ├── test_bagel_text2img.py - ├── test_z_image.py - ├── test_wan22.py - ├── test_zimage_tensor_parallel.py - ├── test_cache_dit.py - ├── test_teacache.py - ├── test_stable_audio_expansion.py - ├── test_diffusion_cpu_offload.py - ├── test_diffusion_layerwise_offload.py - ├── test_diffusion_lora.py - ├── test_sequence_parallel.py - ├── test_qwen_image_edit_expansion.py - └── stage_configs/ (legacy schema, still present - ├── bagel_*.yaml for unmigrated models) - └── npu/, rocm/, etc. - -# Migrated models (qwen3_omni_moe, qwen2_5_omni, qwen3_tts) live under -# vllm_omni/deploy/ instead — see docs/configuration/stage_configs.md. -examples/ tests -│ └── examples -├── online_serving/ → ├── online_serving/ -│ └── {doc_page_title}/README.md │ └── test_{doc_page_title}.py ⬜ -└── offline_inference/ → └── offline_inference/ - └── {doc_page_title}/README.md └── test_{doc_page_title}.py ⬜ -``` - - - -### Naming Conventions - -- **Unit Tests**: Use `test_.py` format. Example: `stage_utils.py` → `test_stage_utils.py` - -- **E2E Tests**: Place in `tests/e2e/offline_inference/` or `tests/e2e/online_serving/` with descriptive names. Example: `tests/e2e/offline_inference/test_qwen3_omni.py`, `tests/e2e/offline_inference/test_diffusion_model.py` - -- **Expansion Tests** - -### Best Practices - -1. **Mirror Source Structure**: Test directories should mirror the source code structure -2. **Test Type Indicators**: Use comments to indicate test types (UT for unit tests, E2E for end-to-end tests) -3. **Shared Resources**: Place shared test configurations (e.g., CI configs) in appropriate subdirectories -4. **Consistent Naming**: Follow the `test_*.py` naming convention consistently across all test files - - -## Test codes requirements - -### Coding style - -1. **File header**: Add SPDX license header to all test files -2. **Imports**: Pls don't use manual `sys.path` modifications, use standard imports instead. -3. **Test type differentiation**: - - - Unit tests: Maintain mock style - - E2E tests for models: Consider using OmniRunner uniformly, avoid decorators - -4. **Documentation**: Add docstrings to all test functions -5. **Environment variables**: Set uniformly in `conftest.py` or at the top of files -6. **Type annotations**: Add type annotations to all test function parameters -7. **Pytest Markers**: Add necessary markers like `@pytest.mark.core_model` and use `@hardware_test` to declare hardware requirements (check detailed in [Markers for Tests](../ci/tests_markers.md)). - -### Template -#### E2E - Online serving - -E2E Online tests for Qwen3-Omni model with mix input and audio+text output. Based on `tests/e2e/online_serving/test_qwen3_omni.py`. - -```python -""" -E2E Online tests for Qwen3-Omni model with mix input and audio+text output. -""" - -import os - -os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" - -import threading -from pathlib import Path - -import openai -import pytest - -from tests.helpers.media import ( - convert_audio_bytes_to_text, - cosine_similarity_text, - generate_synthetic_video, -) -from tests.helpers.runtime import OmniServer, dummy_messages_from_mix_data -from tests.helpers.stage_config import get_deploy_config_path, modify_stage_config -from vllm_omni.platforms import current_omni_platform - -# Edit: model name and stage config path -models = ["Qwen/Qwen3-Omni-30B-A3B-Instruct"] - -#If you use the default configuration file, you can directly use the following address. -def get_default_config(): - return get_deploy_config_path("ci/qwen3_omni_moe.yaml") - -#If you need to modify the configuration file, you can use modify_stage_config. -def get_chunk_config(): - path = modify_stage_config( - get_default_config(), - updates={ - "async_chunk": True, - "stage_args": { - 0: { - "engine_args.custom_process_next_stage_input_func": "vllm_omni.model_executor.stage_input_processors.qwen3_omni.thinker2talker_async_chunk" - }, - 1: { - "engine_args.custom_process_next_stage_input_func": "vllm_omni.model_executor.stage_input_processors.qwen3_omni.talker2code2wav_async_chunk" - }, - }, - }, - deletes={"stage_args": {2: ["custom_process_input_func"]}}, - ) - return path - -stage_configs = [get_default_config(), CHUNK_CONFIG_PATH] - -test_params = [(model, stage_config) for model in models for stage_config in stage_configs] - - -#Please use this method to launch the online instance. -_omni_server_lock = threading.Lock() - -@pytest.fixture(scope="module") -def omni_server(request): - """Start vLLM-Omni server as a subprocess. Use module scope for multi-stage init (10-20+ min).""" - with _omni_server_lock: - model, stage_config_path = request.param - with OmniServer( - model, - ["--stage-configs-path", stage_config_path, "--stage-init-timeout", "120"], - ) as server: - yield server - - -@pytest.fixture -def client(omni_server): - """OpenAI client for the running vLLM-Omni server.""" - return openai.OpenAI( - base_url=f"http://{omni_server.host}:{omni_server.port}/v1", - api_key="EMPTY", - ) - -#Please use function definitions above the test function to define the prompts and other parameters you need. -def get_system_prompt(): - return { - "role": "system", - "content": [ - { - "type": "text", - "text": ( - "You are Qwen, a virtual human developed by the Qwen Team, " - "Alibaba Group, capable of perceiving auditory and visual inputs, " - "as well as generating text and speech." - ), - } - ], - } - -... - -#Please define test case tags according to the instructions in the marker documentation. -@pytest.mark.core_model -@pytest.mark.omni -@pytest.mark.parametrize("omni_server", test_params, indirect=True) -def test_mix_to_text_audio_001(client: openai.OpenAI, omni_server, request) -> None: - # PLEASE FOLLOW THESE TEMPLATE INSTRUCTIONS: - # ============================================================================ - # TEMPLATE USAGE GUIDE: - # 1. Copy this entire function as a starting point for multi-modal tests - # 2. Update the test name to reflect your specific test scenario - # 3. Modify input/output modalities as needed (see OPTIONS section below) - # 4. Adjust assertions based on your expected outcomes - # 5. Add custom validation logic for your specific use case - # ============================================================================ - - #Please list the relevant test points. - """ - Test multi-modal input processing and text/audio output generation via OpenAI API. - Deploy Setting: default yaml - Input Modal: text + audio + video + image - Output Modal: text + audio - Input Setting: stream=True - Datasets: single request - """ - # SECTION 1: TEST SETUP AND INITIALIZATION - # ========================================= - # INSTRUCTIONS: Initialize test variables and prepare test environment - # MODIFY: Add any additional test setup required for your scenario - e2e_list = list() - # SECTION 2: TEST DATA GENERATION - # ================================ - # INSTRUCTIONS: Generate or load test data for each input modality - # MODIFY: Replace synthetic generators with your actual data sources - # VIDEO DATA - Generate synthetic video for testing - # FORMAT: data:video/mp4;base64,{base64_encoded_video} - # PARAMETERS: width, height, duration_frames - video_data_url = f"data:video/mp4;base64,{generate_synthetic_video(224, 224, 300)['base64']}" - # IMAGE DATA - Generate synthetic image for testing - # FORMAT: data:image/jpeg;base64,{base64_encoded_image} - # PARAMETERS: width, height - image_data_url = f"data:image/jpeg;base64,{generate_synthetic_image(224, 224)['base64']}" - # AUDIO DATA - Generate synthetic audio for testing - # FORMAT: data:audio/wav;base64,{base64_encoded_audio} - # PARAMETERS: duration_seconds, channels - audio_data_url = f"data:audio/wav;base64,{generate_synthetic_audio(5, 1)['base64']}" - - # SECTION 3: MESSAGE CONSTRUCTION - # ================================ - # INSTRUCTIONS: Assemble the complete message payload for API request - # MODIFY: Add/remove modalities or change prompt structure as needed - - # USAGE: Construct a message containing all input modalities - # IMPORTANT: Ensure the message structure matches OpenAI API expectations - # CUSTOMIZATION POINTS: - # - system_prompt: Controls the assistant's behavior - # - content_text: The user's text prompt/question - # - *_data_url: URLs for media content (video/image/audio) - messages = dummy_messages_from_mix_data( - system_prompt=get_system_prompt(), - video_data_url=video_data_url, - image_data_url=image_data_url, - audio_data_url=audio_data_url, - content_text=get_prompt("mix"), - ) - - # SECTION 4: API REQUEST EXECUTION - # ================================= - # INSTRUCTIONS: Make the API call and measure performance - # MODIFY: Add timeout, retry logic, or additional parameters - start_time = time.perf_counter() - chat_completion = client.chat.completions.create(model=omni_server.model, messages=messages, stream=True) - - #Call using your preferred method and obtain the final audio and text outputs. - ... - - # SECTION 5: OUTPUT VALIDATION - # ============================= - # INSTRUCTIONS: Verify that outputs meet expected criteria - # MODIFY: Adjust validation logic for your specific requirements - - # ASSERTION 1: E2E Validation - # PURPOSE: Verify that the E2E latency is less than the baseline. - current_e2e = time.perf_counter() - start_time - print(f"the request e2e is: {current_e2e}") - e2e_list.append(current_e2e) - - print(f"the avg e2e is: {sum(e2e_list) / len(e2e_list)}") - - - - # ASSERTION 2: Text Output Validation - # PURPOSE: Verify that text output was generated with keyword content - assert text_content is not None and len(text_content) >= 2, "No text output is generated" - assert any( - keyword in text_content.lower() for keyword in ["square", "quadrate", "sphere", "globe", "circle", "round"] - ), "The output does not contain any of the keywords." - - - # ASSERTION 3: Cross-Modal Consistency - # PURPOSE: Verify text and audio outputs convey the same information - # CUSTOMIZATION: Adjust similarity threshold (0.9) based on accuracy requirements - assert audio_data is not None, "No audio output is generated" - audio_content = convert_audio_bytes_to_text(audio_data) - print(f"text content is: {text_content}") - print(f"audio content is: {audio_content}") - similarity = cosine_similarity_text(audio_content.lower(), text_content.lower()) - print(f"similarity is: {similarity}") - assert similarity > 0.9, "The audio content is not same as the text" -``` - - -#### E2E - Offline inference -```python -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Offline E2E smoke test for an omni model (video → audio). -""" - -import os -from pathlib import Path - -import pytest -from vllm.assets.video import VideoAsset - -from tests.helpers.mark import hardware_test -from ..multi_stages.conftest import OmniRunner - -# Optional: set process start method for workers -os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" - -models = ["{your model name}"] #Edit here to load your model -stage_configs = [str(Path(__file__).parent / "stage_configs" / {your model yaml})] #Edit here to load your model yaml - -# Create parameter combinations for model and stage config -test_params = [(model, stage_config) for model in models for stage_config in stage_configs] - -# function name: test_{input_modality}_to_{output_modality} -# modality candidate: text, image, audio, video, mixed_modalities -@pytest.mark.core_model -@pytest.mark.omni -@hardware_test( - res={"cuda": "L4", "rocm": "MI325", "npu": "A2"}, - num_cards=2, -) -@pytest.mark.parametrize("test_config", test_params) -def test_video_to_audio(omni_runner: type[OmniRunner], model: str) -> None: - """Offline inference: video input, audio output.""" - model, stage_config_path = test_config - with omni_runner(model, seed=42, stage_configs_path=stage_config_path) as runner: - # Prepare inputs - video = VideoAsset(name="sample", num_frames=4).np_ndarrays - - outputs = runner.generate_multimodal( - prompts="Describe this video briefly.", - videos=video, - ) - - # Minimal assertions: got outputs and at least one audio result - assert outputs - has_audio = any(o.final_output_type == "audio" for o in outputs) - assert has_audio -``` diff --git a/docs/contributing/model/adding_diffusion_model.md b/docs/contributing/model/adding_diffusion_model.md index b38805112..7a8d6177f 100644 --- a/docs/contributing/model/adding_diffusion_model.md +++ b/docs/contributing/model/adding_diffusion_model.md @@ -685,7 +685,7 @@ For a fair comparison, keep the same **prompt**, **seed**, **resolution**, **num To ensure project maintainability and sustainable development, please submit test code (unit tests, system tests, or end-to-end tests) alongside their code changes. -For comprehensive testing guidelines and the definition of test levels (L1-L5), please refer to the [Multi-Level Automated Testing System Documentation](../ci/CI_5levels.md). You are at least required to add an L4 *functionality* test described in that document. +For the definition of test levels (L1-L5), see [Test System Overview](../ci/test_system_overview.md). You are at least required to add an L4 *functionality* test as described in [Test Writing Guide](../ci/test_writing_guide.md#l4-level-testing-full-functionality-performance-and-documentation-testing). --- diff --git a/docs/contributing/model/adding_omni_model.md b/docs/contributing/model/adding_omni_model.md index 5032c9805..1aeed0b51 100644 --- a/docs/contributing/model/adding_omni_model.md +++ b/docs/contributing/model/adding_omni_model.md @@ -540,7 +540,7 @@ Do **not** add a no-suffix `thinker2talker` when a `sync_process_input_func` is ## Testing -For comprehensive testing guidelines, please refer to the [Test File Structure and Style Guide](../ci/tests_style.md). +For comprehensive testing guidelines, please refer to the [Test Writing Guide](../ci/test_writing_guide.md). ## Adding a Model Recipe diff --git a/docs/contributing/model/adding_tts_model.md b/docs/contributing/model/adding_tts_model.md index b9bb9484f..7d0968d80 100644 --- a/docs/contributing/model/adding_tts_model.md +++ b/docs/contributing/model/adding_tts_model.md @@ -637,7 +637,7 @@ Key points: ## Testing -For general testing conventions, see [tests_style.md](../ci/tests_style.md). +For general testing conventions, see [Test Writing Guide](../ci/test_writing_guide.md). Recommended test cases for a new TTS model: diff --git a/tests/buildkite/test_skip_ci.py b/tests/buildkite/test_skip_ci.py new file mode 100644 index 000000000..ddefc7271 --- /dev/null +++ b/tests/buildkite/test_skip_ci.py @@ -0,0 +1,370 @@ +# SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations + +import sys +from dataclasses import asdict +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[2] / ".buildkite" / "common" / "scripts")) + +from skip_ci import ( # noqa: E402 + CiDecision, + _diff_only_contains_skip_mark_changes, + resolve_ci_decision, +) + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +def _decision(changed_files: list[str]) -> CiDecision: + return resolve_ci_decision(changed_files) + + +def _is_yaml_gated(decision: CiDecision) -> bool: + return decision.skip_l2_l3 and not decision.skip_all + + +def _all_l23_skipped(decision: CiDecision) -> bool: + checks = ( + ("cuda", "l2"), + ("cuda", "l3"), + ("amd", "l2"), + ("amd", "l3"), + ("intel", "l2"), + ("npu", "l2"), + ) + return all(not decision.is_run(platform, level) for platform, level in checks) + + +def test_docs_only_change() -> None: + assert _decision(["docs/foo.md", "README.md", "mkdocs.yml"]).skip_all + + +def test_docs_only_rejects_product_code() -> None: + assert not _decision(["docs/foo.md", "vllm_omni/foo.py"]).skip_all + + +def test_skip_mark_diff_only() -> None: + diff = """\ +diff --git a/tests/foo/test_bar.py b/tests/foo/test_bar.py +--- a/tests/foo/test_bar.py ++++ b/tests/foo/test_bar.py +@@ -1,3 +1,4 @@ + import pytest ++ ++@pytest.mark.skip(reason="temp") + def test_x(): +""" + assert _diff_only_contains_skip_mark_changes(diff) + + +def test_skip_mark_removal_does_not_qualify() -> None: + """Removing a skip decorator re-enables the test; must not suppress CI.""" + diff = """\ +diff --git a/tests/foo/test_bar.py b/tests/foo/test_bar.py +--- a/tests/foo/test_bar.py ++++ b/tests/foo/test_bar.py +@@ -1,5 +1,4 @@ + import pytest +- +-@pytest.mark.skip(reason="temp") + def test_x(): + assert True +""" + assert not _diff_only_contains_skip_mark_changes(diff) + + +def test_skip_mark_reason_edit_still_qualifies() -> None: + diff = """\ +diff --git a/tests/foo/test_bar.py b/tests/foo/test_bar.py +--- a/tests/foo/test_bar.py ++++ b/tests/foo/test_bar.py +@@ -1,4 +1,4 @@ + import pytest +-@pytest.mark.skip(reason="old") ++@pytest.mark.skip(reason="new") + def test_x(): +""" + assert _diff_only_contains_skip_mark_changes(diff) + + +def test_skip_mark_diff_rejects_assertion_change() -> None: + diff = """\ +diff --git a/tests/foo/test_bar.py b/tests/foo/test_bar.py +--- a/tests/foo/test_bar.py ++++ b/tests/foo/test_bar.py +@@ -1,3 +1,3 @@ + def test_x(): +- assert 1 == 1 ++ assert 2 == 2 +""" + assert not _diff_only_contains_skip_mark_changes(diff) + + +def test_cuda_ready_only_runs_cuda_l2() -> None: + decision = _decision([".buildkite/cuda/test-ready.yml"]) + assert _is_yaml_gated(decision) + assert decision.is_run("cuda", "l2") + assert not decision.is_run("cuda", "l3") + assert _all_l23_skipped(decision) is False + + +def test_cuda_merge_only_runs_cuda_l3() -> None: + decision = _decision([".buildkite/cuda/test-merge.yml"]) + assert _is_yaml_gated(decision) + assert not decision.is_run("cuda", "l2") + assert decision.is_run("cuda", "l3") + assert not decision.is_run("amd", "l3") + + +def test_l4_only_skips_all_l23() -> None: + decision = _decision([".buildkite/cuda/test-nightly.yml"]) + assert _is_yaml_gated(decision) + assert _all_l23_skipped(decision) + assert not decision.is_run("cuda", "l2") + assert not decision.is_run("cuda", "l3") + + +def test_weekly_only_skips_all_l23() -> None: + decision = _decision([".buildkite/cuda/test-weekly.yml"]) + assert _is_yaml_gated(decision) + assert _all_l23_skipped(decision) + assert not decision.is_run("cuda", "l2") + assert not decision.is_run("cuda", "l3") + + +def test_npu_nightly_only_skips_all_l23() -> None: + decision = _decision([".buildkite/npu/test-npu-nightly.yml"]) + assert _is_yaml_gated(decision) + assert _all_l23_skipped(decision) + assert not decision.is_run("npu", "l2") + assert not decision.is_run("npu", "l3") + + +def test_nightly_and_weekly_skips_all_l23() -> None: + decision = _decision( + [ + ".buildkite/cuda/test-nightly.yml", + ".buildkite/cuda/test-weekly.yml", + ], + ) + assert _is_yaml_gated(decision) + assert _all_l23_skipped(decision) + assert not decision.is_run("cuda", "l2") + assert not decision.is_run("cuda", "l3") + + +def test_l2_and_l4_skips_l3() -> None: + files = [ + ".buildkite/cuda/test-ready.yml", + ".buildkite/cuda/test-nightly.yml", + ] + decision = _decision(files) + assert _is_yaml_gated(decision) + assert decision.is_run("cuda", "l2") + assert not decision.is_run("cuda", "l3") + + +def test_l3_and_l4_skips_l2() -> None: + files = [ + ".buildkite/cuda/test-merge.yml", + ".buildkite/cuda/test-nightly.yml", + ] + decision = _decision(files) + assert _is_yaml_gated(decision) + assert not decision.is_run("cuda", "l2") + assert decision.is_run("cuda", "l3") + + +def test_ready_and_merge_runs_cuda_l2_l3() -> None: + decision = _decision( + [ + ".buildkite/cuda/test-ready.yml", + ".buildkite/cuda/test-merge.yml", + ], + ) + assert _is_yaml_gated(decision) + assert decision.is_run("cuda", "l2") + assert decision.is_run("cuda", "l3") + + +def test_cross_platform_cuda_l2_and_amd_l3_preserves_both() -> None: + """CUDA ready + AMD merge must keep cuda/l2 and amd/l3 (not wipe L2 outside L3 platforms).""" + decision = _decision( + [ + ".buildkite/cuda/test-ready.yml", + ".buildkite/amd/test-amd-merge.yml", + ], + ) + assert _is_yaml_gated(decision) + assert decision.is_run("cuda", "l2") + assert not decision.is_run("cuda", "l3") + assert not decision.is_run("amd", "l2") + assert decision.is_run("amd", "l3") + assert not decision.is_run("intel", "l2") + assert not decision.is_run("npu", "l2") + assert not decision.device.skip_cuda_l2 + assert decision.device.skip_cuda_l3 + assert decision.device.skip_amd_l2 + assert not decision.device.skip_amd_l3 + + +def test_amd_ready_only_runs_amd_l2() -> None: + decision = _decision([".buildkite/amd/test-amd-ready.yml"]) + assert _is_yaml_gated(decision) + assert decision.is_run("amd", "l2") + assert not decision.is_run("cuda", "l2") + + +def test_npu_ready_only_runs_npu_l2() -> None: + decision = _decision([".buildkite/npu/test-npu-ready.yml"]) + assert _is_yaml_gated(decision) + assert decision.is_run("npu", "l2") + assert not decision.is_run("cuda", "l2") + assert not decision.is_run("npu", "l3") + + +def test_amd_merge_only_runs_amd_l3() -> None: + decision = _decision([".buildkite/amd/test-amd-merge.yml"]) + assert _is_yaml_gated(decision) + assert decision.is_run("amd", "l3") + assert not decision.is_run("amd", "l2") + assert not decision.is_run("cuda", "l3") + + +def test_intel_pipeline_runs_intel_l2() -> None: + decision = _decision([".buildkite/intel/pipeline-intel.yml"]) + assert _is_yaml_gated(decision) + assert decision.is_run("intel", "l2") + assert not decision.is_run("cuda", "l2") + + +def test_cuda_and_amd_ready_runs_both_l2() -> None: + decision = _decision( + [ + ".buildkite/cuda/test-ready.yml", + ".buildkite/amd/test-amd-ready.yml", + ], + ) + assert _is_yaml_gated(decision) + assert decision.is_run("cuda", "l2") + assert decision.is_run("amd", "l2") + assert not decision.is_run("cuda", "l3") + assert not decision.is_run("amd", "l3") + assert not decision.is_run("intel", "l2") + assert not decision.is_run("npu", "l2") + + +def test_cuda_and_amd_merge_runs_both_l3() -> None: + decision = _decision( + [ + ".buildkite/cuda/test-merge.yml", + ".buildkite/amd/test-amd-merge.yml", + ], + ) + assert _is_yaml_gated(decision) + assert decision.is_run("cuda", "l3") + assert decision.is_run("amd", "l3") + assert not decision.is_run("cuda", "l2") + assert not decision.is_run("amd", "l2") + + +def test_ready_merge_and_nightly_runs_cuda_l2_l3() -> None: + """G7: L2+L3+L45 → same matrix as ready+merge (L45 rescued by L2/L3 YAML).""" + decision = _decision( + [ + ".buildkite/cuda/test-ready.yml", + ".buildkite/cuda/test-merge.yml", + ".buildkite/cuda/test-nightly.yml", + ], + ) + assert _is_yaml_gated(decision) + assert decision.is_run("cuda", "l2") + assert decision.is_run("cuda", "l3") + assert not decision.is_run("amd", "l2") + assert not decision.is_run("amd", "l3") + + +def test_non_whitelist_buildkite_runs_normal_ci() -> None: + assert not _is_yaml_gated(_decision([".buildkite/common/scripts/skip_ci.py"])) + assert not _is_yaml_gated(_decision([".buildkite/cuda/pipeline.yml"])) + + +def test_whitelist_plus_non_whitelist_buildkite_runs_normal_ci() -> None: + assert not _is_yaml_gated( + _decision( + [ + ".buildkite/cuda/test-ready.yml", + ".buildkite/cuda/pipeline.yml", + ], + ), + ) + + +def test_mixed_product_code_disables_targeting() -> None: + assert not _is_yaml_gated( + _decision([".buildkite/cuda/test-ready.yml", "vllm_omni/x.py"]), + ) + + +def test_docs_mixed_with_ci_yaml_follows_yaml_gating() -> None: + decision = _decision(["docs/a.md", ".buildkite/cuda/test-ready.yml"]) + assert _is_yaml_gated(decision) + assert not decision.skip_all + assert decision.is_run("cuda", "l2") + assert not decision.is_run("cuda", "l3") + assert not decision.is_run("amd", "l2") + assert "follow CI YAML gating" in decision.message + + +def test_docs_mixed_with_l45_yaml_skips_all_l23() -> None: + decision = _decision(["docs/a.md", ".buildkite/cuda/test-nightly.yml"]) + assert _is_yaml_gated(decision) + assert not decision.skip_all + assert not decision.is_run("cuda", "l2") + assert not decision.is_run("cuda", "l3") + assert not decision.is_run("amd", "l2") + + +def test_non_whitelist_path_disables_targeting() -> None: + assert not _is_yaml_gated(_decision([".buildkite/unknown/foo.yml"])) + assert not _is_yaml_gated(_decision(["vllm_omni/x.py"])) + + +def test_is_run_l23_when_not_yaml_gated() -> None: + decision = _decision(["vllm_omni/x.py"]) + assert decision.is_run("cuda", "l2") + assert decision.is_run("amd", "l3") + + +def test_is_run_l23_respects_decision() -> None: + decision = _decision([".buildkite/cuda/test-ready.yml"]) + assert _is_yaml_gated(decision) + assert decision.is_run("cuda", "l2") + assert not decision.is_run("cuda", "l3") + assert not decision.is_run("amd", "l2") + + +def test_docs_or_skip_mark_only_without_diff_range_helper() -> None: + assert _decision(["docs/a.md"]).skip_all + assert not _decision(["vllm_omni/a.py"]).skip_all + + +def test_decision_serializes_to_dict() -> None: + decision = _decision([".buildkite/cuda/test-ready.yml"]) + assert _is_yaml_gated(decision) + payload = asdict(decision) + assert payload["skip_l2_l3"] is True + assert payload["device"]["skip_cuda_l2"] is False + assert payload["device"]["skip_cuda_l3"] is True + + +def test_yaml_gated_message_lists_changed_and_targets() -> None: + decision = _decision([".buildkite/cuda/test-ready.yml"]) + assert decision.message.startswith("only CI config YAML changed") + assert "changed: L2=[.buildkite/cuda/test-ready.yml]" in decision.message + assert "run: cuda/l2" in decision.message + assert "skip: cuda/l3" in decision.message + assert "amd/l2" in decision.message diff --git a/tests/buildkite/test_upload_pipeline.py b/tests/buildkite/test_upload_pipeline.py new file mode 100644 index 000000000..24a2a731e --- /dev/null +++ b/tests/buildkite/test_upload_pipeline.py @@ -0,0 +1,141 @@ +# SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations + +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[2] / ".buildkite" / "common" / "scripts")) + +from skip_ci import resolve_ci_decision # noqa: E402 +from upload_pipeline import ( # noqa: E402 + _expand_mirror_hardwares, + _load_bootstrap_steps, + _render_bootstrap_pipeline, + _render_test_pipeline, +) + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + +CUDA_BOOTSTRAP_STEPS = Path(".buildkite/cuda/bootstrap-upload-steps.yml") +BOOTSTRAP_STEPS_TEMPLATE = """steps: + - key: image-build + - key: upload-ready-pipeline + - key: upload-merge-pipeline + - key: upload-nightly-pipeline + - key: upload-weekly-pipeline +""" + + +def _render(changed_files: list[str]) -> str: + decision = resolve_ci_decision(changed_files) + return _render_bootstrap_pipeline( + BOOTSTRAP_STEPS_TEMPLATE, + decision=decision, + path=CUDA_BOOTSTRAP_STEPS, + ) + + +def test_bootstrap_if_injected_by_step_key() -> None: + import yaml + + rendered = _render_bootstrap_pipeline( + BOOTSTRAP_STEPS_TEMPLATE, + decision=resolve_ci_decision([]), + path=Path(".buildkite/npu/bootstrap-upload-steps.yml"), + ) + doc = yaml.safe_load(rendered) + by_key = {step["key"]: step for step in doc["steps"]} + assert "image-build" in by_key + # Unconditional image has no ``if`` (Buildkite rejects YAML bool if: true). + assert "if" not in by_key["image-build"] + assert isinstance(by_key["upload-ready-pipeline"]["if"], str) + assert isinstance(by_key["upload-nightly-pipeline"]["if"], str) + + +def test_bootstrap_steps_loaded_from_file() -> None: + steps = _load_bootstrap_steps(CUDA_BOOTSTRAP_STEPS) + assert "key: image-build" in steps + assert "key: upload-ready-pipeline" in steps + assert "placeholder:" not in steps + + +def test_docs_only_allows_main_scheduled_nightly_weekly_only() -> None: + """skip_all: no PR labels; main + NIGHTLY=1 / WEEKLY=1 still gates scheduled CI.""" + rendered = _render(["docs/foo.md"]) + assert "key: image-build" in rendered + assert "key: upload-nightly-pipeline" in rendered + assert "key: upload-weekly-pipeline" in rendered + # Scheduled nightly also uploads L2/L3 with --e2e + assert "key: upload-ready-pipeline" in rendered + assert "key: upload-merge-pipeline" in rendered + assert 'build.env("NIGHTLY") == "1"' in rendered + assert 'build.env("WEEKLY") == "1"' in rendered + assert "nightly-test" not in rendered + assert "weekly-test" not in rendered + assert "merge-test" not in rendered + assert 'labels includes "ready"' not in rendered + assert "if: false" not in rendered + + +def test_yaml_gated_l45_only_does_not_unconditionally_build_image() -> None: + rendered = _render([".buildkite/cuda/test-nightly.yml"]) + assert "if: true" not in rendered + assert 'build.pull_request.labels includes "nightly-test"' in rendered + assert 'build.pull_request.labels includes "weekly-test"' in rendered + # L2/L3 upload steps are unconditionally disabled → omitted from pipeline + assert "key: upload-ready-pipeline" not in rendered + assert "key: upload-merge-pipeline" not in rendered + assert "key: upload-weekly-pipeline" in rendered + + +def test_yaml_gated_l2_still_enables_image_via_ready_base() -> None: + rendered = _render([".buildkite/cuda/test-ready.yml"]) + assert 'build.pull_request.labels includes "ready"' in rendered + assert "if: true" not in rendered + + +def test_mirror_hardwares_l4_1_expands_to_agents_and_plugins() -> None: + doc = { + "steps": [ + { + "label": "Simple Test", + "mirror_hardwares": "l4_1", + "commands": ["pytest -sv tests/example"], + }, + ], + } + rendered = _render_test_pipeline(doc, changed_files=None) + step = rendered["steps"][0] + assert "mirror_hardwares" not in step + assert step["agents"]["queue"] == "gpu_1_queue" + assert step["plugins"][0]["docker#v5.2.0"]["image"].endswith("$BUILDKITE_COMMIT") + + +def test_mirror_hardwares_conflicts_with_explicit_agents() -> None: + with pytest.raises(ValueError, match="agents/plugins/image"): + _expand_mirror_hardwares( + {"label": "bad", "mirror_hardwares": "l4_1", "agents": {"queue": "gpu_1_queue"}}, + ) + + +def test_mirror_hardwares_a2b3_npu_4_expands_agents_image_and_plugins() -> None: + doc = { + "steps": [ + { + "label": "NPU X2V Test", + "mirror_hardwares": "a2b3_npu_4", + "commands": ["pytest -sv tests/example"], + }, + ], + } + rendered = _render_test_pipeline(doc, changed_files=None) + step = rendered["steps"][0] + assert "mirror_hardwares" not in step + assert step["agents"]["queue"] == "ascend-a2b3" + assert step["agents"]["resource_class"] == "npu-4" + assert step["image"].endswith("${BUILDKITE_COMMIT}") + assert step["plugins"][0]["kubernetes"]["podSpecPatch"]["imagePullSecrets"] == [ + {"name": "swr-secret"}, + ] From 8927ec057c272ca15b11097dc404025b89005055 Mon Sep 17 00:00:00 2001 From: Yang_wdrs <66548561+yangjianjuan@users.noreply.github.com> Date: Fri, 24 Jul 2026 22:42:45 +0800 Subject: [PATCH 086/240] [Refactor][1/N]diffusion/cache: cache-dit (#5226) Signed-off-by: yangjianjuan <66548561+yangjianjuan@users.noreply.github.com> --- .claude/skills/add-diffusion-model/SKILL.md | 3 +- .../references/cache-dit-patterns.md | 20 +- docs/design/feature/cache_dit.md | 20 +- docs/design/module/dit_module.md | 2 +- .../image_to_image/custom_pipeline.py | 4 +- tests/diffusion/cache/test_cache_backends.py | 117 ++++- tests/diffusion/cache/test_cache_dit.py | 92 +++- .../models/cosmos3/test_cosmos3_pipeline.py | 2 +- .../models/ltx2/test_ltx2_pipeline.py | 2 +- .../models/ltx2/test_ltx2_transformer.py | 2 +- .../diffusion/cache/cachedit/__init__.py | 35 ++ vllm_omni/diffusion/cache/cachedit/backend.py | 222 ++++++++++ vllm_omni/diffusion/cache/cachedit/config.py | 96 ++++ .../model_specific.py} | 410 ++---------------- .../diffusion/cache/magcache/__init__.py | 2 + vllm_omni/diffusion/cache/selector.py | 6 +- .../models/bagel/bagel_transformer.py | 2 +- .../models/cosmos3/transformer_cosmos3.py | 2 +- .../diffusion/models/dreamid_omni/fusion.py | 2 +- .../ernie_image/ernie_image_transformer.py | 2 +- .../diffusion/models/flux/flux_transformer.py | 2 +- .../models/flux2/flux2_transformer.py | 2 +- .../flux2_klein/flux2_klein_transformer.py | 2 +- .../models/glm_image/glm_image_transformer.py | 2 +- .../models/helios/helios_transformer.py | 2 +- .../hunyuan_image3_transformer.py | 2 +- .../hunyuan_video_15_transformer.py | 2 +- .../longcat_image_transformer.py | 2 +- .../diffusion/models/ltx2/ltx2_transformer.py | 2 +- .../diffusion/models/sd3/sd3_transformer.py | 2 +- .../sensenova_u1/sensenova_u1_transformer.py | 2 +- .../models/soulx_singer/modules/llama.py | 2 +- .../stable_audio/stable_audio_transformer.py | 2 +- .../worker/diffusion_model_runner.py | 2 +- 34 files changed, 635 insertions(+), 436 deletions(-) create mode 100644 vllm_omni/diffusion/cache/cachedit/__init__.py create mode 100644 vllm_omni/diffusion/cache/cachedit/backend.py create mode 100644 vllm_omni/diffusion/cache/cachedit/config.py rename vllm_omni/diffusion/cache/{cache_dit_backend.py => cachedit/model_specific.py} (67%) diff --git a/.claude/skills/add-diffusion-model/SKILL.md b/.claude/skills/add-diffusion-model/SKILL.md index e06af9f21..63382e76f 100644 --- a/.claude/skills/add-diffusion-model/SKILL.md +++ b/.claude/skills/add-diffusion-model/SKILL.md @@ -418,7 +418,8 @@ def enable_cache_for_your_model(pipeline, cache_config): #### 9d. Register the custom enabler -Add your enabler to `CUSTOM_DIT_ENABLERS` in `vllm_omni/diffusion/cache/cache_dit_backend.py`: +Add your enabler to `CUSTOM_DIT_ENABLERS` in +`vllm_omni/diffusion/cache/cachedit/model_specific.py`: ```python CUSTOM_DIT_ENABLERS = { diff --git a/.claude/skills/add-diffusion-model/references/cache-dit-patterns.md b/.claude/skills/add-diffusion-model/references/cache-dit-patterns.md index d34ce0e0f..2385a70ee 100644 --- a/.claude/skills/add-diffusion-model/references/cache-dit-patterns.md +++ b/.claude/skills/add-diffusion-model/references/cache-dit-patterns.md @@ -29,8 +29,13 @@ vLLM-Omni integrates cache-dit through `CacheDiTBackend`: | `cache_dit.refresh_context()` | Updates cache context when `num_inference_steps` changes | **Source files**: -- `vllm_omni/diffusion/cache/cache_dit_backend.py` — `CacheDiTBackend`, enablers, `CUSTOM_DIT_ENABLERS` -- `vllm_omni/diffusion/cache/` — cache backend implementations +- `vllm_omni/diffusion/cache/cachedit/__init__.py` — public package API +- `vllm_omni/diffusion/cache/cachedit/backend.py` — generic backend lifecycle and integration +- `vllm_omni/diffusion/cache/cachedit/config.py` — Cache-DiT configuration types +- `vllm_omni/diffusion/cache/cachedit/model_specific.py` — model-specific adapters, enablers, and registry + +Outside `vllm_omni/diffusion/cache`, import Cache-DiT symbols only from +`vllm_omni.diffusion.cache.cachedit`. ## Standard Models: Automatic Support @@ -175,7 +180,8 @@ Inspect your block's `forward()` return type and residual connection pattern to ## Registering Custom Enablers -Add your enabler to `CUSTOM_DIT_ENABLERS` in `vllm_omni/diffusion/cache/cache_dit_backend.py`: +Add your enabler to `CUSTOM_DIT_ENABLERS` in +`vllm_omni/diffusion/cache/cachedit/model_specific.py`: ```python CUSTOM_DIT_ENABLERS = { @@ -248,7 +254,7 @@ Models listed in `_NO_CACHE_ACCELERATION` in `vllm_omni/diffusion/registry.py` d | Model | Path | Notes | |-------|------|-------| -| Standard DiT | `cache_dit_backend.py::enable_cache_for_dit` | Default enabler, automatic | -| Wan2.2 | `cache_dit_backend.py::enable_cache_for_wan22` | Dual-transformer, auto-detects mode | -| LongCat | `cache_dit_backend.py::enable_cache_for_longcat_image` | Multi-block-list | -| BAGEL | `cache_dit_backend.py::enable_cache_for_bagel` | Complex omni model | +| Standard DiT | `cachedit.backend::enable_cache_for_dit` | Default enabler, automatic | +| Wan2.2 | `cachedit.model_specific::enable_cache_for_wan22` | Dual-transformer, auto-detects mode | +| LongCat | `cachedit.config::CacheDiTAdapterConfig` | Declarative multi-block-list adapter | +| BAGEL | `cachedit.model_specific::BagelCachedAdapter` | Custom cached adapter | diff --git a/docs/design/feature/cache_dit.md b/docs/design/feature/cache_dit.md index 237a95877..38b597ca3 100644 --- a/docs/design/feature/cache_dit.md +++ b/docs/design/feature/cache_dit.md @@ -32,10 +32,13 @@ The library supports three main caching strategies: vLLM-omni integrates cache-dit through the `CacheDiTBackend` class, which provides a unified interface for managing cache-dit acceleration on diffusion models. +Code outside `vllm_omni/diffusion/cache` must import Cache-DiT symbols from the +package-level API: `vllm_omni.diffusion.cache.cachedit`. + | Method/Class | Purpose | Behavior | |--------------|---------|----------| -| [`CacheDiTBackend`](https://docs.vllm.ai/projects/vllm-omni/en/latest/api/vllm_omni/diffusion/cache/#vllm_omni.diffusion.cache.CacheBackend) | Unified backend interface | Automatically handles enabler selection and cache refresh | -| [`enable_cache_for_dit()`](https://docs.vllm.ai/projects/vllm-omni/en/latest/api/vllm_omni/diffusion/cache/cache_dit_backend/#vllm_omni.diffusion.cache.cache_dit_backend.enable_cache_for_dit) | Apply caching to transformer | Configures DBCache on transformer blocks | +| `CacheDiTBackend` | Unified backend interface | Automatically handles enabler selection and cache refresh | +| `enable_cache_for_dit()` | Apply caching to transformer | Configures DBCache on transformer blocks | **Key APIs from Cache-DiT:** @@ -47,7 +50,7 @@ vLLM-omni integrates cache-dit through the `CacheDiTBackend` class, which provid | `ForwardPattern` | Defines block forward signature patterns: `Pattern_0`, `Pattern_1`, `Pattern_2` | | `ParamsModifier` | Per-transformer or per-block-list cache configuration customization | | `DBCacheConfig` | Configuration for DBCache parameters (warmup steps, cached steps, thresholds) | -| `refresh_context()` | Update cache context | Called when `num_inference_steps` changes | +| `refresh_context()` | Update cache context | Called at every generation boundary | --- @@ -179,7 +182,8 @@ cache_dit.enable_cache( ### Registering Custom Implementations -After writing your custom enabler, register it in `CUSTOM_DIT_ENABLERS` in `vllm_omni/diffusion/cache/cache_dit_backend.py`: +After writing your custom enabler, register it in `CUSTOM_DIT_ENABLERS` in +`vllm_omni/diffusion/cache/cachedit/model_specific.py`: ```python CUSTOM_DIT_ENABLERS = { @@ -266,10 +270,10 @@ Complete examples in the codebase: | Model | Path | Pattern | Notes | |-------|------|---------|-------| -| **Standard DiT** | [`cache_dit_backend.py::enable_cache_for_dit`](https://docs.vllm.ai/projects/vllm-omni/en/latest/api/vllm_omni/diffusion/cache/cache_dit_backend/#vllm_omni.diffusion.cache.cache_dit_backend.enable_cache_for_dit) | Default enabler | Single transformer, automatic | -| **Wan2.2** | [`cache_dit_backend.py::enable_cache_for_wan22`](https://docs.vllm.ai/projects/vllm-omni/en/latest/api/vllm_omni/diffusion/cache/cache_dit_backend/#vllm_omni.diffusion.cache.cache_dit_backend.enable_cache_for_wan22) | Single or dual-transformer | Auto-detects mode based on transformer_2 presence | -| **LongCat** | [`cache_dit_backend.py::enable_cache_for_longcat_image`](https://docs.vllm.ai/projects/vllm-omni/en/latest/api/vllm_omni/diffusion/cache/cache_dit_backend/#vllm_omni.diffusion.cache.cache_dit_backend.enable_cache_for_longcat_image) | Multi-block-list | Two block lists in one transformer | -| **BAGEL** | [`cache_dit_backend.py::enable_cache_for_bagel`](https://docs.vllm.ai/projects/vllm-omni/en/latest/api/vllm_omni/diffusion/cache/cache_dit_backend/#vllm_omni.diffusion.cache.cache_dit_backend.enable_cache_for_bagel) | Omni model | Complex architecture | +| **Standard DiT** | `cachedit.backend::enable_cache_for_dit` | Default enabler | Single transformer, automatic | +| **Wan2.2** | `cachedit.model_specific::enable_cache_for_wan22` | Single or dual-transformer | Auto-detects mode based on transformer_2 presence | +| **LongCat** | `cachedit.config::CacheDiTAdapterConfig` | Declarative block adapter | Two block lists in one transformer | +| **BAGEL** | `cachedit.model_specific::BagelCachedAdapter` | Custom cached adapter | Complex architecture | --- diff --git a/docs/design/module/dit_module.md b/docs/design/module/dit_module.md index 7494ee7b5..ace355427 100644 --- a/docs/design/module/dit_module.md +++ b/docs/design/module/dit_module.md @@ -769,7 +769,7 @@ class TeaCacheBackend(CacheBackend): **2. Cache-DiT Backend** -**Location**: `vllm_omni/diffusion/cache/cache_dit_backend.py` +**Location**: `vllm_omni/diffusion/cache/cachedit/` ```python class CacheDiTBackend(CacheBackend): diff --git a/examples/offline_inference/custom_pipeline/image_to_image/custom_pipeline.py b/examples/offline_inference/custom_pipeline/image_to_image/custom_pipeline.py index fb14d9d33..1d38a598e 100644 --- a/examples/offline_inference/custom_pipeline/image_to_image/custom_pipeline.py +++ b/examples/offline_inference/custom_pipeline/image_to_image/custom_pipeline.py @@ -6,13 +6,13 @@ import logging import torch from vllm_omni.diffusion.data import OmniDiffusionConfig -from vllm_omni.diffusion.models.qwen_image.pipeline_qwen_image_edit import QwenImageEditPipeline +from vllm_omni.diffusion.models.qwen_image.pipeline_qwen_image_edit_plus import QwenImageEditPlusPipeline from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch logger = logging.getLogger(__name__) -class CustomPipeline(QwenImageEditPipeline): +class CustomPipeline(QwenImageEditPlusPipeline): def __init__(self, *, od_config: OmniDiffusionConfig, prefix: str = ""): super().__init__(od_config=od_config, prefix=prefix) diff --git a/tests/diffusion/cache/test_cache_backends.py b/tests/diffusion/cache/test_cache_backends.py index f5981b927..bb7e9f2c9 100644 --- a/tests/diffusion/cache/test_cache_backends.py +++ b/tests/diffusion/cache/test_cache_backends.py @@ -17,13 +17,14 @@ import cache_dit import pytest from cache_dit import ForwardPattern -from vllm_omni.diffusion.cache.cache_dit_backend import ( +from vllm_omni.diffusion.cache.cachedit import ( CacheDiTAdapterConfig, CacheDiTBackend, + CacheDiTConfig, ) -from vllm_omni.diffusion.cache.magcache.backend import MagCacheBackend +from vllm_omni.diffusion.cache.magcache import MagCacheBackend from vllm_omni.diffusion.cache.selector import get_cache_backend -from vllm_omni.diffusion.cache.teacache.backend import TeaCacheBackend +from vllm_omni.diffusion.cache.teacache import TeaCacheBackend from vllm_omni.diffusion.data import DiffusionCacheConfig pytestmark = [pytest.mark.core_model, pytest.mark.cpu] @@ -32,6 +33,19 @@ pytestmark = [pytest.mark.core_model, pytest.mark.cpu] class TestCacheDiTBackend: """Test CacheDiTBackend implementation.""" + def test_config_projection_contains_only_cache_dit_fields(self): + shared_config = DiffusionCacheConfig( + Fn_compute_blocks=4, + max_warmup_steps=8, + rel_l1_thresh=0.3, + ) + + config = CacheDiTConfig.from_diffusion_config(shared_config) + + assert config.Fn_compute_blocks == 4 + assert config.max_warmup_steps == 8 + assert not hasattr(config, "rel_l1_thresh") + def test_init_with_dict(self): """Test initialization with dictionary config.""" config_dict = {"Fn_compute_blocks": 4, "max_warmup_steps": 8} @@ -47,8 +61,8 @@ class TestCacheDiTBackend: assert backend.config.Fn_compute_blocks == 4 assert backend.enabled is False - @patch("vllm_omni.diffusion.cache.cache_dit_backend.BlockAdapter") - @patch("vllm_omni.diffusion.cache.cache_dit_backend.cache_dit") + @patch("vllm_omni.diffusion.cache.cachedit.backend.BlockAdapter") + @patch("vllm_omni.diffusion.cache.cachedit.backend.cache_dit") def test_enable_single_transformer(self, mock_cache_dit, mock_block_adapter): """Test enabling cache-dit on single-transformer pipeline.""" # Mock pipeline @@ -75,10 +89,61 @@ class TestCacheDiTBackend: assert backend._refresh_func is not None mock_cache_dit.enable_cache.assert_called_once() - @patch("vllm_omni.diffusion.cache.cache_dit_backend.BlockAdapter") - @patch("vllm_omni.diffusion.cache.cache_dit_backend.cache_dit") + @patch("vllm_omni.diffusion.cache.cachedit.backend.logger") + @patch("vllm_omni.diffusion.cache.cachedit.backend.cache_dit") + def test_enable_without_model_adapter_uses_cache_dit_registry(self, mock_cache_dit, mock_logger): + """A missing model-declared adapter falls back to Cache-DiT's registry.""" + + class BuiltinTransformer: + pass + + class BuiltinPipeline: + transformer = BuiltinTransformer() + + pipeline = BuiltinPipeline() + backend = CacheDiTBackend({"Fn_compute_blocks": 2}) + + backend.enable(pipeline) + + assert backend.enabled is True + assert backend._refresh_func is not None + assert mock_cache_dit.enable_cache.call_args.args[0] is pipeline.transformer + mock_logger.info.assert_any_call( + "Transformer %s does not declare _cache_dit_adapter_config; " + "falling back to Cache-DiT's built-in adapter registry.", + "BuiltinTransformer", + ) + + @patch("vllm_omni.diffusion.cache.cachedit.backend.cache_dit") + def test_enable_without_compatible_adapter_has_contextual_error(self, mock_cache_dit): + """An unsupported fallback reports both pipeline and transformer names.""" + + class UnsupportedTransformer: + pass + + class UnsupportedPipeline: + transformer = UnsupportedTransformer() + + pipeline = UnsupportedPipeline() + backend = CacheDiTBackend({"Fn_compute_blocks": 2}) + mock_cache_dit.enable_cache.side_effect = ValueError("unsupported") + + with pytest.raises( + ValueError, + match=( + "Failed to enable Cache-DiT for pipeline UnsupportedPipeline with transformer UnsupportedTransformer" + ), + ) as exc_info: + backend.enable(pipeline) + + assert isinstance(exc_info.value.__cause__, ValueError) + assert backend.enabled is False + assert backend._refresh_func is None + + @patch("vllm_omni.diffusion.cache.cachedit.backend.BlockAdapter") + @patch("vllm_omni.diffusion.cache.cachedit.backend.cache_dit") def test_refresh(self, mock_cache_dit, mock_block_adapter): - """Test refreshing cache context with SCM mask policy updates when num_inference_steps changes.""" + """Test refreshing cache context with SCM mask policy updates.""" # Mock pipeline mock_pipeline = Mock() mock_pipeline.__class__.__name__ = "DiTPipeline" @@ -107,8 +172,6 @@ class TestCacheDiTBackend: # First refresh with 50 steps backend.refresh(mock_pipeline, num_inference_steps=50) - assert backend._last_num_inference_steps == 50 - # Verify steps_mask was called with mask policy (not direct steps mask) mock_cache_dit.steps_mask.assert_called_with(mask_policy="fast", total_steps=50) assert mock_cache_dit.steps_mask.call_count == 1 @@ -137,10 +200,28 @@ class TestCacheDiTBackend: call_args = mock_cache_dit.refresh_context.call_args assert call_args[0][0] == mock_transformer assert "cache_config" in call_args[1] - assert backend._last_num_inference_steps == 100 + assert mock_cache_dit.refresh_context.call_count == 1 - @patch("vllm_omni.diffusion.cache.cache_dit_backend.BlockAdapter") - @patch("vllm_omni.diffusion.cache.cache_dit_backend.cache_dit") + @patch("vllm_omni.diffusion.cache.cachedit.backend.BlockAdapter") + @patch("vllm_omni.diffusion.cache.cachedit.backend.cache_dit") + def test_refresh_resets_same_step_count(self, mock_cache_dit, mock_block_adapter): + """Every generation must reset context, even when the step count is unchanged.""" + mock_pipeline = Mock() + mock_pipeline.__class__.__name__ = "DiTPipeline" + mock_pipeline.transformer = Mock() + mock_pipeline.transformer._cache_dit_adapter_config = CacheDiTAdapterConfig( + block_forward_patterns={"layers": ForwardPattern.Pattern_0} + ) + + backend = CacheDiTBackend(DiffusionCacheConfig()) + backend.enable(mock_pipeline) + backend.refresh(mock_pipeline, num_inference_steps=20) + backend.refresh(mock_pipeline, num_inference_steps=20) + + assert mock_cache_dit.refresh_context.call_count == 2 + + @patch("vllm_omni.diffusion.cache.cachedit.backend.BlockAdapter") + @patch("vllm_omni.diffusion.cache.cachedit.backend.cache_dit") def test_enable_hunyuan_pipeline_uses_model_transformer(self, mock_cache_dit, mock_block_adapter): """Test HunyuanImage3 uses pipeline.transformer for cache enable/refresh. @@ -179,8 +260,8 @@ class TestCacheDiTBackend: assert call_args[0][0] is mock_pipeline.model assert call_args[1]["num_inference_steps"] == 12 - @patch("vllm_omni.diffusion.cache.cache_dit_backend.BlockAdapter") - @patch("vllm_omni.diffusion.cache.cache_dit_backend.cache_dit") + @patch("vllm_omni.diffusion.cache.cachedit.backend.BlockAdapter") + @patch("vllm_omni.diffusion.cache.cachedit.backend.cache_dit") def test_enable_dreamid_pipeline_uses_fused_blocks(self, mock_cache_dit, mock_block_adapter): """Test DreamID uses pipeline.transformer for cache enable/refresh. @@ -221,8 +302,8 @@ class TestCacheDiTBackend: assert call_args[1]["num_inference_steps"] == 12 @pytest.mark.parametrize("num_inference_steps", [1, 7]) - @patch("vllm_omni.diffusion.cache.cache_dit_backend.BlockAdapter") - @patch("vllm_omni.diffusion.cache.cache_dit_backend.cache_dit") + @patch("vllm_omni.diffusion.cache.cachedit.backend.BlockAdapter") + @patch("vllm_omni.diffusion.cache.cachedit.backend.cache_dit") def test_refresh_scm_bypassed_for_unsupported_step_counts( self, mock_cache_dit, mock_block_adapter, num_inference_steps ): @@ -362,8 +443,6 @@ class TestCacheSelector: class TestMagCacheBackend: """Test MagCacheBackend implementation.""" - from vllm_omni.diffusion.cache.magcache.backend import MagCacheBackend - def test_init(self): """Test initialization.""" config = DiffusionCacheConfig(mag_threshold=0.1, mag_max_skip_steps=2, mag_calibrate=True) diff --git a/tests/diffusion/cache/test_cache_dit.py b/tests/diffusion/cache/test_cache_dit.py index 22f9b8611..fa224cf20 100644 --- a/tests/diffusion/cache/test_cache_dit.py +++ b/tests/diffusion/cache/test_cache_dit.py @@ -5,15 +5,18 @@ Model specific tests for CacheDiT enablement. """ +import ast import sys +from pathlib import Path from unittest.mock import Mock, patch import pytest import torch from cache_dit.caching.cache_blocks.pattern_0_1_2 import CachedBlocks_Pattern_0_1_2 -import vllm_omni.diffusion.cache.cache_dit_backend as cd_backend -from vllm_omni.diffusion.cache.cache_dit_backend import CacheDiTAdapterConfig, CacheDiTBackend, cache_summary +import vllm_omni.diffusion.cache.cachedit as cd_backend +import vllm_omni.diffusion.cache.cachedit.model_specific as cd_model_specific +from vllm_omni.diffusion.cache.cachedit import CacheDiTAdapterConfig, CacheDiTBackend, cache_summary from vllm_omni.diffusion.data import DiffusionCacheConfig from vllm_omni.diffusion.models.cosmos3.transformer_cosmos3 import Cosmos3VFMTransformer from vllm_omni.diffusion.models.helios.helios_transformer import HeliosTransformer3DModel @@ -42,8 +45,73 @@ SEPARATE_CFG_TRANSFORMERS = [ SAMPLE_CACHE_CONFIG = DiffusionCacheConfig() +def test_custom_cache_dit_enablers_are_registered_explicitly(): + expected_enablers = { + "Wan22Pipeline": cd_model_specific.enable_cache_for_wan22, + "Wan22I2VPipeline": cd_model_specific.enable_cache_for_wan22, + "Wan22TI2VPipeline": cd_model_specific.enable_cache_for_wan22, + "Wan22VACEPipeline": cd_model_specific.enable_cache_for_wan22, + "Wan22S2VPipeline": cd_model_specific.enable_cache_for_wan22_s2v, + "Cosmos3OmniDiffusersPipeline": cd_model_specific.enable_cache_for_cosmos3, + "Cosmos3OmniPipeline": cd_model_specific.enable_cache_for_cosmos3, + "Krea2Pipeline": cd_model_specific.enable_cache_for_krea2, + } + + with patch.dict(cd_backend.CUSTOM_DIT_ENABLERS, {}, clear=True): + cd_model_specific.register_custom_dit_enablers() + assert cd_backend.CUSTOM_DIT_ENABLERS == expected_enablers + + def test_wan22_vace_uses_wan22_custom_cache_dit_enabler(): - assert cd_backend.CUSTOM_DIT_ENABLERS["Wan22VACEPipeline"] is cd_backend.enable_cache_for_wan22 + assert cd_backend.CUSTOM_DIT_ENABLERS["Wan22VACEPipeline"] is cd_model_specific.enable_cache_for_wan22 + + +@pytest.mark.parametrize( + "pipeline_name", + ["Cosmos3OmniDiffusersPipeline", "Cosmos3OmniPipeline"], +) +def test_cosmos3_aliases_use_cosmos3_custom_cache_dit_enabler(pipeline_name: str): + assert cd_backend.CUSTOM_DIT_ENABLERS[pipeline_name] is cd_model_specific.enable_cache_for_cosmos3 + + +def test_cachedit_public_api_is_explicit(): + assert set(cd_backend.__all__) == { + "BagelCachedAdapter", + "CUSTOM_DIT_ENABLERS", + "CacheDiTAdapterConfig", + "CacheDiTBackend", + "CacheDiTConfig", + "SensenovaCachedAdapter", + "cache_summary", + "enable_cache_for_dit", + } + assert not hasattr(cd_backend, "enable_cache_for_wan22") + assert not hasattr(cd_backend, "enable_cache_for_wan22_s2v") + + +def test_cachedit_consumers_use_package_api(): + cache_dir = Path(cd_backend.__file__).resolve().parents[1] + package_root = cache_dir.parents[1] + legacy_module = "vllm_omni.diffusion.cache.cache_dit_backend" + internal_prefix = "vllm_omni.diffusion.cache.cachedit." + + invalid_imports = [] + for source_path in package_root.rglob("*.py"): + if source_path.is_relative_to(cache_dir): + continue + tree = ast.parse(source_path.read_text(encoding="utf-8")) + for node in ast.walk(tree): + modules = [] + if isinstance(node, ast.ImportFrom) and node.module is not None: + modules.append(node.module) + elif isinstance(node, ast.Import): + modules.extend(alias.name for alias in node.names) + + for module in modules: + if module == legacy_module or module.startswith(internal_prefix): + invalid_imports.append(f"{source_path.relative_to(package_root)}:{node.lineno}: {module}") + + assert not invalid_imports, "Cache-DiT consumers must use the package API:\n" + "\n".join(invalid_imports) @pytest.mark.parametrize("transformer_model", SEPARATE_CFG_TRANSFORMERS) @@ -54,12 +122,12 @@ def test_cache_dit_configs_have_separate_cfg(transformer_model): assert transformer_model._cache_dit_adapter_config.has_separate_cfg is True -@patch("vllm_omni.diffusion.cache.cache_dit_backend.BlockAdapter") -@patch("vllm_omni.diffusion.cache.cache_dit_backend.cache_dit") +@patch("vllm_omni.diffusion.cache.cachedit.model_specific.BlockAdapter") +@patch("vllm_omni.diffusion.cache.cachedit.model_specific.cache_dit") def test_separate_wan22_custom_enabler_has_separate_cfg(mock_cache_dit, mock_block_adapter): """Ensure that Wan22, which has a custom enabler, setts custom CFG correctly.""" mock_pipeline = Mock() - cd_backend.enable_cache_for_wan22(mock_pipeline, SAMPLE_CACHE_CONFIG) + cd_model_specific.enable_cache_for_wan22(mock_pipeline, SAMPLE_CACHE_CONFIG) mock_cache_dit.enable_cache.assert_called_once() adapter_kwargs = mock_block_adapter.call_args.kwargs @@ -67,8 +135,8 @@ def test_separate_wan22_custom_enabler_has_separate_cfg(mock_cache_dit, mock_blo @pytest.mark.parametrize("has_transformer_2", [False, True]) -@patch("vllm_omni.diffusion.cache.cache_dit_backend.BlockAdapter") -@patch("vllm_omni.diffusion.cache.cache_dit_backend.cache_dit") +@patch("vllm_omni.diffusion.cache.cachedit.model_specific.BlockAdapter") +@patch("vllm_omni.diffusion.cache.cachedit.model_specific.cache_dit") def test_wan22_custom_enabler_passes_taylorseer_calibrator( mock_cache_dit, mock_block_adapter, @@ -82,7 +150,7 @@ def test_wan22_custom_enabler_passes_taylorseer_calibrator( mock_pipeline.transformer_2 = None cache_config = DiffusionCacheConfig(enable_taylorseer=True, taylorseer_order=1) - cd_backend.enable_cache_for_wan22(mock_pipeline, cache_config) + cd_model_specific.enable_cache_for_wan22(mock_pipeline, cache_config) enable_cache_kwargs = mock_cache_dit.enable_cache.call_args.kwargs calibrator_config = enable_cache_kwargs["calibrator_config"] @@ -94,8 +162,8 @@ def test_wan22_custom_enabler_passes_taylorseer_calibrator( assert modifier._context_kwargs["calibrator_config"] is calibrator_config -@patch("vllm_omni.diffusion.cache.cache_dit_backend.BlockAdapter") -@patch("vllm_omni.diffusion.cache.cache_dit_backend.cache_dit") +@patch("vllm_omni.diffusion.cache.cachedit.backend.BlockAdapter") +@patch("vllm_omni.diffusion.cache.cachedit.backend.cache_dit") def test_cosmos3_cache_dit_wraps_gen_layers(mock_cache_dit, mock_block_adapter): """Cosmos3 should cache only the repeated GEN pathway blocks.""" mock_pipeline = Mock() @@ -103,7 +171,7 @@ def test_cosmos3_cache_dit_wraps_gen_layers(mock_cache_dit, mock_block_adapter): mock_pipeline.transformer.gen_layers = gen_layers mock_pipeline.transformer._cache_dit_adapter_config = Cosmos3VFMTransformer._cache_dit_adapter_config - cd_backend.enable_cache_for_cosmos3(mock_pipeline, SAMPLE_CACHE_CONFIG) + cd_model_specific.enable_cache_for_cosmos3(mock_pipeline, SAMPLE_CACHE_CONFIG) mock_cache_dit.enable_cache.assert_called_once() adapter_kwargs = mock_block_adapter.call_args.kwargs diff --git a/tests/diffusion/models/cosmos3/test_cosmos3_pipeline.py b/tests/diffusion/models/cosmos3/test_cosmos3_pipeline.py index 404979a78..3093e6ceb 100644 --- a/tests/diffusion/models/cosmos3/test_cosmos3_pipeline.py +++ b/tests/diffusion/models/cosmos3/test_cosmos3_pipeline.py @@ -465,7 +465,7 @@ def test_edge_forward_rejects_unsupported_modes( def test_pipeline_registered_and_exported() -> None: - from vllm_omni.diffusion.cache.cache_dit_backend import CUSTOM_DIT_ENABLERS + from vllm_omni.diffusion.cache.cachedit import CUSTOM_DIT_ENABLERS from vllm_omni.diffusion.models import cosmos3 from vllm_omni.diffusion.models.cosmos3.pipeline_cosmos3 import Cosmos3OmniDiffusersPipeline from vllm_omni.diffusion.models.progress_bar import ProgressBarMixin diff --git a/tests/diffusion/models/ltx2/test_ltx2_pipeline.py b/tests/diffusion/models/ltx2/test_ltx2_pipeline.py index 41b1d5a84..3ba36ea57 100644 --- a/tests/diffusion/models/ltx2/test_ltx2_pipeline.py +++ b/tests/diffusion/models/ltx2/test_ltx2_pipeline.py @@ -731,7 +731,7 @@ class TestRegistryIntegration: def test_cache_dit_for_ltx2_does_not_have_custom_enablers_registered(self): """Pipeline variants are *not* registered in CUSTOM_DIT_ENABLERS.""" - from vllm_omni.diffusion.cache.cache_dit_backend import CUSTOM_DIT_ENABLERS + from vllm_omni.diffusion.cache.cachedit import CUSTOM_DIT_ENABLERS # NOTE: We used to have custom enablers for this model, but refactored to handle # it more generically. Now we only need to ensure it has git cache adapter config. diff --git a/tests/diffusion/models/ltx2/test_ltx2_transformer.py b/tests/diffusion/models/ltx2/test_ltx2_transformer.py index 01a1fa3cf..e5ccfd013 100644 --- a/tests/diffusion/models/ltx2/test_ltx2_transformer.py +++ b/tests/diffusion/models/ltx2/test_ltx2_transformer.py @@ -4,7 +4,7 @@ import pytest from torch import nn -from vllm_omni.diffusion.cache.cache_dit_backend import CacheDiTAdapterConfig +from vllm_omni.diffusion.cache.cachedit import CacheDiTAdapterConfig from vllm_omni.diffusion.models.ltx2.ltx2_transformer import LTX2VideoTransformer3DModel, _make_rms_norm pytestmark = [pytest.mark.core_model, pytest.mark.cpu] diff --git a/vllm_omni/diffusion/cache/cachedit/__init__.py b/vllm_omni/diffusion/cache/cachedit/__init__.py new file mode 100644 index 000000000..f3d075489 --- /dev/null +++ b/vllm_omni/diffusion/cache/cachedit/__init__.py @@ -0,0 +1,35 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +"""Public API for the Cache-DiT diffusion cache backend.""" + +from vllm_omni.diffusion.cache.cachedit.backend import ( + CUSTOM_DIT_ENABLERS, + CacheDiTBackend, + cache_summary, + enable_cache_for_dit, +) +from vllm_omni.diffusion.cache.cachedit.config import ( + CacheDiTAdapterConfig, + CacheDiTConfig, +) +from vllm_omni.diffusion.cache.cachedit.model_specific import ( + BagelCachedAdapter, + SensenovaCachedAdapter, +) +from vllm_omni.diffusion.cache.cachedit.model_specific import ( + register_custom_dit_enablers as _register_custom_dit_enablers, +) + +_register_custom_dit_enablers() + +__all__ = [ + "BagelCachedAdapter", + "CUSTOM_DIT_ENABLERS", + "CacheDiTAdapterConfig", + "CacheDiTBackend", + "CacheDiTConfig", + "SensenovaCachedAdapter", + "cache_summary", + "enable_cache_for_dit", +] diff --git a/vllm_omni/diffusion/cache/cachedit/backend.py b/vllm_omni/diffusion/cache/cachedit/backend.py new file mode 100644 index 000000000..ac6f93785 --- /dev/null +++ b/vllm_omni/diffusion/cache/cachedit/backend.py @@ -0,0 +1,222 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +"""Generic Cache-DiT backend lifecycle and integration.""" + +from collections.abc import Callable +from typing import Any, TypeAlias + +import cache_dit +from cache_dit import BlockAdapter, DBCacheConfig +from cache_dit.caching.cache_adapters.cache_adapter import CachedAdapter +from vllm.logger import init_logger + +from vllm_omni.diffusion.cache.base import CacheBackend +from vllm_omni.diffusion.cache.cachedit.config import ( + CacheDiTAdapterConfig, + CacheDiTConfig, +) +from vllm_omni.diffusion.data import DiffusionCacheConfig + +logger = init_logger(__name__) + +RefreshCacheContextFunc: TypeAlias = Callable[[Any, int, bool], None] +CacheDiTEnabler: TypeAlias = Callable[[Any, DiffusionCacheConfig], RefreshCacheContextFunc] + +# Model-specific implementations register themselves when the package loads. +CUSTOM_DIT_ENABLERS: dict[str, CacheDiTEnabler] = {} + + +def cache_summary(pipeline: Any, details: bool = True) -> None: + """Log Cache-DiT statistics for every transformer on the pipeline.""" + + transformers = [ + transformer + for attribute in ("transformer", "transformer_2") + if (transformer := getattr(pipeline, attribute, None)) is not None + ] + for transformer in transformers: + cache_dit.summary(transformer, details=details) + + if not transformers: + logger.warning("CacheDiT summary failed; this pipeline has no defined transformer attribute") + + +def _default_get_pipeline_transformer(pipeline: Any) -> Any: + return pipeline.transformer + + +def _build_cache_context_refresh( + cache_config: DiffusionCacheConfig, + get_pipeline_transformer: Callable[[Any], Any] = _default_get_pipeline_transformer, +) -> RefreshCacheContextFunc: + """Build the cache context refresh callback for one transformer.""" + + projected_config = CacheDiTConfig.from_diffusion_config(cache_config) + + def refresh_cache_context(pipeline: Any, num_inference_steps: int, verbose: bool = True) -> None: + transformer = get_pipeline_transformer(pipeline) + + # Cache-DiT has no predefined SCM mask for these small step counts. + scm_supported_steps = num_inference_steps >= 8 or num_inference_steps in (4, 6) + if projected_config.scm_steps_mask_policy is None or not scm_supported_steps: + cache_dit.refresh_context(transformer, num_inference_steps=num_inference_steps, verbose=verbose) + return + + cache_dit.refresh_context( + transformer, + cache_config=DBCacheConfig().reset( + num_inference_steps=num_inference_steps, + steps_computation_mask=cache_dit.steps_mask( + mask_policy=projected_config.scm_steps_mask_policy, + total_steps=num_inference_steps, + ), + steps_computation_policy=projected_config.scm_steps_policy, + ), + verbose=verbose, + ) + + return refresh_cache_context + + +def enable_cache_for_dit( + pipeline: Any, + cache_config: DiffusionCacheConfig, + block_adapter: BlockAdapter | None = None, + adapter_cls: type[CachedAdapter] | None = None, +) -> RefreshCacheContextFunc: + """Enable Cache-DiT for a standard single-transformer DiT pipeline.""" + + projected_config = CacheDiTConfig.from_diffusion_config(cache_config) + db_cache_config = projected_config.to_db_cache_config() + calibrator_config = projected_config.to_calibrator_config() + + logger.info( + "Enabling cache-dit on transformer: Fn=%s, Bn=%s, W=%s", + db_cache_config.Fn_compute_blocks, + db_cache_config.Bn_compute_blocks, + db_cache_config.max_warmup_steps, + ) + + transformer = _default_get_pipeline_transformer(pipeline) + cache_target = transformer if block_adapter is None else block_adapter + if adapter_cls is not None: + adapter_cls.apply( + cache_target, + cache_config=db_cache_config, + calibrator_config=calibrator_config, + ) + elif block_adapter is None: + try: + cache_dit.enable_cache( + cache_target, + cache_config=db_cache_config, + calibrator_config=calibrator_config, + ) + except ValueError as exc: + raise ValueError( + "Failed to enable Cache-DiT for pipeline " + f"{type(pipeline).__name__} with transformer " + f"{type(transformer).__name__}: no model-declared " + "_cache_dit_adapter_config or compatible Cache-DiT built-in " + "adapter was found." + ) from exc + else: + cache_dit.enable_cache( + cache_target, + cache_config=db_cache_config, + calibrator_config=calibrator_config, + ) + + return _build_cache_context_refresh(cache_config) + + +def _maybe_build_block_adapter(pipeline: Any) -> BlockAdapter | None: + """Build the model-declared block adapter, when one is configured.""" + + transformer = _default_get_pipeline_transformer(pipeline) + adapter_config: CacheDiTAdapterConfig | None = getattr(transformer, "_cache_dit_adapter_config", None) + if adapter_config is None: + logger.info( + "Transformer %s does not declare _cache_dit_adapter_config; " + "falling back to Cache-DiT's built-in adapter registry.", + type(transformer).__name__, + ) + return None + + block_attributes, forward_patterns = zip(*adapter_config.block_forward_patterns.items()) + missing_attributes = [ + block_attribute for block_attribute in block_attributes if not hasattr(transformer, block_attribute) + ] + if missing_attributes: + raise AttributeError(f"Missing Cache-DiT block attributes: {missing_attributes}") + + return BlockAdapter( + transformer=transformer, + blocks=[getattr(transformer, block_attribute) for block_attribute in block_attributes], + forward_pattern=list(forward_patterns), + has_separate_cfg=adapter_config.has_separate_cfg, + check_forward_pattern=adapter_config.check_forward_pattern, + ) + + +def _maybe_get_cached_adapter_cls(pipeline: Any) -> type[CachedAdapter] | None: + """Return the custom cached adapter declared by the transformer.""" + + transformer = _default_get_pipeline_transformer(pipeline) + adapter_config: CacheDiTAdapterConfig | None = getattr(transformer, "_cache_dit_adapter_config", None) + return None if adapter_config is None else adapter_config.cached_adapter_cls + + +class CacheDiTBackend(CacheBackend): + """Manage Cache-DiT through the common diffusion cache lifecycle.""" + + def __init__(self, cache_config: Any = None): + if cache_config is None: + config = DiffusionCacheConfig() + elif isinstance(cache_config, dict): + config = DiffusionCacheConfig.from_dict(cache_config) + else: + config = cache_config + + super().__init__(config) + self._refresh_func: RefreshCacheContextFunc | None = None + + def enable(self, pipeline: Any) -> None: + pipeline_name = type(pipeline).__name__ + custom_enabler = CUSTOM_DIT_ENABLERS.get(pipeline_name) + if custom_enabler is not None: + logger.info("Using custom cache-dit enabler for model: %s", pipeline_name) + self._refresh_func = custom_enabler(pipeline, self.config) + else: + block_adapter = _maybe_build_block_adapter(pipeline) + adapter_cls = _maybe_get_cached_adapter_cls(pipeline) + self._refresh_func = enable_cache_for_dit( + pipeline, + self.config, + block_adapter, + adapter_cls, + ) + + self.enabled = True + logger.info("Cache-dit enabled successfully on %s", pipeline_name) + + def refresh(self, pipeline: Any, num_inference_steps: int, verbose: bool = True) -> None: + if not self.enabled or self._refresh_func is None: + logger.warning("Cache-dit is not enabled. Cannot refresh cache context.") + return + + if verbose: + logger.info( + "Refreshing cache context for transformer with num_inference_steps: %s", + num_inference_steps, + ) + self._refresh_func(pipeline, num_inference_steps, verbose) + + +__all__ = [ + "CUSTOM_DIT_ENABLERS", + "CacheDiTBackend", + "cache_summary", + "enable_cache_for_dit", +] diff --git a/vllm_omni/diffusion/cache/cachedit/config.py b/vllm_omni/diffusion/cache/cachedit/config.py new file mode 100644 index 000000000..8fcaa517b --- /dev/null +++ b/vllm_omni/diffusion/cache/cachedit/config.py @@ -0,0 +1,96 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +"""Configuration types and conversion helpers for Cache-DiT.""" + +from __future__ import annotations + +from dataclasses import dataclass + +from cache_dit import ( + CalibratorConfig, + DBCacheConfig, + ForwardPattern, + TaylorSeerCalibratorConfig, +) +from cache_dit.caching.cache_adapters.cache_adapter import CachedAdapter +from vllm.logger import init_logger + +from vllm_omni.diffusion.data import DiffusionCacheConfig + +logger = init_logger(__name__) + + +@dataclass +class CacheDiTAdapterConfig: + """Describe how Cache-DiT wraps a model's transformer blocks.""" + + block_forward_patterns: dict[str, ForwardPattern] + has_separate_cfg: bool = False + cached_adapter_cls: type[CachedAdapter] | None = None + check_forward_pattern: bool = True + + +@dataclass(frozen=True) +class CacheDiTConfig: + """Cache-DiT-specific projection of the shared diffusion cache config.""" + + Fn_compute_blocks: int = 1 + Bn_compute_blocks: int = 0 + max_warmup_steps: int = 4 + max_cached_steps: int = -1 + residual_diff_threshold: float = 0.24 + max_continuous_cached_steps: int = 3 + enable_taylorseer: bool = False + taylorseer_order: int = 1 + scm_steps_mask_policy: str | None = None + scm_steps_policy: str = "dynamic" + force_refresh_step_hint: int | None = None + force_refresh_step_policy: str = "once" + + @classmethod + def from_diffusion_config(cls, config: DiffusionCacheConfig) -> CacheDiTConfig: + """Project the shared config onto fields consumed by Cache-DiT.""" + + return cls( + Fn_compute_blocks=config.Fn_compute_blocks, + Bn_compute_blocks=config.Bn_compute_blocks, + max_warmup_steps=config.max_warmup_steps, + max_cached_steps=config.max_cached_steps, + residual_diff_threshold=config.residual_diff_threshold, + max_continuous_cached_steps=config.max_continuous_cached_steps, + enable_taylorseer=config.enable_taylorseer, + taylorseer_order=config.taylorseer_order, + scm_steps_mask_policy=config.scm_steps_mask_policy, + scm_steps_policy=config.scm_steps_policy, + force_refresh_step_hint=config.force_refresh_step_hint, + force_refresh_step_policy=config.force_refresh_step_policy, + ) + + def to_db_cache_config(self) -> DBCacheConfig: + """Build the third-party cache configuration used during installation.""" + + return DBCacheConfig( + # The request step count is supplied by ``CacheDiTBackend.refresh``. + num_inference_steps=None, + Fn_compute_blocks=self.Fn_compute_blocks, + Bn_compute_blocks=self.Bn_compute_blocks, + max_warmup_steps=self.max_warmup_steps, + max_cached_steps=self.max_cached_steps, + max_continuous_cached_steps=self.max_continuous_cached_steps, + residual_diff_threshold=self.residual_diff_threshold, + force_refresh_step_hint=self.force_refresh_step_hint, + force_refresh_step_policy=self.force_refresh_step_policy, + ) + + def to_calibrator_config(self) -> CalibratorConfig | None: + """Build the optional TaylorSeer calibrator configuration.""" + + if not self.enable_taylorseer: + return None + + logger.info("TaylorSeer enabled with order=%s", self.taylorseer_order) + return TaylorSeerCalibratorConfig(taylorseer_order=self.taylorseer_order) + + +__all__ = ["CacheDiTAdapterConfig", "CacheDiTConfig"] diff --git a/vllm_omni/diffusion/cache/cache_dit_backend.py b/vllm_omni/diffusion/cache/cachedit/model_specific.py similarity index 67% rename from vllm_omni/diffusion/cache/cache_dit_backend.py rename to vllm_omni/diffusion/cache/cachedit/model_specific.py index 5d63ce3c0..f7af987ce 100644 --- a/vllm_omni/diffusion/cache/cache_dit_backend.py +++ b/vllm_omni/diffusion/cache/cachedit/model_specific.py @@ -1,27 +1,19 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -cache-dit integration backend for vllm-omni. -This module provides a CacheDiTBackend class to enable cache-dit acceleration on diffusion -pipelines in vllm-omni, supporting both single and dual-transformer architectures. -""" +"""Model-specific Cache-DiT adapters and enablers.""" import functools -from collections.abc import Callable from contextlib import ExitStack -from dataclasses import dataclass -from typing import Any, Optional, TypeAlias +from typing import Any import cache_dit import torch from cache_dit import ( BlockAdapter, - CalibratorConfig, DBCacheConfig, ForwardPattern, ParamsModifier, - TaylorSeerCalibratorConfig, ) from cache_dit.caching.block_adapters import FakeDiffusionPipeline from cache_dit.caching.cache_adapters.cache_adapter import CachedAdapter @@ -31,178 +23,18 @@ from cache_dit.caching.cache_contexts import BasicCacheConfig from cache_dit.caching.cache_contexts.cache_manager import CachedContextManager from vllm.logger import init_logger -from vllm_omni.diffusion.cache.base import CacheBackend -from vllm_omni.diffusion.data import DiffusionCacheConfig, OmniDiffusionConfig +from vllm_omni.diffusion.cache.cachedit.backend import ( + CUSTOM_DIT_ENABLERS, + RefreshCacheContextFunc, + _build_cache_context_refresh, + _default_get_pipeline_transformer, + _maybe_build_block_adapter, + enable_cache_for_dit, +) +from vllm_omni.diffusion.cache.cachedit.config import CacheDiTConfig logger = init_logger(__name__) -RefreshCacheContextFunc: TypeAlias = Callable[[Any, int, bool], None] - - -@dataclass -class CacheDiTAdapterConfig: - """Config for creating a Cache DiT's block adapter; to enable CacheDiT, - most models just need to define an instance of this class as a class - var in the DiT. - """ - - block_forward_patterns: dict[str, ForwardPattern] - has_separate_cfg: bool = False - cached_adapter_cls: type[CachedAdapter] | None = None - check_forward_pattern: bool = True - - -# Registry of custom cache-dit enablers for specific models -# Maps pipeline names to their cache-dit enablement functions -CUSTOM_DIT_ENABLERS: dict[str, Callable] = {} - - -# Small helper to centralize cache-dit summaries. -def cache_summary(pipeline: Any, details: bool = True) -> None: - if hasattr(pipeline, "transformer"): - cache_dit.summary(pipeline.transformer, details=details) - if hasattr(pipeline, "transformer_2"): - cache_dit.summary(pipeline.transformer_2, details=details) - if not hasattr(pipeline, "transformer") and not hasattr(pipeline, "transformer_2"): - logger.warning("CacheDiT summary failed; this pipeline has no defined transformer attribute") - - -def default_get_pipeline_transformer(pipeline: Any) -> Any: - return pipeline.transformer - - -def build_cache_context_refresh( - cache_config: DiffusionCacheConfig, - get_pipeline_transformer: Callable[[Any], Any] = default_get_pipeline_transformer, -) -> RefreshCacheContextFunc: - """Build the cache context refresh func for a single Transformer.""" - - def refresh_cache_context(pipeline: Any, num_inference_steps: int, verbose: bool = True) -> None: - """Refresh cache context for the transformer with new num_inference_steps. - - Args: - pipeline: The pipeline instance. - num_inference_steps: New number of inference steps. - """ - transformer = get_pipeline_transformer(pipeline) - - # Bypass SCM for step counts that don't support predefined masks (e.g., vLLM's 1-step dummy run) - scm_supported_steps = num_inference_steps >= 8 or num_inference_steps in (4, 6) - - if cache_config.scm_steps_mask_policy is None or not scm_supported_steps: - cache_dit.refresh_context(transformer, num_inference_steps=num_inference_steps, verbose=verbose) - else: - cache_dit.refresh_context( - transformer, - cache_config=DBCacheConfig().reset( - num_inference_steps=num_inference_steps, - steps_computation_mask=cache_dit.steps_mask( - mask_policy=cache_config.scm_steps_mask_policy, - total_steps=num_inference_steps, - ), - steps_computation_policy=cache_config.scm_steps_policy, - ), - verbose=verbose, - ) - - return refresh_cache_context - - -def _resolve_calibrator_config(cache_config: DiffusionCacheConfig) -> CalibratorConfig | None: - """Resolves the Calibrator subconfig For DiT cache.""" - calibrator = None - if cache_config.enable_taylorseer: - taylorseer_order = cache_config.taylorseer_order - calibrator = TaylorSeerCalibratorConfig(taylorseer_order=taylorseer_order) - logger.info(f"TaylorSeer enabled with order={taylorseer_order}") - # In the future, more calibrators will likely be added to DiT Cache, - # e.g., focal; handle them generically here. - return calibrator - - -def _build_db_cache_config(cache_config: DiffusionCacheConfig) -> DBCacheConfig: - """Build DBCacheConfig with optional SCM (Step Computation Masking) support. - - Args: - cache_config: DiffusionCacheConfig instance. - - Returns: - DBCacheConfig instance with SCM support if configured. - """ - return DBCacheConfig( - # we will refresh the context when we get num_inference_steps in the first inference request - num_inference_steps=None, - Fn_compute_blocks=cache_config.Fn_compute_blocks, - Bn_compute_blocks=cache_config.Bn_compute_blocks, - max_warmup_steps=cache_config.max_warmup_steps, - max_cached_steps=cache_config.max_cached_steps, - max_continuous_cached_steps=cache_config.max_continuous_cached_steps, - residual_diff_threshold=cache_config.residual_diff_threshold, - force_refresh_step_hint=cache_config.force_refresh_step_hint, - force_refresh_step_policy=cache_config.force_refresh_step_policy, - ) - - -def enable_cache_for_dit( - pipeline: Any, - cache_config: Any, - block_adapter: BlockAdapter | None = None, - adapter_cls: type[CachedAdapter] | None = None, -) -> RefreshCacheContextFunc: - """Enable cache-dit for regular single-transformer DiT models. - - Args: - pipeline: The diffusion pipeline instance. - cache_config: DiffusionCacheConfig instance with cache configuration. - block_adapter: Custom block adapters for specific model architectures. - adapter_cls: Custom cached adapter class for specific model architectures. - - Returns: - A refresh function that can be called to update cache context with new num_inference_steps. - """ - # Build DBCacheConfig with optional SCM support - db_cache_config = _build_db_cache_config(cache_config) - - # Build calibrator config if TaylorSeer is enabled - calibrator_config = _resolve_calibrator_config(cache_config) - - logger.info( - f"Enabling cache-dit on transformer: " - f"Fn={db_cache_config.Fn_compute_blocks}, " - f"Bn={db_cache_config.Bn_compute_blocks}, " - f"W={db_cache_config.max_warmup_steps}, " - ) - - # Enable cache-dit on the transformer - transformer = default_get_pipeline_transformer(pipeline) - - # If we have a custom cached adapter subclass, call apply directly - if adapter_cls is not None: - adapter_cls.apply( - transformer if block_adapter is None else block_adapter, - cache_config=db_cache_config, - calibrator_config=calibrator_config, - ) - else: - # Otherwise, call enable cache, which will call CachedAdapter.apply for us - cache_dit.enable_cache( - transformer if block_adapter is None else block_adapter, - cache_config=db_cache_config, - calibrator_config=calibrator_config, - ) - - return build_cache_context_refresh(cache_config) - - -### Complex / custom enablers for DiT cache -# NOTE (Alex): This case is rare; you should only really need to do this if you have a dual transformer -# architecture, since it hasn't been handled generically yet, or if the model class has unique attributes -# that it sets during Cache DiT enablement. -# -# For the vast majority of models, you should only need to add a _cache_dit_adapter_config attribute -# to the Transformer class, which controls the forward pattern, whether or not we have separate CFG, -# and so on. - # from https://github.com/vipshop/cache-dit/pull/542 def _split_wan22_inference_steps(pipeline, num_inference_steps: int) -> tuple[int, int]: @@ -249,9 +81,9 @@ def enable_cache_for_wan22(pipeline: Any, cache_config: Any) -> RefreshCacheCont Returns: A refresh function that can be called to update cache context with new num_inference_steps. """ - # Build DBCacheConfig with optional SCM support - db_cache_config = _build_db_cache_config(cache_config) - calibrator_config = _resolve_calibrator_config(cache_config) + projected_config = CacheDiTConfig.from_diffusion_config(cache_config) + db_cache_config = projected_config.to_db_cache_config() + calibrator_config = projected_config.to_calibrator_config() if getattr(pipeline, "transformer_2", None) is None: logger.info("transformer_2 not found, enabling cache-dit for single transformer mode") @@ -273,7 +105,7 @@ def enable_cache_for_wan22(pipeline: Any, cache_config: Any) -> RefreshCacheCont cache_config=db_cache_config, calibrator_config=calibrator_config, ) - return build_cache_context_refresh(cache_config) + return _build_cache_context_refresh(cache_config) cache_dit.enable_cache( BlockAdapter( @@ -296,8 +128,8 @@ def enable_cache_for_wan22(pipeline: Any, cache_config: Any) -> RefreshCacheCont # high-noise transformer only have 30% steps ParamsModifier( cache_config=DBCacheConfig().reset( - max_warmup_steps=cache_config.max_warmup_steps, - max_cached_steps=cache_config.max_cached_steps, + max_warmup_steps=projected_config.max_warmup_steps, + max_cached_steps=projected_config.max_cached_steps, ), calibrator_config=calibrator_config, ), @@ -315,8 +147,8 @@ def enable_cache_for_wan22(pipeline: Any, cache_config: Any) -> RefreshCacheCont calibrator_config=calibrator_config, ) - refresh_trans_one = build_cache_context_refresh(cache_config) - refresh_trans_two = build_cache_context_refresh(cache_config, lambda pipeline: pipeline.transformer_2) + refresh_trans_one = _build_cache_context_refresh(cache_config) + refresh_trans_two = _build_cache_context_refresh(cache_config, lambda pipeline: pipeline.transformer_2) def refresh_cache_context(pipeline: Any, num_inference_steps: int, verbose: bool = True) -> None: """Refresh cache context for both transformers with new num_inference_steps. @@ -347,12 +179,9 @@ def enable_cache_for_wan22_s2v(pipeline: Any, cache_config: Any) -> RefreshCache permanently replace it with a no-op on the transformer to prevent double injection from the main forward loop. """ - db_cache_config = _build_db_cache_config(cache_config) - calibrator_config = None - if cache_config.enable_taylorseer: - taylorseer_order = cache_config.taylorseer_order - calibrator_config = TaylorSeerCalibratorConfig(taylorseer_order=taylorseer_order) - logger.info(f"TaylorSeer enabled with order={taylorseer_order}") + projected_config = CacheDiTConfig.from_diffusion_config(cache_config) + db_cache_config = projected_config.to_db_cache_config() + calibrator_config = projected_config.to_calibrator_config() # Save the original after_transformer_block before cache-dit wrapping transformer = pipeline.transformer @@ -383,7 +212,7 @@ def enable_cache_for_wan22_s2v(pipeline: Any, cache_config: Any) -> RefreshCache def refresh_cache_context(pipeline: Any, num_inference_steps: int, verbose: bool = True) -> None: """Refresh cache context for the S2V transformer.""" - if cache_config.scm_steps_mask_policy is None: + if projected_config.scm_steps_mask_policy is None: cache_dit.refresh_context( pipeline.transformer, num_inference_steps=num_inference_steps, @@ -395,10 +224,10 @@ def enable_cache_for_wan22_s2v(pipeline: Any, cache_config: Any) -> RefreshCache cache_config=DBCacheConfig().reset( num_inference_steps=num_inference_steps, steps_computation_mask=cache_dit.steps_mask( - mask_policy=cache_config.scm_steps_mask_policy, + mask_policy=projected_config.scm_steps_mask_policy, total_steps=num_inference_steps, ), - steps_computation_policy=cache_config.scm_steps_policy, + steps_computation_policy=projected_config.scm_steps_policy, ), verbose=verbose, ) @@ -971,7 +800,7 @@ def enable_cache_for_cosmos3(pipeline: Any, cache_config: Any) -> RefreshCacheCo # step accounting. Still do both cond/uncond CFG steps when cache-dit is active. # CFG is instead neutralized via scale=1.0 outside the interval. pipeline._cache_dit_requires_paired_cfg = True - block_adapter = CacheDiTBackend.maybe_build_block_adapter(pipeline) + block_adapter = _maybe_build_block_adapter(pipeline) return enable_cache_for_dit(pipeline, cache_config, block_adapter) @@ -996,7 +825,7 @@ def enable_cache_for_krea2(pipeline: Any, cache_config: Any) -> RefreshCacheCont Returns: A refresh function that can be called to update cache context with new num_inference_steps. """ - transformer = default_get_pipeline_transformer(pipeline) + transformer = _default_get_pipeline_transformer(pipeline) block_adapter = BlockAdapter( transformer=transformer, blocks=[transformer.transformer_blocks], @@ -1007,171 +836,28 @@ def enable_cache_for_krea2(pipeline: Any, cache_config: Any) -> RefreshCacheCont return enable_cache_for_dit(pipeline, cache_config, block_adapter) -# Register custom cache-dit enablers after function definitions -CUSTOM_DIT_ENABLERS.update( - { - "Wan22Pipeline": enable_cache_for_wan22, - "Wan22I2VPipeline": enable_cache_for_wan22, - "Wan22TI2VPipeline": enable_cache_for_wan22, - "Wan22VACEPipeline": enable_cache_for_wan22, - "Wan22S2VPipeline": enable_cache_for_wan22_s2v, - "Cosmos3OmniDiffusersPipeline": enable_cache_for_cosmos3, - "Cosmos3OmniPipeline": enable_cache_for_cosmos3, - "Krea2Pipeline": enable_cache_for_krea2, - } -) +def register_custom_dit_enablers() -> None: + """Register model-specific Cache-DiT enablers. - -class CacheDiTBackend(CacheBackend): - """Backend class for cache-dit acceleration on diffusion pipelines. - - This class implements cache-dit acceleration (DBCache, SCM, TaylorSeer) using - the cache-dit library. It inherits from CacheBackend and provides a unified - interface for managing cache-dit acceleration on diffusion models. - - Attributes: - config: Cache configuration (DiffusionCacheConfig instance), inherited from CacheBackend. - enabled: Whether cache-dit is enabled on this pipeline, inherited from CacheBackend. - _refresh_func: Internal refresh function for updating cache context. - _last_num_inference_steps: Last num_inference_steps used for refresh optimization. + This is called explicitly by the package initializer so registration does + not depend on unrelated model-specific symbols being imported for their + side effects. """ - - def __init__(self, cache_config: Any = None): - """Initialize the cache-dit backend. - - Args: - cache_config: Cache configuration (DiffusionCacheConfig instance, dict, or None). - If None or empty, uses default DiffusionCacheConfig(). - """ - # Use default config if cache_config is not provided or is empty - if cache_config is None: - config = DiffusionCacheConfig() - elif isinstance(cache_config, dict): - # Convert dict to DiffusionCacheConfig, using defaults for missing keys - config = DiffusionCacheConfig.from_dict(cache_config) - else: - config = cache_config - - # Initialize base class with normalized config - super().__init__(config) - - # Cache-dit specific attributes - self._refresh_func: Callable[[Any, int, bool], None] | None = None - self._last_num_inference_steps: int | None = None - - @staticmethod - def maybe_build_block_adapter(pipeline) -> BlockAdapter | None: - """If a module defines `_cache_dit_adapter_config`, build the corresponding - block adapter. - """ - transformer = default_get_pipeline_transformer(pipeline) - - adapter_cfg: CacheDiTAdapterConfig | None = getattr(transformer, "_cache_dit_adapter_config", None) - if adapter_cfg is None: - return None - - block_attrs, forward_pattern = zip(*(adapter_cfg.block_forward_patterns).items()) - missing_attrs = [block_attr for block_attr in block_attrs if not hasattr(transformer, block_attr)] - - if missing_attrs: - logger.warning("Missing Cache DiT block attributes: %s", missing_attrs) - - block_adapter = BlockAdapter( - transformer=transformer, - blocks=[getattr(transformer, block_attr) for block_attr in block_attrs], - forward_pattern=list(forward_pattern), - has_separate_cfg=adapter_cfg.has_separate_cfg, - check_forward_pattern=adapter_cfg.check_forward_pattern, - ) - return block_adapter - - @staticmethod - def maybe_get_cached_adapter_cls(pipeline) -> type[CachedAdapter] | None: - """If a module has a custom cached adapter type registered, e.g., SenseNova, retrieve it - from the transformer's CacheDiTAdapterConfig.""" - transformer = default_get_pipeline_transformer(pipeline) - - adapter_cfg: CacheDiTAdapterConfig | None = getattr(transformer, "_cache_dit_adapter_config", None) - if adapter_cfg is None: - return None - return adapter_cfg.cached_adapter_cls - - def enable(self, pipeline: Any) -> None: - """Enable cache-dit on the pipeline if configured. - - This method applies cache-dit acceleration to the appropriate transformer(s) - in the pipeline. It handles both single-transformer and dual-transformer - architectures (e.g., Wan2.2). - - Args: - pipeline: The diffusion pipeline instance. - """ - - # Extract pipeline name from pipeline - pipeline_name = pipeline.__class__.__name__ - # Check if this model has a custom cache-dit enabler - if pipeline_name in CUSTOM_DIT_ENABLERS: - logger.info(f"Using custom cache-dit enabler for model: {pipeline_name}") - self._refresh_func = CUSTOM_DIT_ENABLERS[pipeline_name](pipeline, self.config) - else: - # Common case; either the model doesn't explicitly support dit cache yet, - # Or it defines its _cache_dit_adapter_config, which describes how we should - # create its block adapter. - block_adapter = self.maybe_build_block_adapter(pipeline) - adapter_cls = self.maybe_get_cached_adapter_cls(pipeline) - self._refresh_func = enable_cache_for_dit(pipeline, self.config, block_adapter, adapter_cls) - - self.enabled = True - logger.info(f"Cache-dit enabled successfully on {pipeline_name}") - - def refresh(self, pipeline: Any, num_inference_steps: int, verbose: bool = True) -> None: - """Refresh cache context with new num_inference_steps. - - This method updates the cache context when num_inference_steps changes - during inference. For dual-transformer models (e.g., Wan2.2), it automatically - splits the steps based on boundary_ratio. - - Args: - pipeline: The diffusion pipeline instance. - num_inference_steps: New number of inference steps. - verbose: Whether to log refresh operations. - """ - if not self.enabled or self._refresh_func is None: - logger.warning("Cache-dit is not enabled. Cannot refresh cache context.") - return - - # Only refresh if num_inference_steps has changed - if self._last_num_inference_steps is None or num_inference_steps != self._last_num_inference_steps: - if verbose: - logger.info(f"Refreshing cache context for transformer with num_inference_steps: {num_inference_steps}") - self._refresh_func(pipeline, num_inference_steps, verbose) - self._last_num_inference_steps = num_inference_steps - - def is_enabled(self) -> bool: - """Check if cache-dit is enabled on this pipeline. - - Returns: - True if cache-dit is enabled, False otherwise. - """ - return self.enabled + CUSTOM_DIT_ENABLERS.update( + { + "Wan22Pipeline": enable_cache_for_wan22, + "Wan22I2VPipeline": enable_cache_for_wan22, + "Wan22TI2VPipeline": enable_cache_for_wan22, + "Wan22VACEPipeline": enable_cache_for_wan22, + "Wan22S2VPipeline": enable_cache_for_wan22_s2v, + "Cosmos3OmniDiffusersPipeline": enable_cache_for_cosmos3, + "Cosmos3OmniPipeline": enable_cache_for_cosmos3, + "Krea2Pipeline": enable_cache_for_krea2, + } + ) -def may_enable_cache_dit(pipeline: Any, od_config: OmniDiffusionConfig) -> Optional["CacheDiTBackend"]: - """Enable cache-dit on the pipeline if configured (convenience function). - - This is a convenience function that creates and enables a CacheDiTBackend. - For new code, consider using CacheDiTBackend directly. - - Args: - pipeline: The diffusion pipeline instance. - od_config: OmniDiffusionConfig with cache configuration. - - Returns: - A CacheDiTBackend instance if cache-dit is enabled, None otherwise. - """ - if od_config.cache_backend != "cache-dit" or not od_config.cache_config: - return None - - backend = CacheDiTBackend(od_config.cache_config) - backend.enable(pipeline) - return backend if backend.is_enabled() else None +__all__ = [ + "BagelCachedAdapter", + "SensenovaCachedAdapter", +] diff --git a/vllm_omni/diffusion/cache/magcache/__init__.py b/vllm_omni/diffusion/cache/magcache/__init__.py index 08f0d88fa..c68ebd169 100644 --- a/vllm_omni/diffusion/cache/magcache/__init__.py +++ b/vllm_omni/diffusion/cache/magcache/__init__.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from vllm_omni.diffusion.cache.magcache.backend import MagCacheBackend from vllm_omni.diffusion.cache.magcache.config import MagCacheConfig from vllm_omni.diffusion.cache.magcache.hook import ( MagCacheBlockHook, @@ -21,6 +22,7 @@ __all__ = [ "Flux2MagCacheStrategy", "FluxMagCacheStrategy", "MagCacheBlockHook", + "MagCacheBackend", "MagCacheConfig", "MagCacheHeadHook", "MagCacheState", diff --git a/vllm_omni/diffusion/cache/selector.py b/vllm_omni/diffusion/cache/selector.py index 3b002abe0..d58bada6c 100644 --- a/vllm_omni/diffusion/cache/selector.py +++ b/vllm_omni/diffusion/cache/selector.py @@ -1,10 +1,10 @@ from typing import Any from vllm_omni.diffusion.cache.base import CacheBackend -from vllm_omni.diffusion.cache.cache_dit_backend import CacheDiTBackend -from vllm_omni.diffusion.cache.magcache.backend import MagCacheBackend +from vllm_omni.diffusion.cache.cachedit import CacheDiTBackend +from vllm_omni.diffusion.cache.magcache import MagCacheBackend from vllm_omni.diffusion.cache.stepcache import StepCacheBackend -from vllm_omni.diffusion.cache.teacache.backend import TeaCacheBackend +from vllm_omni.diffusion.cache.teacache import TeaCacheBackend from vllm_omni.diffusion.data import DiffusionCacheConfig diff --git a/vllm_omni/diffusion/models/bagel/bagel_transformer.py b/vllm_omni/diffusion/models/bagel/bagel_transformer.py index 30f4a041b..33f86f34b 100644 --- a/vllm_omni/diffusion/models/bagel/bagel_transformer.py +++ b/vllm_omni/diffusion/models/bagel/bagel_transformer.py @@ -37,7 +37,7 @@ from vllm.transformers_utils.configs.bagel import BagelConfig from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata as DiffusionAttentionMetadata from vllm_omni.diffusion.attention.layer import Attention as DiffusionAttention -from vllm_omni.diffusion.cache.cache_dit_backend import BagelCachedAdapter, CacheDiTAdapterConfig +from vllm_omni.diffusion.cache.cachedit import BagelCachedAdapter, CacheDiTAdapterConfig from vllm_omni.diffusion.data import DiffusionParallelConfig from vllm_omni.diffusion.distributed.cfg_parallel import CFGParallelMixin from vllm_omni.diffusion.distributed.parallel_state import ( diff --git a/vllm_omni/diffusion/models/cosmos3/transformer_cosmos3.py b/vllm_omni/diffusion/models/cosmos3/transformer_cosmos3.py index 88d280bd0..5c7e5bd8f 100644 --- a/vllm_omni/diffusion/models/cosmos3/transformer_cosmos3.py +++ b/vllm_omni/diffusion/models/cosmos3/transformer_cosmos3.py @@ -33,7 +33,7 @@ from vllm.model_executor.layers.quantization.base_config import ( from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata from vllm_omni.diffusion.attention.layer import Attention as FrameworkAttention -from vllm_omni.diffusion.cache.cache_dit_backend import CacheDiTAdapterConfig +from vllm_omni.diffusion.cache.cachedit import CacheDiTAdapterConfig from vllm_omni.diffusion.data import OmniDiffusionConfig from vllm_omni.diffusion.distributed.sp_plan import SequenceParallelInput, SequenceParallelOutput from vllm_omni.diffusion.forward_context import get_forward_context, is_forward_context_available diff --git a/vllm_omni/diffusion/models/dreamid_omni/fusion.py b/vllm_omni/diffusion/models/dreamid_omni/fusion.py index acf24dd2f..8bcbbbb64 100644 --- a/vllm_omni/diffusion/models/dreamid_omni/fusion.py +++ b/vllm_omni/diffusion/models/dreamid_omni/fusion.py @@ -9,7 +9,7 @@ from vllm.logger import init_logger from vllm.model_executor.layers.linear import ColumnParallelLinear from vllm_omni.diffusion.attention.layer import Attention -from vllm_omni.diffusion.cache.cache_dit_backend import CacheDiTAdapterConfig +from vllm_omni.diffusion.cache.cachedit import CacheDiTAdapterConfig if TYPE_CHECKING: from vllm.model_executor.layers.quantization.base_config import QuantizationConfig diff --git a/vllm_omni/diffusion/models/ernie_image/ernie_image_transformer.py b/vllm_omni/diffusion/models/ernie_image/ernie_image_transformer.py index 8398cb7ab..244aeeb95 100644 --- a/vllm_omni/diffusion/models/ernie_image/ernie_image_transformer.py +++ b/vllm_omni/diffusion/models/ernie_image/ernie_image_transformer.py @@ -22,7 +22,7 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata from vllm_omni.diffusion.attention.layer import Attention -from vllm_omni.diffusion.cache.cache_dit_backend import CacheDiTAdapterConfig +from vllm_omni.diffusion.cache.cachedit import CacheDiTAdapterConfig from vllm_omni.diffusion.data import DiffusionParallelConfig, OmniDiffusionConfig from vllm_omni.diffusion.distributed.sp_plan import ( SequenceParallelInput, diff --git a/vllm_omni/diffusion/models/flux/flux_transformer.py b/vllm_omni/diffusion/models/flux/flux_transformer.py index 952ee9eb4..db36e7097 100644 --- a/vllm_omni/diffusion/models/flux/flux_transformer.py +++ b/vllm_omni/diffusion/models/flux/flux_transformer.py @@ -26,7 +26,7 @@ from vllm.model_executor.layers.linear import ( from vllm.model_executor.model_loader.weight_utils import default_weight_loader from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata -from vllm_omni.diffusion.cache.cache_dit_backend import CacheDiTAdapterConfig +from vllm_omni.diffusion.cache.cachedit import CacheDiTAdapterConfig from vllm_omni.quantization.component_config import safe_quant_config if TYPE_CHECKING: diff --git a/vllm_omni/diffusion/models/flux2/flux2_transformer.py b/vllm_omni/diffusion/models/flux2/flux2_transformer.py index 2beb1fadb..f05632efd 100644 --- a/vllm_omni/diffusion/models/flux2/flux2_transformer.py +++ b/vllm_omni/diffusion/models/flux2/flux2_transformer.py @@ -26,7 +26,7 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata from vllm_omni.diffusion.attention.layer import Attention -from vllm_omni.diffusion.cache.cache_dit_backend import CacheDiTAdapterConfig +from vllm_omni.diffusion.cache.cachedit import CacheDiTAdapterConfig from vllm_omni.diffusion.data import DiffusionParallelConfig, OmniDiffusionConfig from vllm_omni.diffusion.distributed.sp_plan import ( SequenceParallelInput, diff --git a/vllm_omni/diffusion/models/flux2_klein/flux2_klein_transformer.py b/vllm_omni/diffusion/models/flux2_klein/flux2_klein_transformer.py index e62fccba0..72bed54cb 100644 --- a/vllm_omni/diffusion/models/flux2_klein/flux2_klein_transformer.py +++ b/vllm_omni/diffusion/models/flux2_klein/flux2_klein_transformer.py @@ -41,7 +41,7 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata from vllm_omni.diffusion.attention.layer import Attention -from vllm_omni.diffusion.cache.cache_dit_backend import CacheDiTAdapterConfig +from vllm_omni.diffusion.cache.cachedit import CacheDiTAdapterConfig from vllm_omni.diffusion.data import DiffusionParallelConfig, OmniDiffusionConfig from vllm_omni.diffusion.distributed.sp_plan import ( SequenceParallelInput, diff --git a/vllm_omni/diffusion/models/glm_image/glm_image_transformer.py b/vllm_omni/diffusion/models/glm_image/glm_image_transformer.py index c933d4e7f..75ba60203 100644 --- a/vllm_omni/diffusion/models/glm_image/glm_image_transformer.py +++ b/vllm_omni/diffusion/models/glm_image/glm_image_transformer.py @@ -21,7 +21,7 @@ from vllm.model_executor.layers.linear import ( ) from vllm.model_executor.model_loader.weight_utils import default_weight_loader -from vllm_omni.diffusion.cache.cache_dit_backend import CacheDiTAdapterConfig +from vllm_omni.diffusion.cache.cachedit import CacheDiTAdapterConfig if TYPE_CHECKING: from vllm.model_executor.layers.quantization.base_config import QuantizationConfig diff --git a/vllm_omni/diffusion/models/helios/helios_transformer.py b/vllm_omni/diffusion/models/helios/helios_transformer.py index 730c10f1c..4e3cf5078 100644 --- a/vllm_omni/diffusion/models/helios/helios_transformer.py +++ b/vllm_omni/diffusion/models/helios/helios_transformer.py @@ -26,7 +26,7 @@ from vllm.model_executor.layers.linear import ColumnParallelLinear, QKVParallelL from vllm.model_executor.model_loader.weight_utils import default_weight_loader from vllm_omni.diffusion.attention.layer import Attention -from vllm_omni.diffusion.cache.cache_dit_backend import CacheDiTAdapterConfig +from vllm_omni.diffusion.cache.cachedit import CacheDiTAdapterConfig from vllm_omni.diffusion.distributed.sp_plan import ( SequenceParallelInput, SequenceParallelOutput, diff --git a/vllm_omni/diffusion/models/hunyuan_image3/hunyuan_image3_transformer.py b/vllm_omni/diffusion/models/hunyuan_image3/hunyuan_image3_transformer.py index f38d1ca7c..cae7913f2 100644 --- a/vllm_omni/diffusion/models/hunyuan_image3/hunyuan_image3_transformer.py +++ b/vllm_omni/diffusion/models/hunyuan_image3/hunyuan_image3_transformer.py @@ -60,7 +60,7 @@ from vllm_omni.diffusion.attention.backends.abstract import ( AttentionMetadata, ) from vllm_omni.diffusion.attention.layer import Attention -from vllm_omni.diffusion.cache.cache_dit_backend import CacheDiTAdapterConfig +from vllm_omni.diffusion.cache.cachedit import CacheDiTAdapterConfig from vllm_omni.diffusion.distributed.parallel_state import ( get_cfg_group, get_classifier_free_guidance_rank, diff --git a/vllm_omni/diffusion/models/hunyuan_video/hunyuan_video_15_transformer.py b/vllm_omni/diffusion/models/hunyuan_video/hunyuan_video_15_transformer.py index 9224973dd..04862fe0a 100644 --- a/vllm_omni/diffusion/models/hunyuan_video/hunyuan_video_15_transformer.py +++ b/vllm_omni/diffusion/models/hunyuan_video/hunyuan_video_15_transformer.py @@ -23,7 +23,7 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata from vllm_omni.diffusion.attention.layer import Attention -from vllm_omni.diffusion.cache.cache_dit_backend import CacheDiTAdapterConfig +from vllm_omni.diffusion.cache.cachedit import CacheDiTAdapterConfig from vllm_omni.diffusion.data import OmniDiffusionConfig from vllm_omni.diffusion.distributed.hsdp_utils import is_transformer_block_module from vllm_omni.diffusion.distributed.parallel_state import get_sequence_parallel_world_size diff --git a/vllm_omni/diffusion/models/longcat_image/longcat_image_transformer.py b/vllm_omni/diffusion/models/longcat_image/longcat_image_transformer.py index 744c1806a..3cde2a868 100644 --- a/vllm_omni/diffusion/models/longcat_image/longcat_image_transformer.py +++ b/vllm_omni/diffusion/models/longcat_image/longcat_image_transformer.py @@ -19,7 +19,7 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata from vllm_omni.diffusion.attention.layer import Attention -from vllm_omni.diffusion.cache.cache_dit_backend import CacheDiTAdapterConfig +from vllm_omni.diffusion.cache.cachedit import CacheDiTAdapterConfig from vllm_omni.diffusion.data import DiffusionParallelConfig, OmniDiffusionConfig from vllm_omni.diffusion.distributed.sp_plan import ( SequenceParallelInput, diff --git a/vllm_omni/diffusion/models/ltx2/ltx2_transformer.py b/vllm_omni/diffusion/models/ltx2/ltx2_transformer.py index 6072c0456..3d1a2d663 100644 --- a/vllm_omni/diffusion/models/ltx2/ltx2_transformer.py +++ b/vllm_omni/diffusion/models/ltx2/ltx2_transformer.py @@ -45,7 +45,7 @@ if TYPE_CHECKING: from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata from vllm_omni.diffusion.attention.layer import Attention -from vllm_omni.diffusion.cache.cache_dit_backend import CacheDiTAdapterConfig +from vllm_omni.diffusion.cache.cachedit import CacheDiTAdapterConfig from vllm_omni.diffusion.distributed.hsdp_utils import is_transformer_block_module from vllm_omni.diffusion.distributed.sp_plan import SequenceParallelInput, SequenceParallelOutput from vllm_omni.diffusion.forward_context import get_forward_context, is_forward_context_available diff --git a/vllm_omni/diffusion/models/sd3/sd3_transformer.py b/vllm_omni/diffusion/models/sd3/sd3_transformer.py index d8e037051..5e446a496 100644 --- a/vllm_omni/diffusion/models/sd3/sd3_transformer.py +++ b/vllm_omni/diffusion/models/sd3/sd3_transformer.py @@ -20,7 +20,7 @@ from vllm.model_executor.layers.linear import ( from vllm.model_executor.model_loader.weight_utils import default_weight_loader from vllm_omni.diffusion.attention.layer import Attention -from vllm_omni.diffusion.cache.cache_dit_backend import CacheDiTAdapterConfig +from vllm_omni.diffusion.cache.cachedit import CacheDiTAdapterConfig from vllm_omni.diffusion.data import OmniDiffusionConfig logger = init_logger(__name__) diff --git a/vllm_omni/diffusion/models/sensenova_u1/sensenova_u1_transformer.py b/vllm_omni/diffusion/models/sensenova_u1/sensenova_u1_transformer.py index 843358a70..d2db1f489 100644 --- a/vllm_omni/diffusion/models/sensenova_u1/sensenova_u1_transformer.py +++ b/vllm_omni/diffusion/models/sensenova_u1/sensenova_u1_transformer.py @@ -32,7 +32,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import ( from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata from vllm_omni.diffusion.attention.layer import Attention -from vllm_omni.diffusion.cache.cache_dit_backend import CacheDiTAdapterConfig, SensenovaCachedAdapter +from vllm_omni.diffusion.cache.cachedit import CacheDiTAdapterConfig, SensenovaCachedAdapter logger = init_logger(__name__) diff --git a/vllm_omni/diffusion/models/soulx_singer/modules/llama.py b/vllm_omni/diffusion/models/soulx_singer/modules/llama.py index 23a3377f2..b0b6afce3 100644 --- a/vllm_omni/diffusion/models/soulx_singer/modules/llama.py +++ b/vllm_omni/diffusion/models/soulx_singer/modules/llama.py @@ -7,7 +7,7 @@ from transformers import LlamaConfig, LlamaModel from transformers.modeling_outputs import BaseModelOutputWithPast from transformers.models.llama.modeling_llama import LlamaDecoderLayer -from vllm_omni.diffusion.cache.cache_dit_backend import CacheDiTAdapterConfig +from vllm_omni.diffusion.cache.cachedit import CacheDiTAdapterConfig from vllm_omni.diffusion.distributed.sp_plan import SequenceParallelInput, SequenceParallelOutput diff --git a/vllm_omni/diffusion/models/stable_audio/stable_audio_transformer.py b/vllm_omni/diffusion/models/stable_audio/stable_audio_transformer.py index c7bc9b2d4..ad7c27e1b 100644 --- a/vllm_omni/diffusion/models/stable_audio/stable_audio_transformer.py +++ b/vllm_omni/diffusion/models/stable_audio/stable_audio_transformer.py @@ -16,7 +16,7 @@ from vllm.model_executor.layers.linear import ReplicatedLinear from vllm.model_executor.model_loader.weight_utils import default_weight_loader from vllm_omni.diffusion.attention.layer import Attention -from vllm_omni.diffusion.cache.cache_dit_backend import CacheDiTAdapterConfig +from vllm_omni.diffusion.cache.cachedit import CacheDiTAdapterConfig from vllm_omni.diffusion.data import OmniDiffusionConfig from vllm_omni.diffusion.distributed.hsdp_utils import is_transformer_block_module from vllm_omni.diffusion.layers.fourier import GaussianFourierProjection diff --git a/vllm_omni/diffusion/worker/diffusion_model_runner.py b/vllm_omni/diffusion/worker/diffusion_model_runner.py index 44a873e56..ced51ba3d 100644 --- a/vllm_omni/diffusion/worker/diffusion_model_runner.py +++ b/vllm_omni/diffusion/worker/diffusion_model_runner.py @@ -22,7 +22,7 @@ from vllm.config import LoadConfig, VllmConfig from vllm.logger import init_logger from vllm.utils.mem_utils import DeviceMemoryProfiler, GiB_bytes -from vllm_omni.diffusion.cache.cache_dit_backend import cache_summary +from vllm_omni.diffusion.cache.cachedit import cache_summary from vllm_omni.diffusion.cache.prompt_embed_cache import ( install_prompt_embed_cache, resolve_prompt_embed_cache_config, From 84bbd57e4d2f1210527587b8ba53e168902506cb Mon Sep 17 00:00:00 2001 From: Honghan Zhu Date: Fri, 24 Jul 2026 22:45:26 +0800 Subject: [PATCH 087/240] =?UTF-8?q?[Feature]=20Add=20AllGather-KV=20sequen?= =?UTF-8?q?ce-parallel=20attention=20str=E2=80=A6=20(#4968)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zhuhh97 Signed-off-by: dph97 Signed-off-by: zhuhh97 Co-authored-by: Claude Opus 4.7 Co-authored-by: dengyunyang <584797741@qq.com> --- tests/config/test_omni_config.py | 30 ++ .../diffusion/attention/test_attention_sp.py | 288 +++++++++++++++++- tests/diffusion/attention/test_flash_attn.py | 30 ++ .../attention/test_piecewise_attn.py | 87 ++++++ tests/diffusion/attention/test_sdpa.py | 57 ++++ .../test_parallel_state_sp_groups.py | 39 ++- .../test_image_kv_cache_manager.py | 2 + .../test_hunyuan_image3_pixel_accuracy.py | 48 ++- tests/test_config_factory.py | 25 ++ vllm_omni/config/omni_config.py | 8 +- vllm_omni/config/stage_config.py | 8 +- vllm_omni/deploy/hunyuan_image3_dit.yaml | 1 + .../diffusion/attention/backends/abstract.py | 9 + .../attention/backends/cudnn_attn.py | 6 +- .../attention/backends/flash_attn.py | 41 ++- .../attention/backends/flash_attn_hub.py | 8 +- .../diffusion/attention/backends/sdpa.py | 21 +- .../backends/utils/attn_runtime_selector.py | 40 +++ .../backends/utils/piecewise_attn.py | 91 ++++-- vllm_omni/diffusion/attention/layer.py | 17 ++ .../attention/parallel/allgather_kv.py | 182 +++++++++++ .../diffusion/attention/parallel/factory.py | 32 +- vllm_omni/diffusion/data.py | 29 +- .../distributed/group_coordinator.py | 20 +- .../diffusion/distributed/parallel_state.py | 90 ++++-- vllm_omni/diffusion/distributed/sp_plan.py | 20 +- .../hunyuan_image3_transformer.py | 31 +- vllm_omni/diffusion/registry.py | 29 +- vllm_omni/diffusion/utils/kv_utils.py | 13 + .../diffusion/worker/diffusion_worker.py | 1 + vllm_omni/engine/arg_utils.py | 1 + vllm_omni/engine/async_omni_engine.py | 4 +- vllm_omni/entrypoints/cli/serve.py | 8 + 33 files changed, 1181 insertions(+), 135 deletions(-) create mode 100644 tests/diffusion/attention/test_sdpa.py create mode 100644 vllm_omni/diffusion/attention/backends/utils/attn_runtime_selector.py create mode 100644 vllm_omni/diffusion/attention/parallel/allgather_kv.py diff --git a/tests/config/test_omni_config.py b/tests/config/test_omni_config.py index 1da71b21d..523542232 100644 --- a/tests/config/test_omni_config.py +++ b/tests/config/test_omni_config.py @@ -431,6 +431,7 @@ def test_sub_config_fields_match_rfc_scopes(): "sequence_parallel_size", "ulysses_degree", "ring_degree", + "allgather_degree", "ulysses_mode", "cfg_parallel_size", "vae_patch_parallel_size", @@ -547,11 +548,40 @@ def test_from_pipeline_config_derives_sequence_parallel_size_from_degrees(tmp_pa assert stage.parallel_config.world_size == 6 +def test_from_pipeline_config_derives_sequence_parallel_size_from_allgather_degree(tmp_path): + deploy_path = tmp_path / "dreamzero_allgather_parallel.yaml" + deploy_path.write_text( + "\n".join( + [ + "pipeline: dreamzero", + "async_chunk: false", + "stages:", + " - stage_id: 0", + " parallel_config:", + " sequence_parallel_size: 99", + " allgather_degree: 2", + ] + ) + ) + + stage = _from_pipeline_key("dreamzero", deploy_config_path=str(deploy_path)).stage_by_id(0) + + assert isinstance(stage, VllmOmniDiffusionStageConfig) + assert stage.parallel_config.allgather_degree == 2 + assert stage.parallel_config.sequence_parallel_size == 2 + assert stage.parallel_config.world_size == 2 + + def test_diffusion_parallel_config_rejects_cfg_parallel_size_outside_current_bound(): with pytest.raises(ValidationError): OmniStageDiffusionParallelConfig(cfg_parallel_size=4) +def test_diffusion_parallel_config_rejects_allgather_with_ulysses_or_ring(): + with pytest.raises(ValidationError): + OmniStageDiffusionParallelConfig(allgather_degree=2, ulysses_degree=2) + + def test_stage_realizations_use_stage_specific_parallel_config_types(): qwen_config = _from_pipeline_key("qwen3_tts") ar_stage = qwen_config.stage_by_id(0) diff --git a/tests/diffusion/attention/test_attention_sp.py b/tests/diffusion/attention/test_attention_sp.py index e5fb93d71..e32c4c5e4 100644 --- a/tests/diffusion/attention/test_attention_sp.py +++ b/tests/diffusion/attention/test_attention_sp.py @@ -31,7 +31,11 @@ import tempfile import pytest import torch +from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata, QueryRange from vllm_omni.diffusion.attention.layer import Attention +from vllm_omni.diffusion.attention.parallel.allgather_kv import ( + AllGatherKVParallelAttention, +) from vllm_omni.diffusion.config import set_current_diffusion_config from vllm_omni.diffusion.data import ( DiffusionParallelConfig, @@ -155,14 +159,267 @@ class TestMultiLayerAttentionModel(torch.nn.Module): return hidden_states +class _MockAllGatherSPGroup: + def __init__(self, *, rank: int, gather_chunks: list[list[torch.Tensor]]) -> None: + self.allgather_group = object() + self.allgather_world_size = len(gather_chunks[0]) + self.allgather_rank = rank + self._gather_chunks = list(gather_chunks) + self.gathered_input_shapes: list[tuple[int, ...]] = [] + + def all_gather(self, input_: torch.Tensor, dim: int = 0, separate_tensors: bool = False, group=None): + assert not separate_tensors + assert group is self.allgather_group + self.gathered_input_shapes.append(tuple(input_.shape)) + chunks = self._gather_chunks.pop(0) + return torch.cat(chunks, dim=dim) + + +def test_allgather_kv_slices_full_dense_mask_to_local_query_rows(): + rank = 1 + joint_len = 1 + img_seq_local = 2 + img_seq_full = 4 + key_chunks = [ + torch.full((1, img_seq_local, 1, 1), 10.0), + torch.full((1, img_seq_local, 1, 1), 20.0), + ] + value_chunks = [ + torch.full((1, img_seq_local, 1, 1), 30.0), + torch.full((1, img_seq_local, 1, 1), 40.0), + ] + strategy = AllGatherKVParallelAttention( + _MockAllGatherSPGroup(rank=rank, gather_chunks=[key_chunks, value_chunks]), + ) + + query = torch.zeros((1, img_seq_local, 1, 1)) + key = key_chunks[rank] + value = value_chunks[rank] + joint = torch.ones((1, joint_len, 1, 1)) + mask = torch.arange((joint_len + img_seq_full) * (joint_len + img_seq_full)) + mask = (mask % 2 == 0).view(1, 1, joint_len + img_seq_full, joint_len + img_seq_full) + metadata = AttentionMetadata( + attn_mask=mask, + joint_query=joint, + joint_key=joint, + joint_value=joint, + joint_strategy="front", + ) + + q_local, k_full, v_full, metadata_local, _ = strategy.pre_attention(query, key, value, metadata) + + expected_rows = torch.cat( + [ + mask[..., :joint_len, :], + mask[ + ..., + joint_len + rank * img_seq_local : joint_len + (rank + 1) * img_seq_local, + :, + ], + ], + dim=-2, + ) + assert q_local.shape[1] == joint_len + img_seq_local + assert k_full.shape[1] == joint_len + img_seq_full + assert v_full.shape[1] == joint_len + img_seq_full + torch.testing.assert_close(k_full[:, joint_len:], torch.cat(key_chunks, dim=1)) + torch.testing.assert_close(v_full[:, joint_len:], torch.cat(value_chunks, dim=1)) + assert metadata_local is not None + assert torch.equal(metadata_local.attn_mask, expected_rows) + + +def test_allgather_kv_slices_rear_joint_dense_mask(): + rank = 1 + joint_len = 1 + img_seq_local = 2 + img_seq_full = 4 + key_chunks = [torch.zeros((1, img_seq_local, 1, 1)) for _ in range(2)] + value_chunks = [torch.zeros_like(chunk) for chunk in key_chunks] + strategy = AllGatherKVParallelAttention( + _MockAllGatherSPGroup(rank=rank, gather_chunks=[key_chunks, value_chunks]), + ) + query = torch.zeros((1, img_seq_local, 1, 1)) + joint = torch.ones((1, joint_len, 1, 1)) + mask = torch.arange((img_seq_full + joint_len) ** 2) + mask = (mask % 2 == 0).view(1, 1, img_seq_full + joint_len, img_seq_full + joint_len) + metadata = AttentionMetadata( + attn_mask=mask, + joint_query=joint, + joint_key=joint, + joint_value=joint, + joint_strategy="rear", + ) + + q_local, _, _, metadata_local, _ = strategy.pre_attention(query, key_chunks[rank], value_chunks[rank], metadata) + + img_start = rank * img_seq_local + expected_rows = torch.cat( + [ + mask[..., img_start : img_start + img_seq_local, :], + mask[..., img_seq_full : img_seq_full + joint_len, :], + ], + dim=-2, + ) + assert q_local.shape[1] == img_seq_local + joint_len + assert metadata_local is not None + assert torch.equal(metadata_local.attn_mask, expected_rows) + + +def test_allgather_kv_rejects_invalid_joint_strategy(): + chunks = [[torch.zeros((1, 2, 1, 1)) for _ in range(2)] for _ in range(2)] + strategy = AllGatherKVParallelAttention(_MockAllGatherSPGroup(rank=0, gather_chunks=chunks)) + joint = torch.ones((1, 1, 1, 1)) + metadata = AttentionMetadata( + joint_query=joint, + joint_key=joint, + joint_value=joint, + joint_strategy="back", + ) + + with pytest.raises(ValueError, match="Unsupported joint_strategy"): + strategy.pre_attention(torch.zeros((1, 2, 1, 1)), chunks[0][0], chunks[1][0], metadata) + + +def test_allgather_kv_preserves_global_spans_and_sets_query_ranges(): + strategy = AllGatherKVParallelAttention( + _MockAllGatherSPGroup( + rank=1, + gather_chunks=[ + [torch.zeros((1, 2, 1, 1)), torch.zeros((1, 2, 1, 1))], + [torch.zeros((1, 2, 1, 1)), torch.zeros((1, 2, 1, 1))], + ], + ), + ) + query = torch.zeros((1, 2, 1, 1)) + key = torch.zeros((1, 2, 1, 1)) + value = torch.zeros((1, 2, 1, 1)) + joint = torch.ones((1, 1, 1, 1)) + metadata = AttentionMetadata( + joint_query=joint, + joint_key=joint, + joint_value=joint, + full_attn_spans=[ + [ + (0, 1), # joint span: kept on every rank. + (2, 5), # image span crossing rank 0 and rank 1 image shards. + ] + ], + ) + + _, _, _, metadata_out, _ = strategy.pre_attention(query, key, value, metadata) + + assert metadata_out is not None + assert metadata_out.full_attn_spans == [[(0, 1), (2, 5)]] + assert metadata_out.query_ranges == ( + QueryRange(0, 1, 0), + QueryRange(1, 3, 3), + ) + + +def test_allgather_kv_query_ranges_include_reused_prefix_offset(): + strategy = AllGatherKVParallelAttention( + _MockAllGatherSPGroup( + rank=1, + gather_chunks=[ + [torch.zeros((1, 2, 1, 1)), torch.zeros((1, 2, 1, 1))], + [torch.zeros((1, 2, 1, 1)), torch.zeros((1, 2, 1, 1))], + ], + ), + ) + query = torch.zeros((1, 2, 1, 1)) + key = torch.zeros((1, 2, 1, 1)) + value = torch.zeros((1, 2, 1, 1)) + joint_query = torch.ones((1, 1, 1, 1)) + # K/V retain two reused AR-prefix tokens that have already been removed + # from Q and from the attention mask's query rows. + joint_key = torch.ones((1, 3, 1, 1)) + joint_value = torch.ones((1, 3, 1, 1)) + mask = torch.arange(5 * 7).view(1, 1, 5, 7) + metadata = AttentionMetadata( + attn_mask=mask, + joint_query=joint_query, + joint_key=joint_key, + joint_value=joint_value, + full_attn_spans=[[(0, 7)]], + ) + + _, _, _, metadata_out, _ = strategy.pre_attention(query, key, value, metadata) + + assert metadata_out is not None + assert metadata_out.query_ranges == ( + QueryRange(0, 1, 2), + QueryRange(1, 3, 5), + ) + expected_mask = torch.cat([mask[..., :1, :], mask[..., 3:5, :]], dim=-2) + assert torch.equal(metadata_out.attn_mask, expected_mask) + + +def test_allgather_kv_allows_empty_full_attn_spans(): + strategy = AllGatherKVParallelAttention( + _MockAllGatherSPGroup( + rank=0, + gather_chunks=[ + [torch.zeros((1, 2, 1, 1)), torch.zeros((1, 2, 1, 1))], + [torch.zeros((1, 2, 1, 1)), torch.zeros((1, 2, 1, 1))], + ], + ), + ) + query = torch.zeros((1, 2, 1, 1)) + key = torch.zeros((1, 2, 1, 1)) + value = torch.zeros((1, 2, 1, 1)) + metadata = AttentionMetadata(full_attn_spans=[[]]) + + _, _, _, metadata_out, _ = strategy.pre_attention(query, key, value, metadata) + + assert metadata_out is not None + assert metadata_out.full_attn_spans == [[]] + assert metadata_out.query_ranges == (QueryRange(0, 2, 0),) + + +def test_allgather_kv_keeps_gathered_kv_compressed_for_gqa(): + rank = 0 + img_seq_local = 2 + kv_heads = 2 + repeat_num = 2 + q_heads = kv_heads * repeat_num + key_chunks = [ + torch.arange(1 * img_seq_local * kv_heads * 1, dtype=torch.float32).view(1, img_seq_local, kv_heads, 1), + torch.arange(100, 100 + 1 * img_seq_local * kv_heads * 1, dtype=torch.float32).view( + 1, img_seq_local, kv_heads, 1 + ), + ] + value_chunks = [chunk + 1000 for chunk in key_chunks] + sp_group = _MockAllGatherSPGroup(rank=rank, gather_chunks=[key_chunks, value_chunks]) + strategy = AllGatherKVParallelAttention(sp_group) + + query = torch.zeros((1, img_seq_local, q_heads, 1)) + _, k_full, v_full, _, _ = strategy.pre_attention(query, key_chunks[rank], value_chunks[rank], AttentionMetadata()) + + assert sp_group.gathered_input_shapes == [ + (1, img_seq_local, kv_heads, 1), + (1, img_seq_local, kv_heads, 1), + ] + assert k_full.shape == (1, img_seq_local * 2, kv_heads, 1) + assert v_full.shape == (1, img_seq_local * 2, kv_heads, 1) + expected_key = torch.cat(key_chunks, dim=1) + expected_value = torch.cat(value_chunks, dim=1) + torch.testing.assert_close(k_full, expected_key) + torch.testing.assert_close(v_full, expected_value) + + @pytest.mark.parametrize( "test_model_cls", [ TestMultiLayerAttentionModel, ], ) -@pytest.mark.parametrize("ulysses_degree", [2]) -@pytest.mark.parametrize("ring_degree", [2]) +@pytest.mark.parametrize( + ("ulysses_degree", "ring_degree", "allgather_degree"), + [ + pytest.param(2, 2, 1, id="ulysses-ring"), + pytest.param(1, 1, 2, id="allgather-kv"), + ], +) @pytest.mark.parametrize("batch_size", [2]) @pytest.mark.parametrize("seq_len", [16]) @pytest.mark.parametrize("num_heads", [8]) @@ -175,6 +432,7 @@ class TestMultiLayerAttentionModel(torch.nn.Module): def test_sequence_parallel( ulysses_degree: int, ring_degree: int, + allgather_degree: int, test_model_cls: type[torch.nn.Module], dtype: torch.dtype, causal: bool, @@ -186,8 +444,8 @@ def test_sequence_parallel( num_heads: int, head_size: int, ): - """Test Ulysses attention by comparing with and without SP enabled.""" - sequence_parallel_size = ulysses_degree * ring_degree + """Compare Ulysses/Ring and AllGather-KV SP against a single-rank run.""" + sequence_parallel_size = allgather_degree if allgather_degree > 1 else ulysses_degree * ring_degree # Skip if not enough GPUs available available_gpus = current_omni_platform.get_device_count() @@ -223,6 +481,7 @@ def test_sequence_parallel( use_compile, 1, # ulysses_degree = 1 1, # ring_degree = 1 + 1, # allgather_degree = 1 (baseline: no AllGather-KV SP) 1, # sequence_parallel_size = 1 baseline_output_file, model_state_file, @@ -233,7 +492,10 @@ def test_sequence_parallel( ) # Step 2: Run with SP enabled - print(f"\n[SP Test] Running with SP (ulysses_degree={ulysses_degree}, ring_degree={ring_degree})...") + print( + "\n[SP Test] Running with SP " + f"(ulysses={ulysses_degree}, ring={ring_degree}, allgather={allgather_degree})..." + ) torch.multiprocessing.spawn( ulysses_attention_on_test_model, args=( @@ -250,6 +512,7 @@ def test_sequence_parallel( use_compile, ulysses_degree, ring_degree, + allgather_degree, sequence_parallel_size, sp_output_file, model_state_file, @@ -345,6 +608,7 @@ def ulysses_attention_on_test_model( use_compile: bool, ulysses_degree: int, ring_degree: int, + allgather_degree: int, sequence_parallel_size: int, output_file: str, model_state_file: str, @@ -356,7 +620,11 @@ def ulysses_attention_on_test_model( RANDOM_SEED = 42 seed_everything(RANDOM_SEED) - mode_str = "Baseline (no SP)" if is_baseline else f"SP (ulysses={ulysses_degree}, ring={ring_degree})" + if allgather_degree > 1: + sp_kind = f"allgather={allgather_degree}" + else: + sp_kind = f"ulysses={ulysses_degree}, ring={ring_degree}" + mode_str = "Baseline (no SP)" if is_baseline else f"SP ({sp_kind})" print(f"\n[{mode_str}] Rank {local_rank}/{world_size} - Random seed set to {RANDOM_SEED}") device = torch.device(f"{current_omni_platform.device_type}:{local_rank}") @@ -384,6 +652,7 @@ def ulysses_attention_on_test_model( sequence_parallel_size=sequence_parallel_size, ulysses_degree=ulysses_degree, ring_degree=ring_degree, + allgather_degree=allgather_degree, cfg_parallel_size=1, ) @@ -400,6 +669,7 @@ def ulysses_attention_on_test_model( sequence_parallel_size=sequence_parallel_size, ulysses_degree=ulysses_degree, ring_degree=ring_degree, + allgather_degree=allgather_degree, tensor_parallel_size=1, pipeline_parallel_size=1, ) @@ -519,7 +789,11 @@ def ulysses_attention_on_test_model( with open(output_file, "wb") as f: pickle.dump(output_np, f) - mode_str = "baseline (no SP)" if is_baseline else f"SP (ulysses={ulysses_degree}, ring={ring_degree})" + if allgather_degree > 1: + sp_kind = f"allgather={allgather_degree}" + else: + sp_kind = f"ulysses={ulysses_degree}, ring={ring_degree}" + mode_str = "baseline (no SP)" if is_baseline else f"SP ({sp_kind})" print( f"\n[{mode_str}] ✓ Saved output with shape {full_output.shape}:\n" f" - batch_size={batch_size}, seq_len={seq_len}\n" diff --git a/tests/diffusion/attention/test_flash_attn.py b/tests/diffusion/attention/test_flash_attn.py index a6fcf8fcb..5ffd6684a 100644 --- a/tests/diffusion/attention/test_flash_attn.py +++ b/tests/diffusion/attention/test_flash_attn.py @@ -297,6 +297,36 @@ def test_flash_attn_func_preferred_over_varlen(): print("✓ flash_attn_func forward works correctly!") +def test_piecewise_flash_attn_uses_varlen_fallback(monkeypatch): + calls = [] + + def fake_varlen_func(q, k, v, **kwargs): + assert q.shape[1] == 4 + assert k.shape[1] == 2 + assert v.shape[1] == 2 + calls.append(kwargs) + return q + + monkeypatch.setattr(fa, "flash_attn_func", None) + monkeypatch.setattr(fa, "flash_attn_varlen_func", fake_varlen_func) + monkeypatch.setattr(fa, "HAS_FLASH_ATTN", True) + + impl = FlashAttentionImpl(num_heads=4, num_kv_heads=2, head_size=4, softmax_scale=0.5, causal=False) + query = torch.randn(1, 3, 4, 4) + key = torch.randn(1, 3, 2, 4) + value = torch.randn(1, 3, 2, 4) + metadata = AttentionMetadata(full_attn_spans=[[(0, 3)]]) + + output = impl.forward_cuda(query, key, value, metadata) + + assert output.shape == query.shape + assert len(calls) == 1 + assert calls[0]["max_seqlen_q"] == 3 + assert calls[0]["max_seqlen_k"] == 3 + assert calls[0]["causal"] is False + assert calls[0]["softmax_scale"] == 0.5 + + if __name__ == "__main__": print("Running FlashAttention Padding Tests...") print("=" * 60) diff --git a/tests/diffusion/attention/test_piecewise_attn.py b/tests/diffusion/attention/test_piecewise_attn.py index 656087623..00b98ac64 100644 --- a/tests/diffusion/attention/test_piecewise_attn.py +++ b/tests/diffusion/attention/test_piecewise_attn.py @@ -19,7 +19,11 @@ import pytest import torch import torch.nn.functional as F +import vllm_omni.diffusion.attention.backends.utils.piecewise_attn as piecewise_module +from vllm_omni.diffusion.attention.backends.abstract import QueryRange from vllm_omni.diffusion.attention.backends.utils.piecewise_attn import ( + Segment, + build_segments, piecewise_attn, ) @@ -71,6 +75,12 @@ BATCH_CASES = [ ] +def test_build_segments_full_span_retains_global_kv_end(): + segments = build_segments([(2, 7)], query_offset=4, query_len=2) + + assert segments == [Segment(q_start=4, q_end=6, kv_end=7, mode="full")] + + @pytest.mark.parametrize("global_spans", SPAN_CASES) @pytest.mark.parametrize("q_range", Q_RANGE_CASES) @pytest.mark.parametrize("batch_size", BATCH_CASES) @@ -124,3 +134,80 @@ def test_piecewise_span_fully_before_qstart(): ) expected = _full_reference(query, key, value, global_spans, q_start, q_end, softmax_scale) torch.testing.assert_close(got, expected, atol=1e-5, rtol=1e-5) + + +def test_piecewise_noncontiguous_query_ranges_match_full_mask(): + torch.manual_seed(0) + B, H, D, Sk = 1, 2, 8, 8 + key = torch.randn(B, Sk, H, D, device=DEVICE) + value = torch.randn(B, Sk, H, D, device=DEVICE) + query = torch.randn(B, 4, H, D, device=DEVICE) + spans = [(2, 7)] + query_ranges = ( + QueryRange(0, 1, 0), + QueryRange(1, 4, 4), + ) + softmax_scale = D**-0.5 + + got = piecewise_attn( + query, + key, + value, + full_attn_spans=[spans], + softmax_scale=softmax_scale, + attn_func=_sdpa_attn_func, + query_ranges=query_ranges, + ) + expected = torch.cat( + [ + _full_reference(query[:, :1], key, value, spans, 0, 1, softmax_scale), + _full_reference(query[:, 1:], key, value, spans, 4, 7, softmax_scale), + ], + dim=1, + ) + torch.testing.assert_close(got, expected, atol=1e-5, rtol=1e-5) + + +def test_piecewise_rejects_incomplete_query_ranges(): + query = torch.zeros(1, 3, 1, 2) + key = torch.zeros(1, 3, 1, 2) + value = torch.zeros_like(key) + + with pytest.raises(ValueError, match="cover local query contiguously"): + piecewise_attn( + query, + key, + value, + full_attn_spans=[[]], + softmax_scale=1.0, + attn_func=_sdpa_attn_func, + query_ranges=(QueryRange(1, 3, 1),), + ) + + +def test_piecewise_combines_noncontiguous_query_ranges_with_one_cat(monkeypatch): + query = torch.randn(1, 4, 2, 8) + key = torch.randn(1, 8, 2, 8) + value = torch.randn_like(key) + cat_calls = 0 + original_cat = torch.cat + + def counted_cat(tensors, dim=0): + nonlocal cat_calls + cat_calls += 1 + return original_cat(tensors, dim=dim) + + monkeypatch.setattr(piecewise_module.torch, "cat", counted_cat) + + output = piecewise_attn( + query, + key, + value, + full_attn_spans=[[(0, 8)]], + softmax_scale=1.0, + attn_func=lambda q, k, v, **kwargs: q, + query_ranges=(QueryRange(0, 1, 0), QueryRange(1, 4, 4)), + ) + + assert cat_calls == 1 + torch.testing.assert_close(output, query) diff --git a/tests/diffusion/attention/test_sdpa.py b/tests/diffusion/attention/test_sdpa.py new file mode 100644 index 000000000..d02642517 --- /dev/null +++ b/tests/diffusion/attention/test_sdpa.py @@ -0,0 +1,57 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import pytest +import torch + +import vllm_omni.diffusion.attention.backends.sdpa as sdpa_backend +from vllm_omni.diffusion.attention.backends.sdpa import SDPAImpl + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +def test_sdpa_expands_kv_when_native_gqa_kernel_is_unavailable(monkeypatch): + calls = [] + + def fake_sdpa(query, key, value, **kwargs): + calls.append((query.shape, key.shape, value.shape, kwargs)) + return query + + monkeypatch.setattr(sdpa_backend, "can_sdpa_use_fused_gqa", lambda *args: False) + monkeypatch.setattr(torch.nn.functional, "scaled_dot_product_attention", fake_sdpa) + + impl = SDPAImpl(num_heads=4, num_kv_heads=2, head_size=8, softmax_scale=0.5) + output = impl.forward_cuda( + torch.randn(1, 3, 4, 8), + torch.randn(1, 3, 2, 8), + torch.randn(1, 3, 2, 8), + ) + + query_shape, key_shape, value_shape, kwargs = calls[0] + assert query_shape == key_shape == value_shape == (1, 4, 3, 8) + assert kwargs["enable_gqa"] is False + assert output.shape == (1, 3, 4, 8) + + +def test_sdpa_keeps_compressed_kv_when_native_gqa_kernel_is_available(monkeypatch): + calls = [] + + def fake_sdpa(query, key, value, **kwargs): + calls.append((query.shape, key.shape, value.shape, kwargs)) + return query + + monkeypatch.setattr(sdpa_backend, "can_sdpa_use_fused_gqa", lambda *args: True) + monkeypatch.setattr(torch.nn.functional, "scaled_dot_product_attention", fake_sdpa) + + impl = SDPAImpl(num_heads=4, num_kv_heads=2, head_size=8, softmax_scale=0.5) + output = impl.forward_cuda( + torch.randn(1, 3, 4, 8), + torch.randn(1, 3, 2, 8), + torch.randn(1, 3, 2, 8), + ) + + query_shape, key_shape, value_shape, kwargs = calls[0] + assert query_shape == (1, 4, 3, 8) + assert key_shape == value_shape == (1, 2, 3, 8) + assert kwargs["enable_gqa"] is True + assert output.shape == (1, 3, 4, 8) diff --git a/tests/diffusion/distributed/test_parallel_state_sp_groups.py b/tests/diffusion/distributed/test_parallel_state_sp_groups.py index 9dda5610b..c12882a22 100644 --- a/tests/diffusion/distributed/test_parallel_state_sp_groups.py +++ b/tests/diffusion/distributed/test_parallel_state_sp_groups.py @@ -40,7 +40,7 @@ def test_set_seq_parallel_pg_uses_explicit_sp_groups(rank, expected_ulysses, exp # tp=2, sp=2 -> SP groups are non-contiguous: [0,2] and [1,3] sp_group_ranks = RankGenerator(2, 2, 1, 1, 1, "tp-sp-pp-cfg-dp").get_ranks("sp") - ulysses_pg, ring_pg = set_seq_parallel_pg( + ulysses_pg, ring_pg, allgather_pg = set_seq_parallel_pg( sp_ulysses_degree=2, sp_ring_degree=1, rank=rank, @@ -50,6 +50,43 @@ def test_set_seq_parallel_pg_uses_explicit_sp_groups(rank, expected_ulysses, exp assert ulysses_pg.ranks == expected_ulysses assert ring_pg.ranks == expected_ring + assert allgather_pg.ranks == [rank] + + +@pytest.mark.cpu +@pytest.mark.parametrize("rank", [0, 1, 2, 3]) +def test_set_seq_parallel_pg_builds_independent_allgather_group(rank, monkeypatch): + created_groups: list[SimpleNamespace] = [] + monkeypatch.setattr(torch.distributed, "new_group", _fake_new_group_factory(created_groups)) + sp_group_ranks = [[0, 2], [1, 3]] + + ulysses_pg, ring_pg, allgather_pg = set_seq_parallel_pg( + sp_ulysses_degree=1, + sp_ring_degree=1, + sp_allgather_degree=2, + rank=rank, + world_size=4, + sp_group_ranks=sp_group_ranks, + ) + + assert ulysses_pg.ranks == [rank] + assert ring_pg.ranks == [rank] + assert allgather_pg.ranks == ([0, 2] if rank % 2 == 0 else [1, 3]) + + +@pytest.mark.cpu +def test_set_seq_parallel_pg_rejects_allgather_with_ulysses_or_ring(monkeypatch): + created_groups: list[SimpleNamespace] = [] + monkeypatch.setattr(torch.distributed, "new_group", _fake_new_group_factory(created_groups)) + + with pytest.raises(ValueError, match="mutually exclusive"): + set_seq_parallel_pg( + sp_ulysses_degree=2, + sp_ring_degree=1, + sp_allgather_degree=2, + rank=0, + world_size=4, + ) @pytest.mark.cpu diff --git a/tests/diffusion/models/hunyuan_image3/test_image_kv_cache_manager.py b/tests/diffusion/models/hunyuan_image3/test_image_kv_cache_manager.py index 8a35d8aaa..543b93a25 100644 --- a/tests/diffusion/models/hunyuan_image3/test_image_kv_cache_manager.py +++ b/tests/diffusion/models/hunyuan_image3/test_image_kv_cache_manager.py @@ -45,6 +45,8 @@ def patched_mgr_env(sp_size=1): target = _TRANSFORMER_MODULE patches = [ patch(f"{target}.get_sequence_parallel_world_size", return_value=sp_size), + patch(f"{target}.get_allgather_parallel_world_size", return_value=sp_size), + patch(f"{target}.get_ulysses_parallel_world_size", return_value=1, create=True), patch(f"{target}.get_sequence_parallel_rank", return_value=0), patch(f"{target}.Attention", MockAttention), ] diff --git a/tests/e2e/accuracy/test_hunyuan_image3_pixel_accuracy.py b/tests/e2e/accuracy/test_hunyuan_image3_pixel_accuracy.py index 3f799c038..74fb14086 100644 --- a/tests/e2e/accuracy/test_hunyuan_image3_pixel_accuracy.py +++ b/tests/e2e/accuracy/test_hunyuan_image3_pixel_accuracy.py @@ -3,6 +3,7 @@ from __future__ import annotations import base64 import copy import io +import logging import os import tempfile from pathlib import Path @@ -18,6 +19,8 @@ from tests.helpers.runtime import OmniServer pytestmark = [pytest.mark.full_model, pytest.mark.diffusion] +logger = logging.getLogger(__name__) + MODEL_NAME = "tencent/HunyuanImage-3.0-Instruct" SEED = 42 NUM_INFERENCE_STEPS = 50 @@ -53,11 +56,12 @@ _DEPLOY_CONFIG = { "parallel_config": { "pipeline_parallel_size": 1, "data_parallel_size": 1, - "tensor_parallel_size": 4, + "tensor_parallel_size": 1, "enable_expert_parallel": True, "sequence_parallel_size": 1, "ulysses_degree": 1, "ring_degree": 1, + "allgather_degree": 4, "cfg_parallel_size": 1, "vae_patch_parallel_size": 1, "use_hsdp": False, @@ -83,9 +87,47 @@ def _devices() -> str: def _write_deploy_config(path: Path) -> None: config = copy.deepcopy(_DEPLOY_CONFIG) devices = _devices() + num_devices = len(devices.split(",")) config["stages"][0]["devices"] = devices - config["stages"][0]["parallel_config"]["tensor_parallel_size"] = len(devices.split(",")) - path.write_text(yaml.dump(config, default_flow_style=False, sort_keys=False)) + parallel_config = config["stages"][0]["parallel_config"] + if parallel_config.get("allgather_degree", 1) > 1: + parallel_config["allgather_degree"] = num_devices + elif parallel_config.get("tensor_parallel_size", 1) > 1: + parallel_config["tensor_parallel_size"] = num_devices + + sequence_parallel_size = ( + parallel_config.get("allgather_degree", 1) + if parallel_config.get("allgather_degree", 1) > 1 + else parallel_config.get("ulysses_degree", 1) * parallel_config.get("ring_degree", 1) + ) + configured_world_size = ( + parallel_config.get("pipeline_parallel_size", 1) + * parallel_config.get("data_parallel_size", 1) + * parallel_config.get("tensor_parallel_size", 1) + * sequence_parallel_size + * parallel_config.get("cfg_parallel_size", 1) + ) + assert configured_world_size == num_devices, ( + f"Configured parallel world size ({configured_world_size}) does not match " + f"the number of devices ({num_devices}): {parallel_config}" + ) + + config_text = yaml.dump(config, default_flow_style=False, sort_keys=False) + path.write_text(config_text) + logger.info( + "HunyuanImage3 pixel accuracy deploy config:\n" + " path=%s\n" + " CUDA_VISIBLE_DEVICES=%s\n" + " devices=%s\n" + " tensor_parallel_size=%s\n" + " allgather_degree=%s\n%s", + path, + os.environ.get("CUDA_VISIBLE_DEVICES", ""), + devices, + parallel_config.get("tensor_parallel_size", 1), + parallel_config.get("allgather_degree", 1), + config_text, + ) def _run_vllm_omni_hunyuan_image3_online(*, model: str, deploy_config: str, output_path: Path) -> Image.Image: diff --git a/tests/test_config_factory.py b/tests/test_config_factory.py index 10f423c84..aa3ab604e 100644 --- a/tests/test_config_factory.py +++ b/tests/test_config_factory.py @@ -336,6 +336,31 @@ class TestStageConfig: assert "ring_degree" not in omega_config.engine_args assert "sequence_parallel_size" not in omega_config.engine_args + def test_to_omegaconf_diffusion_parallel_allgather_override_recomputes_sequence_parallel_size(self): + config = StageConfig( + stage_id=1, + model_stage="diffusion", + stage_type=StageType.DIFFUSION, + yaml_engine_args={ + "parallel_config": { + "sequence_parallel_size": 1, + "ulysses_degree": 1, + "ring_degree": 1, + "allgather_degree": 1, + } + }, + runtime_overrides={ + "allgather_degree": 2, + }, + ) + + omega_config = config.to_omegaconf() + + assert omega_config.engine_args.parallel_config.allgather_degree == 2 + assert omega_config.engine_args.parallel_config.sequence_parallel_size == 2 + assert "allgather_degree" not in omega_config.engine_args + assert "sequence_parallel_size" not in omega_config.engine_args + def test_to_omegaconf_diffusion_parallel_explicit_sequence_parallel_size_is_preserved(self): config = StageConfig( stage_id=1, diff --git a/vllm_omni/config/omni_config.py b/vllm_omni/config/omni_config.py index be89e1cae..cdcbafa79 100644 --- a/vllm_omni/config/omni_config.py +++ b/vllm_omni/config/omni_config.py @@ -109,6 +109,7 @@ class _ParallelConfigEngineOverrides(TypedDict, total=False): sequence_parallel_size: int ulysses_degree: int ring_degree: int + allgather_degree: int ulysses_mode: str cfg_parallel_size: int vae_patch_parallel_size: int @@ -360,6 +361,7 @@ class OmniStageDiffusionParallelConfig(OmniStageParallelConfig): sequence_parallel_size: int = Field(default=1, ge=1, init=False) ulysses_degree: int = Field(default=1, ge=1) ring_degree: int = Field(default=1, ge=1) + allgather_degree: int = Field(default=1, ge=1) ulysses_mode: str = "strict" cfg_parallel_size: int = Field(default=1, ge=1, le=3) vae_patch_parallel_size: int = Field(default=1, ge=1) @@ -370,7 +372,11 @@ class OmniStageDiffusionParallelConfig(OmniStageParallelConfig): hsdp_replicate_size: int = Field(default=1, ge=1) def __post_init__(self) -> None: - self.sequence_parallel_size = self.ulysses_degree * self.ring_degree + self.sequence_parallel_size = ( + self.allgather_degree if self.allgather_degree > 1 else self.ulysses_degree * self.ring_degree + ) + if self.allgather_degree > 1 and (self.ulysses_degree > 1 or self.ring_degree > 1): + raise ValueError("allgather_degree > 1 is mutually exclusive with ulysses_degree/ring_degree > 1") if self.ulysses_mode not in {"strict", "advanced_uaa"}: raise ValueError("ulysses_mode must be 'strict' or 'advanced_uaa'") if self.vae_parallel_mode not in {"tile", "spatial_shard_height", "spatial_shard_width"}: diff --git a/vllm_omni/config/stage_config.py b/vllm_omni/config/stage_config.py index 2a8efe824..2f9749420 100644 --- a/vllm_omni/config/stage_config.py +++ b/vllm_omni/config/stage_config.py @@ -149,14 +149,17 @@ def _apply_diffusion_parallel_runtime_overrides( continue if parallel_config_dict is None: parallel_config_dict = {} - if key in ("ulysses_degree", "ring_degree"): + if key in ("ulysses_degree", "ring_degree", "allgather_degree"): degree_overridden = True parallel_config_dict[key] = runtime_overrides.pop(key) if parallel_config_dict is not None and degree_overridden and not sequence_parallel_explicit: ulysses_degree = parallel_config_dict.get("ulysses_degree") or 1 ring_degree = parallel_config_dict.get("ring_degree") or 1 - parallel_config_dict["sequence_parallel_size"] = ulysses_degree * ring_degree + allgather_degree = parallel_config_dict.get("allgather_degree") or 1 + parallel_config_dict["sequence_parallel_size"] = ( + allgather_degree if allgather_degree > 1 else ulysses_degree * ring_degree + ) if parallel_config_dict is not None: engine_args["parallel_config"] = parallel_config_dict @@ -366,6 +369,7 @@ class StageDeployConfig: ulysses_degree: int | None = None ulysses_mode: str | None = None ring_degree: int | None = None + allgather_degree: int | None = None sequence_parallel_size: int | None = None cfg_parallel_size: int | None = None vae_patch_parallel_size: int | None = None diff --git a/vllm_omni/deploy/hunyuan_image3_dit.yaml b/vllm_omni/deploy/hunyuan_image3_dit.yaml index 3c0ba1901..537afadc0 100644 --- a/vllm_omni/deploy/hunyuan_image3_dit.yaml +++ b/vllm_omni/deploy/hunyuan_image3_dit.yaml @@ -24,6 +24,7 @@ stages: sequence_parallel_size: 1 ulysses_degree: 1 ring_degree: 1 + allgather_degree: 1 cfg_parallel_size: 1 vae_patch_parallel_size: 1 use_hsdp: false diff --git a/vllm_omni/diffusion/attention/backends/abstract.py b/vllm_omni/diffusion/attention/backends/abstract.py index cd0408f3c..43c6f525c 100644 --- a/vllm_omni/diffusion/attention/backends/abstract.py +++ b/vllm_omni/diffusion/attention/backends/abstract.py @@ -14,6 +14,7 @@ class AttentionBackend(ABC): """Abstract class for diffusion attention backends.""" accept_output_buffer: bool = False + supports_piecewise_spans: bool = False @classmethod def supports_attention_mask(cls) -> bool: @@ -51,6 +52,13 @@ class AttentionBackend(ABC): return (not supported_head_sizes) or head_size in supported_head_sizes +@dataclass(frozen=True, slots=True) +class QueryRange: + local_start: int + local_end: int + global_start: int + + @dataclass class AttentionMetadata: attn_mask: torch.Tensor | None = None @@ -75,6 +83,7 @@ class AttentionMetadata: # Piecewise attention metadata (mixed causal/full masks). # full_attn_spans: per-sample [start, end) spans in global coordinates using full attention. full_attn_spans: list[list[tuple[int, int]]] | None = None + query_ranges: tuple[QueryRange, ...] | None = None T = TypeVar("T", bound=AttentionMetadata) diff --git a/vllm_omni/diffusion/attention/backends/cudnn_attn.py b/vllm_omni/diffusion/attention/backends/cudnn_attn.py index 357a2c0fd..78e428796 100644 --- a/vllm_omni/diffusion/attention/backends/cudnn_attn.py +++ b/vllm_omni/diffusion/attention/backends/cudnn_attn.py @@ -51,7 +51,6 @@ class CuDNNAttentionImpl(AttentionImpl): ) -> None: self.causal = causal self.softmax_scale = softmax_scale - self.requires_gqa = num_heads != num_kv_heads def forward_cuda( self, @@ -64,6 +63,7 @@ class CuDNNAttentionImpl(AttentionImpl): if attn_metadata: attention_mask = _maybe_reshape_attn_mask(query, key, attn_metadata.attn_mask, mask_mode="broadcast_k") + enable_gqa = query.shape[2] != key.shape[2] query, key, value = (x.permute(0, 2, 1, 3) for x in (query, key, value)) # Pin cuDNN exclusively. A priority list like [CUDNN, FLASH, MATH] hits a # PyTorch SDPA dispatch quirk: when FLASH rejects a non-None attn_mask, @@ -85,7 +85,7 @@ class CuDNNAttentionImpl(AttentionImpl): dropout_p=0.0, is_causal=self.causal, scale=self.softmax_scale, - enable_gqa=self.requires_gqa, + enable_gqa=enable_gqa, ) except RuntimeError as e: if "No available kernel" not in str(e): @@ -102,6 +102,6 @@ class CuDNNAttentionImpl(AttentionImpl): dropout_p=0.0, is_causal=self.causal, scale=self.softmax_scale, - enable_gqa=self.requires_gqa, + enable_gqa=enable_gqa, ) return output.permute(0, 2, 1, 3) diff --git a/vllm_omni/diffusion/attention/backends/flash_attn.py b/vllm_omni/diffusion/attention/backends/flash_attn.py index 19ec49417..82d61c25d 100644 --- a/vllm_omni/diffusion/attention/backends/flash_attn.py +++ b/vllm_omni/diffusion/attention/backends/flash_attn.py @@ -21,6 +21,7 @@ logger = init_logger(__name__) class FlashAttentionBackend(AttentionBackend): accept_output_buffer: bool = True + supports_piecewise_spans: bool = True @classmethod def supports_attention_mask(cls) -> bool: @@ -82,6 +83,28 @@ class FlashAttentionImpl(AttentionImpl): def _flash_wrapper(q, k, v, *, attn_func, **kwargs): return FlashAttentionImpl._unwrap_flash_output(attn_func(q, k, v, **kwargs)) + @staticmethod + def _flash_varlen_wrapper(q, k, v, *, attn_func, causal, softmax_scale, **kwargs): + """Call a varlen-only FlashAttention backend for a dense segment.""" + del kwargs + batch_size, q_len = q.shape[:2] + k_len = k.shape[1] + cu_seqlens_q = torch.arange(0, (batch_size + 1) * q_len, q_len, dtype=torch.int32, device=q.device) + cu_seqlens_k = torch.arange(0, (batch_size + 1) * k_len, k_len, dtype=torch.int32, device=q.device) + out = attn_func( + q=q.flatten(0, 1), + k=k.flatten(0, 1), + v=v.flatten(0, 1), + cu_seqlens_q=cu_seqlens_q, + cu_seqlens_k=cu_seqlens_k, + max_seqlen_q=q_len, + max_seqlen_k=k_len, + causal=causal, + softmax_scale=softmax_scale, + ) + out = FlashAttentionImpl._unwrap_flash_output(out) + return out.reshape(batch_size, q_len, *out.shape[1:]) + def _forward_varlen_masked( self, query: torch.Tensor, @@ -171,6 +194,7 @@ class FlashAttentionImpl(AttentionImpl): from vllm_omni.diffusion.attention.backends.utils.fa import ( HAS_FLASH_ATTN, flash_attn_func, + flash_attn_varlen_func, ) if not HAS_FLASH_ATTN: @@ -186,10 +210,18 @@ class FlashAttentionImpl(AttentionImpl): # Try piecewise attention if full_attn_spans is not None: logger.debug("Using piecewise Flash Attention for mixed causal/full mask") - attn_func = partial( - FlashAttentionImpl._flash_wrapper, - attn_func=flash_attn_func, - ) + if flash_attn_func is not None: + attn_func = partial( + FlashAttentionImpl._flash_wrapper, + attn_func=flash_attn_func, + ) + elif flash_attn_varlen_func is not None: + attn_func = partial( + FlashAttentionImpl._flash_varlen_wrapper, + attn_func=flash_attn_varlen_func, + ) + else: + raise ImportError("Piecewise FlashAttention requires a dense or varlen FlashAttention function") return piecewise_attn( query, @@ -198,6 +230,7 @@ class FlashAttentionImpl(AttentionImpl): full_attn_spans, self.softmax_scale, attn_func, + query_ranges=attn_metadata.query_ranges, ) if attention_mask is not None and torch.any(~attention_mask): diff --git a/vllm_omni/diffusion/attention/backends/flash_attn_hub.py b/vllm_omni/diffusion/attention/backends/flash_attn_hub.py index bce607888..73faa57a9 100644 --- a/vllm_omni/diffusion/attention/backends/flash_attn_hub.py +++ b/vllm_omni/diffusion/attention/backends/flash_attn_hub.py @@ -12,9 +12,7 @@ from vllm_omni.diffusion.attention.backends.abstract import ( AttentionImpl, AttentionMetadata, ) -from vllm_omni.diffusion.attention.backends.utils.piecewise_attn import ( - piecewise_attn, -) +from vllm_omni.diffusion.attention.backends.utils.piecewise_attn import piecewise_attn logger = init_logger(__name__) @@ -78,6 +76,7 @@ def _run_varlen_dense( class FlashAttentionHubBackend(AttentionBackend): accept_output_buffer: bool = True + supports_piecewise_spans: bool = True @classmethod def supports_attention_mask(cls) -> bool: @@ -214,6 +213,7 @@ class FlashAttentionHubImpl(AttentionImpl): full_attn_spans, self.softmax_scale, attn_func, + query_ranges=attn_metadata.query_ranges, ) if attention_mask is not None and torch.any(~attention_mask): @@ -243,6 +243,7 @@ class FlashAttentionHubImpl(AttentionImpl): class FlashAttention3HubBackend(AttentionBackend): accept_output_buffer: bool = True + supports_piecewise_spans: bool = True @classmethod def supports_attention_mask(cls) -> bool: @@ -379,6 +380,7 @@ class FlashAttention3HubImpl(AttentionImpl): full_attn_spans, self.softmax_scale, attn_func, + query_ranges=attn_metadata.query_ranges, ) if attention_mask is not None and torch.any(~attention_mask): diff --git a/vllm_omni/diffusion/attention/backends/sdpa.py b/vllm_omni/diffusion/attention/backends/sdpa.py index f8d607e48..aec8eb50b 100644 --- a/vllm_omni/diffusion/attention/backends/sdpa.py +++ b/vllm_omni/diffusion/attention/backends/sdpa.py @@ -11,6 +11,7 @@ from vllm_omni.diffusion.attention.backends.abstract import ( AttentionImpl, AttentionMetadata, ) +from vllm_omni.diffusion.attention.backends.utils.attn_runtime_selector import can_sdpa_use_fused_gqa logger = init_logger(__name__) @@ -85,7 +86,6 @@ class SDPAImpl(AttentionImpl): ) -> None: self.causal = causal self.softmax_scale = softmax_scale - self.requires_gqa = num_heads != num_kv_heads if backend_kwargs: logger.warning("SDPAImpl ignoring backend_kwargs: %s", list(backend_kwargs.keys())) @@ -103,7 +103,24 @@ class SDPAImpl(AttentionImpl): if attn_metadata: attention_mask = _maybe_reshape_attn_mask(query, key, attn_metadata.attn_mask, mask_mode=mask_mode) + enable_gqa = query.shape[2] != key.shape[2] query, key, value = (x.permute(0, 2, 1, 3) for x in (query, key, value)) + # Only the PyTorch SDPA backend needs this dispatch check. If SDPA + # cannot select a fused GQA kernel for the runtime shape/mask, expand + # K/V locally so it can use the better-supported equal-head path. + if enable_gqa and not can_sdpa_use_fused_gqa(query, key, value, attention_mask, self.causal): + if query.shape[1] % key.shape[1] != 0: + raise ValueError( + "GQA requires query heads to be a multiple of KV heads, " + f"got q_heads={query.shape[1]} and kv_heads={key.shape[1]}." + ) + repeat_num = query.shape[1] // key.shape[1] + key = key.repeat_interleave(repeat_num, dim=1) + value = value.repeat_interleave(repeat_num, dim=1) + enable_gqa = False + logger.debug( + "CUDA SDPA cannot use a fused native-GQA kernel for this shape; expanding K/V heads before SDPA." + ) output = torch.nn.functional.scaled_dot_product_attention( query, key, @@ -112,7 +129,7 @@ class SDPAImpl(AttentionImpl): dropout_p=0.0, is_causal=self.causal, scale=self.softmax_scale, - enable_gqa=self.requires_gqa, + enable_gqa=enable_gqa, ) out = output.permute(0, 2, 1, 3) return out diff --git a/vllm_omni/diffusion/attention/backends/utils/attn_runtime_selector.py b/vllm_omni/diffusion/attention/backends/utils/attn_runtime_selector.py new file mode 100644 index 000000000..84170dcd7 --- /dev/null +++ b/vllm_omni/diffusion/attention/backends/utils/attn_runtime_selector.py @@ -0,0 +1,40 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Runtime kernel-selection helpers for attention backends.""" + +import torch + + +def can_sdpa_use_fused_gqa( + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + attention_mask: torch.Tensor | None, + causal: bool, +) -> bool: + """Return whether PyTorch SDPA can use a fused kernel for this GQA call. + + This check is specific to the SDPA backend. Native GQA backends such as + FlashAttention should keep compressed K/V heads and do not use this helper. + """ + if not query.is_cuda: + return False + + try: + params = torch.backends.cuda.SDPAParams( + query, + key, + value, + attention_mask, + 0.0, + causal, + True, + ) + can_use_flash = torch.backends.cuda.can_use_flash_attention(params) + can_use_cudnn = getattr(torch.backends.cuda, "can_use_cudnn_attention", None) + return can_use_flash or (can_use_cudnn is not None and can_use_cudnn(params)) + except (AttributeError, RuntimeError, TypeError): + # Older PyTorch versions may not expose GQA-aware SDPA capability checks. + # Returning False lets the SDPA backend expand K/V and use the mature + # equal-head fused-kernel path instead. + return False diff --git a/vllm_omni/diffusion/attention/backends/utils/piecewise_attn.py b/vllm_omni/diffusion/attention/backends/utils/piecewise_attn.py index fbbd3c8e0..5d83d8dbf 100644 --- a/vllm_omni/diffusion/attention/backends/utils/piecewise_attn.py +++ b/vllm_omni/diffusion/attention/backends/utils/piecewise_attn.py @@ -9,17 +9,24 @@ follows FlashAttention's bottom-right convention (``K[:e]`` is attended by Per segment: - causal segment ``[s, e)``: ``attn(Q[:, s:e], K[:, :e], V[:, :e], causal=True)`` - - full-attn span ``[a, e)``: ``attn(Q[:, a:e], K[:, :e], V[:, :e], causal=False)`` + - full-attn span ``[a, b)`` intersecting the query range at ``[s, e)``: + ``attn(Q[:, s:e], K[:, :b], V[:, :b], causal=False)`` """ from __future__ import annotations -from typing import Literal, NamedTuple +from typing import TYPE_CHECKING, Literal, NamedTuple + +import torch + +if TYPE_CHECKING: + from vllm_omni.diffusion.attention.backends.abstract import QueryRange class Segment(NamedTuple): - start: int - end: int + q_start: int + q_end: int + kv_end: int mode: Literal["causal", "full"] @@ -30,30 +37,33 @@ def build_segments(full_attn_spans, query_offset, query_len): query_len: length of the query return: - List[Segment] in global coordinates, clipped to [query_offset, query_offset + query_len) + List[Segment] in global coordinates, clipped to + [query_offset, query_offset + query_len). Full-attention segments retain + the original span end as kv_end so a local query shard can attend past + its own boundary. """ q_start = query_offset q_end = query_offset + query_len - segs: list[Segment] = [] + segments: list[Segment] = [] cur = q_start - for a, e in full_attn_spans: + for span_start, span_end in full_attn_spans: # clip span to query range - a_clipped = max(a, q_start) - e_clipped = min(e, q_end) - if a_clipped >= e_clipped: + overlap_start = max(span_start, q_start) + overlap_end = min(span_end, q_end) + if overlap_start >= overlap_end: continue - if cur < a_clipped: - segs.append(Segment(cur, a_clipped, "causal")) - segs.append(Segment(a_clipped, e_clipped, "full")) - cur = e_clipped + if cur < overlap_start: + segments.append(Segment(cur, overlap_start, overlap_start, "causal")) + segments.append(Segment(overlap_start, overlap_end, span_end, "full")) + cur = overlap_end if cur < q_end: - segs.append(Segment(cur, q_end, "causal")) + segments.append(Segment(cur, q_end, q_end, "causal")) - return segs + return segments def _check_homogeneous( @@ -76,23 +86,40 @@ def piecewise_attn( full_attn_spans: list[list[tuple[int, int]]], softmax_scale: float, attn_func, + query_ranges: tuple[QueryRange, ...] | None = None, ): - B, Sq, H, D = query.shape _check_homogeneous(full_attn_spans) - - query_offset = key.shape[1] - Sq spans = full_attn_spans[0] - out = query.new_zeros(B, Sq, H, D) + if query_ranges is None: + query_len = query.shape[1] + ranges = ((0, query_len, key.shape[1] - query_len),) + else: + ranges = tuple((r.local_start, r.local_end, r.global_start) for r in query_ranges) - for s, e, mode in build_segments(spans, query_offset, Sq): - q_s = s - query_offset - q_e = e - query_offset - out_seg = attn_func( - query[:, q_s:q_e], - key[:, :e], - value[:, :e], - causal=(mode == "causal"), - softmax_scale=softmax_scale, - ) - out[:, q_s:q_e] = out_seg - return out + outputs = [] + covered = 0 + for local_start, local_end, global_start in ranges: + query_len = local_end - local_start + if local_start != covered or query_len < 0: + raise ValueError("query_ranges must cover local query contiguously") + for segment in build_segments(spans, global_start, query_len): + q_start = local_start + segment.q_start - global_start + q_end = local_start + segment.q_end - global_start + outputs.append( + attn_func( + query[:, q_start:q_end], + key[:, : segment.kv_end], + value[:, : segment.kv_end], + causal=(segment.mode == "causal"), + softmax_scale=softmax_scale, + ) + ) + covered = local_end + + if covered != query.shape[1]: + raise ValueError("query_ranges must cover the full local query") + if not outputs: + return torch.empty_like(query) + if len(outputs) == 1: + return outputs[0] + return torch.cat(outputs, dim=1) diff --git a/vllm_omni/diffusion/attention/layer.py b/vllm_omni/diffusion/attention/layer.py index 0ea122273..a11aecbb8 100644 --- a/vllm_omni/diffusion/attention/layer.py +++ b/vllm_omni/diffusion/attention/layer.py @@ -140,6 +140,7 @@ class Attention(nn.Module): scatter_idx=scatter_idx, gather_idx=gather_idx, use_sync=use_sync, + causal=causal, ) # Fallback strategy when SP is not active (outside sharded regions) self._no_parallel_strategy = NoParallelAttention() @@ -282,6 +283,8 @@ class Attention(nn.Module): return out def _run_local_attention(self, query, key, value, attn_metadata): + self._assert_piecewise_compatible(attn_metadata) + if query.dtype == torch.float32: logger.warning_once( f"Only SDPA supports float32. Overriding user config {type(self.attention)} " @@ -292,6 +295,20 @@ class Attention(nn.Module): # Fallback to standard attention return self.attention.forward(query, key, value, attn_metadata) + def _assert_piecewise_compatible(self, attn_metadata: AttentionMetadata | None) -> None: + if attn_metadata is None or attn_metadata.full_attn_spans is None: + return + if attn_metadata.attn_mask is not None and attn_metadata.attn_mask.ndim == 4: + return + backend_name = self.attn_backend.get_name() + if not self.attn_backend.supports_piecewise_spans: + raise ValueError( + f"Attention backend '{backend_name}' does not support " + f"piecewise attention (full_attn_spans without a 4D attn_mask). " + f"Use a Flash backend (FLASH_ATTN / FLASH_ATTN_HUB / FLASH_ATTN_3_HUB), " + f"or provide a 4D attn_mask that encodes the mixed causal/full pattern." + ) + def _run_ring_attention(self, query, key, value, attn_metadata): # Delegate to RingParallelAttention strategy if available if self.ring_runner is not None: diff --git a/vllm_omni/diffusion/attention/parallel/allgather_kv.py b/vllm_omni/diffusion/attention/parallel/allgather_kv.py new file mode 100644 index 000000000..51a463bb1 --- /dev/null +++ b/vllm_omni/diffusion/attention/parallel/allgather_kv.py @@ -0,0 +1,182 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from __future__ import annotations + +from dataclasses import dataclass, replace +from typing import TYPE_CHECKING + +import torch + +from vllm_omni.diffusion.attention.backends.abstract import QueryRange +from vllm_omni.diffusion.attention.parallel.base import ParallelAttentionContext +from vllm_omni.diffusion.distributed.group_coordinator import ( + SequenceParallelGroupCoordinator, +) + +if TYPE_CHECKING: + from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata + + +@dataclass(frozen=True, slots=True) +class _AllGatherKVCtx(ParallelAttentionContext): + pass + + +class AllGatherKVParallelAttention: + """Compute local Q against AllGathered K/V.""" + + def __init__( + self, + sp_group: SequenceParallelGroupCoordinator, + ) -> None: + self._sp_group = sp_group + self._allgather_group = sp_group.allgather_group + self._sp_size = sp_group.allgather_world_size + self._sp_rank = sp_group.allgather_rank + + @property + def enabled(self) -> bool: + return True + + @property + def name(self) -> str: + return "allgather_kv" + + def pre_attention( + self, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + attn_metadata: AttentionMetadata | None, + ): + joint_q = joint_k = joint_v = None + joint_strategy = "front" + if attn_metadata is not None: + joint_q = attn_metadata.joint_query + joint_k = attn_metadata.joint_key + joint_v = attn_metadata.joint_value + joint_strategy = attn_metadata.joint_strategy or "front" + if joint_strategy not in {"front", "rear"}: + raise ValueError(f"Unsupported joint_strategy: {joint_strategy!r}") + + k_img_full = self._sp_group.all_gather(key, dim=1, group=self._allgather_group) + v_img_full = self._sp_group.all_gather(value, dim=1, group=self._allgather_group) + + if joint_k is not None: + if joint_k.shape[2] != key.shape[2]: + raise ValueError( + "AllGather-KV SP expects joint_k to be GQA-compressed with the " + f"same num_kv_heads as the image shard, got joint_k.heads=" + f"{joint_k.shape[2]} vs key.heads={key.shape[2]}." + ) + if joint_strategy == "front": + k_full = torch.cat([joint_k, k_img_full], dim=1) + v_full = torch.cat([joint_v, v_img_full], dim=1) + else: + k_full = torch.cat([k_img_full, joint_k], dim=1) + v_full = torch.cat([v_img_full, joint_v], dim=1) + else: + k_full, v_full = k_img_full, v_img_full + + joint_len = joint_q.shape[1] if joint_q is not None else 0 + logical_q_full_len = joint_len + k_img_full.shape[1] + query_global_offset = k_full.shape[1] - logical_q_full_len + if query_global_offset < 0: + raise ValueError( + "AllGather-KV SP global K length is shorter than the logical Q length: " + f"k_len={k_full.shape[1]}, logical_q_len={logical_q_full_len}." + ) + + if joint_q is not None: + if joint_strategy == "front": + q_local = torch.cat([joint_q, query], dim=1) + else: + q_local = torch.cat([query, joint_q], dim=1) + else: + q_local = query + + attn_metadata = self._slice_attn_metadata_for_local_query( + attn_metadata, + q_local_len=q_local.shape[1], + img_seq_local=query.shape[1], + img_seq_full=k_img_full.shape[1], + joint_len=joint_len, + joint_strategy=joint_strategy, + query_global_offset=query_global_offset, + ) + + return q_local, k_full, v_full, attn_metadata, _AllGatherKVCtx(name=self.name) + + def _slice_attn_metadata_for_local_query( + self, + attn_metadata: AttentionMetadata | None, + *, + q_local_len: int, + img_seq_local: int, + img_seq_full: int, + joint_len: int, + joint_strategy: str, + query_global_offset: int, + ) -> AttentionMetadata | None: + """Slice global query metadata for the local query shard.""" + if attn_metadata is None: + return None + + img_start = self._sp_rank * img_seq_local + img_end = img_start + img_seq_local + if img_end > img_seq_full: + raise ValueError( + "AllGather-KV SP local image query range exceeds gathered image length: " + f"rank={self._sp_rank}, img_start={img_start}, img_end={img_end}, img_seq_full={img_seq_full}." + ) + + if joint_strategy == "front": + ranges = ( + *((QueryRange(0, joint_len, query_global_offset),) if joint_len else ()), + QueryRange(joint_len, q_local_len, query_global_offset + joint_len + img_start), + ) + else: + ranges = ( + QueryRange(0, img_seq_local, query_global_offset + img_start), + *((QueryRange(img_seq_local, q_local_len, query_global_offset + img_seq_full),) if joint_len else ()), + ) + + if attn_metadata.attn_mask is None: + return replace(attn_metadata, query_ranges=ranges) + + mask = attn_metadata.attn_mask + if mask.ndim != 4: + return replace(attn_metadata, query_ranges=ranges) + + if mask.shape[-2] == q_local_len: + local_mask = mask + else: + q_full_len = joint_len + img_seq_full + if mask.shape[-2] != q_full_len: + raise ValueError( + "AllGather-KV SP received an attention mask with incompatible Q length: " + f"mask_q={mask.shape[-2]}, expected local_q={q_local_len} or full_q={q_full_len} " + f"(joint_len={joint_len}, img_seq_local={img_seq_local}, img_seq_full={img_seq_full})." + ) + + parts = [ + mask[..., local_start : local_start + (r.local_end - r.local_start), :] + for r in ranges + for local_start in (r.global_start - query_global_offset,) + ] + local_mask = torch.cat(parts, dim=-2) + + if local_mask.shape[-2] != q_local_len: + raise ValueError( + "AllGather-KV SP produced an attention mask with incompatible local Q length: " + f"mask_q={local_mask.shape[-2]}, q_local={q_local_len}." + ) + return replace(attn_metadata, attn_mask=local_mask.contiguous(), query_ranges=ranges) + + def post_attention( + self, + attn_output: torch.Tensor, + ctx: ParallelAttentionContext | None, + ) -> torch.Tensor: + return attn_output diff --git a/vllm_omni/diffusion/attention/parallel/factory.py b/vllm_omni/diffusion/attention/parallel/factory.py index ce95d409a..76f7a78da 100644 --- a/vllm_omni/diffusion/attention/parallel/factory.py +++ b/vllm_omni/diffusion/attention/parallel/factory.py @@ -5,10 +5,19 @@ from __future__ import annotations from vllm.logger import init_logger -from vllm_omni.diffusion.attention.parallel.base import NoParallelAttention, ParallelAttentionStrategy +from vllm_omni.diffusion.attention.parallel.allgather_kv import ( + AllGatherKVParallelAttention, +) +from vllm_omni.diffusion.attention.parallel.base import ( + NoParallelAttention, + ParallelAttentionStrategy, +) from vllm_omni.diffusion.attention.parallel.ring import RingParallelAttention from vllm_omni.diffusion.attention.parallel.ulysses import UlyssesParallelAttention -from vllm_omni.diffusion.distributed.parallel_state import get_sequence_parallel_world_size, get_sp_group +from vllm_omni.diffusion.distributed.parallel_state import ( + get_sequence_parallel_world_size, + get_sp_group, +) from vllm_omni.diffusion.forward_context import get_forward_context logger = init_logger(__name__) @@ -19,6 +28,7 @@ def build_parallel_attention_strategy( scatter_idx: int, gather_idx: int, use_sync: bool, + causal: bool = False, ) -> ParallelAttentionStrategy: """Select a parallel attention strategy based on current diffusion config. @@ -36,6 +46,7 @@ def build_parallel_attention_strategy( ulysses_degree = getattr(p, "ulysses_degree", 1) ring_degree = getattr(p, "ring_degree", 1) + allgather_degree = getattr(p, "allgather_degree", 1) try: sp_group = get_sp_group() @@ -44,13 +55,26 @@ def build_parallel_attention_strategy( return NoParallelAttention() except Exception as e: # Log warning if SP is configured but group is not available - if ulysses_degree > 1 or ring_degree > 1: + if ulysses_degree > 1 or ring_degree > 1 or allgather_degree > 1: logger.warning( - f"SP configured (ulysses={ulysses_degree}, ring={ring_degree}) but SP group not available: {e}. " + f"SP configured (ulysses={ulysses_degree}, ring={ring_degree}, " + f"allgather={allgather_degree}) but SP group not available: {e}. " f"Falling back to NoParallelAttention. This may cause incorrect results." ) return NoParallelAttention() + if allgather_degree > 1: + if causal: + raise ValueError("AllGather-KV SP only supports non-causal attention.") + if ulysses_degree > 1 or ring_degree > 1: + raise ValueError( + f"AllGather-KV SP is mutually exclusive with Ulysses/Ring in v1 " + f"(got ulysses_degree={ulysses_degree}, ring_degree={ring_degree}, " + f"allgather_degree={allgather_degree})." + ) + logger.debug(f"Using AllGatherKVParallelAttention (allgather_degree={allgather_degree})") + return AllGatherKVParallelAttention(sp_group=sp_group) + # Ulysses (or Hybrid Ulysses+Ring) if ulysses_degree > 1: logger.debug(f"Using UlyssesParallelAttention (ulysses_degree={ulysses_degree})") diff --git a/vllm_omni/diffusion/data.py b/vllm_omni/diffusion/data.py index e131b833b..2d8ed2410 100644 --- a/vllm_omni/diffusion/data.py +++ b/vllm_omni/diffusion/data.py @@ -156,7 +156,9 @@ class DiffusionParallelConfig: """Enable expert parallelism for MoE layers (TP is still used for non-MoE layers).""" sequence_parallel_size: int | None = None - """Number of sequence parallel groups. sequence_parallel_size = ring_degree * ulysses_degree""" + """Number of sequence parallel groups. + sequence_parallel_size = ulysses_degree * ring_degree, or allgather_degree + when AllGather-KV is enabled.""" ulysses_degree: int = 1 """Number of GPUs used for ulysses sequence parallelism.""" @@ -164,6 +166,9 @@ class DiffusionParallelConfig: ring_degree: int = 1 """Number of GPUs used for ring sequence parallelism.""" + allgather_degree: int = 1 + """Number of GPUs used for AllGather-KV sequence parallelism (causal=False only).""" + ulysses_mode: str = "strict" """Ulysses sequence-parallel mode. @@ -225,6 +230,7 @@ class DiffusionParallelConfig: assert self.sequence_parallel_size > 0, "Sequence parallel size must be > 0" assert self.ulysses_degree > 0, "Ulysses degree must be > 0" assert self.ring_degree > 0, "Ring degree must be > 0" + assert self.allgather_degree > 0, "AllGather degree must be > 0" assert self.cfg_parallel_size > 0, "CFG parallel size must be > 0" assert self.cfg_parallel_size in [1, 2, 3], ( f"CFG parallel size must be 1, 2, or 3, but got {self.cfg_parallel_size}" @@ -234,9 +240,17 @@ class DiffusionParallelConfig: "vae_parallel_mode must be one of {'tile', 'spatial_shard_height', 'spatial_shard_width'}, " f"but got {self.vae_parallel_mode!r}." ) - assert self.sequence_parallel_size == self.ulysses_degree * self.ring_degree, ( - "Sequence parallel size must be equal to the product of ulysses degree and ring degree," - f" but got {self.sequence_parallel_size} != {self.ulysses_degree} * {self.ring_degree}" + if self.allgather_degree > 1: + assert self.ulysses_degree == 1 and self.ring_degree == 1, ( + "AllGather-KV (allgather_degree>1) is mutually exclusive with Ulysses/Ring in v1. " + f"Got ulysses_degree={self.ulysses_degree}, ring_degree={self.ring_degree}, " + f"allgather_degree={self.allgather_degree}." + ) + expected_sp_size = ( + self.allgather_degree if self.allgather_degree > 1 else self.ulysses_degree * self.ring_degree + ) + assert self.sequence_parallel_size == expected_sp_size, ( + f"Sequence parallel size must be {expected_sp_size}, but got {self.sequence_parallel_size}" ) assert self.ulysses_mode in {"strict", "advanced_uaa"}, ( f"ulysses_mode must be one of {{'strict','advanced_uaa'}}, but got {self.ulysses_mode!r}." @@ -250,15 +264,16 @@ class DiffusionParallelConfig: def __post_init__(self) -> None: if self.sequence_parallel_size is None: - self.sequence_parallel_size = self.ulysses_degree * self.ring_degree + self.sequence_parallel_size = ( + self.allgather_degree if self.allgather_degree > 1 else self.ulysses_degree * self.ring_degree + ) # Calculate world_size from other parallelism dimensions other_parallel_world_size = ( self.pipeline_parallel_size * self.data_parallel_size * self.tensor_parallel_size - * self.ulysses_degree - * self.ring_degree + * self.sequence_parallel_size * self.cfg_parallel_size ) diff --git a/vllm_omni/diffusion/distributed/group_coordinator.py b/vllm_omni/diffusion/distributed/group_coordinator.py index 0a67fd941..14847007f 100644 --- a/vllm_omni/diffusion/distributed/group_coordinator.py +++ b/vllm_omni/diffusion/distributed/group_coordinator.py @@ -199,9 +199,15 @@ class GroupCoordinator: return input_ def all_gather( - self, input_: torch.Tensor, dim: int = 0, separate_tensors: bool = False + self, + input_: torch.Tensor, + dim: int = 0, + separate_tensors: bool = False, + group: ProcessGroup | None = None, ) -> torch.Tensor | list[torch.Tensor]: - world_size = self.world_size + if group is None: + group = self.device_group + world_size = torch.distributed.get_world_size(group) # Bypass the function if we are using only 1 GPU. if world_size == 1: return input_ @@ -214,7 +220,7 @@ class GroupCoordinator: input_size[0] *= world_size output_tensor = torch.empty(input_size, dtype=input_.dtype, device=input_.device) # All-gather. - torch.distributed.all_gather_into_tensor(output_tensor, input_.contiguous(), group=self.device_group) + torch.distributed.all_gather_into_tensor(output_tensor, input_.contiguous(), group=group) if dim != 0: input_size[0] //= world_size output_tensor = output_tensor.reshape( @@ -1002,6 +1008,7 @@ class SequenceParallelGroupCoordinator(GroupCoordinator): ulysses_group = kwargs.get("ulysses_group", None) ring_group = kwargs.get("ring_group", None) + allgather_group = kwargs.get("allgather_group", None) if ulysses_group is None: raise RuntimeError( "Please pass argument 'ulysses_group' when calling init func of SequenceParallelGroupCoordinator" @@ -1010,10 +1017,17 @@ class SequenceParallelGroupCoordinator(GroupCoordinator): raise RuntimeError( "Please pass argument 'ring_group' when calling init func of SequenceParallelGroupCoordinator" ) + if allgather_group is None: + raise RuntimeError( + "Please pass argument 'allgather_group' when calling init func of SequenceParallelGroupCoordinator" + ) self.ulysses_group = ulysses_group self.ring_group = ring_group + self.allgather_group = allgather_group self.ulysses_world_size = torch.distributed.get_world_size(self.ulysses_group) self.ulysses_rank = torch.distributed.get_rank(self.ulysses_group) self.ring_world_size = torch.distributed.get_world_size(self.ring_group) self.ring_rank = torch.distributed.get_rank(self.ring_group) + self.allgather_world_size = torch.distributed.get_world_size(self.allgather_group) + self.allgather_rank = torch.distributed.get_rank(self.allgather_group) diff --git a/vllm_omni/diffusion/distributed/parallel_state.py b/vllm_omni/diffusion/distributed/parallel_state.py index 7b38d3775..4b531fec9 100644 --- a/vllm_omni/diffusion/distributed/parallel_state.py +++ b/vllm_omni/diffusion/distributed/parallel_state.py @@ -303,6 +303,14 @@ def get_ring_parallel_rank(): return get_sp_group().ring_rank +def get_allgather_parallel_world_size(): + return get_sp_group().allgather_world_size + + +def get_allgather_parallel_rank(): + return get_sp_group().allgather_rank + + def get_expert_parallel_group_ranks() -> list[list[int]]: assert vllm_parallel_state._EP is not None, "expert parallel group is not initialized" assert _EXPERT_PARALLEL_GROUP_RANKS is not None, "expert parallel group ranks are not initialized" @@ -548,13 +556,12 @@ def set_seq_parallel_pg( world_size: int, use_ulysses_low: bool = True, sp_group_ranks: list[list[int]] | None = None, -) -> tuple[torch.distributed.ProcessGroup, torch.distributed.ProcessGroup]: + sp_allgather_degree: int = 1, +) -> tuple[torch.distributed.ProcessGroup, torch.distributed.ProcessGroup, torch.distributed.ProcessGroup]: """ - Initialize sequence-parallel Ulysses and Ring process groups. + Initialize Ulysses, Ring, and AllGather-KV process groups. - This builds sequence-parallel (SP) subgroups inside each data-parallel (DP) - slice. The SP group size is sp_ulysses_degree * sp_ring_degree, and - world_size must be divisible by that size. + AllGather-KV is mutually exclusive with Ulysses and Ring. Args: sp_ulysses_degree: Size of each Ulysses subgroup. @@ -569,7 +576,7 @@ def set_seq_parallel_pg( ranges. Returns: - ulyssess_pg (torch.distributed.ProcessGroup): The Ulysses process group + ulysses_pg (torch.distributed.ProcessGroup): The Ulysses process group for this rank. ring_pg (torch.distributed.ProcessGroup): The Ring process group for this rank. @@ -586,6 +593,31 @@ def set_seq_parallel_pg( - If sp_group_ranks is None, groups are auto-generated within each DP slice using offsets of size sp_size. """ + if sp_allgather_degree > 1: + if sp_ulysses_degree > 1 or sp_ring_degree > 1: + raise ValueError("AllGather-KV is mutually exclusive with Ulysses and Ring") + sp_size = sp_allgather_degree + if sp_group_ranks is None: + sp_group_ranks = [list(range(offset, offset + sp_size)) for offset in range(0, world_size, sp_size)] + if len(sp_group_ranks) * sp_size != world_size or any(len(ranks) != sp_size for ranks in sp_group_ranks): + raise ValueError(f"Invalid sp_group_ranks: expected {world_size // sp_size} groups of size {sp_size}.") + + ulysses_pg = ring_pg = allgather_pg = None + for group_ranks in sp_group_ranks: + group = torch.distributed.new_group(group_ranks) + if rank in group_ranks: + allgather_pg = group + for singleton_rank in range(world_size): + group = torch.distributed.new_group([singleton_rank]) + if rank == singleton_rank: + ulysses_pg = group + for singleton_rank in range(world_size): + group = torch.distributed.new_group([singleton_rank]) + if rank == singleton_rank: + ring_pg = group + assert ulysses_pg is not None and ring_pg is not None and allgather_pg is not None + return ulysses_pg, ring_pg, allgather_pg + sp_size = sp_ring_degree * sp_ulysses_degree dp_size = world_size // sp_size @@ -619,7 +651,7 @@ def set_seq_parallel_pg( ulysses_ranks = group_ranks[i * sp_ulysses_degree : (i + 1) * sp_ulysses_degree] group = torch.distributed.new_group(ulysses_ranks) if rank in ulysses_ranks: - ulyssess_pg = group + ulysses_pg = group local_ulysses = list(ulysses_ranks) for i in range(num_ring_pgs): ring_ranks = group_ranks[i::num_ring_pgs] @@ -639,7 +671,7 @@ def set_seq_parallel_pg( ulysses_ranks = group_ranks[i::num_ulysses_pgs] group = torch.distributed.new_group(ulysses_ranks) if rank in ulysses_ranks: - ulyssess_pg = group + ulysses_pg = group local_ulysses = list(ulysses_ranks) if local_sp_group is not None: logger.info( @@ -662,7 +694,7 @@ def set_seq_parallel_pg( ) group = torch.distributed.new_group(ulysses_ranks) if rank in ulysses_ranks: - ulyssess_pg = group + ulysses_pg = group for i in range(num_ring_pgs): ring_ranks = list(range(i + offset, sp_size + offset, num_ring_pgs)) @@ -683,9 +715,15 @@ def set_seq_parallel_pg( ulysses_ranks = list(range(i + offset, sp_size + offset, num_ulysses_pgs)) group = torch.distributed.new_group(ulysses_ranks) if rank in ulysses_ranks: - ulyssess_pg = group + ulysses_pg = group - return ulyssess_pg, ring_pg + allgather_pg = None + for singleton_rank in range(world_size): + group = torch.distributed.new_group([singleton_rank]) + if rank == singleton_rank: + allgather_pg = group + assert allgather_pg is not None + return ulysses_pg, ring_pg, allgather_pg def initialize_model_parallel( @@ -694,6 +732,7 @@ def initialize_model_parallel( sequence_parallel_size: int | None = None, ulysses_degree: int = 1, ring_degree: int = 1, + allgather_degree: int = 1, tensor_parallel_size: int = 1, pipeline_parallel_size: int = 1, fully_shard_degree: int = 1, @@ -710,9 +749,12 @@ def initialize_model_parallel( data_parallel_size: number of data parallelism groups. cfg_parallel_size: number of GPUs used for Classifier Free Guidance (CFG) parallelism. sequence_parallel_size: number of GPUs used for sequence parallelism. - sequence_parallel_size = ulysses_degree * ring_degree + Uses allgather_degree when AllGather-KV is enabled, otherwise + ulysses_degree * ring_degree. ulysses_degree: number of GPUs used for ulysses sequence parallelism. ring_degree: number of GPUs used for ring sequence parallelism. + allgather_degree: number of GPUs used for AllGather-KV sequence parallelism + (causal=False only). Mutually exclusive with ulysses/ring in v1. tensor_parallel_size: number of GPUs used for tensor parallelism. pipeline_parallel_size: number of GPUs used for pipeline parallelism. fully_shard_degree: number of GPUs used for fully sharded data parallelism (HSDP shard dimension). @@ -749,16 +791,22 @@ def initialize_model_parallel( world_size: int = torch.distributed.get_world_size() backend = backend or torch.distributed.get_backend(get_world_group().device_group) - if sequence_parallel_size is None: - sequence_parallel_size = ring_degree * ulysses_degree - logger.info( - f"sequence_parallel_size is not provided, using ring_degree * ulysses_degree = {sequence_parallel_size}" + if allgather_degree > 1: + assert ulysses_degree == 1 and ring_degree == 1, ( + "AllGather-KV (allgather_degree>1) is mutually exclusive with Ulysses/Ring in v1. " + f"Got ulysses_degree={ulysses_degree}, ring_degree={ring_degree}, " + f"allgather_degree={allgather_degree}." ) - if sequence_parallel_size != ring_degree * ulysses_degree: + expected_sequence_parallel_size = allgather_degree if allgather_degree > 1 else ring_degree * ulysses_degree + if sequence_parallel_size is None: + sequence_parallel_size = expected_sequence_parallel_size + logger.info("sequence_parallel_size is not provided, using %d", sequence_parallel_size) + + if sequence_parallel_size != expected_sequence_parallel_size: raise ValueError( - "sequence_parallel_size is not equal to ring_degree * ulysses_degree," - f" but got {sequence_parallel_size} != {ring_degree} * {ulysses_degree}" + f"sequence_parallel_size must be {expected_sequence_parallel_size} for the configured SP mode, " + f"but got {sequence_parallel_size}" ) dit_parallel_size = ( @@ -835,9 +883,10 @@ def initialize_model_parallel( global _SP assert _SP is None, "sequence parallel group is already initialized" - ulysses_pg, ring_pg = set_seq_parallel_pg( + ulysses_pg, ring_pg, allgather_pg = set_seq_parallel_pg( sp_ulysses_degree=ulysses_degree, sp_ring_degree=ring_degree, + sp_allgather_degree=allgather_degree, rank=get_world_group().rank_in_group, world_size=dit_parallel_size, sp_group_ranks=sp_group_ranks, @@ -849,6 +898,7 @@ def initialize_model_parallel( parallel_mode="sequence", ulysses_group=ulysses_pg, ring_group=ring_pg, + allgather_group=allgather_pg, ) if use_moe_parallel_mapping: # Diffusion normally uses its own SP group. Map it to vLLM PCP only for diff --git a/vllm_omni/diffusion/distributed/sp_plan.py b/vllm_omni/diffusion/distributed/sp_plan.py index ee7b1d66b..b8d248865 100644 --- a/vllm_omni/diffusion/distributed/sp_plan.py +++ b/vllm_omni/diffusion/distributed/sp_plan.py @@ -77,6 +77,7 @@ class SequenceParallelConfig: ulysses_degree: int = 1 ring_degree: int = 1 + allgather_degree: int = 1 convert_to_fp32: bool = True # Internal state - populated by setup() @@ -85,18 +86,18 @@ class SequenceParallelConfig: _device: torch.device | None = None def __post_init__(self) -> None: - if self.ulysses_degree < 1 or self.ring_degree < 1: - raise ValueError("`ulysses_degree` and `ring_degree` must be >= 1.") + if self.ulysses_degree < 1 or self.ring_degree < 1 or self.allgather_degree < 1: + raise ValueError("SP degrees must be >= 1.") - if self.ulysses_degree == 1 and self.ring_degree == 1: - raise ValueError( - "At least one of `ulysses_degree` or `ring_degree` must be > 1 to use sequence parallelism." - ) + if self.allgather_degree > 1 and (self.ulysses_degree > 1 or self.ring_degree > 1): + raise ValueError("AllGather-KV is mutually exclusive with Ulysses and Ring.") + if self.ulysses_degree == self.ring_degree == self.allgather_degree == 1: + raise ValueError("At least one SP degree must be > 1.") @property def sequence_parallel_size(self) -> int: """Total sequence parallel world size.""" - return self.ulysses_degree * self.ring_degree + return self.allgather_degree if self.allgather_degree > 1 else self.ulysses_degree * self.ring_degree def get_world_size(self) -> int: """Get the sequence parallel world size from parallel state. @@ -178,13 +179,12 @@ class SequenceParallelConfig: self._world_size = world_size self._device = device - expected_sp_size = self.ulysses_degree * self.ring_degree + expected_sp_size = self.sequence_parallel_size actual_sp_size = self.get_world_size() if expected_sp_size != actual_sp_size: raise ValueError( - f"Configuration mismatch: ulysses_degree ({self.ulysses_degree}) * " - f"ring_degree ({self.ring_degree}) = {expected_sp_size}, but " + f"Configuration mismatch: expected SP size {expected_sp_size}, but " f"actual sequence parallel world size is {actual_sp_size}." ) diff --git a/vllm_omni/diffusion/models/hunyuan_image3/hunyuan_image3_transformer.py b/vllm_omni/diffusion/models/hunyuan_image3/hunyuan_image3_transformer.py index cae7913f2..5f72a607d 100644 --- a/vllm_omni/diffusion/models/hunyuan_image3/hunyuan_image3_transformer.py +++ b/vllm_omni/diffusion/models/hunyuan_image3/hunyuan_image3_transformer.py @@ -62,6 +62,7 @@ from vllm_omni.diffusion.attention.backends.abstract import ( from vllm_omni.diffusion.attention.layer import Attention from vllm_omni.diffusion.cache.cachedit import CacheDiTAdapterConfig from vllm_omni.diffusion.distributed.parallel_state import ( + get_allgather_parallel_world_size, get_cfg_group, get_classifier_free_guidance_rank, get_classifier_free_guidance_world_size, @@ -78,6 +79,7 @@ from vllm_omni.diffusion.forward_context import set_forward_context_denoise_step from vllm_omni.diffusion.layers.norm import RMSNorm from vllm_omni.diffusion.layers.rope import RotaryEmbedding from vllm_omni.diffusion.models.hunyuan_image3.hunyuan_fused_moe import HunyuanFusedMoE +from vllm_omni.diffusion.utils.kv_utils import repeat_kv from vllm_omni.model_executor.layers.timestep_embedding import timestep_embedding logger = logging.getLogger(__name__) @@ -426,22 +428,6 @@ def apply_rotary_pos_emb( return q_embed, k_embed -def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor: - """ - Equivalent to torch.repeat_interleave(x, dim=2, repeats=n_rep). - Input: (batch, seqlen, num_key_value_heads, head_dim) - Output: (batch, seqlen, num_attention_heads, head_dim) - """ - batch, slen, num_key_value_heads, head_dim = hidden_states.shape - - if n_rep == 1: - return hidden_states - - hidden_states = hidden_states[:, :, :, None, :].expand(batch, slen, num_key_value_heads, n_rep, head_dim) - - return hidden_states.reshape(batch, slen, num_key_value_heads * n_rep, head_dim) - - def default(value, default_value): return value if value is not None else default_value @@ -957,6 +943,7 @@ class ImageKVCacheManager: self._injected_ar_kv: list[tuple[torch.Tensor, torch.Tensor]] | None = None self.sp_size = get_sequence_parallel_world_size() + self.allgather_size = get_allgather_parallel_world_size() self.sp_rank = get_sequence_parallel_rank() self.attn = Attention( num_heads=self.num_heads, @@ -1130,6 +1117,7 @@ class ImageKVCacheManager: head_num_per_rank = query.shape[1] kv_head_num_per_rank = key.shape[1] repeat_num = head_num_per_rank // kv_head_num_per_rank + keep_kv_compressed = self.allgather_size > 1 head_dim = query.shape[2] query = query.reshape(bs, q_len, head_num_per_rank, head_dim) @@ -1171,11 +1159,12 @@ class ImageKVCacheManager: joint_text_query = query[:, :0, :, :] joint_text_key, joint_text_value = self._reuse_prompt_kv(key, value, seq_len, bs, shard_image_size) - key = repeat_kv(key, repeat_num) - value = repeat_kv(value, repeat_num) - if self.sp_size > 1: - joint_text_key = repeat_kv(joint_text_key, repeat_num) - joint_text_value = repeat_kv(joint_text_value, repeat_num) + if not keep_kv_compressed: + key = repeat_kv(key, repeat_num) + value = repeat_kv(value, repeat_num) + if self.sp_size > 1: + joint_text_key = repeat_kv(joint_text_key, repeat_num) + joint_text_value = repeat_kv(joint_text_value, repeat_num) attention_mask = attention_mask.contiguous() diff --git a/vllm_omni/diffusion/registry.py b/vllm_omni/diffusion/registry.py index 41412424d..a0b1e9a7b 100644 --- a/vllm_omni/diffusion/registry.py +++ b/vllm_omni/diffusion/registry.py @@ -464,18 +464,25 @@ def _apply_sequence_parallel_if_enabled(model, od_config: OmniDiffusionConfig) - if plan is None: continue - # Create SP config - sp_config = SequenceParallelConfig( - ulysses_degree=od_config.parallel_config.ulysses_degree, - ring_degree=od_config.parallel_config.ring_degree, - ) + # AllGather-KV reuses the Ulysses sequence-sharding hooks. + allgather_degree = getattr(od_config.parallel_config, "allgather_degree", 1) + if allgather_degree > 1: + sp_config = SequenceParallelConfig( + allgather_degree=allgather_degree, + ) + mode = "allgather_kv" + else: + sp_config = SequenceParallelConfig( + ulysses_degree=od_config.parallel_config.ulysses_degree, + ring_degree=od_config.parallel_config.ring_degree, + ) + # Apply hooks according to the plan + mode = ( + "hybrid" + if sp_config.ulysses_degree > 1 and sp_config.ring_degree > 1 + else ("ulysses" if sp_config.ulysses_degree > 1 else "ring") + ) - # Apply hooks according to the plan - mode = ( - "hybrid" - if sp_config.ulysses_degree > 1 and sp_config.ring_degree > 1 - else ("ulysses" if sp_config.ulysses_degree > 1 else "ring") - ) logger.info( f"Applying sequence parallelism to {transformer.__class__.__name__} ({attr}) " f"(sp_size={sp_size}, mode={mode}, ulysses={sp_config.ulysses_degree}, ring={sp_config.ring_degree})" diff --git a/vllm_omni/diffusion/utils/kv_utils.py b/vllm_omni/diffusion/utils/kv_utils.py index 9ba231386..fc6ec6cc1 100644 --- a/vllm_omni/diffusion/utils/kv_utils.py +++ b/vllm_omni/diffusion/utils/kv_utils.py @@ -3,6 +3,19 @@ import torch +def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor: + """Expand GQA KV from num_kv_heads to num_heads. + + Input: (batch, seqlen, num_key_value_heads, head_dim) + Output: (batch, seqlen, num_key_value_heads * n_rep, head_dim) + """ + if n_rep == 1: + return hidden_states + batch, slen, num_key_value_heads, head_dim = hidden_states.shape + hidden_states = hidden_states[:, :, :, None, :].expand(batch, slen, num_key_value_heads, n_rep, head_dim) + return hidden_states.reshape(batch, slen, num_key_value_heads * n_rep, head_dim) + + def left_pad_stack( tensors: list[torch.Tensor], ) -> tuple[torch.Tensor, torch.Tensor | None]: diff --git a/vllm_omni/diffusion/worker/diffusion_worker.py b/vllm_omni/diffusion/worker/diffusion_worker.py index df3ee5910..948406c73 100644 --- a/vllm_omni/diffusion/worker/diffusion_worker.py +++ b/vllm_omni/diffusion/worker/diffusion_worker.py @@ -315,6 +315,7 @@ class DiffusionWorker: sequence_parallel_size=parallel_config.sequence_parallel_size, ulysses_degree=parallel_config.ulysses_degree, ring_degree=parallel_config.ring_degree, + allgather_degree=parallel_config.allgather_degree, tensor_parallel_size=parallel_config.tensor_parallel_size, pipeline_parallel_size=parallel_config.pipeline_parallel_size, fully_shard_degree=parallel_config.hsdp_shard_size if parallel_config.use_hsdp else 1, diff --git a/vllm_omni/engine/arg_utils.py b/vllm_omni/engine/arg_utils.py index 19db55c63..2d049d73f 100644 --- a/vllm_omni/engine/arg_utils.py +++ b/vllm_omni/engine/arg_utils.py @@ -459,6 +459,7 @@ class OrchestratorArgs: ulysses_degree: int | None = None ulysses_mode: str = "strict" ring_degree: int | None = None + allgather_degree: int | None = None diffusion_quantization_config: str | None = None use_hsdp: bool = False hsdp_shard_size: int = -1 diff --git a/vllm_omni/engine/async_omni_engine.py b/vllm_omni/engine/async_omni_engine.py index dbebd7e6d..7740f11e7 100644 --- a/vllm_omni/engine/async_omni_engine.py +++ b/vllm_omni/engine/async_omni_engine.py @@ -933,6 +933,7 @@ class AsyncOmniEngine: if parallel_config is None: ulysses_degree = normalized_kwargs.get("ulysses_degree") or 1 ring_degree = normalized_kwargs.get("ring_degree") or 1 + allgather_degree = normalized_kwargs.get("allgather_degree") or 1 ulysses_mode = normalized_kwargs.get("ulysses_mode") or "strict" sequence_parallel_size = normalized_kwargs.get("sequence_parallel_size") pipeline_parallel_size = normalized_kwargs.get("pipeline_parallel_size") or 1 @@ -947,7 +948,7 @@ class AsyncOmniEngine: hsdp_shard_size = normalized_kwargs.get("hsdp_shard_size", -1) hsdp_replicate_size = normalized_kwargs.get("hsdp_replicate_size", 1) if sequence_parallel_size is None: - sequence_parallel_size = ulysses_degree * ring_degree + sequence_parallel_size = allgather_degree if allgather_degree > 1 else ulysses_degree * ring_degree parallel_config = DiffusionParallelConfig( pipeline_parallel_size=pipeline_parallel_size, @@ -957,6 +958,7 @@ class AsyncOmniEngine: sequence_parallel_size=sequence_parallel_size, ulysses_degree=ulysses_degree, ring_degree=ring_degree, + allgather_degree=allgather_degree, ulysses_mode=ulysses_mode, cfg_parallel_size=cfg_parallel_size, vae_patch_parallel_size=vae_patch_parallel_size, diff --git a/vllm_omni/entrypoints/cli/serve.py b/vllm_omni/entrypoints/cli/serve.py index 71483445a..d807c22ed 100644 --- a/vllm_omni/entrypoints/cli/serve.py +++ b/vllm_omni/entrypoints/cli/serve.py @@ -473,6 +473,14 @@ class OmniServeCommand(CLISubcommand): help="Ring Sequence Parallelism degree for diffusion models. " "Equivalent to setting DiffusionParallelConfig.ring_degree.", ) + omni_config_group.add_argument( + "--allgather-degree", + dest="allgather_degree", + type=int, + default=None, + help="AllGather-KV Sequence Parallelism degree for non-causal diffusion attention. " + "Equivalent to setting DiffusionParallelConfig.allgather_degree.", + ) omni_config_group.add_argument( "--diffusion-quantization-config", type=json.loads, From 93786c9cd3edcb0d812051adea1968346001b9b2 Mon Sep 17 00:00:00 2001 From: TaffyOfficial <2587297563@qq.com> Date: Fri, 24 Jul 2026 23:22:06 +0800 Subject: [PATCH 088/240] [perf] Add configurable diffusion compile granularity (#4603) Signed-off-by: TaffyOfficial <2324465096@qq.com> Co-authored-by: TaffyOfficial <2324465096@qq.com> --- docs/.nav.yml | 1 + .../diffusion/regional_compilation.md | 38 +++++++++ tests/config/test_omni_config.py | 41 +++++++++ tests/diffusion/test_compile.py | 20 +++++ tests/diffusion/test_diffusion_engine.py | 51 +++++++++++ .../diffusion/test_diffusion_model_runner.py | 84 ++++++++++++++++++- .../test_async_omni_diffusion_config.py | 30 +++++++ vllm_omni/config/omni_config.py | 4 +- vllm_omni/config/stage_config.py | 2 + vllm_omni/diffusion/compile.py | 22 +++-- vllm_omni/diffusion/data.py | 28 +++++++ .../worker/diffusion_model_runner.py | 24 +++++- vllm_omni/engine/arg_utils.py | 2 + vllm_omni/engine/async_omni_engine.py | 8 ++ vllm_omni/entrypoints/cli/serve.py | 19 +++++ 15 files changed, 359 insertions(+), 15 deletions(-) create mode 100644 docs/user_guide/diffusion/regional_compilation.md diff --git a/docs/.nav.yml b/docs/.nav.yml index 6bf161b1b..e7157b573 100644 --- a/docs/.nav.yml +++ b/docs/.nav.yml @@ -56,6 +56,7 @@ nav: - Diffusion Features: - Overview: user_guide/diffusion_features.md - Feature Compatibility: user_guide/feature_compatibility.md + - Regional Compilation: user_guide/diffusion/regional_compilation.md - Cache Acceleration: - TeaCache: user_guide/diffusion/cache_acceleration/teacache.md - Cache-DiT: user_guide/diffusion/cache_acceleration/cache_dit.md diff --git a/docs/user_guide/diffusion/regional_compilation.md b/docs/user_guide/diffusion/regional_compilation.md new file mode 100644 index 000000000..469a62b2c --- /dev/null +++ b/docs/user_guide/diffusion/regional_compilation.md @@ -0,0 +1,38 @@ +# Regional Compilation + +Regional compilation applies `torch.compile` to the repeated transformer blocks +declared by a diffusion model. It is the default compilation scope when +diffusion inference runs without `--enforce-eager`. + +## Configuration + +Dynamic compilation is enabled by default so the compiled regions can handle +mixed resolutions. For a fixed-shape workload, disable it explicitly: + +```bash +vllm serve --omni --no-diffusion-compile-dynamic +``` + +The equivalent per-stage deploy configuration is: + +```yaml +stages: + - stage_id: 0 + diffusion_compile_granularity: regional + diffusion_compile_dynamic: false +``` + +For an experimental whole-transformer compile scope, set +`--diffusion-compile-granularity full` or use +`diffusion_compile_granularity: full` in the stage configuration. Full scope may +still contain graph breaks; it does not force one graph. It is rejected when +HSDP, sequence parallelism, CPU offload, or layerwise offload is enabled. Use +regional scope with those features. + +These settings control the generic model-runner compilation path. Pipelines +that provide their own `setup_compile()` implementation manage their compilation +policy independently. Compilation is lazy, so backend or graph errors can first +surface on the initial request. + +Use `--enforce-eager` to disable the model runner's generic compile setup. +Pipelines that compile internally define their own eager-mode behavior. diff --git a/tests/config/test_omni_config.py b/tests/config/test_omni_config.py index 523542232..f8fb3ed7f 100644 --- a/tests/config/test_omni_config.py +++ b/tests/config/test_omni_config.py @@ -628,6 +628,47 @@ def test_from_pipeline_config_preserves_diffusion_parallel_mask_sp_padding(tmp_p assert stage.parallel_config.mask_sp_padding is True +def test_from_pipeline_config_routes_regional_compile_dynamic(tmp_path): + deploy_path = tmp_path / "dreamzero_compile.yaml" + deploy_path.write_text( + "\n".join( + [ + "pipeline: dreamzero", + "async_chunk: false", + "stages:", + " - stage_id: 0", + " diffusion_compile_granularity: regional", + " diffusion_compile_dynamic: false", + ] + ) + ) + + configured_stage = _from_pipeline_key("dreamzero", deploy_config_path=str(deploy_path)).stage_by_id(0) + overridden_stage = _from_pipeline_key( + "dreamzero", + deploy_config_path=str(deploy_path), + cli_overrides={ + "diffusion_compile_granularity": "full", + "diffusion_compile_dynamic": True, + }, + ).stage_by_id(0) + + assert configured_stage.diffusion_config.diffusion_compile_granularity == "regional" + assert configured_stage.diffusion_config.diffusion_compile_dynamic is False + assert overridden_stage.diffusion_config.diffusion_compile_granularity == "full" + assert overridden_stage.diffusion_config.diffusion_compile_dynamic is True + + +def test_structured_diffusion_config_rejects_non_boolean_compile_dynamic(): + with pytest.raises(ValidationError, match="diffusion_compile_dynamic"): + omni_config_module._DiffusionConfigProjection(diffusion_compile_dynamic="false") + + +def test_structured_diffusion_config_rejects_invalid_compile_granularity(): + with pytest.raises(ValidationError, match="diffusion_compile_granularity"): + omni_config_module._DiffusionConfigProjection(diffusion_compile_granularity="block") + + def test_from_pipeline_config_matches_stage_config_to_omegaconf_behavior_for_representative_stage(): pipeline = _resolve_pipeline_or_skip("qwen3_tts") legacy_stage = merge_pipeline_deploy(pipeline, _load_default_deploy(pipeline))[0] diff --git a/tests/diffusion/test_compile.py b/tests/diffusion/test_compile.py index ca898da95..447d8c75f 100644 --- a/tests/diffusion/test_compile.py +++ b/tests/diffusion/test_compile.py @@ -54,3 +54,23 @@ def test_regionally_compile_matches_wrapped_blocks_by_declared_container_attr(mo assert all(not block.compile_called for block in model.transformer_blocks) assert not model.other_blocks[0].compile_called assert model.transformer_blocks[0].forward("ok") == "compiled:ok" + + +def test_regionally_compile_does_not_partially_mutate_on_setup_failure(monkeypatch): + model = _ModelWithWrappedRepeatedBlocks() + original_forwards = [block.forward.__func__ for block in model.transformer_blocks] + compile_calls = 0 + + def _compile(fn, *args, **kwargs): + nonlocal compile_calls + compile_calls += 1 + if compile_calls == 2: + raise RuntimeError("compile setup failed") + return lambda *fn_args, **fn_kwargs: fn(*fn_args, **fn_kwargs) + + monkeypatch.setattr(compile_module.torch, "compile", _compile) + + with pytest.raises(RuntimeError, match="compile setup failed"): + regionally_compile(model, dynamic=True) + + assert [block.forward.__func__ for block in model.transformer_blocks] == original_forwards diff --git a/tests/diffusion/test_diffusion_engine.py b/tests/diffusion/test_diffusion_engine.py index 01c901d60..e44766feb 100644 --- a/tests/diffusion/test_diffusion_engine.py +++ b/tests/diffusion/test_diffusion_engine.py @@ -392,6 +392,57 @@ class TestRequestBatchCapability: warmup.assert_called_once_with() +class TestDiffusionCompileConfig: + pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu] + + def test_config_defaults_to_dynamic_regional_compile(self) -> None: + config = OmniDiffusionConfig(model="test") + + assert config.diffusion_compile_granularity == "regional" + assert config.diffusion_compile_dynamic is True + + def test_from_kwargs_preserves_compile_controls(self) -> None: + config = OmniDiffusionConfig.from_kwargs( + model="test", + diffusion_compile_granularity="full", + diffusion_compile_dynamic=False, + ) + + assert config.diffusion_compile_granularity == "full" + assert config.diffusion_compile_dynamic is False + + def test_config_rejects_invalid_compile_granularity(self) -> None: + with pytest.raises(ValueError, match="diffusion_compile_granularity"): + OmniDiffusionConfig(model="test", diffusion_compile_granularity="block") + + def test_config_rejects_non_boolean_compile_dynamic(self) -> None: + with pytest.raises(TypeError, match="diffusion_compile_dynamic"): + OmniDiffusionConfig(model="test", diffusion_compile_dynamic="false") + + @pytest.mark.parametrize( + "kwargs, feature", + [ + ({"parallel_config": {"ulysses_degree": 2}, "num_gpus": 2}, "sequence parallelism"), + ( + { + "parallel_config": {"use_hsdp": True, "hsdp_shard_size": 2}, + "num_gpus": 2, + }, + "HSDP", + ), + ({"enable_cpu_offload": True}, "CPU offload"), + ({"enable_layerwise_offload": True}, "layerwise offload"), + ], + ) + def test_full_compile_rejects_incompatible_features(self, kwargs, feature) -> None: + with pytest.raises(ValueError, match=feature): + OmniDiffusionConfig( + model="test", + diffusion_compile_granularity="full", + **kwargs, + ) + + class TestRequestBatchAdmission: pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu] diff --git a/tests/diffusion/test_diffusion_model_runner.py b/tests/diffusion/test_diffusion_model_runner.py index cf9f0d3b4..a7f230d93 100644 --- a/tests/diffusion/test_diffusion_model_runner.py +++ b/tests/diffusion/test_diffusion_model_runner.py @@ -121,6 +121,15 @@ class _FinalOnlyStepPipeline: return DiffusionOutput(output=state.latents.clone()) +class _CompileTrackingModel: + def __init__(self): + self.compile_calls = [] + + def compile(self, *args, **kwargs): + self.compile_calls.append((args, kwargs)) + return self + + def _make_request(): sampling_params = SimpleNamespace( generator=None, @@ -177,10 +186,20 @@ def _make_runner(cache_backend, cache_backend_name: str, enable_cache_dit_summar return runner -def _make_compile_runner(*, use_hsdp: bool): +def _make_compile_runner( + model=None, + *, + compile_granularity: str = "regional", + compile_dynamic: bool = True, + use_hsdp: bool = False, +): runner = object.__new__(DiffusionModelRunner) - runner.pipeline = SimpleNamespace(transformer=SimpleNamespace()) - runner.od_config = SimpleNamespace(parallel_config=SimpleNamespace(use_hsdp=use_hsdp)) + runner.pipeline = SimpleNamespace(transformer=model or SimpleNamespace()) + runner.od_config = SimpleNamespace( + diffusion_compile_granularity=compile_granularity, + diffusion_compile_dynamic=compile_dynamic, + parallel_config=SimpleNamespace(use_hsdp=use_hsdp), + ) return runner @@ -208,6 +227,65 @@ def test_compile_transformer_regionally_compiles_blocks(monkeypatch, use_hsdp): ] +@pytest.mark.core_model +@pytest.mark.cpu +def test_compile_transformer_uses_regional_dynamic_false_config(monkeypatch): + model = _CompileTrackingModel() + runner = _make_compile_runner(model, compile_dynamic=False) + compiled_model = object() + regional_calls = [] + + def _regionally_compile(target, **kwargs): + regional_calls.append((target, kwargs)) + return compiled_model + + monkeypatch.setattr(model_runner_module, "regionally_compile", _regionally_compile) + + DiffusionModelRunner._compile_transformer(runner, "transformer") + + assert regional_calls == [(model, {"dynamic": False})] + assert model.compile_calls == [] + assert runner.pipeline.transformer is compiled_model + + +@pytest.mark.core_model +@pytest.mark.cpu +def test_compile_transformer_uses_full_granularity(monkeypatch): + model = _CompileTrackingModel() + runner = _make_compile_runner(model, compile_granularity="full", compile_dynamic=False) + regional_calls = [] + + monkeypatch.setattr( + model_runner_module, + "regionally_compile", + lambda *args, **kwargs: regional_calls.append((args, kwargs)), + ) + + DiffusionModelRunner._compile_transformer(runner, "transformer") + + assert model.compile_calls == [((), {"dynamic": False})] + assert regional_calls == [] + assert runner.pipeline.transformer is model + + +@pytest.mark.core_model +@pytest.mark.cpu +def test_compile_transformer_falls_back_after_synchronous_setup_failure(monkeypatch, caplog): + model = _CompileTrackingModel() + runner = _make_compile_runner(model) + + def _regionally_compile(*args, **kwargs): + raise RuntimeError("compile setup failed") + + monkeypatch.setattr(model_runner_module, "regionally_compile", _regionally_compile) + + DiffusionModelRunner._compile_transformer(runner, "transformer") + + assert runner.pipeline.transformer is model + assert "failed before activation" in caplog.text + assert "lazy compilation errors" in caplog.text + + @pytest.mark.core_model @pytest.mark.cpu def test_execute_stepwise_streaming_returns_chunks_at_boundaries(monkeypatch): diff --git a/tests/entrypoints/test_async_omni_diffusion_config.py b/tests/entrypoints/test_async_omni_diffusion_config.py index d6b33dc89..8987ae1d8 100644 --- a/tests/entrypoints/test_async_omni_diffusion_config.py +++ b/tests/entrypoints/test_async_omni_diffusion_config.py @@ -85,6 +85,8 @@ def test_default_stage_config_defaults_nullified_parallel_size_kwargs(): "tensor_parallel_size": None, "enable_expert_parallel": None, "enforce_eager": None, + "diffusion_compile_granularity": None, + "diffusion_compile_dynamic": None, } )[0] @@ -94,6 +96,8 @@ def test_default_stage_config_defaults_nullified_parallel_size_kwargs(): assert parallel_config.tensor_parallel_size == 1 assert parallel_config.enable_expert_parallel is False assert stage_cfg["engine_args"]["enforce_eager"] is False + assert stage_cfg["engine_args"]["diffusion_compile_granularity"] == "regional" + assert stage_cfg["engine_args"]["diffusion_compile_dynamic"] is True def test_default_stage_config_propagates_ulysses_mode(): @@ -258,6 +262,32 @@ def test_serve_cli_accepts_diffusion_pipeline_profiler_flag(): assert stage_cfg["engine_args"]["enable_diffusion_pipeline_profiler"] is True +def test_serve_cli_accepts_diffusion_compile_controls(): + """Ensure both compile controls reach the diffusion stage.""" + parser = TrackingArgumentParser() + subparsers = parser.add_subparsers(dest="command") + OmniServeCommand().subparser_init(subparsers) + + args = parser.parse_args( + [ + "serve", + "Lightricks/LTX-Video-0.9.8-13B-distilled", + "--omni", + "--diffusion-compile-granularity", + "full", + "--no-diffusion-compile-dynamic", + ] + ) + + explicit_kwargs = args.get_explicit_kwargs_dict() + stage_cfg = AsyncOmniEngine._create_default_diffusion_stage_cfg(explicit_kwargs)[0] + + assert args.diffusion_compile_granularity == "full" + assert args.diffusion_compile_dynamic is False + assert stage_cfg["engine_args"]["diffusion_compile_granularity"] == "full" + assert stage_cfg["engine_args"]["diffusion_compile_dynamic"] is False + + def test_serve_cli_accepts_diffusion_attention_backend(): """Ensure diffusion serve CLI exposes the shorthand backend flag.""" parser = TrackingArgumentParser() diff --git a/vllm_omni/config/omni_config.py b/vllm_omni/config/omni_config.py index cdcbafa79..14d27ca71 100644 --- a/vllm_omni/config/omni_config.py +++ b/vllm_omni/config/omni_config.py @@ -14,7 +14,7 @@ import copy from collections.abc import Mapping from dataclasses import dataclass, field, fields from pathlib import Path -from typing import Any, TypeAlias, TypedDict, cast +from typing import Any, Literal, TypeAlias, TypedDict, cast from pydantic import ConfigDict, Field from vllm.config.utils import config @@ -470,6 +470,8 @@ class _DiffusionConfigProjection: enable_cpu_offload: bool = False enable_layerwise_offload: bool = False pin_cpu_memory: bool = True + diffusion_compile_granularity: Literal["regional", "full"] = "regional" + diffusion_compile_dynamic: bool = Field(default=True, strict=True) vae_use_slicing: bool = False vae_use_tiling: bool = False mask_strategy_file_path: str | None = None diff --git a/vllm_omni/config/stage_config.py b/vllm_omni/config/stage_config.py index 2f9749420..e275751fe 100644 --- a/vllm_omni/config/stage_config.py +++ b/vllm_omni/config/stage_config.py @@ -388,6 +388,8 @@ class StageDeployConfig: diffusion_attention_config: dict[str, Any] | None = None # Diffusion execution, cache, and VAE behavior. + diffusion_compile_granularity: str | None = None + diffusion_compile_dynamic: bool | None = None cache_backend: str | None = None cache_config: dict[str, Any] | None = None enable_cache_dit_summary: bool | None = None diff --git a/vllm_omni/diffusion/compile.py b/vllm_omni/diffusion/compile.py index 45216febc..227d4f330 100644 --- a/vllm_omni/diffusion/compile.py +++ b/vllm_omni/diffusion/compile.py @@ -53,23 +53,29 @@ def regionally_compile( repeated_block_attrs = getattr(model, "_layerwise_offload_blocks_attrs", []) - # Check if we have modules with the specified class names - has_compiled_region = False - compiled_region_count = 0 + # Build all compiled callables before mutating the model. This keeps setup + # failures atomic: callers can safely continue with the uncompiled model if + # torch.compile raises synchronously for any repeated block. + compiled_forwards: list[tuple[nn.Module, Any]] = [] for name, submod in model.named_modules(): if _matches_repeated_block(name, submod, repeated_blocks, repeated_block_attrs): # Compile the block compute while keeping nn.Module.__call__ hooks # outside the compiled graph. - submod.forward = torch.compile(submod.forward, *compile_args, **compile_kwargs) - has_compiled_region = True - compiled_region_count += 1 + compiled_forwards.append( + ( + submod, + torch.compile(submod.forward, *compile_args, **compile_kwargs), + ) + ) - if not has_compiled_region: + if not compiled_forwards: logger.warning(f"Regional compilation skipped because {repeated_blocks} classes are not found in the model.") else: + for submod, compiled_forward in compiled_forwards: + submod.forward = compiled_forward logger.info( "Regional compilation applied to %d module(s) for repeated blocks %s.", - compiled_region_count, + len(compiled_forwards), repeated_blocks, ) diff --git a/vllm_omni/diffusion/data.py b/vllm_omni/diffusion/data.py index 2d8ed2410..078354d96 100644 --- a/vllm_omni/diffusion/data.py +++ b/vllm_omni/diffusion/data.py @@ -684,6 +684,10 @@ class OmniDiffusionConfig: # Compilation enforce_eager: bool = False + # Controls the generic compilation path used when a pipeline does not + # provide its own setup_compile() implementation. + diffusion_compile_granularity: str = "regional" + diffusion_compile_dynamic: bool = True # Parallel weight loading (for faster diffusion model startup) enable_multithread_weight_load: bool = True @@ -880,6 +884,13 @@ class OmniDiffusionConfig: ) def __post_init__(self): + if self.diffusion_compile_granularity not in {"regional", "full"}: + raise ValueError( + "diffusion_compile_granularity must be 'regional' or 'full', " + f"got {self.diffusion_compile_granularity!r}" + ) + if not isinstance(self.diffusion_compile_dynamic, bool): + raise TypeError(f"diffusion_compile_dynamic must be a bool, got {type(self.diffusion_compile_dynamic)!r}") self.master_port = self._resolve_master_port() self.request_batch_max_wait_ms = float(self.request_batch_max_wait_ms or 0.0) if self.request_batch_max_wait_ms < 0: @@ -915,6 +926,23 @@ class OmniDiffusionConfig: f"num_gpus ({self.num_gpus}) < parallel_config.world_size ({self.parallel_config.world_size})" ) + if self.diffusion_compile_granularity == "full": + incompatible_features = [] + if self.parallel_config.use_hsdp: + incompatible_features.append("HSDP") + if self.parallel_config.sequence_parallel_size > 1: + incompatible_features.append("sequence parallelism") + if self.enable_cpu_offload: + incompatible_features.append("CPU offload") + if self.enable_layerwise_offload: + incompatible_features.append("layerwise offload") + if incompatible_features: + features = ", ".join(incompatible_features) + raise ValueError( + "diffusion_compile_granularity='full' is incompatible with " + f"{features}; use 'regional' compilation instead" + ) + # Convert string dtype to torch.dtype if needed if isinstance(self.dtype, str): dtype_map = { diff --git a/vllm_omni/diffusion/worker/diffusion_model_runner.py b/vllm_omni/diffusion/worker/diffusion_model_runner.py index ced51ba3d..ee1b61c21 100644 --- a/vllm_omni/diffusion/worker/diffusion_model_runner.py +++ b/vllm_omni/diffusion/worker/diffusion_model_runner.py @@ -151,15 +151,33 @@ class DiffusionModelRunner(OmniConnectorModelRunnerMixin): if model is None: return + compile_granularity = self.od_config.diffusion_compile_granularity + compile_dynamic = self.od_config.diffusion_compile_dynamic try: - setattr(self.pipeline, attr_name, regionally_compile(model, dynamic=True)) - logger.info("Model runner: %s compiled with torch.compile.", attr_name) + if compile_granularity == "full": + model.compile(dynamic=compile_dynamic) + compiled_model = model + else: + compiled_model = regionally_compile(model, dynamic=compile_dynamic) + setattr(self.pipeline, attr_name, compiled_model) except Exception as e: logger.warning( - "Model runner: torch.compile for %s failed: %s. Using eager mode.", + "Model runner: %s torch.compile setup for %s failed before activation: %s. " + "Continuing with the uncompiled model; lazy compilation errors can still " + "surface on the first request.", + compile_granularity, attr_name, e, ) + return + + logger.info( + "Model runner: %s configured for lazy %s torch.compile with dynamic=%s; " + "compilation errors may surface on the first request.", + attr_name, + compile_granularity, + compile_dynamic, + ) def load_model( self, diff --git a/vllm_omni/engine/arg_utils.py b/vllm_omni/engine/arg_utils.py index 2d049d73f..4d7622b91 100644 --- a/vllm_omni/engine/arg_utils.py +++ b/vllm_omni/engine/arg_utils.py @@ -466,6 +466,8 @@ class OrchestratorArgs: hsdp_replicate_size: int = 1 diffusion_attention_backend: str | None = None diffusion_attention_config: str | None = None + diffusion_compile_granularity: str | None = None + diffusion_compile_dynamic: bool | None = None cache_backend: str = "none" cache_config: str | None = None enable_cache_dit_summary: bool = False diff --git a/vllm_omni/engine/async_omni_engine.py b/vllm_omni/engine/async_omni_engine.py index 7740f11e7..9deabf6d3 100644 --- a/vllm_omni/engine/async_omni_engine.py +++ b/vllm_omni/engine/async_omni_engine.py @@ -999,6 +999,14 @@ class AsyncOmniEngine: "enable_cpu_offload": kwargs.get("enable_cpu_offload", False), "enable_layerwise_offload": kwargs.get("enable_layerwise_offload", False), "enforce_eager": False if kwargs.get("enforce_eager") is None else kwargs.get("enforce_eager"), + "diffusion_compile_granularity": ( + "regional" + if kwargs.get("diffusion_compile_granularity") is None + else kwargs["diffusion_compile_granularity"] + ), + "diffusion_compile_dynamic": ( + True if kwargs.get("diffusion_compile_dynamic") is None else kwargs["diffusion_compile_dynamic"] + ), "boundary_ratio": kwargs.get("boundary_ratio", None), "flow_shift": kwargs.get("flow_shift", None), "diffusion_load_format": kwargs.get("diffusion_load_format", "default"), diff --git a/vllm_omni/entrypoints/cli/serve.py b/vllm_omni/entrypoints/cli/serve.py index d807c22ed..98c086d86 100644 --- a/vllm_omni/entrypoints/cli/serve.py +++ b/vllm_omni/entrypoints/cli/serve.py @@ -423,6 +423,25 @@ class OmniServeCommand(CLISubcommand): "custom_pipeline, dummy, or diffusers for the HF diffusers adapter." ), ) + omni_config_group.add_argument( + "--diffusion-compile-granularity", + choices=["regional", "full"], + default=None, + help=( + "Compilation scope for the generic diffusion model runner. " + "'regional' compiles repeated blocks (default); 'full' compiles the whole transformer and is " + "incompatible with HSDP, sequence parallelism, CPU offload, and layerwise offload." + ), + ) + omni_config_group.add_argument( + "--diffusion-compile-dynamic", + action=argparse.BooleanOptionalAction, + default=None, + help=( + "Use dynamic shapes for the selected generic diffusion compile scope. " + "Disable for fixed-shape workloads with --no-diffusion-compile-dynamic." + ), + ) omni_config_group.add_argument( "--diffusers-load-kwargs", dest="diffusers_load_kwargs", From ff0a76e31cbdb1f3bc5b0cb11f75ee63d5dbd3ca Mon Sep 17 00:00:00 2001 From: wangyu <53896905+yenuo26@users.noreply.github.com> Date: Sat, 25 Jul 2026 00:34:30 +0800 Subject: [PATCH 089/240] [CI/Bugfix] Move CI editable APP_DIR to /opt/vllm-omni for k8s package discovery (#5364) (#5368) Signed-off-by: wangyu <410167048@qq.com> --- docker/Dockerfile.ci | 7 +- .../test_vllm_omni_package_discovery.py | 88 +++++++++++++++++++ 2 files changed, 93 insertions(+), 2 deletions(-) create mode 100644 tests/buildkite/test_vllm_omni_package_discovery.py diff --git a/docker/Dockerfile.ci b/docker/Dockerfile.ci index 4171b063e..84ce5aa41 100644 --- a/docker/Dockerfile.ci +++ b/docker/Dockerfile.ci @@ -10,7 +10,10 @@ ARG VLLM_BASE_TAG=v0.25.0 # everything except the actual project dir, since that should not # change much across branches. FROM ${VLLM_BASE_IMAGE}:${VLLM_BASE_TAG} AS deps -ARG APP_DIR=/workspace/vllm-omni +# Keep APP_DIR outside /workspace so k8s Buildkite workspace mounts cannot +# shadow the editable install target (#5364). Docker WORKDIR creates /opt if +# needed. +ARG APP_DIR=/opt/vllm-omni WORKDIR ${APP_DIR} RUN ln -sf /usr/bin/python3 /usr/bin/python @@ -47,7 +50,7 @@ RUN uv pip install --system -e ".[dev]" # the heavy base image to build it. Editable install above already # registered APP_DIR, so we must not re-pip-install after COPY. FROM deps -ARG APP_DIR=/workspace/vllm-omni +ARG APP_DIR=/opt/vllm-omni WORKDIR ${APP_DIR} COPY --link . . diff --git a/tests/buildkite/test_vllm_omni_package_discovery.py b/tests/buildkite/test_vllm_omni_package_discovery.py new file mode 100644 index 000000000..118aedb15 --- /dev/null +++ b/tests/buildkite/test_vllm_omni_package_discovery.py @@ -0,0 +1,88 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Guard package discovery outside repo-root ``sys.path[0]``. + +Covers the same failure class as: +- ``vllm ... --omni`` (upstream ``find_spec("vllm_omni")``) — see #5364 +- ``python examples/.../end2end.py`` / multimodal client subprocesses, where + ``sys.path[0]`` is the script directory (not the repo root), so a cwd-only + layout can start OmniServer via ``python -m vllm_omni...`` and still fail + example scripts with ``ModuleNotFoundError: No module named 'vllm_omni'``. +""" + +from __future__ import annotations + +import subprocess +import sys +from importlib.util import find_spec +from pathlib import Path + +import pytest + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +def _assert_subprocess_ok(proc: subprocess.CompletedProcess[str], *, detail: str) -> None: + assert proc.returncode == 0, f"{detail} (exit={proc.returncode})\nstdout:\n{proc.stdout}\nstderr:\n{proc.stderr}" + + +def test_find_spec_vllm_omni() -> None: + """Same probe as upstream ``vllm`` CLI before delegating to omni_main.""" + assert find_spec("vllm_omni") is not None + assert find_spec("vllm_omni.entrypoints.cli.main") is not None + + +def test_find_spec_vllm_omni_from_non_repo_cwd(tmp_path: Path) -> None: + """Console-script / ``python -c`` style: discovery when cwd is not the repo root.""" + code = ( + "from importlib.util import find_spec; " + "assert find_spec('vllm_omni') is not None; " + "assert find_spec('vllm_omni.entrypoints.cli.main') is not None" + ) + proc = subprocess.run( + [sys.executable, "-c", code], + cwd=tmp_path, + capture_output=True, + text=True, + check=False, + ) + _assert_subprocess_ok(proc, detail=f"find_spec failed from cwd={tmp_path}") + + +def test_python_script_off_repo_root_sys_path(tmp_path: Path) -> None: + """``python path/to/script.py``: ``sys.path[0]`` is the script dir, not the repo root. + + One case covers the end2end / online-client / orphan-dir layouts — they all + share this discovery constraint. Use ``find_spec`` only (no heavy imports) + so Simple Test stays fast. + """ + script = tmp_path / "examples" / "offline_inference" / "hunyuan_image3" / "end2end_probe.py" + script.parent.mkdir(parents=True, exist_ok=True) + script.write_text( + "\n".join( + [ + "from importlib.util import find_spec", + "import sys", + 'print("initial_sys_path0", sys.path[0])', + 'assert find_spec("vllm_omni") is not None, (sys.path[0], sys.path[:5])', + 'assert find_spec("vllm_omni.entrypoints.cli.main") is not None', + 'print("ok")', + "", + ] + ), + encoding="utf-8", + ) + cwd = tmp_path / "run_cwd" + cwd.mkdir() + proc = subprocess.run( + [sys.executable, str(script)], + cwd=cwd, + capture_output=True, + text=True, + check=False, + ) + _assert_subprocess_ok( + proc, + detail=f"find_spec failed for python {script} with cwd={cwd}", + ) + assert f"initial_sys_path0 {script.parent}" in proc.stdout From d6240742f365cd813991b540b8ea797347aac72e Mon Sep 17 00:00:00 2001 From: akshatvishu Date: Fri, 24 Jul 2026 23:48:32 +0530 Subject: [PATCH 090/240] [BugFix][CI] Upgrade ROCm and CUDA docker base images to v0.25.0 (#5115) Signed-off-by: akshatvishu --- docker/Dockerfile.ci | 5 +- docker/Dockerfile.rocm | 27 ++++++- tools/install_torchcodec_rocm.sh | 125 +++++++++++++++++++++++++++++++ 3 files changed, 152 insertions(+), 5 deletions(-) create mode 100755 tools/install_torchcodec_rocm.sh diff --git a/docker/Dockerfile.ci b/docker/Dockerfile.ci index 84ce5aa41..caae57abf 100644 --- a/docker/Dockerfile.ci +++ b/docker/Dockerfile.ci @@ -1,6 +1,5 @@ -# Target vllm version: v0.24.0 (releases/v0.24.0 branch). -# The pinned VLLM_PRECOMPILED_WHEEL_COMMIT and VLLM_BASE_TAG below must -# be updated on every rebase to match the new upstream vllm commit/tag. +# Target vllm version: v0.25.0 (releases/v0.25.0 branch). +# Keep VLLM_BASE_TAG aligned with the target vLLM release on every rebase. ARG VLLM_BASE_IMAGE=vllm/vllm-openai ARG VLLM_BASE_TAG=v0.25.0 diff --git a/docker/Dockerfile.rocm b/docker/Dockerfile.rocm index 49c6f02aa..9be3e5fb5 100644 --- a/docker/Dockerfile.rocm +++ b/docker/Dockerfile.rocm @@ -17,7 +17,7 @@ ENV PYTORCH_ROCM_ARCH=${ARG_PYTORCH_ROCM_ARCH:-${PYTORCH_ROCM_ARCH}} ARG COMMON_WORKDIR=/app WORKDIR ${COMMON_WORKDIR} -# Step 1: Setup - Install system dependencies +# Step 1: Setup - Install system dependencies. # Need to include ffmpeg because vllm rocm upstream docker image # does not include it. RUN apt-get update && \ @@ -38,6 +38,26 @@ RUN if [ "${USE_NIGHTLY_BUILD}" = "1" ]; then \ rm -rf vllm/.git; \ fi +# Build TorchCodec after any optional nightly vLLM reinstall so it targets the +# final torch/ROCm ABI. Keep this layer before COPY . so source changes do not +# invalidate the expensive native build. FFmpeg development headers are +# removed after the wheel is built. +COPY tools/install_torchcodec_rocm.sh /tmp/install_torchcodec_rocm.sh +RUN --mount=type=cache,target=/root/.cache/pip \ + --mount=type=cache,target=/root/.cache/torchcodec-wheels \ + apt-get update && \ + apt-get install -y --no-install-recommends \ + pkg-config \ + libavcodec-dev libavformat-dev libavutil-dev \ + libswscale-dev libavdevice-dev libavfilter-dev libswresample-dev && \ + bash /tmp/install_torchcodec_rocm.sh && \ + apt-get purge -y \ + libavcodec-dev libavformat-dev libavutil-dev \ + libswscale-dev libavdevice-dev libavfilter-dev libswresample-dev && \ + rm -f /tmp/install_torchcodec_rocm.sh && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + # Step 3: Copy vllm-omni code and install without uv RUN mkdir -p ${COMMON_WORKDIR}/vllm-omni COPY . ${COMMON_WORKDIR}/vllm-omni @@ -59,7 +79,10 @@ RUN printf '%s\n' \ ' os._exit(_exit_code)' \ > ${COMMON_WORKDIR}/vllm-omni/conftest.py -RUN cd ${COMMON_WORKDIR}/vllm-omni && uv pip install --python "$(python3 -c 'import sys; print(sys.executable)')" --no-cache-dir ".[dev]" --no-build-isolation +RUN cd ${COMMON_WORKDIR}/vllm-omni && \ + uv pip install --python "$(python3 -c 'import sys; print(sys.executable)')" \ + --no-cache-dir ".[dev]" --no-build-isolation && \ + python3 -c "from torchcodec.decoders import VideoDecoder" RUN ln -sf /usr/bin/python3 /usr/bin/python diff --git a/tools/install_torchcodec_rocm.sh b/tools/install_torchcodec_rocm.sh new file mode 100755 index 000000000..787079603 --- /dev/null +++ b/tools/install_torchcodec_rocm.sh @@ -0,0 +1,125 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +# Build TorchCodec against the ROCm PyTorch installation. The PyPI wheel is +# built against upstream PyTorch and is not ABI-compatible with ROCm builds. + +set -euo pipefail + +TORCHCODEC_REPO="${TORCHCODEC_REPO:-https://github.com/pytorch/torchcodec.git}" +# Keep this pinned for ROCm/PyTorch ABI compatibility. When rebasing the ROCm +# base image or vLLM, revalidate this version against the active torch build. +TORCHCODEC_BRANCH="${TORCHCODEC_BRANCH:-v0.10.0}" +TORCHCODEC_WHEEL_CACHE="${TORCHCODEC_WHEEL_CACHE:-/root/.cache/torchcodec-wheels}" + +echo "=== TorchCodec ROCm installation ===" + +verify_torchcodec() { + python3 -c "from torchcodec.decoders import VideoDecoder" +} + +if verify_torchcodec >/dev/null 2>&1; then + echo "TorchCodec is already installed and working." + exit 0 +fi + +install_system_deps() { + if ! command -v apt-get >/dev/null 2>&1; then + echo "TorchCodec build dependencies are missing and apt-get is unavailable." + return 1 + fi + apt-get update + apt-get install -y --no-install-recommends \ + pkg-config \ + ffmpeg libavcodec-dev libavformat-dev libavutil-dev \ + libswscale-dev libavdevice-dev libavfilter-dev libswresample-dev +} + +if ! command -v pkg-config >/dev/null 2>&1 || \ + ! pkg-config --exists libavcodec libavformat libavutil libswscale \ + libavdevice libavfilter libswresample 2>/dev/null; then + echo "Installing missing FFmpeg development dependencies..." + install_system_deps +fi + +if ! python3 -c "import packaging, pybind11" >/dev/null 2>&1; then + python3 -m pip install packaging pybind11 setuptools wheel +fi + +PYBIND11_DIR=$(python3 -c "import pybind11; print(pybind11.get_cmake_dir())") +export pybind11_DIR="$PYBIND11_DIR" +export CMAKE_PREFIX_PATH="${PYBIND11_DIR}:${CMAKE_PREFIX_PATH:-}" + +safe_component() { + printf '%s' "$1" | sed 's/[^A-Za-z0-9_.-]/_/g' +} + +PYTHON_SOABI=$(python3 -c \ + "import sysconfig; print(sysconfig.get_config_var('SOABI') or 'unknown')") +TORCH_VERSION=$(python3 -c "import torch; print(torch.__version__)") +ROCM_VERSION=$(python3 -c "import torch; print(torch.version.hip or 'none')") +FFMPEG_VERSION=$(pkg-config --modversion libavcodec 2>/dev/null || echo unknown) +ARCH_TAG="${PYTORCH_ROCM_ARCH:-all}" + +CACHE_DIR="${TORCHCODEC_WHEEL_CACHE}/$(safe_component "$TORCHCODEC_BRANCH")/$(safe_component "$PYTHON_SOABI")/$(safe_component "$TORCH_VERSION")/$(safe_component "$ROCM_VERSION")/$(safe_component "$FFMPEG_VERSION")/$(safe_component "$ARCH_TAG")" +CACHED_WHEEL="" +if [ -d "$CACHE_DIR" ]; then + CACHED_WHEEL=$(find "$CACHE_DIR" -maxdepth 1 -type f -name 'torchcodec-*.whl' -print -quit) +fi + +validate_wheel_filename() { + WHEEL_PATH="$1" python3 - <<'PY' +from os import environ +from pathlib import Path + +from packaging.utils import parse_wheel_filename + +wheel = Path(environ["WHEEL_PATH"]) +parse_wheel_filename(wheel.name) +PY +} + +if [ -n "$CACHED_WHEEL" ] && validate_wheel_filename "$CACHED_WHEEL"; then + echo "Trying cached wheel: $CACHED_WHEEL" + if python3 -m pip install --no-deps --force-reinstall "$CACHED_WHEEL" && \ + verify_torchcodec; then + echo "Installed TorchCodec from cache." + exit 0 + fi + echo "Cached wheel is incompatible; rebuilding TorchCodec." + rm -f "$CACHED_WHEEL" +fi + +BUILD_DIR=$(mktemp -d -t torchcodec-XXXXXX) +trap 'rm -rf "$BUILD_DIR"' EXIT + +cd "$BUILD_DIR" +echo "Cloning TorchCodec $TORCHCODEC_BRANCH..." +git clone --depth 1 --branch "$TORCHCODEC_BRANCH" "$TORCHCODEC_REPO" torchcodec +cd torchcodec + +export TORCHCODEC_CMAKE_BUILD_DIR="${PWD}/build" +export TORCHCODEC_DISABLE_COMPILE_WARNING_AS_ERROR=1 +export I_CONFIRM_THIS_IS_NOT_A_LICENSE_VIOLATION=1 +export CMAKE_GENERATOR=Ninja +export MAX_JOBS="${MAX_JOBS:-$(nproc)}" + +if command -v ccache >/dev/null 2>&1; then + export CMAKE_C_COMPILER_LAUNCHER=ccache + export CMAKE_CXX_COMPILER_LAUNCHER=ccache +fi + +echo "Building TorchCodec (MAX_JOBS=$MAX_JOBS)..." +python3 -m pip wheel . --no-build-isolation --no-deps -w "$BUILD_DIR/dist" +BUILT_WHEEL=$(find "$BUILD_DIR/dist" -maxdepth 1 -type f -name 'torchcodec-*.whl' -print -quit) +test -n "$BUILT_WHEEL" +validate_wheel_filename "$BUILT_WHEEL" + +python3 -m pip install --no-deps --force-reinstall "$BUILT_WHEEL" +verify_torchcodec + +mkdir -p "$CACHE_DIR" +cp "$BUILT_WHEEL" "$CACHE_DIR/$(basename "$BUILT_WHEEL")" +echo "Cached TorchCodec wheel in $CACHE_DIR" +echo "=== TorchCodec ROCm installation complete ===" From e043818abfc060903a188bde01b9ce648e634054 Mon Sep 17 00:00:00 2001 From: Sy03 <1370724210@qq.com> Date: Sat, 25 Jul 2026 09:20:47 +0800 Subject: [PATCH 091/240] [Bugfix][Frontend] Remove async-chunk guard from realtime API (#5383) Signed-off-by: Sy03 <1370724210@qq.com> --- .../sched/test_omni_ar_scheduler_streaming.py | 29 ++++++++-- ...mni_generation_scheduler_update_session.py | 56 +++++++++++++++++++ .../test_orchestrator_stage_input_bridge.py | 36 ++++++++++++ .../test_qwen3_omni_realtime_websocket.py | 6 -- vllm_omni/core/sched/omni_ar_scheduler.py | 8 ++- .../core/sched/omni_generation_scheduler.py | 5 ++ vllm_omni/engine/orchestrator.py | 6 +- vllm_omni/entrypoints/openai/api_server.py | 16 ------ 8 files changed, 133 insertions(+), 29 deletions(-) diff --git a/tests/core/sched/test_omni_ar_scheduler_streaming.py b/tests/core/sched/test_omni_ar_scheduler_streaming.py index e2ad98f56..9c0533059 100644 --- a/tests/core/sched/test_omni_ar_scheduler_streaming.py +++ b/tests/core/sched/test_omni_ar_scheduler_streaming.py @@ -57,13 +57,22 @@ def _make_update(prompt_token_ids: list[int] | None = None) -> StreamingUpdate: ) -def _run_resumable_segment_stop(session: Request) -> None: +def _run_resumable_segment_stop( + session: Request, + *, + session_finished: bool = False, +): sched = MagicMock() sched.requests = {session.request_id: session} sched.perf_metrics = None sched.structured_output_manager.should_advance.return_value = False - sched._update_request_with_output.return_value = ([42], True) - sched._handle_stopped_request.return_value = False + + def stop_request(request: Request, _token_ids: list[int]): + request.status = RequestStatus.FINISHED_STOPPED + return [42], True + + sched._update_request_with_output.side_effect = stop_request + sched._handle_stopped_request.return_value = session_finished sched.chunk_transfer_adapter = None sched.running = [session] sched.waiting_for_transfer_free = set() @@ -91,7 +100,7 @@ def _run_resumable_segment_stop(session: Request) -> None: model_runner_output.req_id_to_index = {session.request_id: 0} model_runner_output.routed_experts = None - OmniARScheduler.update_from_output(sched, scheduler_output, model_runner_output) + return OmniARScheduler.update_from_output(sched, scheduler_output, model_runner_output) @pytest.mark.parametrize("outstanding_async_tokens", [0, 1, 2]) @@ -116,6 +125,18 @@ def test_resumable_segment_stop_reconciles_async_placeholders( assert session._output_token_ids == [] +def test_resumable_session_terminal_is_not_marked_as_segment_boundary() -> None: + session = _make_request() + session.status = RequestStatus.RUNNING + session.resumable = True + + outputs = _run_resumable_segment_stop(session, session_finished=True) + + output = outputs[session.client_index].outputs[0] + assert output.finish_reason is not None + assert output.is_segment_finished is False + + def test_stage0_streaming_update_discards_outstanding_async_placeholder_token() -> None: sched = _make_scheduler(stage_id=0) session = _make_request() diff --git a/tests/core/sched/test_omni_generation_scheduler_update_session.py b/tests/core/sched/test_omni_generation_scheduler_update_session.py index d94996014..28083349a 100644 --- a/tests/core/sched/test_omni_generation_scheduler_update_session.py +++ b/tests/core/sched/test_omni_generation_scheduler_update_session.py @@ -9,6 +9,8 @@ update payloads early. from __future__ import annotations from dataclasses import replace +from types import SimpleNamespace +from unittest.mock import MagicMock import pytest @@ -17,7 +19,9 @@ import pytest # isort: off import vllm_omni # noqa: F401 - import for side effects (patch vLLM) from vllm.sampling_params import SamplingParams +from vllm.v1.core.sched.output import SchedulerOutput from vllm.v1.engine import EngineCoreEventType +from vllm.v1.outputs import ModelRunnerOutput from vllm.v1.request import Request, RequestStatus, StreamingUpdate from vllm_omni.core.sched.omni_generation_scheduler import OmniGenerationScheduler from vllm_omni.core.sched.omni_scheduler_mixin import OmniSchedulerMixin @@ -77,6 +81,58 @@ def _make_update(**kwargs) -> StreamingUpdate: return StreamingUpdate(**defaults) +def test_resumable_generation_stop_marks_segment_boundary() -> None: + session = _make_request(request_id="req-generation-segment") + session.status = RequestStatus.RUNNING + session.resumable = True + session.num_computed_tokens = len(session.prompt_token_ids) + + sched = MagicMock() + sched.requests = {session.request_id: session} + sched.perf_metrics = None + sched.chunk_transfer_adapter = SimpleNamespace( + is_done_receiving_chunks=lambda _request_id: True, + segment_finished_requests={session.request_id}, + ) + sched._handle_stopped_request.return_value = False + sched.running = [session] + sched.waiting = MagicMock() + sched.skipped_waiting = MagicMock() + sched.structured_output_manager.should_advance.return_value = False + sched._pending_finish_reqs = [] + sched.recompute_kv_load_failures = False + sched.connector = None + sched.kv_cache_manager.take_events.return_value = None + sched.finished_req_ids_dict = {} + sched.make_stats.return_value = None + + scheduler_output = MagicMock(spec=SchedulerOutput) + scheduler_output.num_scheduled_tokens = {session.request_id: 1} + scheduler_output.scheduled_spec_decode_tokens = {} + scheduler_output.num_invalid_spec_tokens = 0 + + model_runner_output = MagicMock(spec=ModelRunnerOutput) + model_runner_output.sampled_token_ids = [[]] + model_runner_output.logprobs = None + model_runner_output.prompt_logprobs_dict = {} + model_runner_output.pooler_output = None + model_runner_output.num_nans_in_logits = None + model_runner_output.kv_connector_output = None + model_runner_output.cudagraph_stats = None + model_runner_output.req_id_to_index = {session.request_id: 0} + model_runner_output.routed_experts = None + + outputs = OmniGenerationScheduler.update_from_output( + sched, + scheduler_output, + model_runner_output, + ) + + output = outputs[session.client_index].outputs[0] + assert output.finish_reason is not None + assert output.is_segment_finished is True + + class TestReplaceSessionWithStreamingUpdate: def test_resets_tokens_and_prompt_from_update(self) -> None: sched = _SchedulerStub() diff --git a/tests/engine/test_orchestrator_stage_input_bridge.py b/tests/engine/test_orchestrator_stage_input_bridge.py index e74b9ea18..0c5bb217d 100644 --- a/tests/engine/test_orchestrator_stage_input_bridge.py +++ b/tests/engine/test_orchestrator_stage_input_bridge.py @@ -3,6 +3,7 @@ from __future__ import annotations +import asyncio import queue from types import SimpleNamespace from typing import Any @@ -241,3 +242,38 @@ async def test_async_route_forwards_to_outgoing_only_stage() -> None: await orchestrator._route_output(0, 0, output, req_state, None) orchestrator._forward_to_next_stage.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_streaming_segment_does_not_complete_final_output_stage() -> None: + orchestrator = object.__new__(Orchestrator) + orchestrator.async_chunk = True + orchestrator._pd_pair = None + orchestrator._cfg_tracker = SimpleNamespace( + is_companion=lambda _request_id: False, + has_companions=lambda _request_id: False, + cleanup_parent=lambda _request_id: [], + ) + orchestrator.stage_pools = [SimpleNamespace(final_output=True)] + orchestrator.output_async_queue = asyncio.Queue() + orchestrator._cleanup_request_ids = AsyncMock() + + req_state = OrchestratorRequestState( + request_id="req-segment-final-output", + sampling_params_list=[SamplingParams(max_tokens=1)], + final_stage_id=0, + final_output_stage_ids={0}, + ) + req_state.streaming.enabled = True + req_state.streaming.segment_finished = True + output = SimpleNamespace( + request_id=req_state.request_id, + finished=True, + ) + + await orchestrator._route_output(0, 0, output, req_state, None) + + assert req_state.finished_final_output_stage_ids == set() + orchestrator._cleanup_request_ids.assert_not_awaited() + routed = orchestrator.output_async_queue.get_nowait() + assert routed.finished is False diff --git a/tests/entrypoints/openai_api/test_qwen3_omni_realtime_websocket.py b/tests/entrypoints/openai_api/test_qwen3_omni_realtime_websocket.py index d935949b3..50f8eef91 100644 --- a/tests/entrypoints/openai_api/test_qwen3_omni_realtime_websocket.py +++ b/tests/entrypoints/openai_api/test_qwen3_omni_realtime_websocket.py @@ -234,12 +234,6 @@ class TestQwen3OmniRealtimeWebSocket: @pytest.mark.omni @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2) @pytest.mark.parametrize("omni_server", realtime_async_chunk_server_params, indirect=True) - @pytest.mark.skip( - reason=( - "/v1/realtime rejects async_chunk since #3907 (server closes with 1000 after " - "unsupported error); tracked in #5363" - ) - ) def test_streaming_audio_input_pcm_output_async_chunk(self, omni_server) -> None: """Merge CI: async_chunk on, paced upload, full accuracy check.""" pcm16 = _synthetic_pcm16_input() diff --git a/vllm_omni/core/sched/omni_ar_scheduler.py b/vllm_omni/core/sched/omni_ar_scheduler.py index a9702eeb3..7dc2255ec 100644 --- a/vllm_omni/core/sched/omni_ar_scheduler.py +++ b/vllm_omni/core/sched/omni_ar_scheduler.py @@ -429,8 +429,10 @@ class OmniARScheduler(OmniSchedulerMixin, VLLMScheduler): # Capture finish_reason BEFORE _handle_stopped_request, which may # reset the status to WAITING for streaming requests that continue. finish_reason = request.get_finished_reason() - is_segment_finished = True finished = self._handle_stopped_request(request) + is_segment_finished = not finished + if finished: + request.resumable = False if not finished: # for streaming input request only if self.chunk_transfer_adapter: @@ -494,7 +496,9 @@ class OmniARScheduler(OmniSchedulerMixin, VLLMScheduler): # Invariant: EngineCore returns no partial prefill outputs. assert not prompt_logprobs_tensors - if self.chunk_transfer_adapter is not None and (inter_stage_output is not None or is_segment_finished): + if self.chunk_transfer_adapter is not None and ( + inter_stage_output is not None or is_segment_finished or finished + ): self.chunk_transfer_adapter.save_async( inter_stage_output, request, diff --git a/vllm_omni/core/sched/omni_generation_scheduler.py b/vllm_omni/core/sched/omni_generation_scheduler.py index 3882fd42d..534019513 100644 --- a/vllm_omni/core/sched/omni_generation_scheduler.py +++ b/vllm_omni/core/sched/omni_generation_scheduler.py @@ -480,6 +480,7 @@ class OmniGenerationScheduler(OmniSchedulerMixin, VLLMScheduler): mm_output = mm_outputs[req_index] if mm_outputs else None status_before_stop = request.status finish_reason = None + is_segment_finished = False routed_experts = None # One-shot generation request: finish after its current input unit @@ -503,6 +504,7 @@ class OmniGenerationScheduler(OmniSchedulerMixin, VLLMScheduler): routed_experts = omni_routed_experts_for_request(model_runner_output.routed_experts, request) finish_reason = request.get_finished_reason() finished = self._handle_stopped_request(request) + is_segment_finished = not finished if not finished: # for streaming input request only if self.chunk_transfer_adapter: @@ -556,6 +558,7 @@ class OmniGenerationScheduler(OmniSchedulerMixin, VLLMScheduler): trace_headers=request.trace_headers, routed_experts=routed_experts, num_nans_in_logits=request.num_nans_in_logits, + is_segment_finished=is_segment_finished, ) ) else: @@ -570,6 +573,7 @@ class OmniGenerationScheduler(OmniSchedulerMixin, VLLMScheduler): request.status = RequestStatus.FINISHED_STOPPED finish_reason = request.get_finished_reason() finished = self._handle_stopped_request(request) + is_segment_finished = not finished kv_transfer_params = None if finished: kv_transfer_params = self._free_request(request) @@ -587,6 +591,7 @@ class OmniGenerationScheduler(OmniSchedulerMixin, VLLMScheduler): events=request.take_events(), kv_transfer_params=kv_transfer_params, trace_headers=request.trace_headers, + is_segment_finished=is_segment_finished, ) ) stopped_running_reqs.add(request) diff --git a/vllm_omni/engine/orchestrator.py b/vllm_omni/engine/orchestrator.py index e37111063..9a1d63170 100644 --- a/vllm_omni/engine/orchestrator.py +++ b/vllm_omni/engine/orchestrator.py @@ -1183,7 +1183,11 @@ class Orchestrator: return request_finished = False - if finished and self.stage_pools[stage_id].final_output: + if ( + finished + and self.stage_pools[stage_id].final_output + and not (req_state.streaming.enabled and req_state.streaming.segment_finished) + ): req_state.finished_final_output_stage_ids.add(stage_id) final_output_stage_ids = req_state.final_output_stage_ids or {req_state.final_stage_id} request_finished = final_output_stage_ids.issubset(req_state.finished_final_output_stage_ids) diff --git a/vllm_omni/entrypoints/openai/api_server.py b/vllm_omni/entrypoints/openai/api_server.py index 56c7ccf46..5338761ad 100755 --- a/vllm_omni/entrypoints/openai/api_server.py +++ b/vllm_omni/entrypoints/openai/api_server.py @@ -1635,22 +1635,6 @@ async def realtime_websocket(websocket: WebSocket): await duplex_handler.handle_realtime_session(websocket) return - engine_client = getattr(websocket.app.state, "engine_client", None) - if engine_client is not None and getattr(engine_client, "async_chunk", False): - await websocket.accept() - await websocket.send_json( - { - "type": "error", - "error": ( - "The /v1/realtime API is not supported when async_chunk is enabled on the server. " - "Use a stage configuration with async_chunk disabled and restart the server before using " - "this endpoint." - ), - "code": "unsupported", - } - ) - await websocket.close() - return serving = getattr(websocket.app.state, "openai_serving_realtime", None) if serving is None: await websocket.accept() From 7afd3a0dcb603be01a84f511946e2df3f1ea6147 Mon Sep 17 00:00:00 2001 From: wangyu <53896905+yenuo26@users.noreply.github.com> Date: Sat, 25 Jul 2026 13:03:19 +0800 Subject: [PATCH 092/240] [CI/Bugfix] Pin dev deps and downgrade mooncake for connector CI (#5389) (#5392) --- .buildkite/cuda/test-ready.yml | 1 - pyproject.toml | 66 +++++++++++++++++----------------- 2 files changed, 33 insertions(+), 34 deletions(-) diff --git a/.buildkite/cuda/test-ready.yml b/.buildkite/cuda/test-ready.yml index d622aecbe..90e39b12a 100644 --- a/.buildkite/cuda/test-ready.yml +++ b/.buildkite/cuda/test-ready.yml @@ -8,7 +8,6 @@ steps: - group: ":card_index_dividers: Simple Test" depends_on: upload-ready-pipeline steps: - - label: "Simple · Diffusion & Model Executor Test" commands: - "timeout 20m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml" diff --git a/pyproject.toml b/pyproject.toml index 4a39c45c2..c71e17b4e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,41 +36,41 @@ classifiers = [ [project.optional-dependencies] dev = [ - "pytest>=7.0.0", - "pytest-asyncio>=0.21.0", - "pytest-cov>=4.0.0", - "pytest-mock>=3.10.0", - "websockets>=15.0.1", - "pytest-subtests>=0.11.0", - "datasets>=2.14.0", + "pytest==9.1.1", + "pytest-asyncio==1.4.0", + "pytest-cov==7.1.0", + "pytest-mock==3.15.1", + "websockets==16.1", + "pytest-subtests==0.15.0", + "datasets==3.6.0", "mypy==1.11.1", "pre-commit==4.0.1", - "openai-whisper>=20250625", - "psutil>=7.2.0", - "soundfile>=0.13.1", - "imageio[ffmpeg]>=0.6.0", - "opencv-python>=4.12.0.88", - "mooncake-transfer-engine-cuda13>=0.3.9", - "av", # for ComfyUI tests - "openpyxl>=3.0.0", # for nightly CI - "pyttsx3>=2.99", - "opencc>=1.2.0", - "mistune>=3.2.0", # for example tests - "torchmetrics>=1.4.0", # for accuracy similarity metrics - "ray[default]", # for RL test - "jiwer>=3.0.0", - "zhon>=2.0.0", - "zhconv>=1.4.2", - "scipy>=1.10.0", - "funasr>=1.0.0", - "lpips>=0.1.4", # for quantization fp8 - "orjson>=3.11.7", # to make v1/embeddings API fast - "voxcpm>=2.0.2", # for VoxCPM2 path-locating in tests - "torchdiffeq>=0.2.5", # for covo_audio Test - "FlagEmbedding", # for hunyuan_image3 tests - "kernels<0.15.0", # for accuracy tests comparison with diffusers output (0.15.0 incompatible with diffusers 0.38.0) - "s3tokenizer>=0.3.0", # for tts test - "step-audio2>=1.0.0", # Step-Audio2 Token2Wav (flashcosyvoice / hyperpyyaml) + "openai-whisper==20250625", + "psutil==7.2.2", + "soundfile==0.14.0", + "imageio[ffmpeg]==2.37.4", + "opencv-python==5.0.0.93", + "mooncake-transfer-engine-cuda13==0.3.11.post1", + "av==18.0.0", # for ComfyUI tests + "openpyxl==3.1.5", # for nightly CI + "pyttsx3==2.99", + "opencc==1.4.1", + "mistune==3.3.4", # for example tests + "torchmetrics==1.9.0", # for accuracy similarity metrics + "ray[default]==2.56.1", # for RL test + "jiwer==4.0.0", + "zhon==2.1.1", + "zhconv==1.4.3", + "scipy==1.18.0", + "funasr==1.3.29", + "lpips==0.1.4", # for quantization fp8 + "orjson==3.11.9", # to make v1/embeddings API fast + "voxcpm==2.0.3", # for VoxCPM2 path-locating in tests + "torchdiffeq==0.2.5", # for covo_audio Test + "FlagEmbedding==1.4.0", # for hunyuan_image3 tests + "kernels==0.14.1", # for accuracy tests comparison with diffusers output (0.15.0 incompatible with diffusers 0.38.0) + "s3tokenizer==0.3.0", # for tts test + "step-audio2==1.0.0", # Step-Audio2 Token2Wav (flashcosyvoice / hyperpyyaml) ] demo = [ From af2f84582bafdfa02a23ecf8bf6a8081252f9e72 Mon Sep 17 00:00:00 2001 From: Ma1oneZhang <73012001+Ma1oneZhang@users.noreply.github.com> Date: Sat, 25 Jul 2026 16:09:19 +0800 Subject: [PATCH 093/240] perf(qwen3-tts): cache decoder masks and compile pre-transformer (#5107) Signed-off-by: Ziyang Zhang Co-authored-by: Sy03 <1370724210@qq.com> Co-authored-by: Hongsheng Liu --- .../test_qwen3_tts_customvoice.py | 43 +++++- .../test_qwen3_tts_tokenizer_mask_cache.py | 125 ++++++++++++++++++ .../modeling_qwen3_tts_tokenizer_v2.py | 121 +++++++++++------ 3 files changed, 249 insertions(+), 40 deletions(-) create mode 100644 tests/model_executor/models/qwen3_tts/test_qwen3_tts_tokenizer_mask_cache.py diff --git a/tests/e2e/online_serving/test_qwen3_tts_customvoice.py b/tests/e2e/online_serving/test_qwen3_tts_customvoice.py index 2aab84840..62528c272 100644 --- a/tests/e2e/online_serving/test_qwen3_tts_customvoice.py +++ b/tests/e2e/online_serving/test_qwen3_tts_customvoice.py @@ -12,15 +12,21 @@ import os os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" import pytest +from vllm.platforms import current_platform from tests.helpers.mark import hardware_test from tests.helpers.runtime import OmniServerParams -from tests.helpers.stage_config import get_deploy_config_path, modify_stage_config +from tests.helpers.stage_config import ( + get_deploy_config_path, + get_deploy_config_stage, + modify_stage_config, +) MODEL = "Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice" +_DEFAULT_STAGE_CONFIG = get_deploy_config_path("qwen3_tts.yaml") _STAGE_CONFIG = modify_stage_config( - get_deploy_config_path("qwen3_tts.yaml"), + _DEFAULT_STAGE_CONFIG, updates={"stages": {0: {"default_sampling_params.max_tokens": 500}}}, ) @@ -50,6 +56,39 @@ tts_server_params = [ ) ] +default_tts_server_params = [ + pytest.param( + OmniServerParams( + model=MODEL, + stage_config_path=_DEFAULT_STAGE_CONFIG, + server_args=["--trust-remote-code"], + ), + id="async_chunk", + ) +] + + +@pytest.mark.core_model +@pytest.mark.advanced_model +@pytest.mark.tts +@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA Graph startup test requires CUDA") +@hardware_test(res={"cuda": "L4"}, num_cards=1) +@pytest.mark.parametrize("omni_server", default_tts_server_params, indirect=True) +def test_default_cuda_graph_startup(omni_server) -> None: + """Verify both stages start with the shipped CUDA Graph configuration. + + The fixture reaching this test is the smoke assertion: it waits for the + server to become ready after both stages finish model initialization and + CUDA Graph capture. The regression covered here exited during stage 1 + capture, before fixture setup could complete. + """ + for stage_id in (0, 1): + stage = get_deploy_config_stage("qwen3_tts.yaml", stage_id=stage_id) + assert stage.get("enforce_eager", False) is False + + assert omni_server.proc is not None + assert omni_server.proc.poll() is None + @pytest.mark.core_model @pytest.mark.advanced_model diff --git a/tests/model_executor/models/qwen3_tts/test_qwen3_tts_tokenizer_mask_cache.py b/tests/model_executor/models/qwen3_tts/test_qwen3_tts_tokenizer_mask_cache.py new file mode 100644 index 000000000..aeb538adc --- /dev/null +++ b/tests/model_executor/models/qwen3_tts/test_qwen3_tts_tokenizer_mask_cache.py @@ -0,0 +1,125 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import inspect + +import pytest +import torch +from transformers.masking_utils import create_sliding_window_causal_mask + +from vllm_omni.model_executor.models.qwen3_tts.tokenizer_12hz.configuration_qwen3_tts_tokenizer_v2 import ( + Qwen3TTSTokenizerV2DecoderConfig, +) +from vllm_omni.model_executor.models.qwen3_tts.tokenizer_12hz.modeling_qwen3_tts_tokenizer_v2 import ( + Qwen3TTSTokenizerV2DecoderTransformerModel, +) + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +def _make_config(attention_implementation: str) -> Qwen3TTSTokenizerV2DecoderConfig: + config = Qwen3TTSTokenizerV2DecoderConfig( + hidden_size=32, + latent_dim=16, + max_position_embeddings=512, + num_attention_heads=4, + num_key_value_heads=4, + intermediate_size=64, + num_hidden_layers=1, + num_quantizers=2, + decoder_dim=32, + upsample_rates=(2,), + upsampling_ratios=(2,), + ) + config._attn_implementation = attention_implementation + return config + + +def _make_model( + attention_implementation: str, + dtype: torch.dtype = torch.float32, +) -> Qwen3TTSTokenizerV2DecoderTransformerModel: + model = Qwen3TTSTokenizerV2DecoderTransformerModel._from_config(_make_config(attention_implementation)) + return model.to(dtype=dtype).eval() + + +def _create_reference_mask( + model: Qwen3TTSTokenizerV2DecoderTransformerModel, + inputs_embeds: torch.Tensor, +) -> torch.Tensor | None: + sequence_length = inputs_embeds.shape[1] + mask_kwargs = { + "config": model.config, + "attention_mask": None, + "past_key_values": None, + "position_ids": None, + } + sig = inspect.signature(create_sliding_window_causal_mask) + if "input_embeds" in sig.parameters: + mask_kwargs["input_embeds"] = inputs_embeds[:1] + mask_kwargs["cache_position"] = torch.arange(sequence_length) + else: + mask_kwargs["inputs_embeds"] = inputs_embeds[:1] + return create_sliding_window_causal_mask(**mask_kwargs) + + +@pytest.mark.parametrize("sequence_length", [17, 72, 73, 169, 325]) +@pytest.mark.parametrize( + ("attention_implementation", "dtype"), + [("sdpa", torch.float32), ("eager", torch.float32), ("eager", torch.float16)], +) +def test_cached_sliding_mask_matches_transformers( + sequence_length: int, + attention_implementation: str, + dtype: torch.dtype, +): + model = _make_model(attention_implementation, dtype) + inputs_embeds = torch.empty(2, sequence_length, model.config.hidden_size, dtype=dtype) + + actual = model._get_sliding_attention_mask(inputs_embeds) + expected = _create_reference_mask(model, inputs_embeds) + + if expected is None: + assert actual is None + assert model._get_sliding_attention_mask(inputs_embeds) is None + return + + assert actual is not None + torch.testing.assert_close(actual, expected, atol=0, rtol=0) + assert actual.shape == (1, 1, sequence_length, sequence_length) + assert actual.is_contiguous() + assert model._get_sliding_attention_mask(inputs_embeds) is actual + + +def test_cached_eager_mask_tracks_compute_dtype(): + model = _make_model("eager") + + fp32_inputs = torch.empty(1, 73, model.config.hidden_size) + fp32_mask = model._get_sliding_attention_mask(fp32_inputs) + assert fp32_mask is not None + assert fp32_mask.dtype == torch.float32 + + model.half() + fp16_inputs = fp32_inputs.half() + fp16_mask = model._get_sliding_attention_mask(fp16_inputs) + assert fp16_mask is not None + assert fp16_mask.dtype == torch.float16 + assert torch.isfinite(fp16_mask).all() + assert fp16_mask.min() == torch.finfo(torch.float16).min + + +@pytest.mark.parametrize( + ("kwargs", "message"), + [ + ({"use_cache": True}, "use_cache=True"), + ({"past_key_values": object()}, "past_key_values"), + ({"cache_position": torch.arange(1, 18)}, "zero-based cache_position"), + ({"position_ids": torch.arange(1, 18).unsqueeze(0)}, "zero-based position_ids"), + ], +) +def test_cached_mask_rejects_unsupported_cache_and_positions(kwargs: dict, message: str): + model = _make_model("sdpa") + inputs_embeds = torch.randn(1, 17, model.config.latent_dim) + + with pytest.raises(ValueError, match=message): + model(inputs_embeds=inputs_embeds, **kwargs) diff --git a/vllm_omni/model_executor/models/qwen3_tts/tokenizer_12hz/modeling_qwen3_tts_tokenizer_v2.py b/vllm_omni/model_executor/models/qwen3_tts/tokenizer_12hz/modeling_qwen3_tts_tokenizer_v2.py index 447823abb..9d6eb7aaa 100644 --- a/vllm_omni/model_executor/models/qwen3_tts/tokenizer_12hz/modeling_qwen3_tts_tokenizer_v2.py +++ b/vllm_omni/model_executor/models/qwen3_tts/tokenizer_12hz/modeling_qwen3_tts_tokenizer_v2.py @@ -25,12 +25,9 @@ from torch import nn from torch.nn import functional as F from transformers import MimiConfig, MimiModel from transformers.activations import ACT2FN -from transformers.cache_utils import Cache, DynamicCache +from transformers.cache_utils import Cache from transformers.integrations import use_kernel_forward_from_hub -from transformers.masking_utils import ( - create_causal_mask, - create_sliding_window_causal_mask, -) +from transformers.masking_utils import create_sliding_window_causal_mask from transformers.modeling_flash_attention_utils import FlashAttentionKwargs from transformers.modeling_layers import GradientCheckpointingLayer from transformers.modeling_outputs import BaseModelOutputWithPast @@ -523,15 +520,64 @@ class Qwen3TTSTokenizerV2DecoderTransformerModel(Qwen3TTSTokenizerV2DecoderPreTr self.norm = Qwen3TTSTokenizerV2DecoderRMSNorm(config.hidden_size, eps=config.rms_norm_eps) self.rotary_emb = Qwen3TTSTokenizerV2DecoderRotatoryEmbedding(config=config) self.gradient_checkpointing = False - self.has_sliding_layers = "sliding_attention" in self.config.layer_types self.window_size = config.sliding_window self.input_proj = nn.Linear(config.latent_dim, config.hidden_size) self.output_proj = nn.Linear(config.hidden_size, config.latent_dim) + # Code2Wav is cache-free and every decoder layer uses sliding attention. + # Cache an exact, compact mask per warmup shape instead of retaining a + # max_position_embeddings square buffer. CUDA Graph warmup populates + # these entries before capture, and dtype/device are part of the key. + self._sliding_attention_mask_cache: dict[tuple[int, torch.dtype, torch.device, str], torch.Tensor | None] = {} + # Initialize weights and apply final processing self.post_init() + def _apply(self, fn, recurse=True): + # Cached masks are keyed by dtype/device and should not retain storage + # from the model's previous placement. + self._sliding_attention_mask_cache.clear() + return super()._apply(fn, recurse=recurse) + + def _get_sliding_attention_mask(self, inputs_embeds: torch.Tensor) -> torch.Tensor | None: + sequence_length = int(inputs_embeds.shape[1]) + max_position_embeddings = int(self.config.max_position_embeddings) + if sequence_length > max_position_embeddings: + raise ValueError( + f"Input length {sequence_length} exceeds max_position_embeddings {max_position_embeddings}" + ) + + attention_implementation = str(self.config._attn_implementation) + cache_key = ( + sequence_length, + inputs_embeds.dtype, + inputs_embeds.device, + attention_implementation, + ) + if cache_key not in self._sliding_attention_mask_cache: + # A batch-one mask broadcasts across requests and avoids caching a + # duplicate tensor for every batch size. + mask_inputs = inputs_embeds[:1] + mask_kwargs = { + "config": self.config, + "attention_mask": None, + "past_key_values": None, + "position_ids": None, + } + sig = inspect.signature(create_sliding_window_causal_mask) + if "input_embeds" in sig.parameters: + mask_kwargs["input_embeds"] = mask_inputs + mask_kwargs["cache_position"] = torch.arange(sequence_length, device=inputs_embeds.device) + else: + mask_kwargs["inputs_embeds"] = mask_inputs + sliding_attention_mask = create_sliding_window_causal_mask(**mask_kwargs) + if sliding_attention_mask is not None: + sliding_attention_mask = sliding_attention_mask.contiguous() + self._sliding_attention_mask_cache[cache_key] = sliding_attention_mask + + return self._sliding_attention_mask_cache[cache_key] + # Note: @check_model_inputs decorator removed for vLLM compatibility # The decorator causes "unexpected keyword argument 'inputs_embeds'" error @auto_docstring @@ -554,57 +600,56 @@ class Qwen3TTSTokenizerV2DecoderTransformerModel(Qwen3TTSTokenizerV2DecoderPreTr raise ValueError("input_ids is not expected") if (input_ids is None) ^ (inputs_embeds is not None): raise ValueError("You must specify exactly one of input_ids or inputs_embeds") + if use_cache: + raise ValueError("Code2Wav sliding-mask caching does not support use_cache=True") + if past_key_values is not None: + raise ValueError("Code2Wav sliding-mask caching does not support past_key_values") + if attention_mask is not None and not isinstance(attention_mask, dict): + raise ValueError("Code2Wav cached masks require attention_mask to be None or a prepared mask mapping") if inputs_embeds is None: inputs_embeds = self.embed_tokens(input_ids) inputs_embeds = self.input_proj(inputs_embeds) - if use_cache and past_key_values is None: - past_key_values = DynamicCache(config=self.config) + sequence_length = inputs_embeds.shape[1] + expected_cache_position = torch.arange(sequence_length, device=inputs_embeds.device) if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position = torch.arange( - past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device - ) + cache_position = expected_cache_position + elif attention_mask is None and ( + cache_position.shape != expected_cache_position.shape + or cache_position.device != expected_cache_position.device + or not torch.equal(cache_position, expected_cache_position) + ): + raise ValueError("Cached Code2Wav masks require contiguous zero-based cache_position") if position_ids is None: position_ids = cache_position.unsqueeze(0) - - # It may already have been prepared by e.g. `generate` - if not isinstance(causal_mask_mapping := attention_mask, dict): - # Prepare mask arguments - mask_kwargs = { - "config": self.config, - "attention_mask": attention_mask, - "past_key_values": past_key_values, - "position_ids": position_ids, - } - # Handle API changes across transformers versions - sig = inspect.signature(create_causal_mask) - if "input_embeds" in sig.parameters: - mask_kwargs["input_embeds"] = inputs_embeds - mask_kwargs["cache_position"] = cache_position - else: - mask_kwargs["inputs_embeds"] = inputs_embeds - # Create the masks - causal_mask_mapping = { - "full_attention": create_causal_mask(**mask_kwargs), - } - # The sliding window alternating layers are not always activated depending on the config - if self.has_sliding_layers: - causal_mask_mapping["sliding_attention"] = create_sliding_window_causal_mask(**mask_kwargs) + elif attention_mask is None: + expected_position_ids = expected_cache_position.unsqueeze(0) + if ( + position_ids.ndim != 2 + or position_ids.shape[-1] != sequence_length + or position_ids.device != expected_position_ids.device + or not torch.equal(position_ids, expected_position_ids.expand_as(position_ids)) + ): + raise ValueError("Cached Code2Wav masks require contiguous zero-based position_ids") hidden_states = inputs_embeds + if isinstance(attention_mask, dict): + sliding_attention_mask = attention_mask["sliding_attention"] + else: + sliding_attention_mask = self._get_sliding_attention_mask(inputs_embeds) + # create position embeddings to be shared across the decoder layers position_embeddings = self.rotary_emb(hidden_states, position_ids) for decoder_layer in self.layers[: self.config.num_hidden_layers]: hidden_states = decoder_layer( hidden_states, - attention_mask=causal_mask_mapping[decoder_layer.attention_type], + attention_mask=sliding_attention_mask, position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, @@ -617,7 +662,7 @@ class Qwen3TTSTokenizerV2DecoderTransformerModel(Qwen3TTSTokenizerV2DecoderPreTr hidden_states = self.output_proj(hidden_states) return BaseModelOutputWithPast( last_hidden_state=hidden_states, - past_key_values=past_key_values if use_cache else None, + past_key_values=None, ) From 76e4a78afc17d5fac34c8e5be83531b9e5daf432 Mon Sep 17 00:00:00 2001 From: boatman <109857087+sphinxkkkbc@users.noreply.github.com> Date: Sat, 25 Jul 2026 23:25:35 +0800 Subject: [PATCH 094/240] [Bugfix][Higgs-Audio-V3] Propagate Sampling Parameters to Sampler (#5406) Signed-off-by: boatman <109857087+sphinxkkkbc@users.noreply.github.com> --- .../higgs_audio_v3/test_higgs_audio_v3.py | 93 ++++++++++++++++++- tests/worker/test_gpu_ar_model_runner.py | 80 ++++++++++++++++ .../entrypoints/openai/serving_speech.py | 3 + .../higgs_audio_v3/higgs_audio_v3_talker.py | 60 ++++++++---- 4 files changed, 215 insertions(+), 21 deletions(-) diff --git a/tests/unit/higgs_audio_v3/test_higgs_audio_v3.py b/tests/unit/higgs_audio_v3/test_higgs_audio_v3.py index b92b0095b..1d4bdb48d 100755 --- a/tests/unit/higgs_audio_v3/test_higgs_audio_v3.py +++ b/tests/unit/higgs_audio_v3/test_higgs_audio_v3.py @@ -250,6 +250,7 @@ class TestSamplerMethods: t = FakeTalker() t._sample_audio_codes = mod.HiggsAudioV3TalkerForConditionalGeneration._sample_audio_codes.__get__(t) + t._expand_audio_sampling_params = mod.HiggsAudioV3TalkerForConditionalGeneration._expand_audio_sampling_params return t def _make_batched_sampler_talker(self, num_rows=4): @@ -300,16 +301,30 @@ class TestSamplerMethods: "_audio_seed_mask_from_step_input", ): setattr(t, name, getattr(cls, name).__get__(t)) + t._expand_audio_sampling_params = cls._expand_audio_sampling_params t._device_cache_key = cls._device_cache_key return t + @staticmethod + def _sampling_metadata(temperature=1.0, top_k=50, top_p=0.95): + return type( + "SamplingMetadata", + (), + { + "temperature": temperature, + "top_k": top_k, + "top_p": top_p, + "all_greedy": temperature is None, + }, + )() + def test_sample_respects_mask(self): """Tokens masked to -inf must never be sampled.""" t = self._make_batched_sampler_talker(num_rows=1) logits = torch.full((10, 1026), float("-inf")) # Only token 500 is allowed for each row logits[:, 500] = 0.0 - result = t._sample_audio_codes(logits) + result = t._sample_audio_codes(logits, self._sampling_metadata(), num_codebooks=1) assert result.shape == (10,) assert (result == 500).all(), f"Expected all 500, got {result.tolist()}" @@ -320,10 +335,76 @@ class TestSamplerMethods: # Row 0: all masked # Row 1: only token 42 allowed logits[1, 42] = 0.0 - result = t._sample_audio_codes(logits) + result = t._sample_audio_codes(logits, self._sampling_metadata(), num_codebooks=1) assert result.shape == (2,) assert result[1].item() == 42 + def test_top_k_minus_one_disables_top_k_filtering(self, monkeypatch): + t = self._make_minimal_talker() + seen_probs = [] + + def fake_multinomial(probs, num_samples): + seen_probs.append(probs.clone()) + return probs.argmax(dim=-1, keepdim=True) + + monkeypatch.setattr(torch, "multinomial", fake_multinomial) + + logits = torch.tensor([[4.0, 3.0, 2.0, 1.0]]) + metadata = self._sampling_metadata( + temperature=1.0, + top_k=-1, + top_p=1.0, + ) + + t._sample_audio_codes(logits, metadata, num_codebooks=1) + + assert len(seen_probs) == 1 + assert torch.count_nonzero(seen_probs[0][0]).item() == 4 + + def test_sampling_metadata_invalid_temperature_is_sanitized(self, monkeypatch): + t = self._make_minimal_talker() + seen = [] + monkeypatch.setattr( + torch, "multinomial", lambda probs, num_samples: seen.append(probs) or probs.argmax(-1, True) + ) + logits = torch.tensor([[0.0, 2.0], [3.0, 0.0]]) + + result = t._sample_audio_codes( + logits, + self._sampling_metadata(torch.tensor([float("nan"), float("inf")])), + num_codebooks=1, + ) + + assert len(seen) == 1 + assert torch.isfinite(seen[0]).all() + assert result.tolist() == [1, 0] + + def test_sampling_metadata_is_expanded_per_request_and_vq(self, monkeypatch): + t = self._make_minimal_talker() + seen = [] + monkeypatch.setattr( + torch, "multinomial", lambda probs, num_samples: seen.append(probs) or probs.argmax(-1, True) + ) + cb_logits = torch.tensor([[[4.0, 3.0, 2.0, 1.0]] * 2] * 2) # [2 requests, 2 VQs, 4 vocab] + logits = cb_logits.reshape(-1, cb_logits.shape[-1]) + metadata = self._sampling_metadata( + temperature=torch.tensor([0.5, 1.0]), + top_k=torch.tensor([1, 4]), + top_p=torch.tensor([1.0, 0.8]), + ) + + temperatures, top_ks, top_ps = t._expand_audio_sampling_params(metadata, logits, num_codebooks=2) + t._sample_audio_codes(logits, metadata, num_codebooks=2) + + torch.testing.assert_close(temperatures, torch.tensor([0.5, 0.5, 1.0, 1.0])) + assert top_ks.tolist() == [1, 1, 4, 4] + torch.testing.assert_close(top_ps, torch.tensor([1.0, 1.0, 0.8, 0.8])) + assert len(seen) == 1 + torch.testing.assert_close(seen[0][0], seen[0][1]) + torch.testing.assert_close(seen[0][2], seen[0][3]) + assert torch.count_nonzero(seen[0][0]).item() == 1 + assert torch.count_nonzero(seen[0][2]).item() > 1 + def test_delay_masking_forces_boc_during_delay(self): """During delay phase, codebooks beyond delay_count must have only BOC allowed.""" from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_talker import BOC_ID @@ -468,7 +549,9 @@ class TestSamplerMethods: t._fast_audio_sampler_gpu_fallback_reason = lambda **kwargs: None t._audio_codebook_logits_from_rows = lambda hidden, rows, all_rows=False: torch.zeros(2, 8, 1026) t._apply_delay_pattern_masking_batched = lambda cb_logits, audio_rows, all_rows=False: None - t._sample_audio_codes = lambda logits_2d: torch.zeros(int(logits_2d.shape[0]), dtype=torch.long) + t._sample_audio_codes = lambda logits_2d, *args, **kwargs: torch.zeros( + int(logits_2d.shape[0]), dtype=torch.long + ) t._update_delay_state_batched = lambda *args, **kwargs: None t.sample = mod.HiggsAudioV3TalkerForConditionalGeneration.sample.__get__(t) @@ -499,7 +582,9 @@ class TestSamplerMethods: t._fast_audio_sampler_gpu_fallback_reason = lambda **kwargs: None t._audio_codebook_logits_from_rows = lambda hidden, rows, all_rows=False: torch.zeros(2, 8, 1026) t._apply_delay_pattern_masking_batched = lambda cb_logits, audio_rows, all_rows=False: None - t._sample_audio_codes = lambda logits_2d: torch.zeros(int(logits_2d.shape[0]), dtype=torch.long) + t._sample_audio_codes = lambda logits_2d, *args, **kwargs: torch.zeros( + int(logits_2d.shape[0]), dtype=torch.long + ) t._update_delay_state_batched = lambda *args, **kwargs: None t.sample = mod.HiggsAudioV3TalkerForConditionalGeneration.sample.__get__(t) diff --git a/tests/worker/test_gpu_ar_model_runner.py b/tests/worker/test_gpu_ar_model_runner.py index 83d270da4..daab10ba0 100644 --- a/tests/worker/test_gpu_ar_model_runner.py +++ b/tests/worker/test_gpu_ar_model_runner.py @@ -1,10 +1,17 @@ +import asyncio from collections.abc import Sequence from types import SimpleNamespace import numpy as np import pytest import torch +from vllm.sampling_params import SamplingParams +from vllm.v1.worker import gpu_input_batch +from vllm.v1.worker.gpu_input_batch import CachedRequestState, InputBatch +from vllm_omni.entrypoints.openai.protocol.audio import OpenAICreateSpeechRequest +from vllm_omni.entrypoints.openai.serving_speech import OmniOpenAIServingSpeech +from vllm_omni.entrypoints.openai.tts_adapters.base import PreparedRequest from vllm_omni.outputs import OmniModelRunnerOutput from vllm_omni.worker.gpu_ar_model_runner import ( ExecuteModelState, @@ -25,6 +32,79 @@ def _make_runner(engine_output_type: str | None, downstream_req_ids: set[str]) - return runner +def test_speech_extra_params_reach_model_sampler_as_sampling_metadata(monkeypatch): + requested = {"temperature": 0.7, "top_p": 0.8, "top_k": 17} + request = OpenAICreateSpeechRequest.model_validate( + {"input": "Hello", "extra_params": requested}, + ) + + class Adapter: + def validate(self, request): + return None + + async def build(self, request, sampling_params_list, has_inline_ref_audio): + return PreparedRequest(prompt={"prompt": request.input}, tts_params={}, model_type="higgs_audio_v3") + + engine = SimpleNamespace( + errored=False, + default_sampling_params_list=[SamplingParams(temperature=1.0, top_p=0.95, top_k=50)], + generate=lambda **kwargs: kwargs["sampling_params_list"], + ) + serving = object.__new__(OmniOpenAIServingSpeech) + serving.engine_client = engine + serving.model_config = SimpleNamespace(async_chunk=True) + serving._tts_model_type = "higgs_audio_v3" + serving._get_tts_adapter = lambda: Adapter() + serving._track_ref_audio_artifact_warmup = lambda *args, **kwargs: None + + _, stage_sampling_params, _ = asyncio.run(serving._prepare_speech_generation(request, request_id="speech-test")) + stage0_params = stage_sampling_params[0] + + monkeypatch.setattr(gpu_input_batch, "PIN_MEMORY", False) + input_batch = InputBatch( + max_num_reqs=1, + max_model_len=8, + max_num_batched_tokens=8, + device=torch.device("cpu"), + vocab_size=1024, + block_sizes=[1], + kernel_block_sizes=[1], + ) + input_batch.add_request( + CachedRequestState( + req_id="speech-test", + prompt_token_ids=[1], + mm_features=[], + sampling_params=stage0_params, + generator=None, + block_ids=([],), + num_computed_tokens=0, + output_token_ids=[], + ) + ) + input_batch.sampling_metadata = input_batch._make_sampling_metadata() + + received = [] + runner = object.__new__(GPUARModelRunner) + runner.input_batch = input_batch + runner.model = SimpleNamespace( + prefer_model_sampler=True, + skips_model_sampler_output_token_history=True, + sample=lambda logits, metadata: received.append(metadata) or "model-sampler", + ) + runner.sampler = SimpleNamespace() + logits = torch.zeros((1, 4)) + + output = runner._sample(logits, spec_decode_metadata=None) + + assert output == "model-sampler" + assert len(received) == 1 + sampling_metadata = received[0] + torch.testing.assert_close(sampling_metadata.temperature, torch.tensor([requested["temperature"]])) + torch.testing.assert_close(sampling_metadata.top_p, torch.tensor([requested["top_p"]])) + assert sampling_metadata.top_k.tolist() == [requested["top_k"]] + + def test_resolve_pooler_payload_req_ids_audio_terminal_stage_keeps_payload(): runner = _make_runner(engine_output_type="audio", downstream_req_ids=set()) diff --git a/vllm_omni/entrypoints/openai/serving_speech.py b/vllm_omni/entrypoints/openai/serving_speech.py index ca3b8c21f..2bb402caf 100644 --- a/vllm_omni/entrypoints/openai/serving_speech.py +++ b/vllm_omni/entrypoints/openai/serving_speech.py @@ -3183,6 +3183,9 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): import copy sampling_params_list = copy.deepcopy(sampling_params_list) + for name in ("temperature", "top_p", "top_k"): + if (value := request.extra_params.get(name)) is not None: + setattr(sampling_params_list[0], name, value) if sampling_params_list[0].extra_args is None: sampling_params_list[0].extra_args = {} sampling_params_list[0].extra_args.update(request.extra_params) diff --git a/vllm_omni/model_executor/models/higgs_audio_v3/higgs_audio_v3_talker.py b/vllm_omni/model_executor/models/higgs_audio_v3/higgs_audio_v3_talker.py index f51adafa9..579d0cd13 100644 --- a/vllm_omni/model_executor/models/higgs_audio_v3/higgs_audio_v3_talker.py +++ b/vllm_omni/model_executor/models/higgs_audio_v3/higgs_audio_v3_talker.py @@ -35,6 +35,8 @@ from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead from vllm.model_executor.models.qwen3 import Qwen3Model from vllm.platforms import current_platform from vllm.v1.outputs import SamplerOutput +from vllm.v1.sample.metadata import SamplingMetadata +from vllm.v1.sample.ops.topk_topp_sampler import apply_top_k_top_p from vllm_omni.model_executor.models.output_templates import OmniOutput from vllm_omni.transformers_utils.configs.higgs_audio_v3 import ( @@ -983,7 +985,11 @@ class HiggsAudioV3TalkerForConditionalGeneration(nn.Module): # Sample per-codebook cb_logits_2d = cb_logits.reshape(-1, cb_logits.shape[-1]) - codes_2d = self._sample_audio_codes(cb_logits_2d) + codes_2d = self._sample_audio_codes( + cb_logits_2d, + sampling_metadata, + int(cb_logits.shape[1]), + ) codes_flat = codes_2d.view(cb_logits.shape[0], cb_logits.shape[1]).to(torch.long) self._update_delay_state_batched( @@ -1337,23 +1343,42 @@ class HiggsAudioV3TalkerForConditionalGeneration(nn.Module): self._postprocess_audio_rows = num_rows self._postprocess_audio_active_rows = 0 - def _sample_audio_codes(self, logits_2d: torch.Tensor) -> torch.Tensor: + @staticmethod + def _expand_audio_sampling_params( + sampling_metadata: SamplingMetadata, logits_2d: torch.Tensor, num_codebooks: int + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + batch_size, remainder = divmod(int(logits_2d.shape[0]), num_codebooks) + if remainder: + raise ValueError(f"audio codebook logits rows must be divisible by {num_codebooks}") + + def expand(value, default, dtype): + if value is None: + value = torch.full((batch_size,), default, device=logits_2d.device, dtype=dtype) + else: + value = torch.as_tensor(value, device=logits_2d.device, dtype=dtype).expand(batch_size) + return value.repeat_interleave(num_codebooks) + + return ( + expand(sampling_metadata.temperature, 0.0 if sampling_metadata.all_greedy else 1.0, logits_2d.dtype), + expand(sampling_metadata.top_k, logits_2d.shape[-1], torch.long), + expand(sampling_metadata.top_p, 1.0, logits_2d.dtype), + ) + + def _sample_audio_codes( + self, logits_2d: torch.Tensor, sampling_metadata: SamplingMetadata, num_codebooks: int = 8 + ) -> torch.Tensor: """Replicate upstream sampling: temperature → top-k → top-p → multinomial.""" x = logits_2d.float() - top_k = 50 - top_p = 0.95 - if 0 < top_k < x.shape[-1]: - kth = x.topk(top_k, dim=-1).values[..., -1:] - x = x.masked_fill(x < kth, float("-inf")) - if 0.0 < top_p < 1.0: - sorted_logits, sorted_idx = x.sort(dim=-1, descending=True) - cumprobs = sorted_logits.softmax(dim=-1).cumsum(dim=-1) - sorted_mask = cumprobs > top_p - sorted_mask[..., 1:] = sorted_mask[..., :-1].clone() - sorted_mask[..., 0] = False - mask = torch.zeros_like(x, dtype=torch.bool) - mask.scatter_(-1, sorted_idx, sorted_mask) - x = x.masked_fill(mask, float("-inf")) + temperatures, top_ks, top_ps = self._expand_audio_sampling_params(sampling_metadata, x, num_codebooks) + top_ks = torch.where(top_ks <= 0, torch.full_like(top_ks, x.shape[-1]), top_ks) + top_ks = top_ks.clamp(max=x.shape[-1]) + top_ps = top_ps.clamp(min=0.0, max=1.0) + temperatures = torch.where(torch.isfinite(temperatures), temperatures, torch.ones_like(temperatures)) + top_ps = torch.where(torch.isfinite(top_ps), top_ps, torch.ones_like(top_ps)) + greedy = temperatures <= 0 + safe_temperatures = torch.where(greedy, torch.ones_like(temperatures), temperatures) + x.div_(safe_temperatures.unsqueeze(-1)) + x = apply_top_k_top_p(x, top_ks, top_ps) # Detect all-masked rows BEFORE softmax (softmax of all-inf yields NaN). has_finite = torch.isfinite(x).any(dim=-1) all_masked = ~has_finite @@ -1362,7 +1387,8 @@ class HiggsAudioV3TalkerForConditionalGeneration(nn.Module): safe_x = torch.where(all_masked.unsqueeze(-1), torch.zeros_like(x), x) probs = safe_x.softmax(dim=-1) sampled = torch.multinomial(probs, num_samples=1).squeeze(-1) - return torch.where(all_masked, fallback, sampled) + sampled = torch.where(all_masked, fallback, sampled) + return torch.where(greedy, logits_2d.argmax(dim=-1), sampled) # ------------------------------------------------------------------ omni output def make_omni_output(self, model_outputs: torch.Tensor | OmniOutput, **kwargs: Any) -> OmniOutput: From b6c4578dac3100a8494abebbbdf212a1e4331743 Mon Sep 17 00:00:00 2001 From: y-null Date: Sun, 26 Jul 2026 08:00:13 +0800 Subject: [PATCH 095/240] [BugFix]Add MiniCPM-o 4.5 end-to-end test suite (offline, online serving) (#5237) Signed-off-by: y_null Signed-off-by: amy-why-3459 Co-authored-by: amy-why-3459 Co-authored-by: Hongsheng Liu --- .buildkite/cuda/test-merge.yml | 42 ++++ .buildkite/cuda/test-ready.yml | 76 ++++++ .buildkite/npu/test-npu-nightly.yml | 22 ++ .buildkite/npu/test-npu-ready.yml | 21 ++ docker/Dockerfile.npu.ci | 14 ++ docker/Dockerfile.npu.ci.a3 | 14 ++ examples/offline_inference/minicpmo/README.md | 2 +- examples/online_serving/minicpmo/README.md | 5 +- pyproject.toml | 13 +- recipes/OpenBMB/MiniCPM-o-4_5.md | 5 +- .../sched/test_omni_ar_scheduler_streaming.py | 50 ++++ .../test_invalid_minicpmo_4_5_omni.py | 80 ++++++ .../offline_inference/test_minicpmo_4_5.py | 118 +++++++++ tests/e2e/online_serving/test_minicpmo_4_5.py | 233 ++++++++++++++++++ .../test_minicpmo_4_5_expansion.py | 178 +++++++++++++ .../entrypoints/test_stream_finish_reason.py | 84 +++++++ tests/helpers/runtime.py | 25 +- .../minicpmo_4_5/test_code2wav_batching.py | 80 ++++++ .../minicpmo_4_5/test_talker_batching.py | 32 +++ vllm_omni/core/sched/omni_ar_scheduler.py | 1 + vllm_omni/entrypoints/openai/serving_chat.py | 30 ++- .../models/minicpmo_4_5/batched_token2wav.py | 45 +++- .../minicpmo_4_5/minicpmo_4_5_code2wav.py | 94 ++++++- .../minicpmo_4_5/minicpmo_4_5_omni_tts.py | 43 +++- .../minicpmo_4_5/minicpmo_4_5_token2wav.py | 38 +++ .../minicpmo_4_5_omni.py | 10 +- .../platforms/npu/worker/npu_model_runner.py | 29 +++ 27 files changed, 1338 insertions(+), 46 deletions(-) create mode 100644 tests/dfx/reliability/invalid_param_test/test_invalid_minicpmo_4_5_omni.py create mode 100644 tests/e2e/offline_inference/test_minicpmo_4_5.py create mode 100644 tests/e2e/online_serving/test_minicpmo_4_5.py create mode 100644 tests/e2e/online_serving/test_minicpmo_4_5_expansion.py diff --git a/.buildkite/cuda/test-merge.yml b/.buildkite/cuda/test-merge.yml index 46ed531f3..22da9617b 100644 --- a/.buildkite/cuda/test-merge.yml +++ b/.buildkite/cuda/test-merge.yml @@ -123,6 +123,48 @@ steps: - pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py tests/e2e/online_serving/test_qwen3_omni.py -m 'advanced_model and cuda' --run-level 'advanced_model' mirror_hardwares: h100_2 + - label: "Omni · MiniCPM-o 4.5 Test" + source_file_dependencies: + - tests/e2e/offline_inference/test_minicpmo_4_5.py + - tests/e2e/online_serving/test_minicpmo_4_5.py + - vllm_omni/deploy/minicpmo_4_5.yaml + timeout_in_minutes: 50 + commands: + - pytest -s -v tests/e2e/offline_inference/test_minicpmo_4_5.py tests/e2e/online_serving/test_minicpmo_4_5.py -m 'advanced_model and cuda' --run-level 'advanced_model' + agents: + queue: "mithril-h100-pool" + plugins: + - kubernetes: + podSpec: + containers: + - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT + resources: + limits: + nvidia.com/gpu: 2 + volumeMounts: + - name: devshm + mountPath: /dev/shm + - name: hf-cache + mountPath: /root/.cache/huggingface + env: + - name: HF_HOME + value: /root/.cache/huggingface + - name: HF_TOKEN + valueFrom: + secretKeyRef: + name: hf-token-secret + key: token + nodeSelector: + node.kubernetes.io/instance-type: gpu-h100-sxm + volumes: + - name: devshm + emptyDir: + medium: Memory + - name: hf-cache + hostPath: + path: /mnt/hf-cache + type: DirectoryOrCreate + - label: "TTS · VoxCPM2 Test" source_file_dependencies: - tests/e2e/offline_inference/test_voxcpm2_tts.py diff --git a/.buildkite/cuda/test-ready.yml b/.buildkite/cuda/test-ready.yml index 90e39b12a..664c6c711 100644 --- a/.buildkite/cuda/test-ready.yml +++ b/.buildkite/cuda/test-ready.yml @@ -134,6 +134,82 @@ steps: " mirror_hardwares: h100_2 + - label: "Omni · MiniCPM-o 4.5 Offline Test" + source_file_dependencies: + - tests/e2e/offline_inference/test_minicpmo_4_5.py + - vllm_omni/deploy/minicpmo_4_5.yaml + commands: + - | + timeout 20m bash -c " + pytest -s -v tests/e2e/offline_inference/test_minicpmo_4_5.py -m 'core_model' --run-level 'core_model' + " + agents: + queue: "mithril-h100-pool" + plugins: + - kubernetes: + podSpec: + containers: + - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT + resources: + limits: + nvidia.com/gpu: 2 + volumeMounts: + - name: devshm + mountPath: /dev/shm + - name: hf-cache + mountPath: /root/.cache/huggingface + env: + - name: HF_HOME + value: /root/.cache/huggingface + nodeSelector: + node.kubernetes.io/instance-type: gpu-h100-sxm + volumes: + - name: devshm + emptyDir: + medium: Memory + - name: hf-cache + hostPath: + path: /mnt/hf-cache + type: DirectoryOrCreate + + - label: "Omni · MiniCPM-o 4.5 Online Test" + source_file_dependencies: + - tests/e2e/online_serving/test_minicpmo_4_5.py + - vllm_omni/deploy/minicpmo_4_5.yaml + commands: + - | + timeout 20m bash -c " + pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5.py -m 'core_model' --run-level 'core_model' + " + agents: + queue: "mithril-h100-pool" + plugins: + - kubernetes: + podSpec: + containers: + - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT + resources: + limits: + nvidia.com/gpu: 2 + volumeMounts: + - name: devshm + mountPath: /dev/shm + - name: hf-cache + mountPath: /root/.cache/huggingface + env: + - name: HF_HOME + value: /root/.cache/huggingface + nodeSelector: + node.kubernetes.io/instance-type: gpu-h100-sxm + volumes: + - name: devshm + emptyDir: + medium: Memory + - name: hf-cache + hostPath: + path: /mnt/hf-cache + type: DirectoryOrCreate + - label: "TTS · VoxCPM2 Test" source_file_dependencies: - tests/e2e/online_serving/test_voxcpm2_tts.py diff --git a/.buildkite/npu/test-npu-nightly.yml b/.buildkite/npu/test-npu-nightly.yml index 4050cca8d..e05178c52 100644 --- a/.buildkite/npu/test-npu-nightly.yml +++ b/.buildkite/npu/test-npu-nightly.yml @@ -34,3 +34,25 @@ steps: EXIT=$$? buildkite-agent artifact upload "tests/dfx/perf/results/*.json" exit $$EXIT + + - group: ":card_index_dividers: Omni Model Test" + key: nightly-omni-group + depends_on: upload-nightly-pipeline + if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" || build.pull_request.labels includes "omni-test" + steps: + - label: ":robot: Omni · MiniCPM-o 4.5 Test" + agents: + queue: "ascend-a3" + resource_class: "npu-2" + image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}-a3" + plugins: + - kubernetes: + podSpecPatch: + imagePullSecrets: + - name: swr-secret + env: + VLLM_WORKER_MULTIPROC_METHOD: spawn + VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1" + HF_TOKEN: "${HF_TOKEN}" + commands: + - pytest -s -v tests/e2e/offline_inference/test_minicpmo_4_5.py tests/e2e/online_serving/test_minicpmo_4_5.py tests/e2e/online_serving/test_minicpmo_4_5_expansion.py -m 'full_model' --run-level 'full_model' diff --git a/.buildkite/npu/test-npu-ready.yml b/.buildkite/npu/test-npu-ready.yml index 317517398..956e8acd6 100644 --- a/.buildkite/npu/test-npu-ready.yml +++ b/.buildkite/npu/test-npu-ready.yml @@ -1,4 +1,25 @@ steps: + - group: ":card_index_dividers: Omni Model Test" + key: ready-omni-group + depends_on: upload-ready-pipeline + steps: + - label: ":robot: Omni · MiniCPM-o 4.5 Test" + agents: + queue: "ascend-a3" + resource_class: "npu-2" + image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}-a3" + plugins: + - kubernetes: + podSpecPatch: + imagePullSecrets: + - name: swr-secret + env: + VLLM_WORKER_MULTIPROC_METHOD: spawn + VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1" + HF_TOKEN: "${HF_TOKEN}" + commands: + - pytest -s -v tests/e2e/offline_inference/test_minicpmo_4_5.py tests/e2e/online_serving/test_minicpmo_4_5.py -m 'core_model' --run-level 'core_model' + - group: ":card_index_dividers: TTS Model Test" key: ready-tts-group depends_on: upload-ready-pipeline diff --git a/docker/Dockerfile.npu.ci b/docker/Dockerfile.npu.ci index a3cede3a7..c0b4af9b6 100644 --- a/docker/Dockerfile.npu.ci +++ b/docker/Dockerfile.npu.ci @@ -37,6 +37,9 @@ RUN source /usr/local/Ascend/ascend-toolkit/set_env.sh && \ "pyttsx3>=2.99" \ "opencv-python>=4.12.0.88" \ "imageio[ffmpeg]>=0.6.0" \ + "hyperpyyaml>=1.2.0" \ + "onnx>=1.16.0" \ + "librosa>=0.10.0" \ --index-url http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple \ --trusted-host cache-service.nginx-pypi-cache.svc.cluster.local \ --default-timeout=60 && \ @@ -45,6 +48,17 @@ RUN source /usr/local/Ascend/ascend-toolkit/set_env.sh && \ --trusted-host cache-service.nginx-pypi-cache.svc.cluster.local \ --default-timeout=60 +# MiniCPM-o 4.5 Stage-2 vocoder assets: s3tokenizer plus flashcosyvoice, which +# is shipped by step-audio2. --no-deps keeps pip from replacing the Ascend torch +# build and from shadowing onnxruntime-cann with the vanilla onnxruntime wheel; +# the deps these two actually need are installed above. +RUN python3 -m pip install -v --no-deps \ + "s3tokenizer>=0.3.0" \ + "step-audio2>=1.0.0" \ + --index-url http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple \ + --trusted-host cache-service.nginx-pypi-cache.svc.cluster.local \ + --default-timeout=60 + ENV VLLM_WORKER_MULTIPROC_METHOD=spawn ENTRYPOINT [] diff --git a/docker/Dockerfile.npu.ci.a3 b/docker/Dockerfile.npu.ci.a3 index 1369d8c85..b86d7c0f8 100644 --- a/docker/Dockerfile.npu.ci.a3 +++ b/docker/Dockerfile.npu.ci.a3 @@ -37,6 +37,9 @@ RUN source /usr/local/Ascend/ascend-toolkit/set_env.sh && \ "pyttsx3>=2.99" \ "opencv-python>=4.12.0.88" \ "imageio[ffmpeg]>=0.6.0" \ + "hyperpyyaml>=1.2.0" \ + "onnx>=1.16.0" \ + "librosa>=0.10.0" \ --index-url http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple \ --trusted-host cache-service.nginx-pypi-cache.svc.cluster.local \ --default-timeout=60 && \ @@ -45,6 +48,17 @@ RUN source /usr/local/Ascend/ascend-toolkit/set_env.sh && \ --trusted-host cache-service.nginx-pypi-cache.svc.cluster.local \ --default-timeout=60 +# MiniCPM-o 4.5 Stage-2 vocoder assets: s3tokenizer plus flashcosyvoice, which +# is shipped by step-audio2. --no-deps keeps pip from replacing the Ascend torch +# build and from shadowing onnxruntime-cann with the vanilla onnxruntime wheel; +# the deps these two actually need are installed above. +RUN python3 -m pip install -v --no-deps \ + "s3tokenizer>=0.3.0" \ + "step-audio2>=1.0.0" \ + --index-url http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple \ + --trusted-host cache-service.nginx-pypi-cache.svc.cluster.local \ + --default-timeout=60 + ENV VLLM_WORKER_MULTIPROC_METHOD=spawn ENTRYPOINT [] diff --git a/examples/offline_inference/minicpmo/README.md b/examples/offline_inference/minicpmo/README.md index bb6ce2568..b80fc0aa6 100644 --- a/examples/offline_inference/minicpmo/README.md +++ b/examples/offline_inference/minicpmo/README.md @@ -6,7 +6,7 @@ Two-stage pipeline: **thinker** (multimodal understanding) → **talker + Token2 ## Setup -- Install talker deps: `pip install 'vllm-omni[minicpmo]'` (or `stepaudio2-minicpmo`) +- Install talker deps: `pip install stepaudio2-minicpmo` - `--trust-remote-code` is always passed by `end2end.py` (required for MiniCPMO) - See [stage configuration docs](https://docs.vllm.ai/projects/vllm-omni/en/latest/configuration/stage_configs/) for memory tuning diff --git a/examples/online_serving/minicpmo/README.md b/examples/online_serving/minicpmo/README.md index 39bbef09b..148fbb5e2 100644 --- a/examples/online_serving/minicpmo/README.md +++ b/examples/online_serving/minicpmo/README.md @@ -13,10 +13,7 @@ capability boundary, and validation scope, see Install vLLM-Omni with the MiniCPM-o talker dependencies: ```bash -pip install 'vllm-omni[minicpmo]' - -# From a source checkout: -pip install -e '.[minicpmo]' +pip install stepaudio2-minicpmo ``` The `minicpmo` extra installs `stepaudio2-minicpmo` and its audio dependencies, diff --git a/pyproject.toml b/pyproject.toml index c71e17b4e..c442c3164 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -71,6 +71,7 @@ dev = [ "kernels==0.14.1", # for accuracy tests comparison with diffusers output (0.15.0 incompatible with diffusers 0.38.0) "s3tokenizer==0.3.0", # for tts test "step-audio2==1.0.0", # Step-Audio2 Token2Wav (flashcosyvoice / hyperpyyaml) + "stepaudio2-minicpmo", # for MiniCPM-o 4.5 talker vocoder (stepaudio2-minicpmo Token2wav) ] demo = [ @@ -88,18 +89,6 @@ local = [ "FlagEmbedding", ] -# Optional runtime dependencies for the MiniCPM-o 4.5 talker pipeline. -# The talker loads MiniCPMTTS via `trust_remote_code` from the model repo, -# and that module performs `from stepaudio2 import Token2wav` against the -# MiniCPM-o-flavored Token2wav vocoder (PyPI: `stepaudio2-minicpmo`, NOT -# upstream `stepfun-ai/Step-Audio2` — the upstream signature rejects -# `n_timesteps` and crashes at construction). A clean `vllm-omni` install -# does not pull this in by default to avoid forcing the dep onto users -# who only need other models; opt in with `pip install vllm-omni[minicpmo]`. -minicpmo = [ - "stepaudio2-minicpmo", -] - # Optional Blackwell-only fused-bias FP8 GEMM; package is `quack-kernels` (imported # as `quack`), not the unrelated `quack`. See docs/user_guide/quantization/fp8.md. quack = [ diff --git a/recipes/OpenBMB/MiniCPM-o-4_5.md b/recipes/OpenBMB/MiniCPM-o-4_5.md index 73d0fd5a5..dddc9fac4 100644 --- a/recipes/OpenBMB/MiniCPM-o-4_5.md +++ b/recipes/OpenBMB/MiniCPM-o-4_5.md @@ -256,11 +256,10 @@ vllm serve openbmb/MiniCPM-o-4_5 --omni \ rejects `n_timesteps`). Install via the published extra: ```bash - pip install 'vllm-omni[minicpmo]' + pip install stepaudio2-minicpmo ``` - Equivalent direct install: `pip install stepaudio2-minicpmo`. A - missing dep raises `ImportError` at first request with the same + A missing dep raises `ImportError` at first request with the same install hint instead of silently emitting empty audio. - **TTS conditioning**: the MiniCPM stage bridge can condition speech from diff --git a/tests/core/sched/test_omni_ar_scheduler_streaming.py b/tests/core/sched/test_omni_ar_scheduler_streaming.py index 9c0533059..06037d4cf 100644 --- a/tests/core/sched/test_omni_ar_scheduler_streaming.py +++ b/tests/core/sched/test_omni_ar_scheduler_streaming.py @@ -137,6 +137,56 @@ def test_resumable_session_terminal_is_not_marked_as_segment_boundary() -> None: assert output.is_segment_finished is False +def test_running_decode_step_without_inter_stage_payload_does_not_raise() -> None: + """A decode step that neither stops nor carries an inter-stage payload. + + ``finished`` is only assigned when the request stops, yet the async-chunk + save condition reads it for every request, so this step used to raise + ``UnboundLocalError: cannot access local variable 'finished'``. + """ + session = _make_request() + session.status = RequestStatus.RUNNING + + sched = MagicMock() + sched.requests = {session.request_id: session} + sched.perf_metrics = None + sched.structured_output_manager.should_advance.return_value = False + sched._update_request_with_output.return_value = ([42], False) + sched._process_kv_transfer_trigger.return_value = False + sched.chunk_transfer_adapter = MagicMock() + sched.running = [session] + sched.waiting_for_transfer_free = set() + sched.transfer_triggered_requests = set() + sched.active_kv_transfers = set() + sched.pending_stop_after_extraction = set() + sched.connector = None + sched.kv_cache_manager.take_events.return_value = None + sched.finished_req_ids_dict = {} + sched.make_stats.return_value = None + + scheduler_output = MagicMock(spec=SchedulerOutput) + scheduler_output.num_scheduled_tokens = {session.request_id: 1} + scheduler_output.scheduled_spec_decode_tokens = {} + scheduler_output.num_invalid_spec_tokens = 0 + + model_runner_output = MagicMock(spec=ModelRunnerOutput) + model_runner_output.sampled_token_ids = [[42]] + model_runner_output.logprobs = None + model_runner_output.prompt_logprobs_dict = {} + model_runner_output.pooler_output = None + model_runner_output.num_nans_in_logits = None + model_runner_output.kv_connector_output = None + model_runner_output.cudagraph_stats = None + model_runner_output.req_id_to_index = {session.request_id: 0} + model_runner_output.routed_experts = None + model_runner_output.inter_stage_outputs = None + + OmniARScheduler.update_from_output(sched, scheduler_output, model_runner_output) + + # Nothing to hand downstream: no payload, no segment boundary, not finished. + sched.chunk_transfer_adapter.save_async.assert_not_called() + + def test_stage0_streaming_update_discards_outstanding_async_placeholder_token() -> None: sched = _make_scheduler(stage_id=0) session = _make_request() diff --git a/tests/dfx/reliability/invalid_param_test/test_invalid_minicpmo_4_5_omni.py b/tests/dfx/reliability/invalid_param_test/test_invalid_minicpmo_4_5_omni.py new file mode 100644 index 000000000..c71dc2d16 --- /dev/null +++ b/tests/dfx/reliability/invalid_param_test/test_invalid_minicpmo_4_5_omni.py @@ -0,0 +1,80 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Invalid audio format rejection on MiniCPM-o 4.5: ``POST /v1/chat/completions``.""" + +import os + +os.environ.setdefault("VLLM_WORKER_MULTIPROC_METHOD", "spawn") + +import pytest + +from tests.helpers.mark import hardware_test +from tests.helpers.runtime import OmniServerParams, OpenAIClientHandler, dummy_messages_from_mix_data +from tests.helpers.stage_config import get_deploy_config_path + +pytestmark = [pytest.mark.slow, pytest.mark.omni, pytest.mark.full_model] + +_MINICPMO_DEPLOY = get_deploy_config_path("minicpmo_4_5_batching.yaml") + +_MINICPMO_SERVER = [ + pytest.param( + OmniServerParams( + model="openbmb/MiniCPM-o-4_5", + stage_config_path=_MINICPMO_DEPLOY, + use_stage_cli=True, + server_args=[ + "--trust-remote-code", + "--no-async-chunk", + ], + ), + id="minicpmo_4_5", + ) +] + + +def _system_prompt() -> dict[str, object]: + return { + "role": "system", + "content": [ + { + "type": "text", + "text": ( + "You are MiniCPM-o 4.5, a virtual human capable of perceiving " + "auditory and visual inputs, as well as generating text and speech." + ), + } + ], + } + + +def _prompt() -> str: + return "What is the capital of China? Answer in 20 words." + + +@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2) +@pytest.mark.parametrize("omni_server", _MINICPMO_SERVER, indirect=True) +def test_invalid_audio_format_rejected(omni_server: OmniServerParams, openai_client: OpenAIClientHandler) -> None: + """ + Test that invalid audio format is properly rejected. + Deploy Setting: default 2GPU + Input Modal: text + invalid audio format + Output Modal: error + """ + fake_base64 = "AAAA" # not a real audio file + + responses = openai_client.send_chat_completions_http_request( + { + "json": { + "model": omni_server.model, + "messages": dummy_messages_from_mix_data( + system_prompt=_system_prompt(), + audio_data_url=f"data:audio/invalid;base64,{fake_base64}", + content_text=_prompt(), + ), + "stream": True, + }, + }, + err_code=400, + err_message="Invalid or unsupported audio file", + ) + assert not responses[0].success diff --git a/tests/e2e/offline_inference/test_minicpmo_4_5.py b/tests/e2e/offline_inference/test_minicpmo_4_5.py new file mode 100644 index 000000000..ee1dc4d01 --- /dev/null +++ b/tests/e2e/offline_inference/test_minicpmo_4_5.py @@ -0,0 +1,118 @@ +""" +E2E offline tests for MiniCPM-o 4.5 model with multimodal input and audio / text output. +""" + +import os + +os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" + +import pytest + +from tests.helpers.mark import hardware_test +from tests.helpers.media import generate_synthetic_audio, generate_synthetic_image, generate_synthetic_video +from tests.helpers.stage_config import get_deploy_config_path + +models = ["openbmb/MiniCPM-o-4_5"] + +_CI_DEPLOY = get_deploy_config_path("minicpmo_4_5_batching.yaml") + + +test_params = [(model, None, {"deploy_config": _CI_DEPLOY, "trust_remote_code": True}) for model in models] + + +def get_question(prompt_type: str = "text") -> str: + prompts = { + "text": "What is the capital of China? Answer in 20 words.", + "audio": "Describe the audio briefly.", + "image": "What color are the squares in this image?", + "video": "Describe the video briefly.", + "mix": "Describe what is in the image and audio.", + } + return prompts.get(prompt_type, prompts["text"]) + + +@pytest.mark.core_model +@pytest.mark.advanced_model +@pytest.mark.omni +@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2) +@pytest.mark.parametrize("omni_runner", test_params, indirect=True) +def test_text_to_text(omni_runner, omni_runner_handler) -> None: + """Test processing text, generating text output.""" + request_config = {"prompts": get_question("text"), "modalities": ["text"]} + omni_runner_handler.send_omni_request(request_config) + + +@pytest.mark.full_model +@pytest.mark.omni +@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2) +@pytest.mark.parametrize("omni_runner", test_params, indirect=True) +def test_audio_to_text(omni_runner, omni_runner_handler) -> None: + """Test processing audio, generating text output.""" + audio = generate_synthetic_audio(1, 1, 16000)["np_array"] + if len(audio.shape) == 2: + audio = audio.squeeze() + request_config = {"prompts": get_question("audio"), "audios": (audio, 16000), "modalities": ["text"]} + omni_runner_handler.send_omni_request(request_config) + + +@pytest.mark.full_model +@pytest.mark.omni +@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2) +@pytest.mark.parametrize("omni_runner", test_params, indirect=True) +def test_image_to_text(omni_runner, omni_runner_handler) -> None: + """Test processing image, generating text output.""" + image = generate_synthetic_image(16, 16)["np_array"] + request_config = {"prompts": get_question("image"), "images": image, "modalities": ["text"]} + omni_runner_handler.send_omni_request(request_config) + + +@pytest.mark.full_model +@pytest.mark.omni +@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2) +@pytest.mark.parametrize("omni_runner", test_params, indirect=True) +def test_video_to_text(omni_runner, omni_runner_handler) -> None: + """Test processing video, generating text output.""" + video = generate_synthetic_video(24, 24, 20)["np_array"] + request_config = {"prompts": get_question("video"), "videos": video, "modalities": ["text"]} + omni_runner_handler.send_omni_request(request_config) + + +@pytest.mark.full_model +@pytest.mark.omni +@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2) +@pytest.mark.parametrize("omni_runner", test_params, indirect=True) +def test_text_to_audio(omni_runner, omni_runner_handler) -> None: + """Test processing text, generating audio output through the talker token2wav path.""" + request_config = {"prompts": get_question("text"), "modalities": ["audio"]} + omni_runner_handler.send_omni_request(request_config) + + +@pytest.mark.core_model +@pytest.mark.advanced_model +@pytest.mark.omni +@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2) +@pytest.mark.parametrize("omni_runner", test_params, indirect=True) +def test_mix_to_audio(omni_runner, omni_runner_handler) -> None: + """Test processing mixed modalities (image + audio), generating audio output.""" + audio = generate_synthetic_audio(1, 1, 16000)["np_array"] + if len(audio.shape) == 2: + audio = audio.squeeze() + image = generate_synthetic_image(16, 16)["np_array"] + request_config = { + "prompts": get_question("mix"), + "audios": (audio, 16000), + "images": image, + "modalities": ["audio"], + } + omni_runner_handler.send_omni_request(request_config) + + +@pytest.mark.full_model +@pytest.mark.omni +@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2) +@pytest.mark.parametrize("omni_runner", test_params, indirect=True) +def test_video_to_audio(omni_runner, omni_runner_handler) -> None: + """Test processing video, generating audio output.""" + video = generate_synthetic_video(24, 24, 20)["np_array"] + request_config = {"prompts": get_question("video"), "videos": video, "modalities": ["audio"]} + omni_runner_handler.send_omni_request(request_config) diff --git a/tests/e2e/online_serving/test_minicpmo_4_5.py b/tests/e2e/online_serving/test_minicpmo_4_5.py new file mode 100644 index 000000000..7d2c7948e --- /dev/null +++ b/tests/e2e/online_serving/test_minicpmo_4_5.py @@ -0,0 +1,233 @@ +""" +E2E Online tests for MiniCPM-o 4.5 model with multimodal input and audio / text output. + +MiniCPM-o 4.5 has ``async_chunk: false``, ``max_num_seqs: 1`` on both stages, +and the vocoder runs in-process inside the talker stage rather than as a separate +Code2Wav stage. +""" + +import os + +import pytest + +from tests.helpers.mark import hardware_test +from tests.helpers.media import generate_synthetic_audio, generate_synthetic_image, generate_synthetic_video +from tests.helpers.runtime import OmniServerParams, dummy_messages_from_mix_data +from tests.helpers.stage_config import get_deploy_config_path + +os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" + +_MODEL = "openbmb/MiniCPM-o-4_5" +_CI_DEPLOY = get_deploy_config_path("minicpmo_4_5_batching.yaml") + +test_params = [ + pytest.param( + OmniServerParams( + model=_MODEL, + stage_config_path=_CI_DEPLOY, + use_stage_cli=True, + server_args=[ + "--trust-remote-code", + "--no-async-chunk", + ], + ), + id="default", + ) +] + + +def get_system_prompt(): + return { + "role": "system", + "content": [ + { + "type": "text", + "text": ( + "You are MiniCPM-o 4.5, a virtual human capable of perceiving " + "auditory and visual inputs, as well as generating text and speech." + ), + } + ], + } + + +def get_prompt(prompt_type: str = "text_only") -> str: + prompts = { + "text_only": "What is the capital of China? Answer in 20 words.", + "mix": "What is recited in the audio? What is in this image? Describe the video briefly.", + "text_image": "What color are the squares in this image?", + } + return prompts.get(prompt_type, prompts["text_only"]) + + +def get_max_batch_size(size_type="few"): + batch_sizes = {"few": 5, "medium": 100, "large": 256} + return batch_sizes.get(size_type, 5) + + +@pytest.mark.core_model +@pytest.mark.advanced_model +@pytest.mark.omni +@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2) +@pytest.mark.parametrize("omni_server", test_params, indirect=True) +def test_text_to_text_001(omni_server, openai_client) -> None: + """ + Test text-only input generating text output via OpenAI API. + Deploy Setting: default 2GPU + Input Modal: text + Output Modal: text + Input Setting: stream=False + """ + messages = dummy_messages_from_mix_data(system_prompt=get_system_prompt(), content_text=get_prompt()) + + request_config = { + "model": omni_server.model, + "messages": messages, + "stream": False, + "modalities": ["text"], + "key_words": {"text": ["Beijing"]}, + } + + openai_client.send_omni_request(request_config, request_num=get_max_batch_size()) + + +@pytest.mark.full_model +@pytest.mark.omni +@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2) +@pytest.mark.parametrize("omni_server", test_params, indirect=True) +def test_text_to_audio_001(omni_server, openai_client) -> None: + """ + Test text-only input generating text + audio output via OpenAI API. + This exercises the talker TTS region detection and in-process token2wav vocoder. + Deploy Setting: default 2GPU + Input Modal: text + Output Modal: text + audio + Input Setting: stream=True + """ + messages = dummy_messages_from_mix_data(system_prompt=get_system_prompt(), content_text=get_prompt()) + + request_config = { + "model": omni_server.model, + "messages": messages, + "stream": True, + "key_words": {"audio": ["Beijing"]}, + } + + openai_client.send_omni_request(request_config) + + +@pytest.mark.full_model +@pytest.mark.omni +@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2) +@pytest.mark.parametrize("omni_server", test_params, indirect=True) +def test_audio_to_text_audio_001(omni_server, openai_client) -> None: + """ + Test audio input generating text + audio output via OpenAI API. + Deploy Setting: default 2GPU + Input Modal: text + audio + Output Modal: text + audio + Input Setting: stream=True + """ + audio_data_url = f"data:audio/wav;base64,{generate_synthetic_audio(5, 1)['base64']}" + messages = dummy_messages_from_mix_data( + system_prompt=get_system_prompt(), + audio_data_url=audio_data_url, + content_text=get_prompt("mix"), + ) + + request_config = { + "model": omni_server.model, + "messages": messages, + "stream": True, + } + + openai_client.send_omni_request(request_config, request_num=get_max_batch_size()) + + +@pytest.mark.full_model +@pytest.mark.omni +@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2) +@pytest.mark.parametrize("omni_server", test_params, indirect=True) +def test_image_to_text_audio_001(omni_server, openai_client) -> None: + """ + Test image input generating text + audio output via OpenAI API. + Deploy Setting: default 2GPU + Input Modal: text + image + Output Modal: text + audio + Input Setting: stream=True + """ + image_data_url = f"data:image/jpeg;base64,{generate_synthetic_image(24, 24)['base64']}" + messages = dummy_messages_from_mix_data( + system_prompt=get_system_prompt(), + image_data_url=image_data_url, + content_text=get_prompt("text_image"), + ) + + request_config = { + "model": omni_server.model, + "messages": messages, + "stream": True, + } + + openai_client.send_omni_request(request_config, request_num=get_max_batch_size()) + + +@pytest.mark.full_model +@pytest.mark.omni +@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2) +@pytest.mark.parametrize("omni_server", test_params, indirect=True) +def test_video_to_text_audio_001(omni_server, openai_client) -> None: + """ + Test video input generating text + audio output via OpenAI API. + Deploy Setting: default 2GPU + Input Modal: text + video + Output Modal: text + audio + Input Setting: stream=True + """ + video_data_url = f"data:video/mp4;base64,{generate_synthetic_video(24, 24, 20)['base64']}" + messages = dummy_messages_from_mix_data( + system_prompt=get_system_prompt(), + video_data_url=video_data_url, + content_text=get_prompt("mix"), + ) + + request_config = { + "model": omni_server.model, + "messages": messages, + "stream": True, + } + + openai_client.send_omni_request(request_config, request_num=get_max_batch_size()) + + +@pytest.mark.core_model +@pytest.mark.advanced_model +@pytest.mark.omni +@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2) +@pytest.mark.parametrize("omni_server", test_params, indirect=True) +def test_mix_to_text_audio_001(omni_server, openai_client) -> None: + """ + Test multi-modal input (text + audio + video + image) generating text + audio output. + Deploy Setting: default 2GPU + Input Modal: text + audio + video + image + Output Modal: text + audio + Input Setting: stream=True + """ + video_data_url = f"data:video/mp4;base64,{generate_synthetic_video(24, 24, 20)['base64']}" + image_data_url = f"data:image/jpeg;base64,{generate_synthetic_image(24, 24)['base64']}" + audio_data_url = f"data:audio/wav;base64,{generate_synthetic_audio(5, 1)['base64']}" + messages = dummy_messages_from_mix_data( + system_prompt=get_system_prompt(), + video_data_url=video_data_url, + image_data_url=image_data_url, + audio_data_url=audio_data_url, + content_text=get_prompt("mix"), + ) + + request_config = { + "model": omni_server.model, + "messages": messages, + "stream": True, + } + + openai_client.send_omni_request(request_config, request_num=get_max_batch_size()) diff --git a/tests/e2e/online_serving/test_minicpmo_4_5_expansion.py b/tests/e2e/online_serving/test_minicpmo_4_5_expansion.py new file mode 100644 index 000000000..8db6cd92b --- /dev/null +++ b/tests/e2e/online_serving/test_minicpmo_4_5_expansion.py @@ -0,0 +1,178 @@ +""" +E2E Online expansion tests for MiniCPM-o 4.5 covering modality combinations, +in-process token2wav vocoder behavior, max_num_seqs=1 serial execution, +and edge-case validation. +""" + +import os + +import pytest + +from tests.helpers.mark import hardware_test +from tests.helpers.media import generate_synthetic_video +from tests.helpers.runtime import OmniServerParams, dummy_messages_from_mix_data +from tests.helpers.stage_config import get_deploy_config_path + +pytestmark = [pytest.mark.full_model, pytest.mark.omni] + +os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" + +_MODEL = "openbmb/MiniCPM-o-4_5" +_CI_DEPLOY = get_deploy_config_path("minicpmo_4_5_batching.yaml") + +test_params = [ + pytest.param( + OmniServerParams( + model=_MODEL, + stage_config_path=_CI_DEPLOY, + use_stage_cli=True, + server_args=[ + "--trust-remote-code", + ], + ), + id="default", + ) +] + + +def get_system_prompt(): + return { + "role": "system", + "content": [ + { + "type": "text", + "text": ( + "You are MiniCPM-o 4.5, a virtual human capable of perceiving " + "auditory and visual inputs, as well as generating text and speech." + ), + } + ], + } + + +def get_prompt(prompt_type: str = "text_only") -> str: + prompts = { + "text_only": "What is the capital of China? Answer in 20 words.", + "mix": "What is recited in the audio? What is in this image? Describe the video briefly.", + } + return prompts.get(prompt_type, prompts["text_only"]) + + +def get_max_batch_size(size_type="few"): + batch_sizes = {"few": 5, "medium": 100, "large": 256} + return batch_sizes.get(size_type, 5) + + +@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2) +@pytest.mark.parametrize("omni_server", test_params, indirect=True) +def test_text_video_to_text_001(omni_server, openai_client) -> None: + """ + Test text + video input generating text output. + Deploy Setting: default 2GPU + Input Modal: text + video + Output Modal: text + Input Setting: stream=False + """ + video_data_url = f"data:video/mp4;base64,{generate_synthetic_video(24, 24, 20)['base64']}" + messages = dummy_messages_from_mix_data( + system_prompt=get_system_prompt(), + video_data_url=video_data_url, + content_text=get_prompt("mix"), + ) + + request_config = { + "model": omni_server.model, + "messages": messages, + "stream": False, + "modalities": ["text"], + } + + openai_client.send_omni_request(request_config, request_num=get_max_batch_size()) + + +@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2) +@pytest.mark.parametrize("omni_server", test_params, indirect=True) +def test_sequential_requests_independent(omni_server, openai_client) -> None: + """ + Verify that sequential requests produce independent results and state + does not leak between requests. MiniCPM-o 4.5 has max_num_seqs=1 so + requests are processed one-at-a-time; the second request must not + receive the first request's audio or token state. + Deploy Setting: default 2GPU + Input Modal: text (two different prompts) + Output Modal: text + audio (both) + """ + messages_1 = dummy_messages_from_mix_data( + system_prompt=get_system_prompt(), + content_text="What is the capital of France?", + ) + messages_2 = dummy_messages_from_mix_data( + system_prompt=get_system_prompt(), + content_text="What is the capital of China?", + ) + + # Send first request and ensure it completes + openai_client.send_omni_request( + { + "model": omni_server.model, + "messages": messages_1, + "stream": True, + }, + request_num=1, + ) + + # Send second request with different prompt; must produce its own output + openai_client.send_omni_request( + { + "model": omni_server.model, + "messages": messages_2, + "stream": True, + "key_words": {"text": ["Beijing"]}, + }, + request_num=1, + ) + + +@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2) +@pytest.mark.parametrize("omni_server", test_params, indirect=True) +def test_text_to_audio_long_output_001(omni_server, openai_client) -> None: + """ + Test text input generating a longer audio output to exercise the + token2wav decoder across multiple frames. + Deploy Setting: default 2GPU + Input Modal: text (longer prompt) + Output Modal: text + audio + Input Setting: stream=True + """ + messages = dummy_messages_from_mix_data( + system_prompt=get_system_prompt(), + content_text="Tell me a short story about a cat in about 50 words.", + ) + + request_config = { + "model": omni_server.model, + "messages": messages, + "stream": True, + "key_words": {"audio": ["cat"]}, + } + + openai_client.send_omni_request(request_config, request_num=get_max_batch_size()) + + +@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2) +@pytest.mark.parametrize("omni_server", test_params, indirect=True) +def test_chinese_text_to_audio(omni_server, openai_client) -> None: + """ + Test Chinese text input generating audio output. + """ + messages = dummy_messages_from_mix_data( + system_prompt=get_system_prompt(), + content_text="北京,中国的首都,是一座融合了长城等历史地点与现代建筑的国际化大都市,充满了独特的文化与活力。请重复这句话。", + ) + request_config = { + "model": omni_server.model, + "messages": messages, + "stream": True, + "key_words": {"text": ["北京"]}, + } + openai_client.send_omni_request(request_config) diff --git a/tests/entrypoints/test_stream_finish_reason.py b/tests/entrypoints/test_stream_finish_reason.py index ebbca1917..63ec9f858 100644 --- a/tests/entrypoints/test_stream_finish_reason.py +++ b/tests/entrypoints/test_stream_finish_reason.py @@ -403,6 +403,90 @@ async def test_declared_modality_not_produced_text_finish_suppressed(): assert ch["finish_reason"] is None +# --------------------------------------------------------------------------- +# Tests: audio chunks that carry no waveform +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_audio_chunk_without_waveform_keeps_stream_alive(): + """An audio message with no PCM must not break the stream. + + ``_create_audio_choice`` used to return an ErrorResponse here, which the + stream generator then iterated as a list of choices; the pydantic model + yields (field, value) tuples, so the loop raised + ``AttributeError: 'tuple' object has no attribute 'finish_reason'``. + """ + serving_chat = _build_serving_chat() + request = _make_request(modalities=["text", "audio"]) + + empty_audio = _make_audio_omni_output() + empty_audio.request_output.outputs[0].multimodal_output = {"audio": []} + + def create_audio_choice(omni_res, role, request, stream=False): + return OmniOpenAIServingChat._create_audio_choice(serving_chat, omni_res, role, request, stream=stream) + + serving_chat._create_audio_choice = create_audio_choice + + async def result_generator(): + yield _make_text_omni_output(text="hi", token_ids=[10], finish_reason=None) + yield _make_text_omni_output(text="!", token_ids=[11], finish_reason="stop") + yield empty_audio + + raw_lines = await _collect_stream( + serving_chat.chat_completion_stream_generator( + request=request, + result_generator=result_generator(), + request_id="test-req", + model_name="test-model", + conversation=[], + tokenizer=MagicMock(), + request_metadata=MagicMock(), + ) + ) + + assert not any("Error in chat completion stream generator" in line for line in raw_lines) + chunks = _parse_sse_chunks(raw_lines) + finish_reasons = [ch["finish_reason"] for c in chunks for ch in c.get("choices", [])] + assert finish_reasons.count("stop") == 1, f"Expected 1 stop, got {finish_reasons}" + assert finish_reasons[-1] == "stop" + + +@pytest.mark.asyncio +async def test_audio_choice_error_response_is_not_iterated_as_choices(): + """Any ErrorResponse from the audio path is skipped, not iterated.""" + from vllm.entrypoints.openai.engine.protocol import ErrorResponse + + serving_chat = _build_serving_chat() + request = _make_request(modalities=["text", "audio"]) + + serving_chat._create_audio_choice = MagicMock( + side_effect=lambda omni_res, role, request, stream=False: serving_chat._create_error_response("boom") + ) + + async def result_generator(): + yield _make_text_omni_output(text="hi", token_ids=[10], finish_reason="stop") + yield _make_audio_omni_output() + + raw_lines = await _collect_stream( + serving_chat.chat_completion_stream_generator( + request=request, + result_generator=result_generator(), + request_id="test-req", + model_name="test-model", + conversation=[], + tokenizer=MagicMock(), + request_metadata=MagicMock(), + ) + ) + + assert isinstance(serving_chat._create_error_response("boom"), ErrorResponse) + assert not any("AttributeError" in line for line in raw_lines) + chunks = _parse_sse_chunks(raw_lines) + finish_reasons = [ch["finish_reason"] for c in chunks for ch in c.get("choices", [])] + assert finish_reasons.count("stop") == 1, f"Expected 1 stop, got {finish_reasons}" + + # --------------------------------------------------------------------------- # Tests: voice/speaker parameter compatibility # --------------------------------------------------------------------------- diff --git a/tests/helpers/runtime.py b/tests/helpers/runtime.py index 5d1d9a3d1..39baf1ce0 100644 --- a/tests/helpers/runtime.py +++ b/tests/helpers/runtime.py @@ -2424,6 +2424,10 @@ class OmniRunner: video_padding_token = "<|VIDEO|>" image_padding_token = "<|IMAGE|>" audio_padding_token = "<|AUDIO|>" + # Default wrapping for Qwen-style models (bos/eos around the placeholder). + audio_fmt = "<|audio_bos|>{p}<|audio_eos|>" + image_fmt = "<|vision_bos|>{p}<|vision_eos|>" + video_fmt = "<|vision_bos|>{p}<|vision_eos|>" if "Qwen3-Omni-30B-A3B-Instruct" in self.model_name: video_padding_token = "<|video_pad|>" image_padding_token = "<|image_pad|>" @@ -2432,6 +2436,15 @@ class OmniRunner: video_padding_token = "NQUVG zQ&m;B24VbW&!unEDm+M-T}DCw-UM(PG>1X&DJk2GlSK3)VvzUqJL#|ITf8au+w7j% zmVFp{sy%aJYwCXb%gd)+9|%8Cys%Le1rXBKwPYJ2B^}qO7Hs*ZpGxbKvb5oRy1@m< z0=vRj4RxMUdtCt92clphk%r@a^SjxGHP+wq|2RIlG9-H^`W&<;CbDA*2b^3w+U0APIvi`0!W2DJ6>1k2%A;&UO2dAZV5&#qp?qi{;P0r3LYz_9GL^nI(A?# ztltCMY_NXkGDE9ay16;Se(cfcyo`CM5Zn7?HNTuS>V&AdDHXevb{tRwr6^LO<;uVJpEDwXfq zv)~0YGl5YMiDeWAX*Z+en@htEjxz5?T_%?0nH}ZX0crjKWEm>p@jgXag8!Pr`#U_8 zz&vf?F$N~BU1cQFAnj|4xS#$jA>;k1k|?Ji=KA$7U!j>c=>K*Ae}^FaG_a4dP!PPwepQz`N~#BD7ld)D zMoqnQeK`vvI&$H&DXdFX`njL7=FM}3A0gg@4z3D?a5-TaDE#rKTZ2VNcR8J<_o{vS zO8NDDne95hJii1%PaOIy#QXsss+V}R;=nx-}0 zS~B4)Up{a%r!h~jdY_$w`?5sOa_#bY!IeWt`)CY&d<9~^sDkJ>KV3(|yYss9Q?H7(Z#}p5>NV@B(5O^I^@kKp0|MwDA+|&Y zbNn4x&PSlD#}EL=S&eRl{}uhC@`3R$1p$KJmwaw%ChA;nD&DszT0PZnOQGVk+@sY4 zcke6D;()R4Y!Dj+Q#LD2z>Z|Vgg(mjt-P94uvzL!3$nS zrgDk(*ws2Z5Md%cDJ}n&yOEI`I+`fm62A9D;3=CePIWHsv-JUPd6BYfTBuVpA^4%` z@4;Y&f-QDHojxX~Rr@+|oi4n6s`0?G@npE5ejP?5XmBrL@@H3^Lf|~+4Ol9?LVYp- z^7nkWa^S}Z>Vms!PLqwahUPuRKB~=@Do?%*Lk^USH~ofba$cRL=#o6UTGhV*5foOJ zJf3*5VtK1DbIg6VY(=q9z+p&{C&tnO;+NtyL!C;CSr}MdW|b^iS6|PTpuh9J7=vbB zRg_0<9DPYlB}y~~#X$Tf&WyqyCqri!=x&tZ?Hb(d`K|lyL~35WU6Q@@%&po>_@166 zWWD099;r+Fr7+AAU|&gA(;A`Y^;~qcw874VD+1D~xkY>Ua5y4(C>*d&tN?&KM?cwp zB>Pl;qpP;R<@>dDl_s62vf_xQgAV7p1B6#3uk?tEnRRSTApHJdy)5wKIQl8_Y=spk z6}m#cmH#o?M}hyXL;4-f+78Mg5mIA7TRhqgD5hVuXW$45x^ovb5kS<70s z(PB$N*_Ejzdy;KLbBXLUp(x5ITS~^RkV&>mmLftiNFp*5#)X-y-^=@R|L*&HpU-{n z-}(K{`TbKJN3ZL;Ud!|KTprKI18scn_5N-S_vY;l1`lw|6BcK{vr6&fBa<@5TBXP* zY;EM5=1JDG3m?bfEHvx9;oX9YD(gEd8pTaRmy)jOs4a$d+~5jMf02oSic^uDt!PWd z36qcViTaU2o?lc!KDpNuMOw~@<)UKYeIrYqC18&m0Jw8-GP>dNGfFf8cx`WFyn^T^ zd=pdWx6Xe1{9Vj>;Sc(y_`?v12$p+t7oK*oo33GrK;Gi!DI4HCX~JKtEj}E@`6W9P zaHQzg*ik^&^9K802qg&o-?r1_pK&oLFsKXI!n2+DT=27IG8Ma=22owmVz@a9aY>3A z-JZvv#!xG%Z1j4DQ9j-Vs=vo}tGI2YDFd5C|h zhZb>ODiF;jBGwJY_Zy=TUk~h@+vf4-cz_?9fJB%@G1w{eXc0XGJ{v%Jj&{#$H^<8O zx!?U!uKK3SASX&Ae78LNH@7&|m>Y39R}Exq5l%Ar-*{x?FA5FZzu+WtEpVi5w~QJ3 z!Q4QEX4q5OrXS_kl<<(5v9#0?K_6lJHrX$IyCQ3Erv9^_3-3EhyBTK8NAg){0#sm- z>N_eCgDv0byjy{(dVCe2B|cmQQ^tOP`EXeA5{BBsP`py4!s=SDh@dXTVjgM12SE*L zKMXt>wla8Y3L1rgT#44yS)YzbL@KRAlHNGe9G#!0SeNHUN#w=_X`Z{?(>h@h8965I z%hj1;*PjoL6c*u1&UG+>))$#nAuG1jOSiOm1AG_ne^-+^8<%A$#Am{P9C%Ohy^k{? zN`dC00YqQ|*rjMbWS3p|Q^v8z zx*BS3!rHTvfN0*6vzq(YgN4;>f(0AyR!g|tVLQ{=(gJ0I_5XKI9{4YoOMsk_XM>q{ z_xV>5+R5}jF7%zC&l{%knH#B$eTDqL@)2HeTgHvymJx1YYJPVW!xwfah6B2>yy4vJ z!<8|bHKo~4#um^4`Xsi{UiE>uM6dgb7LliKh6hjVo?(iSUSr?`S^LDi5a1Un2RQC31z{R{PVANEskJ;My$`rU3~@Jq+UJNT$YM7&(~H zUq~lW0GcrKY>gkZeo^^9le1l1R`LNG6J|iP)+!r3KJ+kjB-8NQKSp(%W}{A z*n~SQ%!+>2Bwst8vwQO6LN!P8Yuq+phrg;Q8d2cc@2iswB4?GgHvPuVX2~-+udu2 zE}aO9wpLXJAXd~(O zt$$?*LE?N_SJGxw7&6-l&;(eAW^)#HiUOuo z9G7^V)jrw?%J6#AD&f&<_~Y2r2jPwPN@k;HDM3osrz|*s17o-hAMD?*tm7FGwaf>t ze7L_T?57roryk8ehmrjscM`>lR5K9*Pl4CYdvR{|l!i|`PqHA!Y73L_9)OzKq645s zO-U*A^F0b_fCAC5GIkNYb=|*7^VOCozW16qlo8A@3D#)MgrXIxc5K@FFQt8G%+e40 zYQTQsNX~^qvA_+j?Wk*lAC_NFSldDi&7-R@3HXsRPxH_K`a?_m7laQP@s)7NOW*Hj zZrlm?KHmmJqL;Y#vZli;TSg2)W*H*nPh>$bF?xf>r>A5tv8Oo<)bF)UH9sZ6br;%G zH7sdq7noP6hfHkK4Z5w$lPiBjb1MIk>m=UM5+fi`biVG%IweKVj47 zrt!W~V}f7Nyze;c+wlTRAN45J(6&9e7ln;}a#Vny#8N_A-#=#oX|Au(JUQ4tn65DwzO|o-6qyX^MX= zy1OYmV3VN{5-_#QGyi6jE_MF4rZVB$hC}U+r`A0JS9Y-xDt>KXfwg@LVP-X*F?6fC zV5Ta$kpFWUKMc+vvTN=7b>8e>XrZ8vK~;(0#zJf%aUn00AKWe~e{CE=`m~&c5p3b4 zMZdmg2UaOjR=s*8#)RZ*8C|rn8942#NeQ_dc76!ZUfsTOSd_sxLI;+DO$7Aj9m4lx z=qH~;tB3Zo{(73a32pYAsL4MyM3Qf8XiO=2@uI*iMQCr2&Vea2RF)lzE9XW3m=R(- z*G;4=`vZ0a4O9#(4M?v5?>WvaMh8HVV@z|qCiF(o7DI}4+0NAU1oqC1!98Q1oF{sI z?2((QVp%vyI>n3wt2bo_E0Y-_E;Ck4BAy)v{bGYHkdsON8NWOzZT0+lW=Ugi>bI6I zq2nTAqiw335^deIPOLvLWKBc>!;Qtz_09myVbkweOwBbg`NpT|H1Ha6NY5nIb~)}; z3=X7x!@{2LtNb)_vZ^-O?s|D&2Yz9UCtg8y_TG1Phb#*Vyv%<$q0ql5TPh-=wzc{R`Ysyu7v(t*$YN4aFzJ7^OoIk4*xKO5 zeOvnVy+chL?c_uZ-zrO1NU(Rw=&}7M8MhQUl!5Urc;n&5sv#VJ@SOsuVGqOe8kloU zkYFr(#)YB*3myLkLm|Xw(w9@ZIi65KQTLDebvtl)fcF*GjkE($70a{~dep0Xq`v|_ zSCLni*kXA>y&hewKcG326diRt_`F28iWg=7+4*L;Joj%2;M~A3A%NG?9C&)qw>pAP`{w~ZR;+a*$ibW-i5?2U${qCU1HF zLG2?rO2kuq-pU_n*2!CQ&10eJ3e1yr|B!B=oHTVk~J_?+WW?VM_5*HHbO9z zly)Rg3RE_XKSLXutzW@G8=M*GCXIH}xtR~NTGbh@!>dGgn4``8*p+tP`liFYEe8s3 zaT~b`pj;DNXA)39Q7AqP2AbYagcar(2*fxEJpDb;hI~DS@|(DE)F7F>cx?b$Egtz3 z1$4i18NvWRn7;*hIUI~xp8_t#Q?%L6sexto*7`@k#2$}0RBI45t>wOUSWsjtIXV2K z1QkDMpbDCrtMXp20!((P|HU-gruReAEBo*!34B56$#BXzzT# z>`;vzwVqt-KORxGB9{3RRVGX;a5$4`)D!5JQH22}^f z;bI)o^~U(Iv@XNlgOs65rw#=06a?&ictgp4CgKx|x|1Cl{{;h7&cC?OH(!0M2b^E@ z%L9y{$f=(tX_l{`)-0S#Z!J{^gEkhP*hypsRuW)fYf!FSQLOu|~KE zw}>_5TltQI@-5M}b|+hJ9)?O!a&|=Szj&`_??joBn;plVSVTHtQwsu5K$E}6#sHja zb+`Fn)F+!%6qAis#*hLQk1YVc0ZNSBo_m?`F>0D66_dTuPppF=C`hVTE%$2m(JPr< z+`>1_on%m~sjH+>ZiEvAb8HARuEpsoJVUo+-U~zT3^4qbI_mHK{6V9*xR8h1@*iiL zl#9LtRnFc=9zm&l0vFFjGyla2^B)KoluIwV9uPYXBAoeKz6_WAnOx9xAP+}&%4)o< zPW-fdHfQ?xaKODs>=#z>y}B2io!)R<}^fU3XRDMpWkT)a?R#2&uquIpuBJ}oem3Su?H z5E_5eWRQaKC(EAoI0Gks?=1Q$DSEGY=(5A%$%9E(r7oRAADUA^sb&_BurDuuE!c$e1A} z;Y~g&X5?7qL-bDFL8JEgB}JN9L{J%qlWd&>7& zqdxM0J@*0AMsrk5Ic){m1%d^(Qf67JI>@I=w;n_~CnSd%PNY8N;qZ{iX#bI`3EV#n zJ^CfZ<#zNphDz58S%azm2Oa3MeL|6LNOTD_q4ol=nj^_rZ@P!u?_AaJ%**DcoX;1} zR(gr+SXEb_sa#`$8ftl_k})R%B>y3XdUkTm1>}*bFI$9xevcYQdm2_cK&i9^hARwXkC&!K-8WsKc7qvR`)aCng2nDo)LDLm*J=n%TM46x0*ttBW z=0UwxyJZ?+Ds*dCkj9jC;99-`L2Nj}zVy?eP^!EYy~S2`5wtGZb@ju6ldz5!l5W=t z!HPqFQ5)U>U>J!wt~2k08437EUX{5(1}p_zhrq<_WdMT~=Mi&>d8aZAgtAI@s!3tX zd^n87NEz~tnl)D6T6F&U0Y%#==`3qCiL3*&@B5~*53HzG>F_F z0f%%oYn-2Ect@lT+(H{{*O zUIrU@+k-Ke;YU=TZbL$1i@8tHd_#l#ME0T7A7e((J>P$VG!K6W9I#X6IHUdhYl-U) zOok*1X#|vm05=jm_Xfagd{O}w`4eQ|6XOJp1f_(nJU&|C?&o!~dT^Dj%{jr|DBR!b zlec@c#AoW1p3{k!*}!m(qY<8WR2Psg7(3lbmPEMlV=Fk7W3-bb%_lK>&v1L{{WLUv zLKUYjn=dLAY3pb|Q+AZTe30}2qP*)*heV!(MvCQBOzkS(a0r^>YKG5ak}pwvPP#*b zTc>0!Pd3{bDiR2K7LTDAR?HL(rI)~t?nYZ4x2AnFMS)+M~2tSVc&K1CdBoE}*U#dN4py#J;bndG2E@o&cxg$t+PO^ zz$p;~xxat6If&6XFI+?Eyd%wm0wNV|qhY!*ROeS-Q<#*N;3uBAfVK(oPnLkScBN4F7pe#)xCFGDC=qBwqsf|)n1>RVc4OXpG-UpL-9<9`2wP(@lW-!LC^iy$e` zX;Z{}g-sw*qr1E8Qt?9YVX72`lxQ6OeK!imY145!ELb~Q529xNqNcE)@cUGskU zII3KxJImGVvFF$Q&m#f<*O34xuT2Hm7YNYo0GGD9FLP^u%mLcKE{7V3 z3`3t)QWf*255{Y^@qM}An5+|Y3}{@tjl=gy3xfzRC)`duIz7S=Ix!k^g4Xq@Qst(p z;({rfIyzLgBH3<}*khL?b6FqlryPyI6 zlE@I60}~k4!X7Zw>pyd5m|^r32kF&AT24*(e>CSvBz$MTkY%YU?;Vu>fi-CC*<1Vv za3;Mi;}9_hj@sP;-NH7Cv>u>#B|Wu_eggYgQq}xU`|rdQ9Bds5E@*6;U@IcZxEsyV!-j9D?<2#z!oy) zzlG&!Wjj2RaRkF*OWDe}4DogAkNefLM5@cr4qab&?fR_ou(094i#cj#EQVa~?afoM zOF=GvIHy$<_%wm0`(y0jIbhWSv!f85Nu< z-rtp11T8{ybgX&(cr(p6?_8jYOUZkJy#a)vp zhLEzhN?@vziil>f!7nP$VoKJK*PoHDR`q1L1@gOJ3j;2nw@!EAu-fR^d zKp?64&4RmAGx{1q%bF_9)|ET>c+#sanr~Tg--tbFW|9q}g?Q!F{-s@*I#3B7Qy)gI zM$tTC^c^z?6|O#$afqvyK;4y-?JfavRS&tA*iIf5r$(yOy0+v94 zbOb|52N?A|j6m?^Za_w<93X5PuPrCn6Hx%t^-KJ>WWY-n?}-tDMLN!jppfbED;J)s^q<%bLuxbpLO%_j z2VF12VeMbmyfEWTbzLeByTUKQ!!q|N7Q_UkQAf+k*Ak1&N@WZ&}iL)Q~;kps=){BrpC zK4ofab`gBV&%xc(DgzRH@gS`^e!pOIg54eOn#gl3j6TLOIAw;IhBy4&s?4yZou$8= z=8ZsXzInL}Rr1t+>sxwR9ARROiT$~s{dya45qo3K!9;35!Yx6~eS@$9Gf^_%dSIwB zj4a~n-kTh5bl4|kZ>pp5ej$r=KJDTw7Pcq|>z-I!?5DI7diHeOBkKMrJ>|23SEY8; z0>&^AEsFE}aj~(HrgSsQs?9+NSbH1_6{W|>2PSP~QUbq^VFnhYk6{jmrP2E zVK4g`;mQBeCm(3-CC(iiT-Ic>+RthoVtniO)!sklN)9`aFdVj1R*`C%0vsU;*ub;p zQ~h;$XjAQ**6G1Ycd0a+7ZxO31rDPCadQuuQVaM$phRGYT=_>wY~iER z#N^yoff0t@Ga1V_c%_EAEl1~U!{k}F=Z#$|mg!znpYOyT4d(#V0&)Q~3{$ZIIMKWc zAan4E9E>>+oLd}eI&#sC8b9qq(7Ue(X@quQxLYvgzYNpgKV5eTov*Udtq@_~) z;dZ>Th&9y!9}Du!ZNg;W>wWV1>1VNWD0jM4pl-gP0%`;YurYo1YBIT@=fMa@&jZe(Z{T zn4+FmZb6c-WZi#^E$&F=6%3HvEx`)51W_S`bDTCg-QA8A=q<*jwPQG1te^V2KYwvr z>EUea{Sb8>O^U^AxU&S?#W2#;oX7AQQRtTeUV$b{3V-4+nJ|=Vu)f~Nt3IM*Bo?I@ zq4aPx z@KvEZMDdAcnNIVbv6tRXQhmcp$2UMNLm1&Tg`Kwi@n7&t%<@)g{8$J>FSl#(p-qP; zdbmYf&q4X|V$685qQULmGcl~Qx5qsqcJJdNKSWb~v0b3^7ZCunTRRVNw&(%V-`{Sk zMXjuev3=BOu&G}-JbbnH@}E63F_*O~_rKQRnYxDtb;S5MWZTv^0$YZI$N4*mQVWtF z@VjZxLBjWE@TISL5f6j+2znfOB`B^);d8rx3#6OSW!niA%KQF=Cdee{5hU95^0dmW7VDyie1nbzbHlR*#NxWDGuFErR(`A~Wkuuemq0#K7tEc` zyv43w`xito?l0=pVBHJzt}=+dE^z#4|6Vh@GZzb5(CI>TdQ&oNlEFt{1Z)7? z6aS_40OQn4X1(cde18r6%K_|f)&8k)a`HLq`#xvjI|tdEYRID9zP=H2Pa(stwmLFl z{DP|lAZMrYmW!~EJuKcrW+35f^R27T?kmj%Rp3i;!~ivZSzqgqK2)nb_w z*ANh#E{->*5i&||=#d2U?}cr7*EMRMi&MkupDmV9RGbb44U+Szw-hP0DC!*nSGI&9}_)ev&@V2+x*}@l3;A<&Hq4m>G;8tgo|0c z?=|HQ^sDN6-?_cv?97rh1d2y1v*7#-=)puJ%LACk#DDFia=@Vq9O?CT0<18(xb=*o zMjh!$o7@c(It`yL$g0!yIv>>~E?BiIa`HCbm%AaPFE=}SKTYV%*fYAuKWQBnU{X9j z)|EPb){FMI6mmygoGT%z!A?(I8ABmv#qMDgn}(xA3}Xg9#g zT7e~e2Ovn1w_<{y_+PV&*Q^Me$!4AL z=fjlK)hpU|tWdTidFlX_z~nUyMmQ`HJy-|S*TV1vkYqlcm3fyTLKIv(SVv55I@?z^ zlfO$zu>1MfaW}uyX6r0w6XhBmHEdh??;E~87Dez+ShqG4eJ`K4NsG#{`fmUB=j%j{ zM{MCD3yf!ApP1ma8K>cwV8I>^>Gr3z8V-fOjgf$8E)Jkf(^Z1GH-^6}n_qgy{8BnT zc%S0S{0mtTYHVlZpOO|z{?!}3@fR3Bc5sEn@x&N|KfA_omL0!nrOU5nb*a~^{k$t> z6c_YszZ@U`9%mD5h$iBAYSiKmvF%qt7mEC|+W)@TP#r|I3;qI4fj+iex&@WENTB9B zl3PE;90*pkDfhqnJMYhepL;;cmoVEu%~gm0JdxvLk9LYV3}#7&vF&#NR)BWevuRA* zvebgC^vI*ykBjF%ua>twOFq{!T%n;UDt$G9QsDUYh4D_UPSF45y0#}em}4x{?(6N2+1-@}sOH-GD=bWj?dszO2sGHi4G|w|iUyX_6R(B~ zp80)a>+snaLs|i$s4}D6v-gpVGc;&FI1}ZaU`(hxav@VO?`VrY;H@~Fphn{@W9%NS z)EKh84o2EnhXW`Lt5)HfX5}cZlci<{`#3pd6Ul{nds%@WKSn@y5w`U4A3;B^j1Iy>Fz{R2L=Y+Prz9ebBemxlbQ<+b!Q}3133Z4XO}AlZ*Mw`3 z>3}*VZlw42vFhg^I`1{>v{YDg2?{TbWq<4rgBaTcO%E}YIxGJ&@x;!}}@`_cW%7JWMecAlKt-~hbm=fKm0Fh8dmCcr{0 zTLX~8H55Vsjv28A4SF$wmQ^+lOLv6#X%@7q(oC~2Z7NCfg_arYN%Y=sVIvng0< zr2F?K@GQ7KolW`!&H^9!uDEvSld-8Xz)ygbA!)$|*{=Puv(T0qBJBw0)D&h6n$As7 zuOkZfM@AO^vMGIgLOfC|E6O%3+3e3vVsy_tF6*`~)aFDih7$u?BVa{n>D`tL{^O(| z#-BL$jKZ9Yt;;7vY&}3Y!<@#o;hwS9(U{|saV3*yn$0FC;Y&up9G8^W5_A0j2FE{c z1)mr<55D&(9svEcL8Z@Afmw_Q;oQ4V({!62mZm&5E&hvoQT3u};_*^Z+e%ASaq>G< zgSHjB6~`Axmp|tJHeb*`6SiG(jlTp@>WFod43VwZXptDvEf>E{&ys0O;^g_(q)vD5 ziU?>-f28zRppeJ0I9=b2XBW)7W-||;DZhkbdORlIA*2(jk#-Ex?k=)OeZOdoGHo)E zAv%@+?eO;jh8p^2g%$vr1~n<#hK6uvMZWGi^jh6D!5n%+07ooN4{GZpQcqzBXsUfb zfJ~Jw*^z$Af?m)LlS>*bh4Jn5yfd4DD^-7%6|%wNXg9`!K4O#9k(<#YsV5zn(77FR1c z=8}@Y!zni)sw+|8*RUZQmTCu5C1M%ZqePp&`AeQloDljg9#f-pL06*Z00m_|j~!hS zVFBs9YW_B~(02Nvn()SMw5QAP?!zA>!2Xgao)m8L%H-a7e~9;a-a3NBx!sR^-<^rQ zVqr$IC&faP8GcyRpo;)2L5t?4j1x?VsH-<=ED#GqJmN3iwaaYxql%kMim&X(>vPyM-mz-|?(e32XKQZDGWsH4h0RCg9=&anxLdK@L=U}e_#;rm|oj_<%d&o|#L zkF~V|?t$f{wXKWWJ98ENWt`qh9$NkLNg&=r?^W=G1G!S41zSpSBY8r?? zKZ3sSIu3^L48mCwMD3yH3{XBY?50T)p15*oRWvue>uzvpfDcc^y776R9bcN|RDAA< z+-|#rkNX!l&p-S;zyXT@9Ha#LXJ$QKKIQ=PG1(h)a~*%Ec0iZ21@r8uq0oIjsVLcL zo0~8G+%hUve+~AWpN2Tyq#k1IVY(;gG;nZ^QnYw+ z!qa@2)4h=$vis}xhu@6bw#Lm<($nv&YEPfqJvMCBaN=pxkUel_bk5U85|J$t}q?LPRuPG%k>d`lK;!?X(H zKS094>OM4%cw?EMvR^`eHM#Nq5|z8c4#=!ZeFO7(0}i2axKb6-RqN?% z@an@OHoj|Qe%I@dyEdGvl;g}SR?o-rvCQ4(y$&ilJGa0A;0J*@h8K`wk@-Ps7kg{6 z_HsRS>kXOzvd`ymlcSx(KPH#*mr8GFEDc?J2kr4Y8o}b2WhsF?0JWDN{RkhZ6vlz# zS(}9^u*&u=Zd=ayzRB7j&uPMMh(fwd&FDihN}Mn9ZF0J8Y6fQ;n~%mz>3D2BX;*zG zhx%>l1R@L!)l3Ukw@hnsDaAQp&Jkq&ddn9C#<-atC&lOuU6!0bde!t5GD34g-#||k zS4oT0si5P=JNV}2$m`(I_$l)1Nx=pE#ymp2F?-QZm-Psxmis^IY7)Oqn7r2S@_i?P zqNLwAy99j39ONs++G4Y1i*$l6Ua}8*=Yn&@H8P=9@>hZEr{8Nk&$Ahb=Lmar{Sjr4 z+j(0!_9V+|Jw_@n9@!-hCO3r&(O*Ru+JN;0#jhI?!vW7J?oQ3y_L?@+Fzhv@h zR~(d!GaR8u10UTGq6T6Zg1HEn@U~z^o4FkGDddNVy;jyf$pd5B8|fcfRg>F7%KM{& z6?f%)OwS$9`rXY{$nu~j9(ZBYGg|N-JBG+y2LR|`QZc(tc@gM?`+5gIHw>sELg6%v z;g~&vmoKEMC8;c~`0PA%J5Iu?+k!2Q7di#PAzCvv0O19jBJLIA3|#goMxV?~k8zta zy6$Cf6HOF?pLUGnFbyS{l{ar$^K=6U_d<`DoLRHKmK zgMjfOnwExeVz+jU0R?&1LceucO?B_&UJsHJ_G@y*iC`NYDK+_di$*TehHUs*OEnM2AGpbircNT+Ki z_R#RPO4~S!Vp(C5S}wb{gRnHWi|t{)>`9K2EY=e0T$<`4jMapjVYeWuEYH)+>Q0oUgdQ zdiDOn?(g3?&wqAvJ@;JzlompPb=iR0&b96V)N)i~a#epunC|EBUC)VbBXzW3?~9eM zB}#7-v-G61f2yqlJ&I=FQ|tw=eiZV?!g~p6WZ)i*YPk-yNEODIiVYj&8o0|;mYG(E z>eU%2M?eM#_9r-=cv4-6h+XJ?!M<@6Fxkm~sRn114BDjyam&+$L_mKx>Mv;X6nAZ= z&HegQ|NT4Jr*sZ;y%U>#J5ekU1$5*G0qHtbyD!N!)T!P_f!w2JjjF-G`vNP)Xg~G=r(6L`1Rl zUSLvkn3Ty0xBD&q3jH^f{;X3jkCP={_vzgD)cAyTr3c~7V#bnv?O+uERq{q;@PqCm zlA!SDJ=5GT22xa}O5Z|CJkGeej++OEcYP)@dEUM)X@EGLT$6a7ZAy7sZbc)5k;G~O zke1V;mk-9Mh&6{&FuT4j&^M2*TZlAeDZVc{=&bru;h@CpZkd3MgduQP7D=$i+2180 zQQov)?ao(vj9Q#;GfW31+ z82BLR6f6u5eElT- z0EP0jMlr)lcgU1Ra`kd2VH`ke!f7sa$Q_oVivfmS^}1gVC3!6%Ripdr;YKzqI~f8b z`Egxe5AIhZAr4uXhx~9F2g2X+M? zvUbvbW4OWmqvDkf!v2|fR??2oPg1tE)Wl)|Ti14VeaD`MRZk1gez5ovxYx)7 z{Q&eI|C{;v48i(3K*4wG4A20?%Q22H3!s3GPkOyB?Rk>8Br6&ek$;}|W*n=re!sU; z%dYAdU#HD)2c2Je7R(-j!ngl}L&By^Qm?Kv#Cz!5o^m*Ta6*Kc4TxXo!yrBzm7C{i z$}gugj-H5j$XUCr@-oiIF>zYo%E|OYANGfjW}zt8Q6vDY`j<(n`&RUmvKOGw{gAPr z{5JCff0$)q4JzuCGs?{5AfP~7t2M^~&5c|-3GA)Q!CxY$%9ebd!=}7D*1KgTZz1be z^-X?{8}Au)O-aNmaql_DXT`&!kc3H=?aMe$(XX>+cCA5 z8fnebtz|${Gznq(=f1~3zXbx0d=)sHL= zX=nkw8(^OsN*v<$IKV`B4yfy6a#o^b#_Y=Fp7JM;s* zm4DO#sOc&cNJ>IXdFY~SWh2}^BP6^WI<0!QiafEMT$3vUiE+^l(pQ3mpD+-9hNCs@~ zd^KfqZT-!}zK@qzzuDO~XN@=v)^lGleXwfebvq&QH%^mEY}&vDt`(oj{T>zyg(ckq3D`VBMLhvs;Y z!e~3aomu!&i*UNpa_t}{u~h+gy3hTxpa<4)!Q^jyWkdQpzDTo#VFSu3eK4ySps+T zJv}7nvZ8<;^I?bpT10C=6w!;V;=$IfFDV3+;kKrTgw-A=>Ri&!yL^wt(-m6cmjn(= zi?hDZl$CxP^I@+!h!EZa_dFSm@U{Y6mjDbR9iEZ+zo@QQaQ512Ciz>UpF?fJpxy4E zF2oi9i3d;Ha=6w~7d-s3-iF%{V9VcLe}dx5SH7>v2>W0B?Y;Zm4l?`2jOmC}N7eUM z+x@AJNQ+zlU`5$gfNjMipC_x<=DNtEr`tDu+98%0HAz*Ct_2%o~~0#|E^5HR?S0g**eq^&3cv z5&FG_{$js$KNKbx>GZQ59!Ir&tc)~_XGXfrCILkegc}rr*}|FSkgo|AqF)C3FkaVH zC|c<)=B3YjpWh|jsLN15gi7kIOK`{Xf`zQvJOdvoDSY>HGyOH<2DUdyFU3CuEiGW@ ziZ4k;0qRKN>sD=AcZzOwSTIPdyipKs%q}@e(H?(0BGlbvn_h9`JE~npITmB}lhh6H z^VFzyfXwd33e*KdJK-Vpn)=B{ciUFCn^)QM_WlP1?LYp;n1bv5=OiCsege_d zF}rjzPpJWt9_RDf0g_5$*XLQ^CCYGM36yt)0(XWRSQ6 z@*<6vbrS3Bmeih^`%%dJekJucG#ZKYOod%pqh4(*vTic+j>ZSB@VhhALCWn0c;YzF zdN|Avif_F*3~1-&MtD+zRRA2B@aO5F0TSO`bgcT4;!x)^=EKX98#3o1Z7<*3k89cc ztWJf*25g#;Kz5aqga%C++azg&6n9XgbZ<#{^ zSSsL`RSjk;)^4|~x4)}|{y3rk^Jmb1{SXBO*SXNx|L4I#G4}uSGybx=J|Nr%j z|Lqab|BJc$-@fDjc!u*_J8gWkt+imI<&Gx6Qli@r^4yLlYA5&Myc(+D@2gP@SD(tbH^pQ6OB8g%z8wca9h(Y(u!ZH;6 zynKf)fBitZar1(y9ynfO(7FCi-N`g^PA*W$3S-$iYJO$1)@}(65;M)PAOMKkP|#_@ z9#leF*i=-JznZYc*`sxrZ>6&W+I{28Nq57}U2_RFV~2LgDggH-MrK;-FDh45dWQ;7 zbHUAZkYJTyhVc3fCSw^<2JG7j0Ipfvk@u`bLLX6rq>QVVOP zRc{P%p^#%_N+Kdd<46JAF6i1$V&bY=?~wo~^_tVS4rhImnt3qKFZIFw_Q$)WefvJ( zoKAJba*~#gw6s%5ii9J( z_AW`_KBfxS-8r3p1G^7gWR5`L*zOAmJM3t|CtyLE7I0H&xoMYIY1hktB7&(q#vJ)r zfWCT)!kM?)PLA0(veQ#hTffQR4yUa41FHxOIncI|a{jog)&n)4RhThP=zA6#;mT!$m3_d+)C22ME*!xDNh4veeUjH8*Ke|>TksB`&BStkRv}59le05r!aIlT@ zS)F!7;?th0>N@9$2*Jb9j*5`D#Salr-d(tsZH8`>rrHGU22C! zl|irMoOW93nP6w@YC&-jpG9NC<3_N#)arxy4aMJg#nb_I33|?%m#|{7fIv!ifSR}&-N&su7S)u)Rr|=A z%knnMsVMo@5EvME4qm9OY15#K$i1*#pNfNy{e<{}9}gGmw;DCV)&LoG z_I}^bt|_o}na#}mVc#q*nE9<}-_6dO^xTMoU7yZ3)xS*cR@BybaQyAUYf%t7$R6HZ zInn9}2E7NhoyT142c?q5jj6vV0(m!SjW;UBTi1_5;)gHkX03P}k&jYNe)~=m>XbfJ zoc{a0X_!M!I0uCYXOz)-K)<~$+`Iw$7IV$e5Y!gfa%g&6-@Hu;WXgfkoy(Dyp*3z& zqI>GN>pY#LJGh^qs4A!4jVjWzqJrfa5J*AFo{5a4_m98yJd&;<&|_a+0M3q zS=e+g60muNcE}Jct$|+HAKyr>=^8!dly%Hoalan+H245lsO^B>^W!sa!=P>hu~aT= z&P7#L{%#fUt+Cckx6x_VD^~~DH{+~OGjtyyPX z4m$g&-p{*1Fh!d8$(d*DJQho35^bb~YcGQ1(6tfgsbTGzC8MzOB#BtMvhP3GEL<* zz83K(&={}XB8eV)MYH{0%!i;AeSz`N2hua5bn!(d*e&T`(hU(khA(yQ4I~ny2q#ej zv;_RsE&|wqw7#uZS#z916eDHA{K%Gl)hY3aA8+C;7gW0JVkcQ4UlONQEXG$WURIbk z9M@+^(xXm+qdk+OZ_K>-Cs`R!`&Z?%jp_6hm?#K!rg3d~h7# zGnx@kyD&hKIvziDM?ZTZ!2idDQHq{aq?J3!e`s4Ou&XFqbnxu!{_Vpsj6^s=uGeZ1 zX8Erummm0!wd=33qvSWWIB|L;3<%*_d~?O0N10|&$h_|J5jAIOp6d3B*7+tLd6If2 z{dTHO>|R#VV<;71@uI-O_Jcjn4ZJnXO@>3%BTw^EEJJ#*uxiM=K0NM+YdEi2kMvc+ z`cbxOF?(wVc^|1nNIqbz3;wpHi}YdZx-xtK{nSwkM;u;mek}as%fUtC;3*FicPY2j zyV_m34oyW%N|7i&PSH7MBK0^ZY|kcqb)loFZnpigSM1o*6m^bLqmQ{{w7vC~ACZgG z-c`tt0SAE-2E{xP@DF1KYM|xxBTv7^k2o=doPPUyoVR(@k1!?a9jpp!thbKtI4IxR zTn-N?B#q5(Te8%$nkEkJ z!yGi|lv4n(YUmHJjLD$!^T;xJb*0#4c6|x_mI_3JJdpP%L591G^VjBf%^e^Q*va0Y zH%PX6njjmaL&G=Epm)QMzqr~?69g(kto=%N-a6_dVn4N`?xTA8oO-9g6>%a+6$S(Y z%~ZMqAoTXN6OM@lndV>#m`_W@pC9z8gj&`$zHCTbIO6V;QPvat+l`)OrXJFXf(}9C zDoX~@t}S>=%~TrbK0bT}g@tdZZAn9IkBHRJ(~>&8;H~$Q zivztsz8)m>SaGIrl(D;hpUYzqzzPHd59j9OY5_kNG!;EXK&%Q`5#A@DCtC^sjZ9Y! zKyPMDY^!WunxxwB!%F>&DqCoz-JSTFFgnu-=@W%oBxtH*zy2?}-aH)Y_Wd6ok$uU& zQy~h85+OrmNg~B0F_r9Fk&rQO5wdSLN=2d&vXgbPlOlW88GB)vX&G;3>36w5-{W|` zzvuZpe{|UV6SQ z9je4uFO*qbBMlOL$bzOLzJEC0j{MFTDJ>MSKAD-sB$qhsZ6g}sKkW}vN;|;iVyy=w zkb$jiP}71R-6Zh(5^&?tJC?|vzX+=2O^{k22`hsP(WDC?hjBWRVhF^=i7~`Oz3dJN zG+QigiG2Iy1rC-cNL$;{U^@!-ca^Fyhx`PwLdKG-8s^ONZ@;2@}ho*=sC!x z@L)+sClO>P&E?AbkRV)l3EDVMzyn%_IGPXD65j68x%9Hg^d2-n-Na=PTq`!Z)c(^% z>vi3U3%?R3*6ZUjUZhB%et?4FQ0qiWewX1UCC3oUhy`QKMkO82&dwrB5keuWfyVo@ zwI>4nKfk(@Y9OsC46HOn`USy9Cg}3A1c)J>2)xxauW!xj3lI+sCeWY+n)rgGG}?4* z<95q%t)m908b!sSp|%UaaPGQnO!>Z{VlhJX zAL=t+f2hx0c$<SfmL>zJx|y(Fr!1kl783JfOe8J9xp0Onl!|AA zvnQ^^P1!U(_CH&b#2=UPAdUCTE8g}!*pavueHEHFe3B&C{$mkJ$`#d9C$xxI)YMv{ zyBg`OFFi*-xL$i)&->)YV}H>zhs=9eo2MBvKt5TfKx!(I#s|XT%6!n8V~N1vfSm(g z3-pPX0kSjSEIOcFY(@FfC0wK5v>XKub5ZYO`(>2qFB$snS*@I8yMM|;1N}mmclMpO zpKnUCzRT?!d#v))GONEbshV*Jd|ZSeG#o?&u|m{VG;-%~_?azO<0QjMbP-8U*G1E{ zP0_=)7&fq&6X4&1=nPQx5OPnna*$0sao;8=kE>x^*g`yRgMMX!C$q^dROoJ3l~u)| zss~XQs#OvPel?tYltAC9zoNX4n<-192k=*atEUOCGuV$(K{BKbty#WTsa|l4|AS`> z?rEnKTE%q9$Xr`a+qyWb5;1V??I&UTHy-@m;ugJUczU{W|BG7PYZ9u${ftb0B>QLn zJp{ZdoAPGmFMknk0LGnQ3NRT?ewRfs_$NUDkO_v^&Zp_Rj{5zqBNd0NHspA)i@SNa zrVfZ6?^%}OMgRYtd}x&L08JIfk)AImKh7tEZ?uY&ktRn+of_l@okX=CG`UPud8Jvt z>Vjed|BJ}+54a~#J+yI_6yJ@)@wTd9C()c3^#vYHC^b-;s}s$Cs$aR^=6jV=ns_1V z+~u)of<`GjArA4!lVOP?Z1U|~+>XO-T=C)8C$0BL z)V@h~a`eC+AlF70-sAaj-{@qIk8)MFjIqtqP0`CQQqe$nU&9e*$|toLoM4d464YlL zAhKX5mKDH@zo^scE}Z}V`emTI9Kt|8$@|?Y=VRjcQVh?@0$eEtyB06=Oo6EGtIQ@C(gE#T*=pW&TLGuxx+w9TiD>wNCPKgJ3BQz`#5ALIYr_=%Nj_@4) zI|X_K6x@ST5W~D21&5JE=eN7c+3Yb!uiJkP9RAEJ}M8YkbR_QSrn|?BU^t%wv8@nKkO%8Gj=}_wcwiJ7wB?N zQYcOQb{av?U->_&`t$ektUx2GO+Kf@Kew&}zPWquK6uko!P&@ft2u(7{odlWkRH`oQMo{Z44wU7#Ab42AlAYI?S#wR}y)XA8lFr#&TZ$9=8nYy1 z!loR#vsW59V9e-)euv8P1B%953?U{wko-*K&p5mkG#NSu{r>WLVtp)FizXP#;J}kW zA8;fUI{qWKY8wOq`zMsNM8OhtO-po63X7p77X%?l?`^CJ$&X+s=1x1-G}{PHi|99M2UH-1bz(8v>I4P z5c+uhFJ5*JG#lieJ4vNH8G-x)lP3Y0Okremphw}QiZxLuUh>(uh0>)7TF1=BhqX=Q zQY2ZDynb3SIP<_sQ^A&`T2&a|7zg2L(Bi=1gAW_Zn9b(CIm5P$XPE4?tXM>lumH89OB;D4hcGP?5ekLd+Imk}`}uq{dxa|$Lh!D!Zf&;b5Dt?&IQe!V$+i=zod36@_s4lW<|PSG$W)^|aPvTRsp z_-gR_VsaN?A3ABQb<|rkS$Mr}H?aH#;h*F$AMRUj2v;=pP2C#~*SW6T0oFTHZ|HApbtl%c9XC*?C)8EywncujC0+Zr?|rGZ%fSeCLZpEJ z=p08}c5(!yZ}apx1}EITI*;#00?RWW^tjp3HMM0WYQgV`giq8G*I~AEq(P0wXcIe@ zJs(;V*$z#gq#)rx3(+Lx$HA%k`SL`~f?6o_?GR0Iu?%^+@xUSL;iqLQLEpHPR1@}f z$%m}@WCCOb&Yql1btMO^JjYFwrT`x z1nzPWZwQyugg_;#FDy_xF_{RqddcH$iu{Q#GRn;Lfg+QE=o$GOgb)+)AHb(3lqKu} z!WnBDCQ(S32C6)5>vJk(ThN4@K41RGmu699@!A9Q;cskhoC;BsF7tW+UNg~7H+Jy} z7)UGj@=hYK2G&1OKdjmFdlnssB;A*#9Vnqp1F?(4ew||hy^PMPe3s;}?p(r$W`*)# zoiMuz%Z6&R<6C}*aY>$jAin9^)Y< z%|Wr?q`wHe7LWmP-JB+7XDkk?rL(>XV&V7mztlUQq&*Z6Wj1EAFC6)D8Hf|8{SX>H zw^?ntc)!ZIM!P`TWLqvm{ay!Zrkb9Fv7jRM`Ub@S`Q1r!MiQ>H3hBD6G~s6dZ5)RZ~1|Kj^Zw@}?BulPkSzUgy6yj7hLo)?l4OvLF{k z0PrD17mva*q{?|g17kSyG7^kI2nXle(Npx*gV9&X3iN?z1u*0DkL!V5i_Idg4jRKx z^A){AC$H9h)jGg2piJq;X(0(yz*#pX@`Gb(q(kX?SKeSpQ4)b!@fc{q?2kn*m&Z2$ z%wNRqE*2A^B!{OjUuT<&`^za@6_$LOCi?q-ApB^+YmZ#gvtM|w1`3OFY}MrBdAza3 zo;k-_NACxhOZ)h~FIyZ=n+rM2=G-UXc;lV!zTSF)sDoyc^dIH@Rm_8Cx7N!VeDw=c=EPW26>xrPO0S>74L(o1O`56ZpXnvon%c(+NK z2i#p2>W_VGRh#|en+dIO`fyo2c#?1|o|NJyR za|GvrBYfThmjf~y5rx5#?GjL|8eupP`FQG|-OMbW#j=)}irL8!NKNG?F*W^5y2s56 zJBbOtXUO0g!J0v!YsD>0^9^nz%q39Drb=N{Rsf?faFdt0D0KN|VbRPh!cV3J9B|7g z(eObWagxf!7-J~V%aGnU?jNmOW*CKg^F+Z8ey`IHUk()bxs~ z2e7uBTnOyq5Co=TX!B6pX3jCPE@Kq8pLT#DkJC|cLNv9;n`tFet~Q3QCKmF$-As9X zIIDZ1C|`RY?tt>{J#hW60S;AeoheP|dABSt~cuV2Hs&Hf8sGKI}@D#6=z6bV9$+g=y;#Kb@Z`dQJMO zx~2))Kpzixi$9%zR(H#Y(fO4PV7XnxEsLOQ>gmt_A`)=FdqU8i!jK=1WT&=sK8pxW z$`#mcHM;s7DDeHuVY}BSzh4M-o-{o#+GUl1m8&JiwEIQxuEqvBtY_3$kNj3W?oj+T zqjl|AcR}XaC`3RFY6@b92(vg?WgF`57Z2p)V6s^Mj0Q)@-X?!oU4`aIie+tDT_SsT!@#O@u1>IxXcahED?* z+$(l|()T_bdHO4M0ugK`|EMG9b!$*_RrAREg33``#~-dGmPrdl&zogQuzle)evD{=8wK4k{AmYg`N-AT<%Eu;aO;i(z~VXsF8%Z06Ee z%`KUDY5ktI%pell(#_BHmv%$;$`mR_W~3{QPy~TFtJe@u>siJqp1TGYrvz{oKBO`(Kf#NwNva@$0Chm+-;kFnFk%ENJl#a#G^RgWx~?b+b>AeB)3M{9m9^05VX z<2!dR#S~wtc>CdYZDU=i=7*rG^(k*gbo6o$xwoqutQaF02Flw_3?2_SgDh$SuE3Nr z>6H)vg%A!kGGaq&YP3_Twf6apz9t`mKT`|SUVMjb$i`hGUV;)*%O?6_AUkINugLv& z1%?pzZ$1NnlsM_x;FI>`RuJh9B8++lM?X225Ja?EP}S|gs=z;-gIta$-dgrl?67{W z9Iv#LdCm3b`#XMm{mb3dbMN?020$ka_?F|=*4B2{ET*TQ9m)3K`f}>M{maY#Iv@YH<=9G*j?K&PmH% z9`UYuTJ9%fTU96F0+mLvM+(R#=%wj*wQr^-Y4jkuems0%n78&|cRaRe&@IGG$I>In z>!hiQKpyw|gaYB41_vWkOuV*@7+kn271i(Rqnm-Ijqa1Hzf=m-wB-_vS;ev*K#hnQtQ+{sYVZjaxumOu zvfzhvtw(9@^r!mb9uOp;CruXY2A$OR>9?mlcU?176S5O$CNzHhY6}aL+)2?(h%m-W zrSACs^C0*(f|d$vE|d$xk_%f=&2EHT0rdA>z!V~>+oEPSTrM-WG7!9KVtzYQLqf|A z5p4qyY`iGB03i@uuN%ofY= zPP^q#<@jRfsECShLzQ;R7q*s^*nV-KCxD�a)nY`-C9WZy*^g_VmZJFz`1XEX6;VjbaVI zS$NLl(D_hj4()Zt<6cF5#LCY#hr)Uj=V!(yS7e_>;j8c3vI#N$u>=l|Y&#p!qF@Dh z%2POUVNfC%WcpBJel)k@{;=Y;tmJ$5?yjs|tVuJo&<#zqe5geeaO-kXB>B;VAvQ_~ z81Mng=GdlHqKrgMXxV8pA+>B%Q~ThD$5jU|m~*EYufNbS`}8sTAU%%?3~JjuSrRHt znUX`Uge6G<7vPwA-N6GloW5gKE6LdOw`EI$4gav!wKOMPyJJa}HglON^lF?>1|c&` zEij8{9N`^?_Z z*l%Xn37 zA!ST7O)+ABV9mvzE>9GrMYBYvfW9~I>o0JGNPIn#CL~D7CqY?6H>yN%fF(H-&u%zh z7SLX`E|TpskV^^KbW0+-H0Nl)O=JxBUOn)=w}6fJ0@f9@tCs-tQu6#m*$O>rgE(w1 z>=e=}4+j_Ad?Dgfzg(g7VqX{&K!{&tu>ULlfY^gdV(Y}JlcgDDMPCSJ!(Q}DV#CId^AoKK3F^p1sV?`v%+2XQB$AN(Y-x}izs!f4Rb^{f`CmZ3HzWVCa|sHPW} zYaGYw@t?@d_ZHsV%SKWnoMP`*-6>fqEKL6CC*)Wl+FtBYjw6)H*2!)#Ig_dh_>GK3 zJZ!Y7=f|5@yf&v;86~qLFl@%}v-peoaK3n~b4_{;Q<4`sh;djLeEV0vonBFpW7ztV z`xhZrK8ypU^m)HdN2}|wFgfxP$u|!f_4}o+{Yc)I>C#g%0@54Z4!;L@C zl<9REsHd)35wd6^xs-RK5Mc@_0Hnq;a8}m{2j|XfJDjg?Zm9FCp1r|iGi_<{<2DQP zK~WhG!zj0lQv19}i~q9@XxN|`C?l*lxauzKYsMhxL7yOsf)ZQXb0 zd`L%wsrc*w$^I{A0wJp8$UP}^NY=*JekFvj%O$2>ww+CI8yyTDGqAfZFq~JnMdTxM zQBa^~ND-G*T8a@>jQb5ZNxvS(Pg)4iW;~yYjZj`iNXyTmt$hn4tfQhUy% z>p0T8{7D_e4eY1tm5w3DGzv`kyYADj)GfR|OOMbNjbo zrNVan6C4}l3&qQDg67pPM(4&*J-fAim7`^!{l#3HGOraN0Ed=~%dHf{S9~?#i=N|1 z&^=R{aKr><0d{$Xxk@jrMal#R0oem_4!R>pp+0=NpJ(~=1L8q-lY}b8XDw!DP1p}k z%;HFag)|XU$|^k;^4LN!`oLMEy2SN3ZUqpO^c+gyNb@L(-Cd6@zm4&+Y+Dzz%ii1b zCiVqO^N2mTO!x@j-eN%^69OZ0tj&s~j<&|=hN3RA;(1*5$fbG>y$5yMU%%*mNp6gl zIC)YEu^*cbwXNehTji#(fMFM_H5066jb_KwWUIdt_0CiLePnIw#{wN@(t;P^XNyIIAz#0;vcc3_C+8gP`$Ac~%>?*;R*t=RSMy?@5bEa)(`{ixkOnOO zZbbP=&w|_&bs5~-@GVj|l`w}BeL~X=^mxC%EBQ@pYU9*mF@7fO^|jWYMbW`8mRxLi znAs4rS)d4ipqwTW$w=G3h@}1k(%Y>}0gYxrYeiHrCIHy~V0fz#_D&h)u>TDaJq0hW39P@JFz%W_4S^XRH`pd}ND-%my-BM{eyv zHGz(_9hC`{<$@nfO?ub#T5+&A-MP4Q>`~@z3#s(+)Op7u2o9D2bUI#8Uz}zD0a^-X6l_aI zwQcXRJ^9V`L*-ARL)p_a}!(mu^Je!FI{leutZh~H2uXNf7z;1%Bx*b_)gg@P;Qrawax^6F))Vm2Dc z?^frxW13~>sgkkK@l#@lwRB3i>xKpfbuT2Hcm_Hs5gL~)?(J-WuYk478C+*yd5!C= zV`p2Ha3ZMGOWuK87H3p9Piy-fw>=Qi`>lgJ0T4%p2GOJ^Nc9V9QAjX00~l`kVJ+)w zV0z{qrJK)`TKb-E&7*`W<-IVbixtG#W)&?a9I#s9g)-iFKHMz!dF`I<4e*oG3q&$A zD{9gPl`F0$@*Apr?nEH^1ZJTy(N@O0Dv!Kj9js;@bk z&L^eMA6lep)~X(fT7rFu3R+A76$%}t-VgVMXs_UM8W9X&Eyl~?qi#ekaIix3RLpIp zCnRuwfhP!@SlpozpXQ5Xmd4SWIf5ss2G$3vq>5aal@orKkF3$0z^xqvP?zUPZX0N_ zMNbZ-0t`fSvql#yA%I@%R}4Sc+?*-Bo|so1fz&zjeZ58cl8*37xH3&0 ztTz=$s03Z!*hB7fr-3#v3+=4ScFL3P8jSff?SQHtm9y(NQZmb^tLW7}nDiP~4n;d1 zYamZ_KzzY-l(B=)?Y|P$-b5U}&^>Z4O;~&tQKt-_BB?#XIFog|uS0Gf0INd0WjyeU zD@E5*`NBxq=jz!@#9HU`PUi9x+mA>VrlYj8B2GN+R0-@yoG{K)EhgJGW3z#PV;?N?u$ziYXTHSwE{oIfpvY_z z+!gu;(*%B_6!w7@2jlUi3t+;Yham>r{JJqwi$=`!y#1krak{g8yfn#H$iK(4^5fD% zI5Nv???FHeE@W7k=68Clo+H~!Wn}vPmh6HMdE(_IDW(@pKhTA_BrhaQ1fu+*R{+4@ zWC6w}WNlDaUvp@|EUdAP9{JQudTraKdG%=EA$754rLukRu08?+ zquD}JjPeD`(?X#?;;a-J9heVSmEI857FU!apcc1j#vr-6|CQ{o6gBa`2>f#-LmmAJ zO$vp80jt75@ELNB8tWS0yfMjfAoWOV=F(J87^Uxs$i`w+0D!mbHo5mg3M&T25j7R} z4&|*Z$@?MjNG^9j&r6Quae?Tml-z2q$*4gJd8ie?>?TQJfIfm5e9!CxjvHDygoahw z=<;0M(Dn*&XcACs7+lpV5d^LWoC@1INM2}A&M8%+kV&hQ(6Aa0JSp>~DNzPB*Ygb+ z*970`X+6}hK0rVSf{qxlk^6LD_Eon2B7grFRZWAT2d-zuY zpZ7)2cSHVSckW32ibU6D(AJ=~G+FR?y2z`GirpT~;zf#43Z%gIsUoeEB?t2`O3gph zB{|}q&i9zL#RfE#ca^h5p?#VV=bAn@Mi8#08q+QeGk9q{aOgX@uuEc;VfXB{`}kXr zRc}x}KaUV`)wV^X8L*65MWcE2g2?F$yosce`Ix9J01XF6TF3R64VNC$un5pgthF_u z^?gUk6PVExxwHcl4EB7I8z7$|Nl0J%WmyM!iQw`}vCSciM_+S8B<2{#=jy=|Q{5j$ zdcNqz$(%daC1oHBibZY5A;O?lv<&y~Z+tumVJ(+GL#-& z|8+w)V`%Jj9`9XqrsR$KkGX(FEsXvYBPaR@ax0H?k|I6G5K`}=_%L5cn)y_8qSd)* zR+))*=^Az7-gyKh1SC)RXCV<>9Lsf}cLKo|8{T&)YSqRLK0N=v$sBS z!I_!y<_X`KSlhv;VyUUA>1f*_1KGYHnusKWrxZXm5VFU&@e)|U+SSg%#}&`sHctFr zm!lB(3AfEc``q}6^3NZ$s#n(UK*zVfTwpMKRf(Hdh;zf6ESX2G3sQUYGDZ^)7!o3M zQNMue!*&{KWB+aS&AA&#n#5IUMJUw3whN&@TMvCb1)ubp_c>>ja+r(pZurW}*llOO z)Vobr5m>{$0p6_Z=zV&SY&4n_SWgqtqAby$;M{RB_^R@abn;mmmrE$&X)~%cIL7>u z41;3tr{+C8TJ!Ex%Ij&-kK(Q=cjghu^jz|u(-RI2{4ak!tRAeY5_1er_MzURoq%Vx zB5L{PCGk-ryVCVfqAE`(78@0>p#-i!W8+eS;uemcej!k5HZb0^(1n0(AZdX6gs-*B z0xDtz2B|fdR~}O`xnyM5(2@9n5Fc;6Tg?ZZm*X$Czjm7MJEuzf8gr}^mSRH3$o{*Q z|H}dUBT9*RjbH~CdcA@T($JzFZ+9qu{MMz-Uy((0!uXjqI7Smf2B( zS*Ur)q-U#2r82XJ`L49rF7torrf@nq!U_mIl8HP(;DFm3f$^Brqd(IO4O8r#J4OD2 zi@wsycOayu=uUN0LkPj{(+=)?9@EWpJC7F+qP~}5LBP|t4DDG-WAMv2gVBy8HQ;1j zA8Cg~RQ^B$mbZNGNCYNRgnaXv8nUEHZ`&`}%(9X$v4Z(snJnwefcXWaHZ>y|oNiQW zs11CGORlsBz7`d`NEWFl{px%${s60#sbJA^j;m<-xhNCjT(9-J3g%-5XO(v@mj3{B zaT+GbdZ?3IVgxxhdHPU7-M~IeWl^|fi39H#`{X!gG*nQ z-c&sk&Tu}X65OY~8b>cEUxgTa{$L3rs|(w0ehHa#JfrfYvihj4n&&Sc>iUm=ovZzt zaWhCtxymJ6yO&K`iTM}q{~ZQH|Hi>TfNFB2dSO5rJt6`Cowdp^Mvzj6s$`T#Tk(Z! z6C(WoSTv|?Db!L8o#KYsBP*o#^+=(}Vdx0}&n_vt=SjB9?m?|@abc`Uu6eK4P@+2B z`NGkKzPF!F&8(pW+m2r8B(?HftW@ahQ&`5IdOC9{2Xz#Jjjce9dd>WhFY(n)k2gIsV3*2`72ZjK(EF&P5UVAl%Ul^4=)%sM1!RC+ZQICZc( z)sA-V792o!Xv>a%^!{3KBNzgI=n9scdE!@7{>?vjCU)E!><1$Z%lFCzdZ58Ao(#H# z;z@kQum^C+92!Tks==YC(L*D|&(TxuBR2IP8X8iP1hf}ST%8|v>!h%=QAW}a-42EJ zt3n-EX|mmm`^iG?r>|$^YrEFsYR&Uibfn+2_m--`mmwD1_aqp+LymPn#^J_yO3eK@ zZ{$=HVPb=assdM2is5m!GMXI0=crR z7vbbLxys%cJ;}u) zXUMfi4zQKj@B?r4`^7ko1(MExO(h;hiulsLl1^X?7r1PX{I0AMFnj z^euWgSSS&i^jb_N0-Df>xxGLxz#en7j{M+_OtQ=3S>{gCT9t#-4V@l8sT9+<4#*7o zgrWiHeV0GSInX~;_=Kh~ZzVSQ_T>T~f7>l<_K%f|w5dwEk7|>Q_;654;N(O)y2EXQ zA2gkhkU-D*U<~s2svmk*8sXnTaBDl^ug?o#2Ra3XmjSMe?DQ%DPZLB_49IH4 z^W-MO=JET}!Vc2N4$6tmFsI$S{rmH7LvL=I9^6W8vo+}9zMHsJyAuEm2n2nS4&-0AaKu^65ZQJTCyXWX(c z@lbt(hh^<&(aX~x(_;0W868cO7MDC&jPhG-#XUq4pzf_58Kj3Sm1R#JOx9|csH>$I z1z<*{FF3lPVVC*!^}gPdvvckh7V?h~Os;q2Nr=_q(qO;&!45o|V+-)xFdX?FSNdhM z6{;6Wb`84souRddofoHf2ju^7+WB(ENqv#c121KYpgW25( z(7}zHJhr%4*o};t&{xm0AI&l$O+>!x^eDya`6;HWe36RvJCJl#y5J60kTa?CDD4p_ z&V9miK*J(>z!8qomIk87;UKXL-E zO_}czvWWKk#38UA79i4*`Pc)1T?^@HOiyk0qpGjTay4Hr^Z0cm_{g>9tAW#ww;kLS zJ~Y;+jt_jBGOHt>DzbMv!{T2YJY$@WeF3M>Z---C^iWPGNG8ee`ZWeTAK%pjGUjXi-F;`@XutL2*^s9l42O_lO(^6oRj3B4oA~&#&L38zGi)7v#cA&`*u^( zw~v0liY+HT)sZ{XyTawb%w4AD)yb89(JA4*hE2B5VBjHnqvJ2avDuAQql1WH$i4Ur zAxa&g6_5hmQpGla<5L0aEWm-P@;hABk#y=+nj(pkmsgXO<^gKebA?&yrepI-iaBGh zoFxGV1Wm{Tc|kpgMa&BD0ucWg-3)4bivJnA1@zve&lsS&nSNuj1WxH@Vf9H z1Z)}z=Ke(i^+&papi-Xld|oXk=QRB#UstkdaP@;cw|U*B(aNgYiiBpLAcfJ)39glp z+{4C|)%?xX2f1*+z-Ei1(0~Y=!6{hmcp{-Pz<2smzK-D8b`i%?=lCUn zh51kacduRD@i3fuk{%-$jpz>T__@OBK9Aq;~47L85tFLmW)~i18VAvs&ZXf>h|vA51T2u zq=e_8G1oTo5;>pHBAOohs=Rrw0+}PHzF?}!&l&7{OKzpV($~(NQeNoUh@4Nn(iPws z_Li~pI(T>&%bpgT8eujm-Y4}U{l!-{L-4O(!dv7*$_pSGei%vk57pQ`;2J#GImcex zIH5lzkUytA6B|}(YrFrmXVxKGgUi}rsYX{N>@wcEKXhr0fBl0r1=&v-q)bgscME7c zTMJ*ldKVPWn7(r6gd^FIgkR-TTa;(8c^K(AB@A{XOWKpRBWafT_%;25mJ%xt&QO35 zx!|6~;S)1)xD!VB>^7iCG;rSdslo-nE@UiJwc3f6U6(mbVvQv%jn|}>&R7P})3JW} z$)BQ4Z_C-+g+MRU;p~GQk`Bl^+1i*5N2(eqAb@Nons|j2U`9@g-{03zT|c^@OH`af zeTh4NDBoMAhbf)4@0>LSm&!Zz9c&9Y(lkKh^`8m0qEXM1=N))rPF zClbY;yyi8z@s7LlpsY2DgL(-&fa8I@{3fYg_-Z$(i+E}_IH>!&)UNAm0qrR>v!Z^H>cS&wzfibI5TFtS>zrl(F-rE$Y2Qr^l;bV%k5qne}c_)fr#0TJ#r;gQjBRG_HjvI@c9<@ocT^@p%9y{_JXv_Z9y-*5#Y` zzTC5=g51h}NpRZ0iW8c1YyEn11sLCQMKH>wj&3J)vcq+0r~Z#l@(wqLG27UVCY&|7JwjfCs!ia%CgSg4mx1<=J`tnC-fhn4%{8Y zRW0f*BOE6e#b5EUzm8Kv?ENX903#@c44dbRTU0GNFd$ah(pZCu^=+E?mA{CimgH^v zb4Cf!CSQ2Nk%emj$m1l}|{~ z)*VfU$)73L??hCV)bq&E-Zf}DMLURYP;upq9D?W1N1YxbqgF=BecvtcNl>9 zu7ejeoDQM;fDyWdBpD%LRV_dPu58+Y69Oeu1~an#FG3JLvT{aqkk#;heP+DZ-5Z91 zJfjat^3Ol}vl>ZgR6znMJ@bDNNg%(4Ye%wTZcNaQlH@47qk8w|rFE^n!uYy&b32{$ zElN6HG2f%ey}k5R?({<3PrswJFP(1>6+Q*_r!5Kp6NPk#sG@p6;l2?agUt z$4~s(AS}J548I1)b`AC9?!2I!l=(XN~~WQO-zjaNmM1uiu%BQFx{i}diUb#FShg<5@KMk0Z$%>5ZO(kY>M~^HC6BJo9D8AO{*8#W& z{xB+`SJP`JP4hjc3cCU;8d_?3jjvdEXW!^&uDX?Q+kRh~38mX=7)LS#J-Zh3YBXVH z%DYI45xIV&r3iC^e7vb}5E-#~hobNP5bgCZiJq zB2&vWT6Itwp+yyexf+M<4;Zm3JDFI+BlDa~y3;1&5wtSi(RkozaToc1M`zFdZc z^(`32_{nNUI~Ewhp@){x*-<9&9=ZVp8-CJ_QU|WZ9RLqpIeLHAtwY~Vnq*lvg)#Z6 zSQ}dMQqa04;^=$T+xO3lKCuaQI+~(%|Cv_larrpp=h-U^zNaH#-tX`bM0h$fV_&+d zs@MqkR`kmoCXEMAnB7e<;jrFd(&qlww+LnBI`0#Kt|Lk3x8O69dk5FC$lE~GTbHfw z&La(WKhSaT%vKF2i;tHDT4;YNB~qel>wn6d+}{yrckzztRYt!Fq`iaMa%6vIZBgH3 z;~5Dxs=I7igksQ?5v~bb0gYtfDn#vdTF!q4r?Sk0t^E#WhzBP{ zela+V6lua`-nG+yF_}6_nXBiMFDhEc(HvF7M{HRSG&H-^3oIG^Vmn@(DcijtyA|xl zxO$%~N>AVb{Px5$#C=(^)GGZpc<(2HyA@e7GTJFyc7m*5usaxwm2%ffTngtnzEu-^B;zB3ySi8rfjIO1C_@HzBjMY%R&7az z9dyBqjp|Ro4CEH@c5p}WYBZ%9wtWP_%WT%X8%r3(w2H2oAo@`?vE8kRds%-LlE zVD{AaSXi^Fu8*nRk>FXqb?*@j zV!E+nUWs-q3?lFIz_W3j#b4W~{j#c}KJL_#dSs4JiAKhcXt})d_6rCG2;&Hk z(Us*i{(K6SgiCBa1kP3kt2lL(W`b+MC0`<2C2F3%txLE0Bps{rF!(UwaPj`!>MNqY zszV!8OqJ+*lvU!kPaC4u0%fkGc6l=EQ(nKXnf6SX3Q^>)=NDMua%!O2trV1 z!Ym=nQKp>^>Jg}|6b!2a$6Ro6Jd$b@>}*D^kJp5FZqQ3iN16lPJ2?Fy%^v3tVihCa z#>;y%A6c-y#yVxjf=%AbN-v^G*0PU)n`}yCqsqa9(EJv$!|M%SbkBvfb;r>^4pHGA zhYwzG>QWbvb{|v%v|qkr>UNatG_c{wmh$67Bq1sL^^~Lef#(Y{F;eYX$}LFFjIL_oKj-<4pPZQ#>9m%2YdiiK-`#>GLv5+`^)WW|w;Pr-t;P>! zqnFGu9hEm;T8l8hl0O#lkd_a~#GLd&%+uxx?j-}stS`4N%igNYm~mnoqM5#OlC*TY zkp0tKX810X`?v0drs%r{dQy=Q?C+su3doK`6hdtv|KXkcL<4h_{uJkhw6BHB(1*-n}^Eut>RHFRI#pN7RMTMt1=JHGF zXpm@W<1ANG!4WECt2-uV`#91U9=w!_R#a{fCw;m%%HJmaoq2a@&v|(bAuDA!&O!wq z=-Mpe(fM3v$MvuM$5TB@XHR6VX!r8Y^V>&6eDpoI&k*rKDkLc8UpI+lIw-V0t}hNI z?)?5Dm@Gnz=FWS$3C{M5h1|LO2qeZuW zRI)2btJp_B=C1y=%`(Mf-DfJ65uDRwlbjW|`4&+$T>t_HdQ%H>;BruJbw}>lUe6_= z>CPw+j7EAvx5YO#9iGWhF93ce-GR;-S=jgura^Ia2MYHM=pDRH1K<-xIR?jhDtMV~ zCqe3~qn`w@3z#jrx^dhXN6MRt+V#FeqXlGLZFg4-iu>qAmOEWbHJ*=nPv465%W8H2 z*>}JoFQ;b-=HELrC6c?sM0kv>vsa{L&B;;qlimWRDhFc*s?)4gnWma%@t%pFd*tCk z8YC_l5~Q_JOs*U7YH$+6K!s0lyD2!&-svPtv)&ufnIO(0_G-s_Y4ylZ9N{HIx`<9d zL9&efuwUoH?F9LeFF6_SgSW0kFdPN6xT`f&lGroF6-<84-4Ok`ZBkk+U!pj33Je!u zfJNRf4P?y;+Y9FOwjSFd*+Hq!6URDx>6DKVF-MwwJmzlZR9*5RVik!~+VDK7M< zv;)~%F{c7GEL0Kz7|`)=N}0QxH<@x4EK3@C5&MnLn+@@YIqiky(0}6c9XAFis3-yy zKH$lgWXnW4Q>}v?u14CDMnCTnJzxK+Yk5gYQkN)!iaQ@H{v<4+(2Kj+@k`SZ-~U>X z-~X(rw8on41x6M&q@KaH;j3M9Nlo)PvH1YaWg@%=rBrsT!_Uq4Q9<^C$2EvH@Y%^; z!QX!N*;CNV&-`DEffGpDv02SIg2HASU+eLHmq2-y?N*t@DaB*tt|YyRFqCdP zj7!-?FgB1b_-SQ|1HBd_8A4_QNZ|9Y;6<~M^2C?)l~{lD#xEjI!(B3yc$zeisR;xepltp7kS=P>TK;uQuht z_yYQM(7VpfywPvp;??UoTJ4@w*Ez_{JV0`bk!gXu_n4_J>e-s}yWx z@%5DIkujf|GEw7^#oC(cC=(IwyjWW~!$~k8t89Zig#(fT??KTlQn?wU59>Ed7G+sz z#$=PS`4W63oNr5do8d~y4(rx;H~i2TyL?|yf9&JNH8MI6&5l*Y=)+`ElL0(P%I{3| zWJm(O9o9FmJ*`WW08j(7oX<^hUS7@$?t%jb2jdXme`JY}96wU(t>r5(M9C*X=ege( z+>GF}HuZGPFZ$*of!VDq4v}stD^=C}(m5b}IMLA6RWTgFl1@ZtHCoZ6`@m~!8iPCm zu^6z%7>M=Rk=Fac47nTbl#+b$<_@tJ$u;ympZNPJT_5fUJo5j@tj6F3sTI~@MmMr{ zE2ft339hmy`geIG$WGavg-G__>t6HmtzAS$IGA-$+a0{A$Am(La45w5vZC5!`*LN# zFrr%06n+?E%X(Z3lN;qYX5w)YwwnSok)FTY)PU`4}Rk zGK7GLyVaT43vvw6 z8K8eN_K*B&TB#mBGI2bvcIH#mr1N?14Pg`e@TsPM1h|y;_@?-opf(BQkAr)%kDHLC zdE$g{gg@*3Jz$*~nA0P3Yes&eAE%EV4)1*+S65xadb5S;&;dcb=PoBL;1%B{iZTW} z05T(J6kmyprD;*G-*A`6Z4X0r0In&Wq=vvSvAViCj+L$=hna|TI{T_jj|rxkXtB2K zv5ou}r*V+TRO(3t7h^+HeOYUf_A@w1Hz+{YB@Zp=sU)YUS7d9;^>sgUEtVgo$qm7+ z1H20y;h$WnI_u+EKhPKz;Q+4or?!;N&zJ2qFH7uf)5NwyKp__UwK1eI1)6^UX#R)DzSWxnXGyB{Z_N;H-%#@k z%gZbIA;*m{yddFcq%H#)H&1Dxett5opop48(EA1yJ*ux5~lswu#-A|$;eyy zse7#|mVNFF$DBEPTAzV?5U`aaeEk`8DT3}O?ems&I3z;!1lLL4|ZboBN5Zgi49 zazsyU-iLE~aICh)iEw^@;~l2-(KItA1Nk%rOZJ*Fm<;AaH8=emoznE`>l@RwH}$pb z75)!rZypZy-?tCjh02yaM%gN)vXwBcmQ+&Nm#OUQq!LQ=5s_U8MHz*J7`v>~*q4yV zGM1U4DC>+gKAGkFd%Ldt_dNIWyRYXsp5yt0Iy#z}&%Bq{dcMxHRq2_ERld&$iEu{S zog%_&E*;7hu7GA0wO}Pvma^O@Xme}sAC5%ZFAzUQk3E2>#SxIO9fCj1pRyKX%uU-X zrKj!Wu-s79oID@*Fw`|LVuy-@zKeA|&yhjuQ&b~F^{p)}o&+nSj09##>Douj3fN=S z@lx86^}RoBzgOO179QGdYAAexV;Tkyga_&X+jxFsExtvqoW>7O=C;(14FeVtP$!+E zjaYgX^s3$%{USL$uX

>aOxrLF4<27=WybKmhh`DtfIIBC0vTrQn@rU?yGyv)_xT zDmK^dz_cB6LDalxx;p47T@%^oEs>ygWM^-J>&|%XSY-Os;FP{bNsklCub@=l4SwLQ zni@d{67iEM7EN3<`${pikl$Xb)0Ac_dO0Fwdp>1R_zFlAw?N5k+kG4uV~UH@Ex>CE z!uD3f`i$$a@nXnE>1i}*6&s0BG;5p{oN(E6?}@#KF2sqMP1x_{z129EI80glZ=nLj zg#I19Y{v>u*!?8xCG_|p-JX?XxvWGTCU<6QwPp7Slrqo1^A#!h+_EqAohQRWyCQcdTS` zQs;JidET=EYGbxf#5jlT_{Mf1rv?JK&#V++97)xtt;n4)jxCr@+dcN_{?_GEI(jEe z9JEgAy2d+FHIW}JbFsXZyHa}m;FV8BYRLPxzvDqP(=_9PfQS#t7%aWWc)*bVeYfJw zeKf;Mu6ASKct3|1U?u_MVqb)}%-7;TFS!=j_;hQAelDhqM7<1%RCqc6i_|& z^kzyr*vb1kemK8ZgRdzSBtDNPC0G$TE#3ob67QAqT6pE~0k zqheF$&_XRmfcCB0!*F4BS>^u5HphcDl0rmLy#VlRvjc)1!9DLSW*q_RwTBHp*%Uvq zMp=tzg5%9o1*iMc1pa7R`88S1IiIyjVqAM{Ejm|mc;AyYG)F z#&-!%UL#->XqRL6ca?aJ+1@d%P3o0a`s?*cZ%6NfA{98|W`+TH5dc|{v#~X}_HpA~ zm;tC(Dw1hCo89Ogps91ocx5znE`jR@PfTMs|4Ea}oX0;YLsFz_$oa?28^-C7c{+R1 zd_J>NHqBgT!2T7h__DcSnoHvLcWi3hzHeFVflW!QC%GkA5aS?n&)3nywpCYahanho*V1%FZ}6%mKJDqMU;m4bXpY2 z?!XBBvfQJ&??3{8{C<4=MZD(D!xe?}ze=YNGj;9TWDf#t@i4WK76Ya!{51b3kR;Pd zD_T@gZfdsjmU#E26@5fZ#!j>@z(Vp$y)yT2DTsNF7ko)@*774mgaP=_38EDPbluCA+k2VMLg_SXnK4OF2BlVJQD1p9Xpn#pr_Xt_z`> z%5{+0TgUg#Zb@1s?PEo9F;1y`!jw9u5g#gsY_d+l{0@rjz%g4w>4oXV09N>HcYAIdl7vvAMZ@#T9-Wq@Ll5+WJMZty*2 zO60MhLXp7hfJ|(?1r#C3Q&{3}Xb!gma%!GZD!NmcsNI2`oKdU!rn^hh-^YwT_;6SL z%=N1ScNsl#b9ZSU6LQSeR*N~9so=`p!IMeJ$)NL(jyKk42#}E1kX^sr{@44uH%<-*;Pxzsd3Xj8FnA#y!ml|`A1;G7oH)&45u?+vP zi{CT>Q$45@N>vRrUEg~@2#6#@KqA`swWqIdpt7j)efn@QcMYfK36ERmfMC6%K^5bQ6tNzpk&Uhn{tu`rNx$ zJ+<<%j2+kRV~{Xi4j5e!`tkzgYQ()ozyysR*KbEv0*%HX#M7F3t&aFk><4 zzp_`GENO=gyxJ?^6d9R-C{ZT^ueE2Pv6~LuZ;vhmY%w1|tDi>tVu^#TBx$w~dl(@bgp-HtV43Q~vu(t> zxEcXm&GE$FGqG}fjvLK z21dY&Q}&KDnU_u5llpx^Esdo=d_Vup)0^0S)cMol`^lj77I6-&OcA0MwOq;wff4i^ zR*A8Ch{Et`7%GC^)!g(X_g|geTDZ~@`!t>V@92<*)!qzG-Xtk6?)wx5>)$YI3>e_T z5&;HKi{X(>WY8_)Hx%9Mc3I7u%-CwA4C#_b?Hz|R&)QU)o)NTORPJZ?5-wDx3h}jq zo*H;;Z3J8J^+VkF1e`hRNnq?UaO_UI(o##)g#NT=P^CD(n?A~Yrtv=p zs^~xw5-%Mx>2m3sOWlx=Q8f04$<4R}Zud+x5B}CZhEwAROxv6e9b0)svur1E{lnol z2YUr-GJg%#k1=M&z4rQ@^&AlQc+z!Gm$phc4qy#~en*$1XZu=V; zGd_^N5Uc@}rLhipqq2~bbT7D`aV=!9YepDODfAg7&S!)3Cg!}v_ied4dlFWwepuvO zn*Up~Rd2Y~?E8(y&lT0saPzOQie+9VJnawQn+w&aC#cG=E?9Z64xz7fuylsXj?<}>`>6O14>ccNQn#|8vDyM#f1leHUsX(Y8A4|*<-Fi${2&FjLd#?S7t2H!t2ImV`KcGv;B ziIqq+3i}MTn%skG{RReW9$^6~b zz%=^;2sA?KEaOe9hYSw}s&u3`Kfm)~=5D`+RpW&Ly%j;O;y<^hFvYeM1XE@cJdPx* z67P+Tg(Q%kmc)~DT``7_-u!Hlk3@adYH}?eEb`II)Vlr5&^PF0b+qJF;f(gEo9UAo z(~SC43Y_5Ow*chI9+O#U$h9kTHI+*Nx`z?~*OCGBDOULBq*_U$Q{DiRFj}j?iF)X)Z+l$AWJP%{tY2veq1*BO2*;v#q zry0UK`{^5hj>gtNJ9j?~?8khyo#kil-9>&B3@lq@IQ|AUVe7fMLKHNI*!AnkwAr{- zp#QAb{0p%V&a129@E86S?9(uim0GmXjrUMH(|Mz_-`RI~7mgD<#neNTPNj6l9k;f= z!m-Tj1k{R|Qee+tgR4|Ps586!2&fA5ay=QE@T9CvcMPPFU-M}i&n*6`QfZ5Dv3LB^ zl(6>r)AZv{xQhT&VCYNHlS@~hQQiOQ_J14)7yfshysA+RrZQ^{RS~{MiQ}vkWP#mHWH6-zG z!FVQUbb%@?^c3lih1GGLi0Bqk$B>P_ejoY!y`!U?d+!&$ckBvLw3bgBn9PCu!)~J9 zf;rEiKj{FGEIBzFmZ6_=vEmmHri~@4?UWcZs{tifP{@&w!3`8=CK_>_fO>@C<74by zbodWgFFUxAoJ_n%cY(7hlUm|lE^?MTmy8j#N9`G-ckBu}zKz&k8Qgib{JHtv{W^Y6 zj#VCUKX8`q2VCv{_O!rZ{oB>D3jl6n0GoiKo)Mn~PGa9>kUjClQ|r3;9-&%Cy&t{{ zsa*eL3h~Yt*Hw`!dt|?N1q@G_Me@GB=B*NQf{FQW;h*&X9sV(FK@KL*9>dGyZ-Epr zIKfy8Gl0=x6KJ~grG-Dsm#n&8F6TBr@6vd-m%VzUeo{@vy!$&BG6VV#u3HCm;8h?( zJ%C=|5)f1~5DR(V>KbZ5N#LX4_bA8iWZp)B(zkYR)il;Jj$g{>@Dx@p zBCt>=qE@XPAsYg98EaJnNug>4BBX_3UzIVide1auuHj|7_8Y;TLpL8wE9V*L|G|Q| z@Dez2&@y-zJGhE1ILr`hqpDN7l1p`fsrKyGy5e>m8-V@qhJC{Rv!2CCO;W6PpI=kGRP z6FlTV0!8pk1^tDwgVEY5?~=KRmY$GwUFvn4m(2RCGXrt@H}8|hNCnme_6SJ~z+Qdn zu5cm^YPFg)E}>X*E^c+c$V|zMD%13Pmvdxsher0$9ZsHvb%QVbzXRtiVQq1?U}zzX zzg|w2>Oeke$a%W^%`tD;q`X=6Nyqr?8{bWppOwCfP;)07kdH_;nM)WOLqgOuXjo5; zj$=JWqA!WlKeIj<9}F?NMLi$2$y9RCiufadbG*{fR9Ro~Y|4;M3$^On)mAPj^RjpS zjTyQlSr#I=K;$BfAuYOS&L71Vl&9qomuyE=n?BZkHCFMtua{z&kloQVUBpJE~!&pCI~6Lgb}OH=(^HhT9$qLnWIzZtc6#W<2$X2$nr!H zsHJT`p5pock`z)7;5w183c#*7(g$cbrd%KmORt6_X_83Mursi3VCl~m37T6}xJ#HlOCIXD4nnMBXd<8Imn_}f7Smb&czGtXvFgn5N!w2 z3R`F7Se2N4A=+oCrtOzJTY-=s1VGa_p**|EaqqEB$uCvseUHU@-j4eIa#LITnc&+l zSq{kQ&tG?7bmhQ_+0mj4e%r4Z^3q)C0dUG2?->nhQ2y5SEv>l1)S2GJUT?$UL;ZWM z8uXTN5U)q?8W9~NBent0yudF<S=Hy6?|6#V8 zvD33DqSL@oZ$eUAuDi0qF@qA^QXNwP+-hDiRt4SzAscpJdk7K~E|MDl;Rvrri+I&J z%vV8V*40NRtr|YYL8{05M|M|g>@!!@Rn*>FyUf914B!F(sgl6hl?xm*L9hip9*=Yr zc)pXQFmzzCkE~ZicE`{-_s$39-(0^%#3ZYZJ9Sp%1iz@fBbUVnp&jKHo_Y-ssgk;q znJ0;3+cc^`?E!3T38^-dc_tp9mQMeGqkPpmN)5hv-Dj(3TH2)Xx8MFbOMCV@@t)>) zt)|3xSES#_T;+uRu7SBwFNolP{22^`{f+|f^ACsm(#FESBUygtJ@90fa90L9xnt!h}zIZNu=bKD#&nZse&1>*XnJz|{FRA)zf}sWDI1oDyDhbiH8C}2+ z1u$((ohDFEUM7_89XjuQD8R(Sxp(}=oAUZkOJF1t)I_~VD|qTzP*Mz{%|Y}xa1~h| z+1kQIyObO&V~8l3Lb=RLk56?4qWvU&?NXFNm7Ld7&qQ$a@u7YufhaEwgpRRzGN{`T zmDqyGj5Dw!J)f1)q8j2e=2%5-cTpItMo2sN`37j_X3YYE`YdVpTR+O=?PUTJ^FhY7H1 z!w`(aVVtZy!)sI!jf}c*RMYkpe%ksmkRJNz;oYE6?$Axiuga+=MFEYj-B;{iM4!CC zzU%-IKt@9@q}y7#fQXk@23~ZLk@#Xk^R)}=J|SaMf5@qAJVH1Spe?ml(!eoPX@cj# zA7<1{rr0E9uroYsvn@trjj1~;Z!@A&TUg|%GPqhM!L4(oJD}i7Uwzv5TL@mqb>aW$ z21%KGv0OILMHDpv#T2!jO=gOTQ;_ZQXXggnoz?7Uxc26=mkn~VDLo3#Cok0uw~T8( zxoa+KSZPl0aqh9UmfOQ!^Ac}X0+8W5f3rZY=8GhP!>`{7Jwa73bjmw+>474M?k>TF zHUm;D&b|4YXb^HmJCJAWPNTV@kZno8Pd{-CKV`_-2F(?kvhLKq;|^NUou06KoL z{Z!Q4U>h!~p|+la-eteoVAJ^imtH4%!fD#VkE)UK&PhGy6@TYcxOOTpMgVG!gTa=c zyW4?+EuBEsO@ieNY!NVTvg3(T3XxB?yk6Lxv+{F#O(Zaa!S19l)Hg_q$EIM>RXR;KL~WtEu8^qS>mL zZ%nshrGXiSlDvuC2eFBGYAC7``vj^HPGF0(2auK6Cu>cblt%%NRKeDD+UlR#TJ#N- zJ(%xa_hSFS6gi)SXCHf%zwrOC1?P=^gjI|=1{U=+TLixQ3Bk#{1XN>TP-6SGHZyl2 zbw=G+-J;v`V~)9sU`&bZ(eDYb0A~ko#RkuXyorPj^&igZcY<5lMnLjHpg29@*xCR^ zH=)WVy-MH1dh8z;6ak>HzBGm>1mE(b7%#Vi!mY$ms~`{kje$K(vfgpDyDVW|+?{ zdxwBAm!Bz1d+WpA`SGK7X^@O`sH$6idgQ6wy_2{5J|3Bl$miaajbny`WApFxcK=K$*szLMP;V8T z$V394{Sh-RxL)6<>G&Qz&$+%{7JK}X8Z zG9*0|o}i$J*T{$nq!*5BVrKGZju9Xzn3Y%9Wuj{lvx4@|tB-ZJ_OQO4dfeGJxHEj1$7#Wxkk&Y9cB~cobjxSPcjgf6j`P}5$sR}%SUY_07bW`G>X}?`d6Rj? zo!_6S!*MjvWdg$#mI8IN2t#M8T&JXLZSwm|u0`WhA$Pu;`v;1@_?{R&AmeO@2t-Y- zbwY71vakg8VFy;O$&+`g0T~&l3zhi{ji1AvoOEz(Zhs5udR!T}sz(&!xO@y$=R(39 z|FsIEsUaoT(mnyyMcey(CL^i_h7L6>W}Ufj(_ahiu0u-Q;$9xs)|#mtd#~`vK!jp^ z#pa4`8IYYf6r!~vBDVh2MH$O`6Z=W=fBG*FN^ewH^vkk4C+;3V9Wz&Z&DEKm?b8iU z%B8R%d)qnBZDG{ii8R0_4$%k8fx0drou52UozI({HeR&k^HtcD6C4%buj$#8qh9?n zCB~#K;!Ntk?t6mQWA(d$`{xClA6be~VG~HTpuy<|0oD{hD!ZvgaDBxPkk<9iQZ8-| z%3g}USC<|rmLH>!{M?_zu{RtuZUvzREZY#UQ62p`Ql#1)?qVQdaki+W?3b zq|&0}_rv?OP3JkgcOD5$(oSKC!N~Tl;KU}C0(j8W+-BHzIJddRendUyN21%6Phy;H zJHsuFesY`qUo_xM1#l!mb&5a&GSnVgla|1kSD}FB*J0Qdvi<@9$8l?InOk)UtB22j zn=fz=RB4_{DX1%~YWzbSZ z5kd%A=0YeRqIJX&nFKy7S z_^YISNr5hNC_BbRksCGzt+876Bx%D7A&94{T?IVH8-SgK>W&BI<1{QhJJvBsRO&(q z20fB%m10ZReeu)vw;7p?&;HYp6db3MwwnRD5C{&vFpi!Ke0y~!8HUJji?40$po^x49E zurGDea>Q5wV2i^J`p-Ih-+E|otO`f>WL_9%UwZRFbN6Utin)qgrr*B2Yd@xxL{Tvc zz;8#SVrFKJ!L%U8r6pudSF&xFEd=;RX4SPs2mN*S#Tq-EQMHaf`Y|ez>us6DDGou} zA;4=q7>%Uf8DmN|Qjkwdo1`fJa7_AM8`1Q)k8=vawq<$X?lWFgkb)liYYd#2yZdT4 zABV&|7JyQil3H|7Ba4XbBR3`!>t@rV>-F1MCcbMzPTuUj)x-JKIoAZBlsbnl8NV0% z4L1(#=+a&P0~As5Kzt7w0P%q{b{>BV&5aqsNVQby69-cEx*Qx{?&iMyZT3p$Z9lOS ziD94YC0^8J(9(Kv7~C9S7u+sleuUFOY>*C*Y(X)zEj0dCl=xIG?P(PGwp(35#Ma1B zj*F4S-vu~`W|XJUpF)NaW(dKH^dUtyAi78s3$uDJXK?}{N&Nx$N?k9!GC2No=p$q3 zLgMpOH6i7I4@t$nr#My)fb!|~kNPv1LF=NBmcLP5+wlq*4c~I_76ocqvTEn}H9ntc9#p&}+q`^#wD$J?j8ONC zGh#Yd1T@+?_{?!X!0`P+VBMYoGMBgPeIHrRA-CzvtApgSc#Q0(ZH8ntBsJAZ()c0G zA2($7Ik0^9l;LA@(Omuo)0ITvDw647RE+tEaVexs8mvWK99q|YU-bT1vA%0_q z#i#Us-Q~eIUy+#;yuZ)C_-MJ$t?#ZW$_W8G2XCDwQOH+})K$e78W}62S!y_q`e|SOXaiboxj!V0B8v)aFns zT)>OJ-z8?7cB!$l!CO)?p?76}udEFB!S2ql!Y56C0=6f4QYJEk%K&1C{_jIL2s)`8 zK+PPt{Dy|4K#w>6G>$fzQ-3>rt$t|xQc2v|$*zB=nREAb-s%5}CB6WJHHEtqTRFIL zkh<2fRT$pn4T^AE7~I=Mal6F~?T`m^kDA*rUo{&Lm)W_4slK7a24fe`Kt}xtEWQvK zkq8|Dma?=-lS*&1Z=wa%z5li5&zlhZK$n%+u=@P)W@_lr8C6HlNja++b}9I{?^W#G z-Yih3^2P~)B3z4V*yRz7U`Pa77^Xkeyw;l4f-N-qEHz3ScXaGNdrn$iqr%|1-;uxO z(l;#IP`iPjAwHZPOx8(`hJ@Lia2;ed`e*u#y5`|tBfcQ*OG>=fQ@s)_lVawZL*w1B+Ys}65rLPB9ERv z)xoLQ{gMe9jsJ&3RA)3%r4p*KF@Vqnqxkda=M00@H2?P#D!sfZw{ld{xx>wN)kL3C z_N<3WeK?rM!C5-X{sh(Z*D>X|X_Msp1yt9om9*fF$>xWSRQ{)H#N&r+)a4SjE=9~Z zCw=OeZ_e+Ky!4^3;*u)I$n^_Y=3{%F!gg$(+cTFF8?fYxskC4Gm1cf8Ci%Q zAogvk{2j7M&817_=WqI|rf3MP*S%O>?*;H>CDZ%CH;UhXHsi?JCP|9?3b|v6LbJl{ zh{%69s@1~T5LG5u0ym_@rKeMR>rFcU*j(whJHjBb zh5iCJwls<)Cod3?yV;Uq(v{oR*PW*!u4h&t$4GD3GL{L|4yG~^nJk{F91eIWVgs5k)gF}}F1+*wuQv(lZjcxuy?*=QVCmH6r` zyS!H9@VaD+8>j(JLjG%8I>J#XShNVDZLvndM%-f07)xz$2nOmf-p#@>)ya&0E~IFi zp+3=Qt|Z&B&pT&^B09pP{jF%}Jg#!vj2~o5hwzm1VW5)x_QWiXsa6KTajYh=u!kXc zG*OCx0)7)S)S?4he(+-N#u9ocOU|y=HF~piRt)E6LiBc^cFXy5uVC4Co`_!I4NcIn zxBU%~wtHKIpO74rpK&gv>JvaVE@xBZaQr{B+G}O47ZL&jI^W#=UYD`nTl;&*@ZA!T z$dlK@?PHYxAgPxJVMD|m{|bWP1`E@JS@}qJ9N>-TPc}U34LDKWP@VS=4${60f92X41WVb1S4bLlAl0TOeC-F z38pvotG~7jtuVUuf<SacO|FMS$@TB=|6l>Wz70nTgN(SB zG!Z*d8^rWcwI|%RlBO{};wqkX^M|{>&Q+9RFZ^0;! znPT(ARyH|P)Sc6BTd@f+#+(ErhDc_H*2EIK11kdkI-iaaAKTPo0B9)4Zh?!14?t~w z$S8cRFu_+#+FLegYsS0Pu`k{t-aNeXXPLiSlGplfY4z~7Xh6GzrwM5>McUX2t#hE9 zWea@Q-%vAaQvC#x*<^d4&8A=PuUm_bbUeyfS|=6F{#G*?Z-E**otIGUHqn0r0&Act>nMP*a#BcChP|UFMd3wn;X9I%>M`MWb6{fZvZe)WPzCw;EP2MdMQAPVrM&s0OH} zkXiJQdX3%Lh+-M}Ks8$?_-(>Ukq+tOtoViQ6AQDJF`zs$9Q7y}P>iDJ_u$tIBPVvR zf-Za_cba*kVMxjh&0i4F;VzMtdp%A$8J$+nVUq0ON|c;T#Idui)h4#hZF)RT!8efPjD-nuCE1pb#7=_t*-54@)K00Fd-hn|1-F5 z?IE<+_S&?Q^S~53Yj)mTHIsw)%IcZ-f*-j7!@@D!OwME6;t)$e4A`Q`+emT(UZLfn zFIL7_iQSp;)ptgDM1f(KZBD~f6`s~@TVEG?XwgkcsOZyccmDjjOC{C_T7C!;0ex@= z9N7fCl+;|^GM=G9^m5UN>$Pxnz6F9whm801ik`%IiI)18H}CL~j&ik(x(`T4a57iG zot}W!p)3+e-^VK`Yb?F9sBB$UHG2w4HG^O^2*~BOH|PU2s-M^sQFW;4^>L;u_Q@i9!ly*k*h(QOqr-3O*g@8QZLAWWwzoZQc$v26dEBJBulil3a5ON?&4r;f-QqO5^ z8EAqFziXi8ACB&7&OgD-J;`jVN!o9AcU>H`F%*PqbB93DtrEGC#yG+zK_nhBOko@i zGpnGXqtM?oM!hBZlg!+%T2sKNkd0aP-sP0jhd#~mwK&tzOgZ49c}~*`<7z+`5m9mO zsPiK6$ylb9ch2!KjnY(er$!AM`exA{(TH}9z~Z+%54{JyRVE7}0BGC#YxHB%>UKTY z>a|V17JNLDea(^vfd+MFH*$mIiyDEJ&w{t#ZtvU%n`2<;UlMkK^$hZ8MD7fUv>zVr z{O!;%iu-diy>?eq$DYjL&w#n%>+G=>kM`-tTXS=GSW^`Qg?znTj4SrrMPrMverRL0 zchM0Z)Ft26Uz))P4HtQt5virB!M>

tFcS+fdMq=Q79)l(|2V=xW~BEpvHLXZrkL zfhltvTaAdsQly?r3hzn8OaZurq<(zTtIX`K=bo|&(PIHjsEk)TVQh`J#G&~E+HL5? z#T%`+l)Rk+69N3`A?35C68PA6}hHWG|QO8VPLd%F%y+NSFth zvlK*TZVk5*v}FlaxXKqr^|51B{{iJ}@*ARN0jZ<9sum?-L9wApw?*xfTw})2Z1$BuN%Fa9eSC1BTp9GV<5!Lfs{N2^vp1Df?$auB%7 ze*dvM^X5h4;ZumZd$y67X)NGG8wYZPH5AB54PYbqJM9d$`5Hs~tK*ORc{VpSxL^JJ z3muSrF!46Gd7YO=?yD5k5paS3!G#B2?+M2sceN<~l;2@dB5%`qkHB+hhvJ<4^`v%Nw=TrL99-A`;ovf{R5wp%j#ekc zP`!`ex}Aj2FQy5P$Fl`G7*Y`7M3a1Rh#aUJ%ezyAGy`Dg#r0BF|H`+i$7kNB&ac{g zxLw&jqE%yYltc0aeJgXokxQq}C!eeQzXW`z)K&nhkEfUQ6iLo^kbIhpnSeO`!d!CX z8dK+h$&dOL_2XsGq_Xc_PmOD#mg^ZY_>BI&e%C7xB924t8lXRj*wPkCW}a{XYyKmO z0|Sm9fPWt_mVm}b@mtS*f?#{pRPxFt3St~hQCmJgtHr?{#_XXcJ*6VgQK6#pZ}(32 z=|qCIR$lV0(>tvmSA+{H*@fRG+z%VawGXb~;G@IrC@m3)v~3V+L=I1;G_a8>cj-^T zK)~HL{Fc=t)X=UlkPh*f1BHWU`q-+L3Mry(%k3X9PNmK(^|#bLMsaD@CCMK7!W*zi?X;KQot*CvVL=}B|t0IRHnlv-fC zPZNK3*xdjGkPfV9Jh=u%y|f)+4YnJdrRXo`F;2a!nZ2ZM$+(wwxbW?acbzyPRYr_# z@i9;P$sDGClKbFs9*}U!Wq$gHTQbw~J=a~x7{_`JQ1@dPIcPjG4;<({x+L3V_5`4x zvs^9!3%&~L8SApYk@kFGs!@g>?n;bd=%h_K4}9m1n;}YgdmO6w*im>({b*~E?z$=S zyf=8snSVI)-i{)vQ%fjj{3*oS4O?EuG0ZTY3-5=Dcz{?D23_iN6*N;ISPG%%e{H=^ zQR3Nac(wDBp|+B{i<;oiPthD>4^QzzB0y>m`g;LdQ3vVSW+!s%VIyv3vE>}A6e2%C zQZGQ0@pDECfP%rC-K&??75_)hE_dWlqJDc;PR7_B?2BeWvNa4~llVc(&@;jm0nX{sd+66~+$b6p{;2_gaskx)VSH}b zeHe(VI84HsZBnh9=Ni5>IhHKmo}A*{r@}D$F;(99=H$>z0aNRswnODOVE@|y&Lm+J z;%(9uW=e_;UZ@{(bX047hMNSA^&CU^#TWUbQRDH_zkxEn#_R0*$y10f703xm_0eL! zK~M@=PYut5pfA3G$z+3OVWd3r=T#a%i$z=eheH83ojf7CM)z)JWie#}84B|JY>Ag& zZYb_m;TM}N1o_t#4q^slDRHlr@a&HLx- zyc1IY_-6Nb-;qbAxuK zDi!T}9ny@?Yzm(xfjQ5*@lIGUt_C7(1iIIv_Ua~ERodH!m$3^1-8I0 z8h0m3!T2EaZY?c2G0Zeip4&3{bajXh!$`BB?&GtM&(W#DSv&qxtd!XM7tlQj0DqCm zqe;!Ftr*QA;;~hOEB+LgQk(IC(ZX*xzA|m!Ib90%HEOG_GI4w2e@{%?|2V&Ncdl?d zXPcSI9okbQ1w}{L{E0ADe5tDCq(WhB|Zy-&I}d8 z1coPk0TO!mM$Y$32xQXWIN|@eFJINF!YPR}fbZ_$1$I=E66Li%!DY{|pO zHg_Kn|G^&}vTs+9%N(~#i{to$dxj^spw^=S7(j~M10bGTdInqcCK3>+e;Fpl@W6a_ z(Fcf~eT$i+2Kq@l0@aO6(~$0SORkn3_oj9zAGsDevNQ127k+9c3fwiS#~8B!587ds z$Pd6-xL{rIk3ige4l~lab@Lr&5+&SUmqqf_{%K3-_53mYIAKnvO9t0~CqIFx0SZN6 zdtSo}vk0Q&>}FS=mK6MM^Gw>gFDqduJ(VSGM(-M4y}16j?IG0dAl*90dCJI~4CZ9E z4FlAB4Y{o!&OQcmOQ4&IAdvp5=SYIBNa_OWjbiv*Fafy6l;xbm6Vm`baA8FgP_TS^ zAQ_p*JVoOr#nfoZb!7RsZ%Q`2{=RN`zoLhO_~oUW$Ej29yQ&wwTszpGL38hEJDw*j zj9LeQ%*zV}3~yUj<`bPnhm+H;m#ObE?@F+~wg=~1^&A<`etlh1kz%eW{`iza&~{(@ ze-dZnOCeY*m?@%)f;ITwknyqD-56a*_+)2}Xm#2}%~FS^tw*D;R==jy?rR49KK^Ic zF^V@l0X3&$>`hiF@+TbwXxTvo!aTqYc#?O4#^Lk5Vfz42GMGMregWhF?#m)|4F*+# zWg}p&)DX+AWZd}gPdRGMqaT{oyiZ_lMz~@Rua+HFZut1@P{Mwhch8FN*4$5c<;F~MD#vJFEUePCqJ~BbnJZ&Cnr>Be##v~~i%~f^7JcFHv@1x}%*D%qK&%{e zGtO!2GbIqSysfRBIxufhiP@f&ok7ha;g&@NW^w`0i@5h+GcOodh%k*@k4B3^<9~9& zSuw|bU&q%W;X_ZE;&ZJaD94XCw&N>3o*wVeBMOWuo}^i^UoNE)ZOkq zay#UN1Poj?khLGh0$qFLZamc#0=o7?5CIV7*=!eF{~J>tY7AnESh9CGf&YaTg2x9H#?Km+~x#fTh82iRS zp7=?rHXCEX>?_`eCtPPs56wHDAx@M-=dPLWm{8iM5*)72%M+y>&iOYK9B7abJpYIp zK+)}VkSz5J>@i;|NGt_FeSj}1WQ=aWs%FYDu42BBYE)xqHIbhlBs>_1&kM1KLH0q#vnfy}XB0m~Uwbl}*``Y(rs%pk#rW9+Lvlr_fw1|<7c z&;@2XZV}HG>Hxbp?}MV+U=sn|V!p9hSZE~`ir4Aq)zS1Iwd>unbieDmC+(8Zx&3lq zty4oe2hOVRjtYhKhS@t~SjCq{&M@diI85=k)WDIS*MmFgp0Der2ecEu@Go^S<-qMd z;Kss2%ch{`P_(rC;J4QJFx`=AyKKqOgq7&m;ouUIkF5UVjpG|jCs)j$4+cgItQ&vf zm|E`Q&Q)}q)BRPQNekw>v#prDJ(t8^h8ZI5Z4yu;Scc~a!r8^%#$q)2oRZ?D2EEtF z)XcH_UniA?)!*+X@KoLu^6!~W2sb68m>N(Fv|+r2f^|S?Bo6npjxEdy_Uj&I99vUb z*osUdRY?&v(OpC+(*KX9I%=p%snGJhPkHlH#@e1fZ!R4uwbd}g#*OEVyE1Q|a9c8b zZ*3b!kWvVuxqmoz&eZ>_62x$4AlW2jt-@=ts?i~X^p{oe-!d?wu(K*8_%tng!yqan zC#X-;aQ{Wy+bRBqsXAC=AF&T*WifqBP{eNShGW5a;c5*y73O%jnEjm115UtxFhYAg z3jCXcAqYNs7c^TBemf}stI=` zMj#VK0CEc>NcdPEJrn*@!XCql11$4v-`RJ#M`Xw3=-=?mrW4TJ1%g89?l*-FrBm+f zLnk7F6X_CSyvZT1d)$$ESxkQL>1UASm#A7`gh`BxVWzASKx#hmyB*ri)YX6NPR+^o zoO`{04)m~uexd!ec0QY!GO5fCnX7BNOt6A-tj#`0sjlNOfJJqFjV)}>dS`6p4b^tF zp(n6raAAoZEZIkKFN~d~t|WG)2rZ7uW)%6VOhp~}icj&~k-OnJMCXIW-u+moC}F5abijl z94?)4XWcH|Z!2*;EKxp0T=8mJTbGcJVP zrwu-48!`c6vwh{74HX}`vhnCzo2*UVk(c+DvfD6+9n_%P&erTf(W9#qAGLXRE?dr- zG6C(8Ah-)=!>n8+wSNiAj7NbpO!WdjE0f;FN(B?HbDEGN8HD<)o==rFuRm{cTZVo@sLuBY|8EKy=r3aB0af}qhlNgT0#b%~wiM28 zhq5APmShV$-~RMSZx$8I_sZq98?qL)wg$4YkAUIi+ro@M)%vZ#)TyHhr3eB->QCQE zE$NVk$u_oH<-=hegQV=+WDB&$od6r2;_~BDABuaepNlSFWg&7bl4`vcfsm*nJ=wx7 z3(#EKwY?l7C*^|)kZ_F=>cYsLJ5UvIdUCS*?zzRY+|T-6m%U%?#$SR6{CM&+1of&1 z*p@uzA^7ziC{i!Y@`p3^$q2F2vq+I8lBnT6=&eJ%Z?H_9R6~^E->J&t_Ll(`dLuSL z96yodRRj?|N3R3Hh2037e}-u{YTF9>GXoi6M{|kO(wfUTs{=BtZjg||^t$;znc%7M z>rN}@xql0{k!09MLM%}gG|On?Lm}~&Bh`J~WgANjBcG7@+8jOMz3&&vCF%!mm@0<* z(&gB@UxA0T#*j#RAaa2|R1E;cYZDJUk)no5bLic_sN>FIM${iORk{gun+nH%iSK$9 zCnr-+-Sd!Bw)txZRhkw3B~S)IGQgBTcNp(0r*^&sri2JDNDSnKN|ZJ=B)9h43`{GS z?s(@IA0r0Fb`rmuEyf)C+{>xRO;2MBZ8Cf@Ba!S~Gpv^_%J$4#BOhu86Nsi!Puoi^ zFPW7VjAksf4L-V^aO?bfs>Ksq_xM{W&pTVM|HEN~q*g#|5YByJfMUj1H0Wa%2*Za> znPPZ)CkzZF&t90Dp%x|h-)&p$I&{m!oxAoG~HgXY<#m8f7yvniZQmSNh{ z6pB9h8ax>zmK8iKwNxGp);^JHpMPh<)6)Z}f~Wd!9R(i-N(dSFT4XB%RyJMPMXj5I z+A^s6F)apOTd%Q^8%W9g(cp|64=6qGRx0U~>oEs)3&nl*7qj=&4%h}$a0#GZ2j-zj zJRx=0sUD?ZElB^1xc7`|Y75s!K~Or8E;ULKl`36YM5Kun3tfpy2Z3#&Nywr|Zvp}e zLO?)Nnn+VXLKCpin+72v*l3a{h9xQco9ungx#ufm-0#mF_Xpq*lB~7noNs;J=Yi*M zgjEJ=f4uO?ANUnrTe^*Rnx&<`dT?1}h`f@G+m(pSsU&Bg{dW*G5NQgKI?--Za13!| zH*WVPI{nrJe2%PMQu`4e1&n3dbAe|F{v6}o(rQp+2WBIeTlN zi$)bZ`D&-sZqz!>#@`5tY>jvUt3AlPdcYT$vJp3x|Be@EZh(~=24v-zJf;+yvz@38 zeVONMC+qYdNQQ*eU$QPswY1fiOUQV@!NdVh?5+w@cH)9GMDc7;bcm?*7Y}!F2zgx* zARni2r84%^7keI#tln7t)vpb(DtZ7)B{mK25P>g+4rC{os^QE4wu8ZU!G3B2!U0_CGE8IpkY$b;s|XWD-r zC|EiItz;hHJh9yJN%^f0<7TyN5s^wn96m&{6&hU2c=stzCddq)aTJj zJ08x?Zpz}G?o zJ2Purg?49mu9;{~=k*X9g~<8w3IId9Qp>IA0)}c`iLe5|^UU@E_j%4C)AR_Xdd-6> zJ+O_)WXZ3Zs#3nCKmebt?Krch{0xj^q(9Di)Dn05Tv^_`TnWP4A!`9?D}_H1YoNTq z2^v%JPZ(%swXqQC5GdSgq?wA@Htd~ycY~}U7l}AK-dtkJ{lHdrRw6=gC8b;_-S=>P8n7$KJpX3<9s_PC0Ax;L=rKS!2;wYG0<#$kM5dzoz&n4eCh6g+ zTqnmdCi=-Onc6+KoCd0%x?gzJU&y24ZYFJg$wvt@%`aAnXuBT5046!v9xTN#u zF2UiT`@d#?b%0@81+p>|(GJQ4pd3K1*}jm+;!yw?jQ%6G;A&R@dRhT|XyS`jXfC;T z>HLLnEMbLOrdaauz-UVpvV9JxG4cm9j5pb#awxpi1Bf_haDddGmIJSnv(xbsc)x5B zhQoB2#vZpwccSO>u@=5;LzQ5&`ii)3nK(5`e@gAs6Rp#V!cVO>mC=bPioXOyB$PJJ zEHstJoC6S;04R1FlvCCvE>IY9$nnil$>np)?N#%=7nJ(mw&o2ySLJ%u216u> z_MV!&*0WS+y7@7n=x!ct>)Mgd;WGX?T(8Y6miPTRUV5aP*lY_Fn*S5U5CiP)U+7?# z#Em6pu?VFfwR;%ZWEGEwEsTR7oqY7oAqPt4UWqIc{-7qQU|?`0stiH?k;dc&L=>bC zp&%GryZ>~>rAr!SN4-Y#5e%7y{b&J2soS%>?@wf}k+1|AHonBf@f-aadxE|>{8v&$ zKMpMx&!YZ_gW^lt7BB|?nJxX-Fa9fmf~x=P|Nh_q`2T~yh5k+B{(rj&Nc_JZ#s9_K zwV8sdt^`D?gG4caWd{P2Z3bOh9-hQwu6|UBSdE*}x(1^g>KjW*3kQDuJXD6bud*Eh zPMQhuV5L}HAa3j(D$bfkiZ!cb8o9!6B78uRmz$Zm{><&b*joR9w-=|Kpc$1$l_o(ql=plJG`F_8AUskhwO1N!%{(o%Oe=}XJ=w|f#FZ$&`BuJFvp@1NK_N^sQ zb~U$JoQDRa#{wnVA=>nMoYg(`jiYsCnR0(&X8W{_v^(^F0Wj5XOQP))?RQsNK3dJ4NZOp8bSrR(v^Vy!%f-R4KGN}^rLYq^s?nV)w{ zg1?6T&v9@rc)LS*|BWn96NgNQ9TQB|KR?~6wAgV`*=g)%j3snTRlUBd-8MP(10>-{ z-uLGCoED##Vjg`Kor`;{d_ps_#-7={J{ab=`_a zY?BN8n(vESmz&;t+7x@CW-sqz_Ie4M7z6nGo&{9%>>!YC;7!-)e!@~&6@txsSi+Oa zlpk?HH!rm{7ya0sOTJ*$hRH(oT;F*4>}gq|BiC1&-_PpiO0ubvcL~13r z_^nHWVNl)%>cSG)YQQAojGQ=JUJrZq-Qk<$Hw@3IUUAW*72FvVpE_whsTfxn9wh^U z)izof`VEK)(Ek+vcI!|!7@~7a`so;D`aXs^btZ19A;oYoy;C3Pdw2>?>2~m}>8jAv z?^Q$vU%D*NAN~G1)VJTo`NUf({WLc=(__p+RNw}|oOX-(&{&pQNZJ_4-s@kgIM(vv z;AGgouswN~eSgkN9n)8A>}tcCdy|NmIqTO=p~RNEFzM^anB-N zqxec3JZ{^OHDqC~bK}jKhKi7sU$L-u!~SnKGa+|7%GH`ft%>Z;B;~lkdQy3 zu}3*2c$1HwZForcXjEuOh=N8zX3;@HkF^USG49Z9EQoLVssVKbelrsV(hk`7_bAJv z;bQ}=gJAL0(F0lUp?!dGb`sTG-3_7kGEBx=9Qdk}noEoZ2zCSapDfJBJmM?NS5sGT z+ui9?da?~2p7Yba|D2#q+vMcmi zIK3Oo2}XF2uy{TPJS^a=@t52u8us>qp`O#U+o>#|(I^JaFNk&HAHX;kuLb3XCskw3 ziHhtoRK9Jxe_2i$qH?9M4n%4=j$c(^G*j$zD1>(x8!5l@M|0SwLP9pm3%)Qqb+WwTb)?^2Wp7l^h`1~fIwy`O*in+*GTD_mZoGQtBv*Nd>ns2OgH`CU!H0|z7FJmT zr(rXQ`i6BZh-V3rdJce!GZ{AQSw_Py4*9z5IvVnaRKC_V+bTSo&55Yd=1~-2(4gKy zIQf67o&wtBax}#TnzQXkQ?70RwzsC91&-!f>_Lo*VR#2wgU`S15jpyC-2^-u?ALNF z?0WpNb?UJ@{i{Mb89~YrsqOzzB|DC6#Z$bu){)eAaymzo3RDQ1$Q@h_fBhSir83Z)}*>1 zN%Zha(9atz1(4Flzk)^^db1hwk3jLHeOsA@NSbWf16R>a16MB}ZAK$jDJ^3uf3BFD z6mzOgUSoR@)@_;(n;%CK+89FBEM7sB1SK1%*3L{D|97 zL%%EVZ+biM?JMtf7lStgK`Vg_-~$tEHrLHTgdVRI=@=-kZT1ZwyjX-Vl<)Em4*S{u z`FeGuj1*I|Y!|*5IakaQ{0;sxK2n3$NS%#E`EIaBikMT9lNjbx0#slwQ2eCM6H zQ)8VIr}75yM&*~vv5v>zG%Oq96E)^S?Wc5bb1LKk+|v(s!G;){nXR5{LRJ@u?=8(f zd+)_2#iHB++>S_zumVbX(zHGbS1*y^(T&|b!%!j zL~loMmLoMs_mlCJh7fBXc)3(~`BT;cwEODK4#|4o&@`4D;M|k{abm?$TSo9NQQ;W2 zgZP&OHkHpCu$l`)^c))8kteI0CC_|pEK9l$iJ?o;n2A zvi*(SLvM6{XJWw%dKTP6${#~Lvbw&{o)ujS1vyv|sZUra^oZuw7z9H0t@1p!n~mSK2V=^StCC_b;-}$GxXzQ<&KW#c2OC&D{WoQCnp08K%0Bb#b(Z~^4;McW* zf4RP=&475g#rwqz>s0IeKi2g2grs={U6SjtH+1A`1rNpn1k!8xxjs_F`aD&G1~CgU z5ErQg*^5UQdpwSGg?b$-Y@mpkh|6JWEw9IK7U`ahQczN0Tilg<-C9Os4aBTmsX+3& z9`Z-?1Q46`V>>w-U_H8f(&WWSV^rB2uUphpm{*kdlkGJXnt$=$nG4U=&iO+G4=oft zv>i@=F#R7GF5+QLl*3%q8Z@?m_6;6v3{wnYXNg-npBt;$Wgy>~O$sS_9QZezROf+5 z0eWC|ZIm3%8NaX~|I>P4QS)id8|5*ue4snWB@b>Yf`tO#4|m#t0eq+l5!ZaU(~kT~ z&n}RTrm8hv^f2?y?c6oL-|B?7pLBXbH!mBPGO_^R7x%R>#8z2Q3|16c6w?Pd#@T1! zeSWxym*HE!^^{z!Jt&?@S@j^IS} zB6fzmL-dMiM;Fm;P(`K--Gi?1o|wVJts`Fz5`V(2mN%4 zIjo!vVa!1F!gSDwnHcpA3XExZLs$7hM=_KSqK)>A7JgyS+k^i!Wd%{3V^L2)%%Qiz z;_IM-!dVL$fy9V2t1$xQJ>*cHDGEFK#@9RX+wOMaj9An6SDPmGy|MqOE@tMHcaPo+4dW~zHe4H%z4W*``3GGX4 zY3!f+u^`^-ze1$A;QnmFp#6UPJdu%qagZU0f3$KG>aoE}t^E9Ox;2l1XiDOHl#=o} zM>o@{M65CXtzyO<=otGfD$->MP-p*zr<`t>vq6!{|7Lqu-G`q;vi2q{zQF`kIjDV2 zr{RtThXnul+I&xeB%bjuWbJlrxYoA^vmF4oLH~!>VN~zX+wRRH-C5iX?EpRWl?zZ2$a#R9mW5_P;7@4lMzb+Yfqtg%6p|%0 zjJ!Ce*!8NdWgpMCk*0^8$ac|^~xwj|d6RmVsy4@RTX<}wsk zz3}O6s=yn+A~VInPdbsDHNt}DIhedIR1k&&{CA6h1B-W=ZrJzegWX?LhWq%Voc$>; zT_?Useu_14a&#??-DL*_ZnC-TRQAv~_=rmuij;?IMJizJ_uGia(&(U$B0AQ;BazUgQM{kW5&como(CmfL{_p85u!} zFhrZ$JE$D+jV(RG6kmYJXYA>a_m^YtG<%C5v7Z@LM0_UTBk-hI6ON0ZWEO9YolBau z=qRL{_eK2W8WsMY7;x9$g;W`dH1fH7$@@$4V<0vO^Z?~-z_K|`8>62FdO}=4xlKUX z;>Fa#Je!FhjH9~mTMJ&l$x<#}=llA+o;{}KMFz=F?Dqy^Z|KfQ>N6+=bohUr0K^kP zs9muA&)GpeiN3z}!9<lm?#k5(1Z^ezo>sk&!{Ct?s9slWvzX;2Y;MjP95^Eqs4z91OA+a(4`s}AXj<);v$h;GM~?!`sRxqc0Kw{QDNq1tJ2hnENmV$pE<}Hq z5^UXGT0+iLuzC4&y`^|g#mEC^N(?_OG}Ly@XJ4bG3tLVzZpg%zg^&Tu)OsOMUqZ?^ zoAQ)Euz}|5)^WWe)>ls0oL8fjzle)_z`tZ5wyDez+Umm|0Ik}D(@>C6bj%)_jjo1B zWa-6Y!qacsGyDmL5*h5j9!Gw@P;YrA*~x%`EJG|uvjjj)kdTH71S485>$Y{Qx5$== zIE=2+=7J2)eM}b=CtO>KJgPb@pZDlEC>C!)QBR%X_VsYOto?>mAc~)J$>R%9&jsJ9 z;I)I%(V>Zi?iVHUO7d!I^<(c1E|8W?{tDWz1Yz3|i1Zl^t5**;z)B#&i8U1h3^aSj zei)xwl0No3s4lv{VE9H*upfWvnYc?Won|aSJsTkIKn399wxO5L zir136i!$U_-g9(cXx=KC+LvkjI=MTFh>Uco+4_0T7UP-KLc{0l_*Q@K zEw|mfmO_lLLBWTM_l*`P?B2Rb;PaCi;~Iwk`Pgr)7*kPzyZp!9xvX$8q{F}`e`5mN z7wKsipfT*uCM0(gL_HNL19ysO#bg9Op{1_5)z`m{K(Lnm^e66VgV17+Nx^L9-BslVRKt$>(X;zG>fQ z_tip7`IoQtTpt&b$tp^!yy(;}+%dWT<|pkbm(MX^>!VsD7!qFnNKj5EC6^+=HhzfF zgY|{sJo0JrTTpVd$*I-h)`>9byD#7W66+~CG+djpH(28?XTy7|CE~xPZWF}C>3yXK z5?K69<^#2FV-3Q#8baUm78D3CG*7;JnwT?S>U8iV+XIeQF&5appioo=0^9|t$DgGE zjOv9h5zNU6l)nQ_gx#w{nt6u92Ugr)PK`0A*kms zC>Q}NL2Gpsr1==QNuxZwDlmH}CULR;>1aD@uJf`sfjFh7O`$#Deh*+X%|3mtLaRXOrw7D_ovKZ07 z0U;toNiNBqnoa3n5{A-G#>+MqOSEQa*S)oN7P^`?VWV=y^0`iG$08ce1j=~8#htvd zgl7rgvih4%E$A~o4)?Qi_BDboHOi{G<%1P>V$*a6zCw$_8Qpo>n+b`+2zBYW8^pk1vuBY*!h0|k=BGmxzcr_`T zemSzO58oWIcIF1)WZHIm`!OXFtWG)%YN-kvqS$^ zv#4yWI6Ub-9*~COV7<_OWGgBWbC`aV^|}qoZ5&bimL?}f%}&FJ0ea@T_hm=s_)ipde% zi$S(i!Oo;rxxjqeeV4rGf2vSNAY~aw|I->Rx-J&9wt9%fpUD@^S#K>t&7s`xI8qSm zn?sQ%P}?U%L35lkw~S=8HB6&md8_pj@R-|}Pw;+fo`Wo&`=D~d_PtAbdLG2v(sl9z zMvjwJg-YtxNr!!$vo2+vb;_CgbM?M}G>FWlb?IPz_Et(bKsId+7zFF9V0+%ZzyZ5R;NC?@fI;gL*6ipw+@ltTTn-CA1*BNes)1~7h?oe9GKp3^822?r~0FogTaS%nna zrIw8}h7Ea+j%X$N%dt(Wh5C(?g1c-zrPJ>TAG|9(6nCFiPjhm~#C<~}pq{UIip{Ga zbmRDMjbxnEW_onTQ2C4LFa&qNaHw|%wCEmnQYs_&?r~eqEdA@E>y`%R{(!6F@&O_X zL6~SDS2mc8-~1?my$a3{o$d`T$R6d@@Bx%Q+nCMRqnI;oofPM~;9!m0UnoL^ZGyr{SR2w{Xcicn*y3Z17FyiJ+k1P-{w7 zshTeL_}jXEd1&cO+GWvLt~09Fx@M$vq>q>yfh`b-`oe>53uUGi4YDdC@?p2*$PWO* z<;40gac z)+U)u$hTW)TW$G3`++_cw4)mxLGJ>c)+s+HsA2WzzXJ|gKSxH-G!;5bsd<$yL=ujX ziZtKIYxb!eCIKNXxfUf+_Me$sJHzcfJWq&rPXJZ5304ZogViZc$Z`ve3@TD$>~77j z4bt~whlq_yxTgWo^MQ{9iBG>T*p;VBmk(; zX7Lo$jWMStSVu7;%+yZy#ZgiJ=gsZ*ZPorym7BvYPkgsA@(9|CIP)MwYFj;x?H3O4 zky|qFLs!vUgh(#hL-@cVv?24ICei!zZC}40pP`w-*{&;o`38zIa?zV&WHg10f)!@L z)X8Esi~Daj`jPFCG`2pJfT>&?Q2aihNh(o=(R3A%{Rx3%to(sB_~j4e_p`C>-N&En z14q$1P+to0C)udYY(x^YgLGZ1Sv5SwF@pESe?-!JnOVvV*)UMxiy$Q@cM4RX(bDhc z5C~UQ9^)@3{$yvllGwB&*QaJ5ju)DpWfzAcccfTYymoXos7Iq~H!>UDf=p8;wwX2yR;KY_CP-O6ccI=Q|bx z^EX?b7c2pY4!%|cZdV)hh#DJo9r+A=4C0@z#9h{CII1~&?G^b=Lvu8gp4>9kJZ!0y z>^Q!$cNeYjubLNWj*=cpK0aYLeVzGZp5vuZq&N6KR=C$drTrHejj?a|c#BQt*Wixp zODG151V66MIPxxs0p#~!jpm` zk}MFVPfVdh_S`J2rW)EnYV`Jf(9N1@mN*p*l_GLevUpNi_a zwn|8I%N7L^M1+{G{OS%$c9@RDFXN-)ghzIXBH`Z-#g#zk&P<#Xw(QAIa*7+0$H$&x zF?Dc%c6&gx?a+7mSk)bL_ST$^0CHFZn!2X~){Vow@|3Yj*uA9*fvE9g~ zkOu1N5-w$}Q-g%#ULHynYv!$@>@xES*tLYIKQSm{8^>CSxB5zCSmprB{`hYQu!K;- z&GMLlom9{QVx|9WLLnB_0%j^)7-Ew&05gwojO1{mHmtpB2%MSi0%Pw%EK#acrCxho zd>7`)!l9~sDM`&aMXSsCJ_bHV)ES^q5|4sM;E0muYKghMi?zjjr%ix0gZ=(#%y<=R7V5Wd_8gLtItfen)!PRsf z))0$_;+L|}YpYQEz39Z?U;D3DjU9XAIOk%}qmXqw4MeeraSRDMZ5OxGtG9@f;2TMe^CTzIriHH)8xmok71Rtf8NqBKaUu_55My(AL#IdSH*eKz-3( zI08@_~h0^NB=qw3YvhR@h}onJ#==7-L*DWw{-9jo7>0V^6Gh3!W* zX1#m@0~jCkp78E0KLMR3JLa#q~#y#Bp&Spf(jSki#`eN zC^R*p0O>{0!;ZHD@&~kRQjA0I-)s`bNB7h{wg^&A!PqRun5gPd)+9z=1jU?i6IU8> zE70G6BZHS=T5y{~9NOSma_DaOI7b_VO|deJP&jIl2n4^}#b@7Ke$rhS9$|Fq9Va#C zPPA5gso9FzF-kUk4GAcFSviZ8QSkgBW`1{tLEv%`m|jl(bugd4I~GFEXsNh2yiTSu;v7F zQ|N%BD%?CjEYEg7E9m+9qB9|^-*uN$hT*{46<^^&)oZLw7b*`}B|os$s#z0w8oCoK zb#+HO8!Hotv=CvV8X%!f-CA-;Hs!bTqH6ST^%tdm=pkpz`b)*M=J;qQd|^KF*ZAsx zhIkcw*)HXRPNvM|&*(8E!3v_CW)|VEB6rU*jF{Cd9vB*wQma~-CoHoLJae0n>^`D+ z`e!v~7Q$o|zCb}TvJEwuiFNqRz}cm;qK5o?hf~_yIYQSNWvkmn#w8n+M+m@ zufd)sD)ghqrmZ%#qkjLBxAeH{$OXN@{o`{-cYH4HU!O}G{a8UWW8Q{a+boHD)#H3i_i8ZFHh}Sc3MH ziNhRS>IfEJ{5DZR6@1Vn?czgtjc^)T(N2i_t}+yiA|*n2inX-#;|B2WQk;OWtG7$) zz-_@9uKhn_f>wDCagf6ZK4DNn5Fo`k3MiP>@StCBjbk)7 zENma*_>4{>QY`QvIl6Ha+aLfY|FRuK$c_!mJX|sR+h^KS)^hbsK)npd89XK?a%mNgt z(7iCj)BvN!tO5dl5^xqrIa1uU(z(^pw@8XFRW+BucYpj*RPJH#0vpNJ$|LD}f1M^8 zu|7Z)-@tlJ= zC0t*V1E~a@WltVXxDyJd-(Z}9O}H|`c2in5dINqJHz1f#y51Zt7p$%}6PqpEI$0z7 z>)_(Z?EY^Y$ca~MSS{>p6vYqGo&7PJRzv|Gg*?&Pe*)xd?mAYzjdfKwW8S7_XES&x8}?AP3mms}QQ<^$(Pe&p_mL{G+Ry{L zHZL{0h>>Q*Leb<5MIcilajko-4%I>#=^~Ra1eFt9n?Zz8cNCg)h^? zDL$wi>7e_tHI6TAZXZY2an!>&x+8X2^+dxuf^^p5xUmPY1%p95p>#H@8_5Ms_aij) zJR$|?Emgj4N62$zW@{FqdhLfn zHh0zA5Y@5@NDsA_bpXIgXS2cpno0!Y2Gzp}7rJtPxz6a?78H?xL>vo{oy)ahvT&l5CcJ+`h$ng9yGUwBFS5Qt*_B^%QtdlvL~H>A`mZN?4egEo1Ny9ZN~c0Iz1tFzBT7c^gZmbye02Z z=R+%d*PI0>Fx89TQ;)t|oUVW6b%Tt9p+hq*(T}X6L<04~k^EO$OCvcF6tBJCzFi~o z{Iklv&%D-3($WqavZCyuNg$fQ;{HWfV12|^8=x16 zA!YB3_{L+70nnkuij+@wD(i4hHJmi__DhA`iRqj4H@CIowkF@F=@TNAc0-!y*_i<6 z)&o+!EYx$CH6&divXFL}0%d-I?^iO;v~@TD!e0GMPSgEK{@yMAB-?-w?)}fSE&0z% zd~(aUKiBq)_!>{~YU_if0pd9@Vgb0y_>|RKzL|REIz^Vnj~TjYKgknQFOOQNu0p~i zE5%NgLY6Y|Jt9U{py<2j9FU~kN=JF2(f~7xe)?}d;E7BPo3s12W9Hu(yWg@4^v=J+ z^+Uzr)A1$a*>yEGEU2K%gA+H~5C^DMh=fE`5G2AHcZb^giDLeQn==c%R={S_bXzmk z`nBffFLYj;KOIsT6y+%@c+&s*+t;%19T3a<87wu%VfaRvKz6@IYM0!Y@5_u=orWVV zyK7FBk)EeswB@JfToi!hkd*6%{n^OKUo1WqYMv#vF#WfH{bb**;nw0Z_k{~hgVh8+ zqbTEhcOgppD-c2!9E?TP3m|B>VJ>ovC-FFiIEdg{k=VMkBn@@~Ypg@J<6D!0>wdQjCEa@Bjd2oW8<`~*%TEJSV{^7)jt zm~=i~>lmlpZw3=wOb!@aHwGDVv%~=U_H%HYL;02C*Tc?_6@AHk>d9=BrytJ-=zk?! z!HR}cj9`fQ*iSy&852;od3@Xe3_@Of3XF&^N zAfRx-+d(%D1VDw$kNsz*ss|hdzHi@ck9h}KhBVWM&k`0^W=_YzRY_Ks=X$uNIM8zf z|EcNt&(BNnB)PB;nY4ZtCuFMNZ^+&$7`&&YrZ4E*`oMZ=kosE4g{4P}t8TYk)+OmC ztL-ISsShy`6rV5apO*!ysv6SN6bqtU6VIP?vTjltC{}rUo`LKdQs};AyaUpGyz#sq~25m-*9cF zB9edGrc2YM@4k5cs`-9J0neEP#gw;Z&dvZT$>|%5Y67$bTMXfMv|HeOS7YDr#Tdh1 zs3*|u^N}uRrEp0M=%%7`vh{u1pazaVcTkjSjXS`<($kuMc_{Ra1AJSX2J*v7!cPC4 zM-2tW26Sh@#f~6lSz3W09vb~9c+s5f*{Faq8*8o^l@__xM-PG4j65>84E|akzGUMot^00S!I}X-w2?EA z7kVuvub}@nw#i7`>&LC+bp^Unfc~%LTh)X3-E$W`PLE%hyAb(kTrtf;QA_K-RxS7b z1W=w3f=EOnfomxw57?-A?2PHbN!Fp&Gb>2Jn;Q~PFHfY#y*K4ip3-SQj$R$S*E_N< z55;x+D!TmIcpNCP-NM+euZhteo_;^9Hay&4^|`=Ov8hSUEmbi$3P^TrwE%htyg-+t zQa?_Rwg3CA?T`5q!4Fr)=DGR&{Gr3eA>pmsvs@GS1E4PeQLdub&~#_khg)0gYRL61 zKZ@UZ>I>E+6_M2PPCCXcWaHxp+h`7v3+TDdGehTnjSZ6#%};Zru%S@lYj~Fy6btoW zyLOHSpkny7)W%T8YE0=~E{ADbHsdqfUn<3&HBCPRleunZT3N_kcQ|((`54{Q#*mVv zb2kiZ2skWlz()XTUBNQptXm6mm%qDRRpLmS@UE{PLk>)R%~cA<8(9|9xOm4Kg*7%F z9JS)K<+M$-;H@xWYal_3vrs=;mZ3nM$)g%AW~Ya4BKLlG8DH?<9w@v%_Q~BoSw-IP zCpQXb{u+wDbv0YPBvduSt%y_A;YQqXk#P%50Lb0A7}h(90IU!(=pc*HqGHR-gY;Ok zP4UF@6F=M~$yrNho7?U?&E^wDpC3?oaI6eb=>Up8IvfdrixAo|adW`LT~to!j3GIs z6x}Qu>;tHtWvTT_?R!4$<1L5kD{R>K9hq*)p4EAp|ki2v4Z5lq}VXb7cU%#{i zd}Y<|Yx!%fi*{K^hxh((8EnvL9YAa0 z4V0g{_P|iYBda=)oj=MecY9|Luky<4Oxld{-5Sl~6PmkIoDD|)s-)?H(5=sr_5N?R z&1-U?eAwIm4(Mdf{@O`g8!@(U+zomJMiVwcWrT@S>I!>pV|-`(zAksYkt;WS^2AJS zEZYb(leQrGDT)xw@BNK>Jr?x>2Orv6J4!hRblueG@?i7~4gY>RzYM6pc;|cS-7ZF* z4bcC~*E;mr0a>rh1y+9~uL~MnhQWT{n}i)gZWg0?p+%)?z=rs>BNSu=x$vJDa25g? zCU8y^73vp;6WrR-M*wkgdT>(RmP`;L2r?|tqHn6ACsJW9Y#SbQKH_2tj%sI8$yScSA5=UpT?@uCA=bebrjE z%dQo|4%)&e9`bjz+H2W`>;3_O6W4u=HFYZ%Jf?f5qGJq6GSl3)nQHaALrtS!)Adtx z&|^2qcSW07dJ&IB>2W`JLb^BL&#}40l1ire0s6EQiUvz;2F(a>Ckg`b#V(EUGAi=+ z?^U&P4(-s{;D*iB%2LVWN;WPhoSvD<$d#qBI90ajEP-HP!xKQ#7;nLvb#%j6Pmde# ze&|VzR5Sh1@82d+;rV{TBY{nV92UFD4%~DbxdgCxEhtU@2>mU4;>!<>hK7RyN-yqzXc}7UFSyd5>!4V)qdc5& ze#gi65Gfvr#4dhH8;BPLlwY^HGlAOmTu_0lu2JMowG%6czMA#jF6FMgu<6Yi0hJ+U zz+k8gL)won!J;8n|!0+xfC(r3X!prT$l@gSdt-TLV6#kW-J{KOFk|Q)7 zao|uTr&1f{8wCJAxR@yi;p#FgG9Fg(&0CVyr#p+pEM7#MXl(A7e#p<~VWG8hL+A9; zmrso^B}u+r96RNA7P!_yBt&r|rnBUz5QrR4_4P@#V{wbCMjZu_&8z1g5#C=(bl`qu zBReE(D;%`wzymk+FK|+IlQWGa8s?YXyYeGsXNsS=+K<}>YiWAt$XXxijs^HF<`4K> z@3us|vDzb8qnC#AnbgTNl~yyH8}|EImdbhU(Kn@H-ZGz5Wtm(kka-Q)l=ZR(#?hiE z$W)B^<9AWeV_4hvWmiYZ^IyYCmG(adU#i+DPtE!r-z|Cu93K;?=LAF=%Z|mjhOP&Y ziG;jGzjbsiMh`8sa&{HSAAjl+Wf4+aqP|3h{3tK79r8T%HK%iuM^M-*&r8+Upn{qB zFc!5Jl*b?#(#j)g^l4-+NWsa}2HMn2XT)Nr#ChA{bG_nN9x3y{UQM_CLH+pr#f%}5 zD2s(5*)vPN!piGkL><{}6m$BdZB-JTIGc`kvxO;6%!2YJT%|9r(`}cds61$fRl71?g9hd{B7;ywIe-T>pY~0 zh)a)C$?voPf)1gbhXx*kA(u#17$jr^qTC)LuLcOuZ5QiQ7A=yhs=wP^C;G(yh+c)i zmUw+vyqTn=Hp{&mYucyNYWcI4a%$*%PtN$W^I2w`H+9&V%rodiP>%p8s4W2&bYWYP zRd$ym{FLr70~idEb1inKNbiil94ydP|5k87$VK^?*zXu10v8Usd=6#EDga}c%WMFE z0s^`Zb(`3@&{Pcvc~Cz^5VFdbR5G;)tY}v?IZ?Wf^VT&Rt_}X~=8W3orw?)dKxW~n zq5$fGW@Uq?f3z?O^#`kX2(<&}{GRXJU!#{A)Ejka3H z1$XqEr*N(PTOP9mtv|O&>+1XB_o)22S-8jt%GjDv4<9NNFn&|cf)0rCZF~TfDJ6&V zBd2PF>SP86YX&E62iX``+*Ifmk~F1eUVmyc7Z+>VT?Qy5U|j&B?xSC00erVaKazl> zb-_`{IMS4{v~r?OC_mBD+~oJty9l4it^08c28(B|Do6&uANh3XblVRxut(8Jc!&}P zijASPZ_M@`p+|P?G~i8==&Pg-w^YaNrxY|Tc1RChdexZAH%IU9817HW=1cFBWN%Zk zG&t}R|MI*e7quHt36WrkzrJ^v8E<@2>snrYfGhHkK6(nQj|a_yI=hAhb)99{jjoKb zpIOsI?1jzE3rwzjvsDf5hknhE-W;#c+#N4T3+bM3GGHTjr8Q&#cLL~09ntAX5H7>Y z`l%>?d>la!cZ1xK={#vZ9`ak4C|<0#@SE;;*UYU*v$8h$r+82BTi$0}AM*t(uFke& zXn~8T$DZ#odColy9YFr);sH1O@8Y4KqbWff3`pE|hrS~|AA^ExNT@lEbCV3428N!v z1kF>kb+T^l;FQ{i-Hh7Tww5#l_7p z#9B?%lAVg}l$@;oYj^R+O9e%hvSCxCtSTfk>XaI^L@`5Z$U& z+7KELB|0l*UqPAcdlz~uZLx^ww7Ae)q3$CC$Ar4QTBb;`i1}>z80cHPq@M%`Xy`J? z6>&}CxS*$zxE@pO?Ea&Xw*m?dhxsSgy>9B^d&&JAfr?x>Zp1w_67&cD_sj)j2l28Y zf3pQ@eAHM>URQ$_hc=_Y(3r`1r;T2o*qi&RrQ2_%^L7n6+1}P#=RfZ!*}d$Uc^r8k zNeLQch#+bAz>Wz4O-jvV+T-!yP0)`4IBm+x8ThiFO});GOSgMDH>7oZhz0C6Y+lOB z9GeSiJH$s#EWUXVnn5hE15RUg!FOBZD7p=H7{%cYCoe)NE}AO^TZZZtK^)8V&n!4R z>u}tT3xjh1{xTPx*+$c zmdS@+zZ~`_Eq;2M@!&}AEtlWpl*!&aSY&ZDnSlrkhYE$i6O^hl@&+nZGx=2{FCmVf z`X=|g4V@2G^=2qT?Ay**e!M3`8ulZB7Nqco{TRc6YIJs5DpbnT)ht>)C-@OFvGkBy zy;m+*)G+x;uG3$s(usCIRdd*|kI$foBp4R5U}wyN`0nf9Z0~&*vsunAn=0Vm!S?8Z ztgt%Zhdnq0B?UXhH~))94sx25jkJ(Xwc3^KtG2vBp~1mHns|R9C$%p*A?J;+u{LgL z%MaAcA4jzRQy^;is>0&MgYeS;>T$tvfe!;yl9W)IKRjB^FrYwvMCp}Mkx3m}Bu{{0 z@kLp~pFu8NMbY$Q-oFk){_oaKG!As^-QyFW_^aWg(ZA%X{gKhMEO4?0aM`z@eiZM! zdb8mANb|E4&-7|5h;#Uhri7l}89V-T{4K3U_VEaX`p=gTD=TRXP|MyS6ehLcS^@M1 z8USRXej6WV``w}2EeH|+iB$qV!)GEmOgc~aG;P&S29mh*Zr6eR9o9|{?v$Tmn*+?~ zc=DkQSRKzqYOuUC_@STAV^NiVv%zZkwR2Q`7BA=qagM+_bQOm5)YV{yLtWqHh>A>` zw7zS)`I{2)yZU}e8fU8>U9^pl6k`m4y=Z`=cp$emL%|D4zfSGUcd>Aw4uo&x%7#eL zL$J;ubWTRKw7u342-tQ>=`;_G4zBZ9K27v1AdhTEE0^kqc)8iV%zO9lDzWW`m-5OH zwjD-?g!6b(HoCQNjbjN)MK?>VN0F=<(#okit~X^Bdbc93IYjTzx>-_xuX{tvs@p<3 zT_i7-3w00I0%+|))oynT1-&i}Ey;2%j;_59TYAME)HGZPHyxj@1!sGIe&w~BG1P~w{!uM)GX*jSu{hk0)j&t2| zMiZTGwL$H)Cw^MCawXBbXFtl@22_R@dF`_p;w@_u`MHjoDuIqd6q^nB7^tDWTmtWc z?#)hsng<7hQDjsC7}A`xr4&>>Bt|;eOv;U|=-(k!ZEAmTd#zl|XN~kK$g)_tt)le? z3GkjbLKzZJI&F&?j}OWgWWB*P;xd8lxR38Y>m5wStLe{$^ z9pIzLk;^_{uXWty;eRMEBEJ0(y52k<$~SBqACV=?*w+|ElqDrn$dGIa(PB3hLiS21 z%`LKIU%m)sQe-E4*~U(mWZ%tuyr1X&qvi9d+kM^Fa$e_o zoab?zMhqTMEwL|oY=rhe!6%>>K#v|bjUoO3p5bJtBibyDG`zL>=Hswi<*3*0@5q2p z>8IKlUyd1F)Z^*XJ&_JSX-AX2(KO+AT~dmp+UifCJ;C)XZOzDB?wXmpq6&|5(s24caG_mnW?*oaGcrnd)m3jEo(a+EC5gNge`?kM+S! z9jpX4kQHbjBh@MxrWJE4fG2R3Cfq?yrze7jCisJB_r&QygVB)Fi(3f|-VuxUetQVN z2~j`X5F+w3sY}~XF-phxAF1mzf~Tib&N5Cxot6Z$BlK6tm)7kh3X-U<2<$+HB9i$& z^jmmfE%@~6=X$$2R@6Y?qH?`sJ(Fb zeh1;pl$+tf!TIakL6F7B&l-~rV#{D~tkwPTR#pPMfD7EA+$zH!+x*Jr&?rt`;?a?(XLVbt|I~F6(W3GN|1I7hBjMO$RQ@4eDP^+~g~O_c|J|Z&Eyg zQUMM5)~2I)_F<4&S6bYI)=vJLmweOO)2*Say@ZLhSHP>OiCHcusH%H^`l5P9m*4se zM7~Lw=Inv$aWnzkBIxf#?y&zbO4DE53zV+rMvZ5pgyN5l0*#m0hbp3SiEi)b zZA(QSoxN|PefFvW*YNo!+nX0lj;b1SB)jPGvTeQw7y>E^M-BjWc;VD#dL!Q57ocgi z)*IpEZS-Ni+r*DeCraVF-P@unr3;FUvz<)?Ka)Rm#-@rv9X^Hzr0^B!IDYZ41+1J% z7u5hIo#fzrXxkC{3GC8gi0qI!8^z$5r~GI|SJiJ|7UQ+rQrEG(iCvGk;#`I|)fF4_ z($}}YXt6|6#co9&J3FZOxiasOdwb2j1UXs-@X_}_9zd-j_&;$?r$OO!AA(~ThbQ2#1~qETq#78w2nSE_2@0xJCS}?-k1xOMx`W2(X+P z@C%BY69ur;XE-h3r=&9ds~&1-R*xKxVU@)Z?~feF8DB4#oWRzD%a4#uuU{4$<~v_S z{;-N+xZ^XT1eq^FYmY(?(SXw#{t``mg{u*v31%}m_HL7RsCTPs$3m_Vha_g<9D;7n zZNaQ2|LOVJ3pFP@kM_rw{b-8Nahi);#zqb6qJT^FKUwn`Jgl+BU?=R|V0;F0{=Nw~ zcE}_9i3&&1yQTEkfzG#cp~0hbrf-jY_*7-$bLEO>8A75qL)?c!K&jKk%QxaNa zJx62|y0d9vUq4#QQTAryVOgO0+4|mu>lPxJeu*m|B{J?8wTU4h3dRR&`vl~PF)%=# zHh{4NQYaOho_p6qM0}%K#Nfc&MT6BgW6A|Mo1EWfjUSAq?LAy(t2gDW16!F6gqpfy zjVSdr$s6Fo7K%n(fCV4?fLdKiUd25vH1l^R(a(@3o+;O53w&~^fAQ1HHFKEhkhra6 zFQWQ1C znRVX&clXYIFwOtWohMx3t2x&%W%Dwy;+(F7ilM@cFXEr*>5$Ho>= zxSdTgqcf5^4Sju0&tHMfHuTwz;2k;Ko=+VZX<6Tk^j-4I}!zQ&Qwz-3Yk z@+)Z0wZA>lAk5#9BB zLF((ZPCJdbJ0r?!(2v5UGaX)3D?K-^tHzoBM?cp(nv=UaRMMIDP8&cc@O3IQNh7eg zK)*K+f&|o_4DS@sw8&*Scz@ToL8fNpyYYqjuTGth7XH*ymG0s4G1=J0*SkEhlIqq@ayj`(~}cq|r>E^lG0X z__7+@n7u^e$9<;zLx^n$1hH&wK^k3?p#2^5(s%9Cg*p+Qc|X!MG}~=I)yAxw zcuPuq_}99)7~RTSetw6I6@Sok5BOPOfb?k%s5=#~-0${89@v>$3X8CHnG(dDY8o^; zew3*$1i8h`XACqZ%^Q*n<6k3sm2A6De@4ATQv`qx$rdBYmIQ&@ItZAWq(bIh5eBy| z**CUH41A9V*-J`=+?|`>lDqiyE~8DIQ{(xZUH^lZ-*kyS@vX-k{QTQ0MlSy`A_1km z4p7Sk1Zn^{-gafT%>2hC*!GS4t#$DbO_M#tOl>P8FSRL>u27yplb_t2p=s`1NeccI zjsoC|4d@4;7)UH(Xo_WMcpQOU9>%Z(wGOc8Ex2lHnlQ5ikO{>z2Qk}|E#kFR+|}jD zUurey=+&n4f`&g&EMCRQj+FXJAUZ<4G+4cHfnS*n_R-l)i2R5U9^$x!f@R8^0dAiW zsm^V&r_%jbosm!F)X3hbi{6gPF$;w@;!O)r`mvHy2eDZJ zzp7^!rk!-FI{Z0{tmB8DS>LqMvjW0`pa&sW1t^(THEPp29obJ!rgSa{ja@Ssdq_Xf zVKJWdxvO??3^Svm#P>PA;;7Q_nd9dz75G_6`>fS1%#;Wt?g>DL7!qcRVCs{9d~G^N zCXf=_64@)B>j=Zl`sDHm_YpIxRMMuLgrZF>|CfG2g+h+=vd>b2-yn(oT?>2^>rm}$ zg)pi|M*=@wTXvLcdcoRcTv>`UTPu1hFWFQg6!8AT@(!@KRg?f`?*ca^5RhL#U=VM~ ziC`)$Yqp}#K8}3T06onf@Gd)YXM0Gl^^su==b2XrnPp?>7WDUc;&(JG1Lm#=FKC7Y zN!&~D`xiwouPq7}k?2S6PTo|%E+X1c8(f$45p(M2R7n#4;S=!Rt}%E-z}|Gg4x?(R zX?quFO7zf&_-8FLl>ASlY8HGRc5<)YS$%RdH07}CThM?ElFIeG-tRgrZ6S~YiLh2w zTg%C>5V1(g7dUcGXv z-(O*NwsT14!}UuJ>GGinNi!%ICxmwcBx-`>Qf1UqRGSJr{8_C#xn_)liQEj}b9b3t z6^-chU4ed1WzNQJIP41bf5&}@+PFX}yF1#6Jxq_+5nTHliF+V97(vhIXjVN@_TG>y z#qbBhCEyl2&{H^YoSBWm0g=#xL7e;gTRK6gZiZyb@UJonA}*pd;Hmtz+MttO<#$HB zQ*2!0Tiut9ZtmCBu*)269s!W%+bSLDpq+LI!>P?yMl+{StFn9|NP|beU7x5Gv&{7N zR$lbD`_J=((bo;Vj>|5Enqg1E)eNLBJ*@^Obnmfs1p3a_;d?WVb-8MGJ|*(eW!_oT zd57rL)1p(Ij=J6vFC3QS#oi!ivlMqM7#t?R#;7y;fKPp0Wk#PNW=9|)qp?zOHDooq z@`?OPvuh*p!w)kSD~ksQnvQE{^0UuaHhBk_=-lcn{uhz|55k%_NkfSd@uK&fnCY*Y zpn@QKOWef~bqm4idkq6;eJ1JxHBRJTXj1KCom#k2_wjhCo6Nsn`KC6NF-2=PPb>ABpnUK&*es_oYT1<=#kpq+8l~XXX<0hczZ+1T_Nx4 zWnQ@~wgAr{Ob=@7fU1Et#4qaL$@cAdu3v-$aF5Zbd4bzoPYsht4e>EmveiavmxZqQ z9z75rrXR+!jDV1<&J0c=B0(fbf^n#R&&b0`4Vc}$xp|;GsPoR-x5%~$shmGi;oMt~O=J9V?O|v_EuHoC!kk|u#1_wd$zL^pA6*>I{IA1L7!!5|cqIJ1$mFTRS z&+;-KM!pu8?%7B$Iu+ml$LH#@Ol1dG5rRU1FZ6@8#LPS)_H;6n;`R49Xa@rd%Bvx& z#@>lW`)OHs?K$tA1DcgrzIigoiPJ7r`~X@r02)3RM2`fw!7GLkxuOF-lT8h(gPTi& z-Y)GKnURnE%<$Icoi%7VgWU`iOjcx577yioJ@ngXfd&Bd=+^*og?7V+4nf3N5Kz-1 zbW!F47sCC<*1k~LkI;Ix;8DF<`yLPN}fkSF$$+kI%BkUtfUb1nD%tLV8 z!Aujr4x;Mk$Vr_O1I`S0Qr_hG+jiA8+O&yZk>Db*JCUv#{`E8R2^qDOqf z+5ZV@KiZwWnkD>bCZ<@`TuApY6A0P9(fCH&a!yi)XhILD{v%Byq^J5fKh?ssqEL@~E%JsG%`HZ5Pq4Ck1$1fjIe8vA}>6slq!}ch%Hna(x^JgTa5DcuaJejlI z=^S~Skyt!U)2b19>{H-Qw&1ukkh|n3cE2>wWmwm9jgxq2f^#<4zAya!cEp z*;8*$YowR9Wv^%%_by4JVt^$Nunr4xhDeF9GO(>UzM#bRhWJR_q85_J3C8~2{=5`x zRcFR*Q5*c`5#n;JzpwVOf0AS)#`klf+vpZuWkX&s#{;yn9m`zy!My>+A!nV0M z%`io>D!wIA97eT)%3PIThzLcuAV+@DMXtsjVYKQr?$&zK2a)bC}5UU*3ge-OV~$JF+`#Z0B*sPZzHs>)vxm#fu3Ojy(_d zO)f-C%&Lg5jlT*8BK(j$WNy8l|8=0{S2dxJV}Zs!o2pSYclk|TZ-~8INifm=%GQS= zyH5fuR=#h)owYRKrw7P0r(@UhQT{dUxh{67LNh zi@xVm1S}Uq?pNUqh;97P}@H`hbZv`nxOBZb*Crh z(7Gys(P8=E_XJN5m(`Jl?Ir6G*Rc5&<~#kek0yDd%~x$?rFr8Mzbk>M2JgLf_gFR+ z1?!ruTXdKy?nej>up`--_`8+P19622!Iq>7wzYbVyX)T|b8<_BZ$X>V$_=iv!|cUh zZtw;2y;6uH9zv|SSM*|LWf(|idJ<5ZAk3hG6Q6^x}eIgpom}^J}pi?Y--wT%jA#nJ`a)ltw*fwBmQ232eYF&H%7HQCEr? zoNnU&LLLC$e4NHk-b&C`DRlq)Y*<~Z(zR&1iT{s?X4kK-rpCj}RmQF{*S&717~D>3 zotzzCg6i>`U1SV{V{%Ugj-nR9TMq*0vAJ)J4yMH&`-X2y+6*4h!prd!1qv4M!NC`BizRqTnkoIC7I8S%VN4A0&21+` zI1A?I=h6%eRLzu>eqK~4{Cw^Zo9=ODm=0z~7@!Xe>fkSpjG?+5B=A3r`)^aUOvhB- zjg~Ilt*;3SNPiN=&NHC~(WBij8f!2d!AnkDY-YM{TC$Q>kgzA=BasdX= zf$~Rh{)=!J-F?zoooEuN%O$a##-6 zT`YEY+ZT>(Q_%u;S0*UJOv`-H-PulOvC&`y@y-FM<~_&wM>lKRonT@u;Ne?Std*p{y%_?|AuQ|lR?*8 zG_e$q0Cp4@KBkJ6(;*VW>dKr}UjtM=zy<9ymB_Q1XMEf(vfjuR$$EZN-Dba~+bYWp ze;8BSqsuY8Rjya1&#cbej9FX=>eDta;3_Rh(qrL;9$SA08Jybv03w6I36WDVe*Chm1$`)z^^Gu;fZzQpM<+HQ*yTOvUn z)C;hE@p{ASxeUAXf}#+eQMZqcucxZ}l@ggeBwwcN6B$g(!$}q!YkN8&>{)-8h)^qp zV};L}nK?U}&`ywI&Cbn7|Mud~4jLWcY_4oPH|Ib0mCe-vdmC!IiY9)qn6?HT*7wX7 z7=4vMTNTv_SOJfQ>tkF;0@yclM}Ep0q=0&>=Y6a4*2l1xpA1dDXlsy$Xi{B!$AaGV zWEn+i2kadIaubg)P;&*v(^~@eJfYlz_5o(|+r}Nnf``*s_}PHUA;!RxJQTfaAqx9i zPy!<>CVrTYd);Xo{h4empr^oQ#1twmbjwdydwd;nyxeqYs0RrmjB4vPKsbB~>ZQt4W^7&O$saBpZ@F<$jwN!hdWGp0p)}^mbRg~A_Xj3ZnFK`U%@GibnCDa0n(MB`LHxzpuj(R|B zTvNmI{b=wh5Q=Zogz{9i1JAlw8`@bMn{+m5&)^YJ$em*Tl^#QLr{mDXogvVcYqGn7 zn|21y*i(fYTN~s)6)C^|7VA^{QYCkkGmPK%NX)5CROW}%4rw=)n7wDPE&E3XYP*ki zvw&rOrjlY&FQhmov@wen!e<8aVT@s>_q5kLEtiJiWh*F#nRGF(Qg)YFmU%M^2K7K5 z`w(Iv2n{d7q&KiU2y7~zN|r}GrapGf9#WXK_ndj!)!B7bpz8S1bZeiOuL|v@L!1|A1Co!T}>4Hdg<~JSW1!Wya7hC+a z)#fkGVE(!O__v0}%eOK=Uw4jU#x&voFKllA;%$MQ$>AiGHZ;Lo|a#nj49U+pjcBK;DAMMpEXOL zS{yHYw4J){YQX+%B}?wnq2glIAOuruQHwaX4_!rQ^X1cs)8WAmViDw=@`*v8nS#g= zKnl5k`wpj}YuGXG=fx?X+GZ{sXZ`~=MEF;2iNzmC_{7ljkl?2Pmy;B*g7ph#Adi`* zI-j~p4PMMTx0+?2;p9Wxw@PnP-d>lNh`?S{sQz%t3Ir8Awe&Tb6?}w@SxQgy&JFQ` z^?{<2&GcL_^%3^OSLIsvHyIw7)BF$p%4(87BK(ZYPtGzp=zFZw4o1Old;GMcG48Ll zt-8pwrZ)}Qq{F0pr4wqHmhoihB9aZRZpDy|(V8Nk?o;#G*2>!M*F`hh0-aL9 zE*E==r{)q$RnK!F^)dF=fWX~m1Y8pW3q;IBSG81wd|rS^hVa>I$I|29tqrI8m#kZr z)CfmU%}Q-4e@JogL93jmks4RTH|OL5r(mIaw$4eAqc zZrVcRD98jXUy$#0@&ukpzbN4=9Asp3TWNyBe>K}=!o~7%<-iBCxZ7!O16&b`8j}zA zdvy9iOteifF0f^`ZNM?8*b@%X1Gt;;4cG!R1&|BjZw!tluuUfE_1Ht-u}8q0qCzk= zu&XD@+t?^K#L$=DcI}#NW-q~AuIXk(idF1CuJXqZ5#uITe$&pyGPuE4=PeikM+pR6 zv{+ps9ag9>6(fk=@I?PjTT6prTg)uFjR4ElZwR=77p>IMLup8&8M+g75a4M@rTg7k zQLog~{sFahGboSfu+f_0)Y4wV$?GoN<2|jQy*3!7KoldvdC8Z-tp+iyud;RE$*w3` z9)1x`PiowvYp)5>&s%pxB-YQol#Vymt}8vL+*dMc$N6=9!k^Zy(4DZxibUKxlUh z@E5{BN~RBkck|s-uybR?RV-M<;mCJEuMvqwT$l2LMU5z|EbS`UF{irl!kCjVxuu{W z9&N7DZ{$-ie7`sO=Vx|@h%e*qp=|G`5{&)ZEYOqsC?5g)dX1D)Gw=%yR5JHAkg8i*Rv#qd!?K@E9tAwzb z$o!%O1J42*Tbk*mQQ553U{P8Bqe`9C)x5bzvPs98zbhyr7d1MEqA+YUWFhSeT(MLp z)8QmJqCD7LRQ{tO?onNOGEaqU6>oIQ@-rvNj4eH8QRpF#Y=!<~cB5Q$Xt>)LP$%M6rvb-4bkT5)*a_N?%=z%9O zG*uYPJPR+deXcc7jE?L`01yCe!Xle$C%29?kTYl}Rk$mtvawnvHjAtCt5PPcN|y)a zq0c*~y|NGhj^YM)&_6#yc>YBMb`t)^t=H1A)+;y~p9+IxKf`v7o`rUobfbpTBk-Q~ z=mhN9uWGenoNF}yd`~3F{pU@aysYE9@DiUkox@b(HtNf# zVGQ*jdWnuy5S!JQcYIeQZV4$0*{+Q`T$J!g<_{LE`WE*LbQ3NbcH$$dzzXu91=b=i zu4XF=6kPfGhW}+WmpUrfrM+_pSzomuu_d3gfA$EPoOW}bLYO^iMhI<-&_wzfR|$I; zL2PpY2LO~FjMJntrJlsrfdtSZhqeJQ=thy=0XPtzTu$$C8dZTf_=bFzR$0OhEfwyk zNsypdk+AhTunuCi{~}KKj^Q^Ff1v+b7fcRo{)YlGcTC;HmU20oSW~&SZ4_{{If`}? z^h|rl+bP=eI-ML&L-kqpY%@Z?LN{6?CsGjUd?6C(tO9faA5AtIEZ?eNZqf_z?vm2uCyv*0i! z8!wmVy+um76g+q4v=!gHZRMv|Ia*W+2yAtJsO=*_PHHW}f2l;E$d%q*;)C8@Qpd_q zQU!op&;x#~FVv<2d1Im&A`~#4g6O4)g7#o-G(?r;T;!t|B&43ndo~BHt(vX`xfn|K z3KreEnkJ)+8?=D^MQ9S7fL+i877eKa?gG$Wj03%5WDGK}xBf-U?@p4qA+N1{yH9_X zNE2{lh>A6d`QiQ&OJ@bnmvdcyVK$ri|XDo%*d zsTnm@I~1+oC9GM~m;Btn;Dy(XBk_XCN3Q0+@UwY!QYR9w0COM*=HN*P2L;@e?f7_1 z9rz=W1I{)*uCC17>7+Bdv88PK7U~T%c1?3s1s6W6KJ!B8h4PqfR2Z6k6*YySrc-iZ z|2=K&6vRdIUp!d-r21`;MfZfp?~5!7QWym>+qx402P>a_d^}n4pAFU>+%_eF%+=1) z(4K(01zk41QM00CP4g~3Z{FQ*JTCBNVBOZi`I`Pu$6kz;$nX$B7D#HBiFR;u1~7XH zXzxpiNP4_A`max8#Vq;-u-U#e0r?(0uX>{Rca(68Qjp!y^)cB`_2i%7B_7@V=GI<+ zPW_=;8Ou13#ipKh`zZoQd0!#St9}FE87vF*TyMMw{y&)I-Vc zO8KGY&J*6=NNDS_u-Q-}QvyO31R+A87#Z>Q(^d>lxJ8)8O+NO!3kjBm?;+qE`kLPG zHg%#17YMz6U3mA3sQ&76A?!*crdTY?s*(V)erU?U^3cN4YpNtL&MZP=~ z+mP6k_&f{rsB&DKEz_R%$Qp{@QhxvRs0f#waKAT;LYYER>3~>W`MC5M=HK~1qrm~! zotTJWf!t6mpDBveP`HtVXMrD|SPCZsUVSI2=M=JRSy6PB3s(Eo6~ z|0Y2Mu^^)^hCnrCD@htI{KX(h1C8TDUOESqoV0OY1p*AEMP^E;ilTODeLp@vY@e8;% z3lTUhVC>fr^GwL)uQlICf~&S3Dg!%aGcaJ9#BOCG&6O3!;Ftvq0|2e?AZ?93d)T@g zeE=fnrbFUQwz&?kO)rs;5jvYWEfa!C5+lUfsgA={e=cz#zRVRW@f>HSw#T>vY54(P zSgU|>b#PH-PpSq*KcP;`vHq>G_ZSO|Z?(l{_?|*gJuzz2-}hb%Yx8Ky< z)QcSq2`r?Y1pR~_(2kMuFVtS#RJPWeWJr5ZGtzx}%Y|)RUTX4}mW8o#As-{Ce;H{XarqY3uJwe0_sEzhh8({CH=KBj>TW-TNnI%)@i>q1z zt~X!yu*9`7YaWKhX5<28TSj*KYg-8}gAQB0fTjZlTlsvp?Gc(|Ver}ZE>@L@q< z6zF{(unvxDPk&Dw)#f9=YZG^@$&#xV&gXfRgS0<)<&>kI_H+X3q@2s2wu=c+Uu)#i zP;^WKBNi+|&(U;3e9&(NF*S)eL>c`(~@CgIp6F?Dd6ut(vso_gmqE67OHW;k%QfJmJ zPxuYxXL25nH@dF7-aB-V;lHggt(>sZ+DzJ10O+vpo#FN~D1Bo?-E_~+& zLr0%E+aM`SY|i1#fX^CXb@7s=9A~1mh2q0`{Q%g2onrJs+n#uC{D$5zs4tEY-zrH2 zq-PDH#YP1=n<0RTSwMqYgC-#otX%q~q+&g3?eEh#*A(`ns(pRYoPq`{+r|f)V*ei| z_ZviPfVQKyX5<;4vA`g|ImA#Z*=whmN02$Go%9?#Up3`BOZpUiMw*_Zn<%j!R z=%zZAZB)kUv5(8Cx+qnk))J+|EDMd)(|8FCj{R^goSqB}YDGM_xVL8RjqZ1eU>WtL zU1g|_1_VDemSxJCMpqB>qtuEkmGZN1iM4iF(m&ye0wAlj!g>WivoMykYb>N4N z;;V(AM02tM{hvqHGvU>0liga`wlQWWZjE@k-n@nJ`Tgh?a^*}blp)p6ECuaBRNz>J zUqBNxA?B7dKq>TREro3nlORo|zO8@P-l9BKLml;9*1h8CS4Z_X{$5cFv`JsWq^=(aE z|8OEr=2a*p`HSwiuZdfT-+m57;;4ryB8;4qKX!Kh|D7%KzjyW=G|dK@AZLN3J965( z1I-4xv8#b9-)~V5eYMP*O;=J*Ec)F|p_=Tx;EKq)nR@Qni#1)T3WJap;Fg7hQCR{t zRTCXHh&244cR){T#f&NS2^}h+Df)v141NiEP78iJ<5c^4qJ+eaC0N!+l5PNuU3a6H zgYj)?;i^?t-qj~)U_{aK``!q8_W?=* zG&=^Hal0W@+layC{4XN=?Z1dm;GZA-+M{CtG5aK%n1ZXqfqF2KP9=GW>RmftQK4;8 zc#QCZW($NkR>x|uiM&)^N_(4dZsOccTOmF^-5tqjrJp@(e>0LcW^c#+i7F$sk*pzw9Uwrvh*meIdY>qX^S~)&smUD44~Uo#B5NL`(LqFmAf#Ea&1Q&~=Q8}9e@euim* z$j<4F<)-C~;T*9H+eYuRoR9_Ayk0?Vhp0@B;aKvD%}$M$l7G=H)l2s{+T%h3K@oKd zI9DKk`koY|1v-K3p%@DITWI1m4nDfIDFMz*G1<2hCxso$nn+`N!8kHlFe@La7Ms>! zrZdx3e);lm_D79Ai8Ivp#kv{R&3e}ia}d zwd9>PJJ;8P(6XiD`n90diR(2lUa@ATs5-WX*%XKMh$6BOf0r0M>=Z)$FARKkh4CYxH{SN4c0uiGCyaMQO z06ass`%YnqBAu}k1fdY_(V9!cvg2=rKfrNoe<90JTd~;Wdtc&4xYDn-o{*O8pp-C< z90O+s*xF%fav(#5il*H_JoN|n%s|jmcNQpUMN=oUFH^8ZUVqu5e#Mq?Z2sclT^j}A z!i50&Y$oo#jD?ulRJQx!ubyGTPj1O%%HAL7Q5v{!@~dKW-PBmsMv5HSA$ zyfJJfBp}gl!ly#Gr}O4SooRsI2xpLR_k6nSoO$QLml2|f%B3v&Y48AqC(ioLzN0FD zhr+*O>Z~_|{lUs}t+g3+_s`7u_-4d-_!0MPZ>Y)N$ZuQNEB&^3aguE+ zC;|PJa9O#n)_$ES2{((*Nev^JMF!-B0Ra*1c5r z;(By=6lRN8w?`TkZujUN$iY4ruSnIr=!7{lay z@)zn^4(Ax8&RHb?5!e3sVd~JC^kB&{XQhFj6cgE$$CKOtLFWcF$6{T|*&h$AQ{_)C;0}g<0SC8H3uG zxtb?jZ$;BDtlam1{i)mhjIQ{vZ9p{zNQd?raOtpw(cc8%c|0{9KixnhPhQe^WX&yD z^l8|hD~~S?XxQt?G(UB90*C(~GdPDH2_%^EiE7&{kv!~4M zIh_C!X_~MEUWY)ZKFLK>||hrRO~r&DFgMDrjd8gavd2|&)=q$p`v6~C!KS# z(8*L-K9p4#4KOQWg(meAz-3_t{HALiLW>B$?V(ELV#bmwI{cprIQQbqstlkV}N|`l%Mp( zE%Td*<(%;>2iIja7v{bs3u)|is8beCwo~~lD+Ok?ozVlzUfv)-tN_Cav`g79je~5L z>Ns|iir)bAT6fdcr%?=nYu;4H+3Mv#DCpBRV=H6Z*;fb_!%RE@lg5QTih?yj#Zc0_ zCAn;X!989B0;_;B8or@Sl_~lc;RSg;&5)=M*75zRJ?G{-RE<`~0`BYnAkhDP5bq$+ zk}uE!GgGo;5ovAPDDFwqscv|x?Vyn~TDE<(&-u#;M6&(otSU3#D{-1PyjA`KD-X{v zA^C#9?Kl>|Nf00m20wl3H)^IG@3-=L{9LsU zxiP#;2eGt+om4i-Tk4m}rANbkso6y*WxPYvC4^6LwB9i$qP|$nWHY!$z;k&vjIRsO zNz{RdTZe*r+BL+-CN)K(%eBqC3rqu83I?QHo5~vGt-3^A;`=VAD+^_b|3ye*Hq969%u` zy~M_Iy;3pBKQ*D2Hb40b9Bnof7|AilS*YzM+7(LVgdu7mh3B z-i}pkufw72Y}@&mW>i&T0?ikk2G-jlPGi!P^y1X%9~xN!uht4LOL1yUJp8^eE%E54 zoQbQx=%35I#JJOJ%Mkn=6iyg{o=@SxD8c*bHp-1Xl|5~!?GP4{g%ug>oQoLvm`WVu zv`1aeX-Zks%}=jF8m^yd801074^9b+Xbq?u+o!d zc;b5$A5Gb}Y`*8cQwV2EW7BBL%5#Asp6WbV)uvmUZU#94H)Ry$k`6G}g{0w$BY^+P zdxxzZ#m1L|q(SmjZBR2j{?{9P(-73gf(8zg9Mm&(7(0%sohW#_rkPVj{1b8_7rthM zPoNF5Oy0CkS?JgHuDVcmH#zUtnX_wRbrFt4)X*eNP|ZyULRA>+e%=;0}VtwgBJiq(raMIRg4FzS^LvEFn39O1eZERw9ArkV~(YFY&+AQOcFKKILt5rvhp1aYc415fZpa-3GSx|RT zsa;S4!7^L8TJ{H*-=*oh$w_i1O!rN+K@(O2;QFwwCekE}Dm=49XneRXXew)P9yhCv zd22UMPbaJT=MS__c%vO>b!egf2|g?~X(fJ9oyB zobAz@!?O6emnX};5BH{jX-$NbOnwLM)Fc^q`_p#*MqWX68X<5KX<;o_s|-NskXgesCVoF;3Bggi4^cuTvGt!%_)9HBq*Ogey zG<)-ins~Xp2?r!j<)#PcDZD)RKDnV$0Ms5&plAZ1#}kHvrUrpkk_<5bzx&dh;Ie|5 zpq?cp^?gCvf(I4vUQ7NpznPX_HB-Wh^5BEYUb!wy=y@DIxpLz!?fw{WPKODwQddsC zzsGHIM_Y!UfmWV|$x@@5Z4y6o#9crk^3#<8WzrxZW_{oZu3wta`yc=7`h_tiwZHzc z=Bjj1yFV*B5Y2PA=q|W_KQ5I)*M;C@@B~o&YeCmxXaXJ7>Iu9jdItKrbMfoKWQT~j zToh?u=33&_CtEc&-m+W3Yd&Ui9MP1n^q47i&sX;#bRHr<`O;X9n$aRck>=F_^ekf><8>o7w6XFPmzxk>+hqmrHR)e0RRaiPZ2VZjLIKRy)%G$*{({R52oqUYZRxc`QT z$D;;2NA8Q5(_Z%7b=1Lo5|1v)vASP@9(H$~u32mWcgg>D5rCWVpDqHXnjr<&L6l|y z=qcfkRlq5(Cy1$yQLyg{Vt-Q(OhJ`ZQP;#fAGg+aZ+4^3p7wDC!Z#$Ivq{M1cDyGT z`De*M`$Xy>+(CNzE4y~|E__>PZ07OYsb}BJ4Cntp&fYv8%J%&q9$S)S>_Ut}lC6{_ z$xzvnw25p}wR!S2EV3>QJxR$M zmf*$b(Ij}qA*?KTo*!UT>(FXCTZ+Qj4XXlSR_7Lmsq(?tGvfyDA@dyC*c8Q<*P8cK z75jDi<98kA+Kv{+Z?Q=@1{)7+m2%ZQ>5GW06Mz*6)gTx90SOSa{IVk@;Pv8gsng92 zLabMXkINhxFAHl2{P1cBSG^y5f1UrV|3AD?)|=og9t8Tv$5z=~OsqdP4ciDh*udvd zcL2GrCmu6rnI}zhkkp6@3!c=qkrAAx#)B)d2Xu{IqV_M!?{1qV4WVI!K3ZcBJ{o{t zL;T@k^gq!coAp(wZY=kcH8gqb;Wpy@$>FQ(yK`$QH@i^nJH&%xw0@ZZ&Ic+t79x>Dbh?kmPuAABVh&(&|yjZSpLfS=K-5;cliy-hZV)%>3dVH?54&>Akd3qxY8SY*I+eF(l7Ge0{$^mi;zON_~fL4pS!-EzVku0r)I&zrNa+BFNn-o z?6kOk*%pQQpvN(rL7;7*8nN9DHv!|y83Oqyt`Q9G4C6?z0j#=?Bgo92cUrY)o>=Y$ zyhxJg=;qC>!mo70X9NnKKlkSc*FUc`>(4$^%)2c*6B>tH)7k(9e(WAIL5}ONsRWa~ z5Kv1~Iy%}}pw%#3HzewpxBq(V+2%Tv=q4LmmDlM*P~w5;#$cHbUoN4r`^@yAj&{(s zqefFanjF+zMm7!MUD0%fX;R6RG`<5Kwop~JD zWn`OQ71*>>;M!Cbe#LeSC-%*42W~X6knyAGX*l(KF2Wx(-@ZOCGM#*L@3v;5 zYfIBV)>HFf@-#XbfC3v3f|WI^inQ zbfGahw18G|)yM9Hl>^Utl;9zxP-~M59{bR28|MOi5yXoGm_#2+^xz*mSf$t7{8*te z&Oq@2Ik@~{ip@^_&V`*Z{fZSLKh7TFQ&rhv)P;fd5;!stbw00KN0H7;_yi&zbtU!a z;o{=(>~1L!XOFl;`-X%Y)KcRulp<)cn^gBssx&oxaXv4>)W6(9+R3Ch);lUl8jR7i zy==REkJ^*MSG~5zYq*X-U3&32akB#)i(y1MQj-yfkm(NMz7kX4nhhWFWargkr5n#;h8W)P? zwxE%1>Fg6AUj^cB1!Ukne28)4Q`jU^CE9WdSCM^cc98eOwx_i!s-d4|?r88wXINSM z8Tbn-hW(6ex^*7}tCPUY9B8ZQ&M*?32e~rT-bjW^uqmi}C*bv-UC^swet^M|mVZwDYv#ZxPY z5YBmSDXi9~Ip=Z1AH$z@PO|I0XXqiuUS6gz_{A+gOW_3cwu%Ec1|U zXP9~IWB_aUK{BvZR{@YcPpAxcFlYRU3Gu1aksP%r7fc*-cTCEP;3r(k+OJld@#f4K zxp(9%5UfYy$PnlAR<$;JipdQ?`WWYpPSl~dGA8f+enI;6>8OyKrP{fp@AS7db!lF> z(50n?vMvB;#hOfsUvRK5%8J?;Fi;E0q42l+$@i&SRz zE2f=)QmLfqS6u!Vvl6uF*;a+wXTzoD(H&S`Uu8As?kc+9naj_XG6S;AvxL-c*z!oI zT8D-1JR|Wi5un^$5ME;&$+Q)t$w~~Zsu&$ zLtV8-$VTM*v@>tajI`*x8{Mm?v5MGv78SUl4V%86A&z!61ebF`sEeI36OO#_8Xjdp ztk?$d3VJbB9Cfp}V7krIPySn;27Gzq|LhVJx+*{W=8lL=Lzd$mjz%GRRT$aoz`h*$ z5BCi??73tSAOn@&nCs-BJx&|DUVnQTQpBqVPzKL5)m)s`H_tlW=oIzZePzbcN;)b8 z2g-qt#{YPl&9cA%biM^Mi`(-P;(yO8lgezL`DR9hNx07WPKI4rS(?yiyjNVguK%M; zKqdN*0ZN7wLYTlYa3DfjcLNTnbuZz-DApgN5Po!I0rxs>u**Ya;yLtk^yasJJf(vy zw!LcK7OUO!@R9brciqI53*@`x8NSV+HoeFqGKWB^ow^xWjOgx|?mWA)C56YXSMHg{ z+st0d_{MBDG0CbKArHB>i7qSa?z8-K8U@t*Bh>s=Ygc5l}|P#hHufE5?;{;mKlbC`a_qA(U$ zD%b}(w@o-lic{qxqKka?{p9la3YiWdQ@TBZZj*M=`Cp1$Pu;7K1p0sLHtq}jGe zr_Iz}zylU2iQEmgt@`~uH5tJ$m0MZcYnln!(B&61Ztim@KgXr*SvWQ*(ezLv?C{gZ z86qQs)PYN;fSI4TF069B1Fx?{xRD1O0u0YX^=GZ$k>7QH@CFUcCpCNR6#cKu^5?u2 zL}4TWAQDKXkrbkSncT%O&gdnN9F(j~QR++a&NAF<(WNCRf9eqbr8H46O?-1KjvYi7 zT#W_`2E(@OlaY?`9j4X`dRu0E72b~OygO~sF;KfpGW5sH8~Jv%1wIrcenD1_QuWKk zBhL$rtaw);)7m1DmvwRbowD*nn}bF^XX7YG<7SY7jCs?&eTny zX*nCDD!@Dj&z{vr1n?Yo-Xafq6fy#VQu(i*OOZ}YP&?j1WZ}F%lk?((Ay2=_n`$w|zDH>}jhgzsu#41*nXxx< zxPGx#r06N1YD}et;61`AFp9xv0i0PJpl!wBWK4HLh2daS(cN5=52-I^9+cL-PJKDt zTyR&J3#3c z`$M4I&8%JJi@{Ru#>Oawyh(4^gwKAGXG#~KWb#$~R4ILJRi5Ujs8!|HjuSN)5?_*~ z+`rEHIXgXNA0#0DyfC(BoPfCvUjf z;!EKI*YI85|MYk!>r1Nf&z~`OM4?bJTu`)j*qSMXFD_s?QiBVmPE5R+a^3}|MlwM> zJl*!=!Llosw+ClZ_33*qpZ&Z?N3}erTP$K1IUyA?pN!P3o`YR`Ow{Kdy?~Y)Z@7Fn z2`#$*amdf~IsjTz+d&7tW6m9wn5}@$JVSyFKwrNQct$~Z6?|80N25N6IwUZcWI)`by=9l15NS9Q9jL%v*eP`y#1_Y033DCH^0H*g;KmHS%)#b0)q=L`s)EjXm!9`PgU0S9fjF%?`ugk-iZ6~ z^|U!w{eOl5rJN^xipSPo+({+qKNjsl2%tA+f}#c)eu(g z;Jd1IW4<7HRkdtR$mDO}J-1YXKF})0F+@mU-d%&fxLF~5T87$okNs}lJ9;ibL(IbqmS+cF^PZ3I-Kxg->&Abplk#rfGbdtc^W>RxKsFK}?)+pX)Q6d#bZQul zJQq~%e61nB?xVbNhwkV7{idoaLOH>&5_j5u0A<3v|K$4rBh??V=0Hjs3@3VHpB}8b zmTHP+B!KXei2s!uW;YQctOJUNYb*ZG^@5Bo){T$~;JM=sT)qIDocGQp7@dq>SvN7vF9nty)7bNHc1$ZB?>vP+Jlswhe5TEP3F)osShy z<{{JGl4Gbcb&wtmoQ|>qiPu*au)G2CQ1a~-x7$^P{H7sy>4$1!0d7#Tuj$TSVgC;n ze1E?Fuk`u9(`f7iGHeWl;%tF!?t;U>cPr-F$|?gWUNA9TYuh3Z-?lfh9c_^*8}ww6 z4L{5+EjD>wf3NA291~bxj)M5K9X3UbHznAe(i6;pW?Gos>LL?;(Zs4qW|HNQ7^e%COYw(Sy(qw~M|6$#LCX+};b!UDjYQ3wUEa- z{4a!JxiQDEq9K~Up!i$vy$=^wkB-Ya|2q7r;>cd%phHJPi{In5kDl_Rz7pP&wU0Vf zkAhBtQqT?uYyDYGVrn6xlYoTh8+mH7Zb%~164o=UefrG=skruGdROCCo~4%x;mhkT6`0A2PBCi&HocWYXf{`4k#s<5QZ@v}10 z-f<|DvCPI2u=H?l8sxJKh5)1hzux6c5cad9qmw6_kw(V4RlZExqTA@=NZnhN*YSob zD>dMI=@!5TK z+X+YI-7~cKHDSmR@Tve!Jf;~Mu`<(z=DmQ?_Bbssa-cEf7wJTv?+Vb}7QeFafWc0eFSD*Y$4f@R1$EqFW_Hua?&dg= zJJ)sH`DD{9ai3f^`B6a-EQJVzy8(eAz^s=T?3_c7uC|H2Um(hdQ!aiPx#?5S=jEgH zq{*_tCxHKkc*isIGU+T^Q=WXzEC4a+W*X3li76H|gel_@i!z7?Lw~iQ+DUGT833+E zQuc0^8$b9Ep+{^&q$ya!2S{I=0=E)Tgm3+^Zk!NNXlTyOBQG}l`g6P8g0XdHE?}5^UPH8?Y1>KAKyU;_U?JTYMuhGowk2V#Kn#{( z4zdhWIu-O%em^*v7->-iSKC{;`VA(@YDIlCsJnO%dOJ|rQB%0c&xd3SL%cM*gJxFhz}Vj|WTERKgl>%^o(zLt`r)vLcQ?9;@+ z7>(de=_@a^VPh0c>JA}}G)ROGvys!wWze+EM?m`2EaRg?aGEMTjH_A<(mzl~e+6%# zz|{szrvTzwU?jMm5J+dprF?1=e@{!_4$FR>Vc$NU#+h?5s2a{&89f$wMp)|e3Hfa? zCAVY149a@;FtB(d=q-~C+BBv#I>{SGj$pXV$){3T#Xk%8k}UbYX%?FPJBCc#APkcnuWhip&8|%3Z2glj z&2C5C(R+H-++KVOuP$%94b}z3*a0o*Dv7O9AP)Mp3`(Me%y)nTnx0f34eKy!yUE)* zVZquan7b|5h|1*yuANshb#yL8eD$&zd}%WkwtQQ01%CfTa3VV8E~tr$ey5iqV^DD3z@ECM#l@xEt#x;5B*c?GV?wpnEy*r zVmo0-xYw#PjSo;x6Y0nvaB@w4p*e@7fZP8mN&rpV>JY7D z^CAz76nr(22Y?Sur(A*Z8+$8v#U2xZg}<>5Lwz`l->{mBVf#wpUYU@uUt?m4pSU)1T57EK=^_rv!0~k454l<-Nu6T0_^m-xo?ojeZU>f;ZIm*1eB@Uc08^JQcWlL$) zBPn+5&yxSH&Woas*BExDl_qoKn1URP@WR&L%VUg2j#S@?M<-pIwmdN` zVs5edp)A@r=p&&$?KnWIsn{o|B6)~3H3VxBIZrEC_W^Mp4Hn|SCnqN0c#LasB)kzZ zUqFG8Il#(m1|`iERl<8RgT%3oD6xy0_xaTtRIlYXACRiv9em)JL|hDNAh6?QdqyB= zIqPtMXse$vEM@L1({t>-k=wc1Qhh?SN5#+K1lNg{6C0P15vyI-ic|5WZpxX8nNoqp zyub)Rv_^6*2>@2GS~WQ50#_zXG1WeEI~a4sU0`Nj;8FGFgtKYRRg@7~ z4cGrMoZW}%eNR~0R80I?LI!rB@B{mcUGh-?_;$e_&guRc;3!r;?k_mNwAuwB7Nk~) z7LR2F{<#a$USh>tTp4;jSl%lP!8FrDPa9gjyu8aBioBo_wF%bG{>8d855C&|*tJFf zwO8he?0P^pQV+>1J;IBD-jiWnh_gd)ezSl|Gu;XN>AxsmFXo>#Q@aw;fFA?vj{ar? zeJAvX_;1Kh91_VkW!Ugv4wN<2?%JbB!^3HM^mMK&$sA<0nIO2UOlE_Fu?+pp#wXxd zN42rPepfmz^q?Rw3}ClzE=}GYGA(lh;>a}GFydRGsQD+iPn{`*1H$>kZut5KsTbDf zMDI;*-1jzh&%mnvV1^Z>1JRAslc>Ma(t%rzHfRB0*W*Jt|kA zy<4>kEVi`q!_#1$n=ir;y{)x~Bfw2qN~spit7p%u$KK)?(XL= z&()T$UNJQP45%3RzgomPEH|Y9WT zKDieq*A#vv(Y?o5-)}TmNi6FY?d`c?_(OG+OsmH+g5?shv>d3;KYgW< zvBtV#hGiZ!Iq8y+R5JFvchtYq^T{#SXD43{T)okBl_oH7FFONE1oNVZoRTiq4RBIB zivC3{xj2mw8DWB)=eK)sEcI)c4XZtT%l>Etoy2U(|CA4$tk9xEiDjw5nCI!-KDuaXE@AiB zE_=VT!<=CC@$srxmy~UaZXdrWZ@tq_H|XBvB9^lQ__)nftKc5JMf-aL8&cM7K zfJMXV=__3{hg27Amlj!ZnJjIS*G`dc+%hoddfqOgTBC{R`5?3>NOo-NyCIeBr~~Nt z=>J!8^`G48#pa$u{W(ZC#Mgxyz8|$C-x<1YLbrU?*EnpgTZ#=lN<>- zM9iAe$C1fiNNZ#p!V6$JDjpSHGijZD;p60~3D+}qZS{%eAC#?X9ALLkk+;MKxm4^`bTqkUZ#ZSJ-KD2Dt_qHIV;){g)vfY7nDMDkL<_IEuWy&M zWCI`dYeI~{upH{q zTwQ6NVW>L@l7^O)ZV4hgfaCwW3 z7urp{!KYLD<_EuLerqKA4FC2wT%q8m_s@?U8VlIEpqO3Me#G41rv~TfPGB0q)fLJ0 zQDF}UFg9v(X;qr13;BSlP3TCU)}r;sG5BbL(N(5L=J9A3&<12Syq~ycs~VDJeducN zpj7^kJx?KUQ7^~>ms+t`!IEVY>l1oE;TP7r_3qyK`mj1Yd0|zd&O??dohzI5!Fp(4 zoSg3);={yVTa6u))Unfo?bMn^jw1Nw{Kec1g7G)d^48^W_tso79gRM5bwsYmY-jfA z`bHw`E#ADUzw?XpqR;0W0#WNAn{g6g+avEFB0u@MUd*kHy7M&kIaP2z!uG zqVO8A`<8vSizvP;zryE_Tt?6Iv;rLJ7Mu&`98;%(%s%aF67`Fq(&9RU7X9wBLJ6sR z8itms%(!SP)n1j8S{!yX`a~@+B*ZNQ?+!>u1BVGmK##91(*eoXt$?o=qQ4IzTRj$l5Y=;t5V>K0HIH$$L;j&^zVk7EwB+GE z_XWAOq|Z|S2L+0#k31go>_}kshoCD%5Uh#s)PiDh_32$jl}jvzn7Ji|)W=`JozKY! z5=~VvNln(vx^X3Q9U)^SH2tK75_v%vnX(A}pulzkIXftR2LZlE8 z;mg2!UUQS%^gc!MbDv}VTIIS5^j_A*FP#3kFcKpC*I&|+y8XfQ%)qO|yC;9nCQN;s z32n~KVF)9_@4=6zb4wAuYY65}Ah6df^q{OC71^uiYAj>O!+i@Wk7p3LxK^#T~Qn99PKtuinu z(L@%2QJjZ{XEyZz^%P)Rh8TZUkxmNXs?RI8p$M!*$DTzlB0?AKW)BA>i z-{wdjX8i-VR!0iL(`=t>x15qW24O7sFuR7~ki+xdL3Ia?j!;|%CA$Y60vW-`BmhSwCKtSWNPqXX?FP#b|L0$nbt?+GO06LSZVDn4%=Y8p z>R}=F0m9@hy+jT!%E#aXv8XM(%-Jq9qxvsDh!E;I<3@*4g`yJI8h!UG5m%ppQ?AX8 zC({ZVHq@vMMyu#&uqMX}`dOXi7wi)f7IagU?{=l(D?63c8$20hZW#x%1R!N9!<~zdLxDTS_HGoxWI*b zI@|>{sWRyPPQHbYv6~u;=nk7g zjP+iVn;(TA(AWL(h5p0#{2vF3?sGqfKRex(3wC>WApGui& z(qPLH7kn0CsJCT*F1RU`K~0_a9;3GS^^$4*eD)kys9XHe&l@4i*xD`yxO+;%uL&@|((;=s=i^AGyy{p{=D z|8D{6#@Ly~h1AJaDQ4HaRh1w8iPiM3@oJ7vb=%|o*cZkH(3N}Q>*7cAzIRyNh`Z?Y$4Moz|e+29oXbsay?JvXaLy=Nui>@3*GiE4S&=l=`m%u_eQ}@u{U0)r~Ior z_Udn?x5^>H!Hjf7NQ)`V@*N3rfVGSg4^q+LuBRqF)4TFF-kZAIHwriu^O4f0_Crbf zp@c-9Z@?v9=m*BfX&soCAUNx9|3$Hlp(#ucX2a-Yp~w*I4dy!(BWctg0JYK`5GSWq zr?c+foWO#kaKMgA;QaZ-Sl01UTAm9uJ!tC zVUC5*kn) zfDL0r%7O3d;cFdj0!(j?%tEiBvdlw+L#w88)y{5iJ(cczdp&MLUb1KGOdHDFmHLnE zxM7OZKiF>j7kH}w$8ClGFSoVs03X34XZ;z=$Xojaq@X^T69y1sP;s+Nxk<=jq??7Q zNy}smb~ckb{9QXI7~!|=KAybcv*-}so%_cy+sZ~QUb3X~m9{J4tF|pn0)YhhzyD7p z8HH%mxW5*C?dcngN$f9BOFGR-^Q;4etVBNcS2sR)hLOje-P(?B!clZchRDs=X5<54yG$+SA zm1A6eSx z!jJqTL&+hNW1cTF6aPio#2-SPe}FpL-+%g@mDtvGG#EwNPKH(FQox*TqaHHjf?nc< zIdNvx#1oTiVYk$KR&M1MU%gjsB;?8e0UB2I@{EucS#$KPi{t8(K2s4fVDI;b!Yh#O zDn9V(J*X&e8Z-^4{I!pIn@vri)pM2Zr#4EOJN&DBO*j{BG-U>6w=`%f96!ZZ+gx_m z)=OS>gO}l*z)_vYZe1d5XNYBAzFf4l5ox$_BsWh|y-xOR^glK^W~ZagQDV^kdrk}; z+zgH*F@&5s!Zw1YB^g7=qf3S8XV;zr4N722+2WhE_?7H#;)dVrLNpJP#MuG7F~1WG zEr_0LZ&T;K`6W^IjdFLwsk5t#QvaFv+T-HC!7~)obGo!Z<{MQp$3&IMGD0s{3ZOcc z0Qv%YHX6HtM-^m21;d=n)!*`OQNNoU50%Sx((-lFc$_(ROzK4P7svd49U__WGp)fK zD<>+I;YZ?>y3)KSyU9RHtB!{a;0=LVmquYGv<&IxGoWDUrO~Qo{CQW!LiE$D9;odLMuVY#C89O#gK;-eRk;?|nM+P9`u9O1Rp*M^fq+1CC%%G@x)`G{glI9$0XTs?rX^>9$lQP4{dp@961PG6S`vq~S0|Yx-3iYKPM+)sR!NCEp z+x|DPZj(jMy1%L7zWz{{G+(yJS>ty$eF8CD`j%TxeC#V=P8yD3=P;AK@Zkm9Ch{_D zTQLO}yB#-~MWiQ*r{a;24*_1u29(j)DyEO+D8gPf4xi4S7K9el+W|}tx^X1frksBm zK=X4*n1h-8o$Ao#ME~RPl4+fEh5RyN+{P;R)8zFx+Y`1;D`wjAiztJP@eu<;wk|XI zXyo`1Pzz`!Bwsf*DxoY}p|>qhth-s4*Yyb7@?)>0*?vyEci>K(^s3qmRc!F&dNlC& z06**><3J#lt%)jwmI4b!@&D7`|L|VluCsy#D z_WC-S{~Rz__;6~ZB+TaLtA6#A9bd`u+qiR#ek_x(G_mPl)Wc}(?<4hv=utg5DFN9U z0kWz2=g_=jH9!{6iAKq=6@ex())N%L8nN{_WkMfOsg9)R)2v+B;L=eOb30y^0} z8(q(|^GT%@3erXIdgOPray)Jzixq~ga^}li9jnNb**7N@) zCqr?KLRYZatyA7_P_9x+sU6LexgU==g8Zs!=C0B(@_eXk4ymbfLO&}!YEDA z;~WO>8c$X*wfFJ~9G>x`=*1m`SMld>W~sj!Q+aZpC3!sc`%|(snbwMBIO+jI?Kp|` zr%HWVBADi{&zm~M)(<{8Fkg`4e;_K$&gRj8N#faZ7rU(ZOCR3ksGUcohzVtU8(>8e zCxEgW^%tI~45$E)hOvMBt@DV9d_WlKav8wy7V{*M_+-F^l0qLxl8vOUFHjMwTN?kO zbSnPFgQl5MBAn`p2t@&Xy1&dd&^p#C^)D)S4C?m7FznVgjE2%7*!~-l(?I&F0K{Vo zAi6<1tp10qxY$SL?Ed9|xSj#%aV1b!BY&fs9RaAO(NpLrh5W#G&;Ry#qW}M&59p>I zIfe+~!=gk>hT2JK6R!sMKFd1?ds}O6)exB4UH;Ded*}<}9lYVBVSw=?+xXa^9=0$Q zH0s7r%Ppj3FNm5AclwG`*5ZHkBweZq^4EX%QI=vOmXi6FHpv+BV1g}!zJPEQ%>la? za~&B|5L{JcICVP(mXv%x13x;R=Ks>~7XE?AMAi#|Q(rzzs@%W6Z|SYzA0f+2k(%&B z+FB|`m~)7kF#okywzx2ybhK{bp#4$QjIO`#W!s{hFjU~%?^fofzhubfJIKAD-561Z ze=~4BVE(eHYgB}PNjNmxxe*`tVs>d{*Q_Nz!e6~5UCN{;iPxsbP77uJRy7&Hb8x^L zIRG=U^f$d&?g9(5@?kBg$g=ERz7C7@;yWuZ%rNHY&a80N(dRZ^S12w9xsF!q$J4ef za1R3Sl#F8>!@veJ3yWa3ChOBE*&j_IFAUu^&?0VGnuL64?*-?-jE{G^f0}b~yXc_4 z(wLheoouX;eI-cKJIm5@m-g;MWWQ^q&wmgx6R=G22qlclxPVV0Z>vkE9P4S8ONbPJ zwq(w?s&=`o_C67j{YsI4(A>WK)n-Ulay9#&7@#C9Z%amYm~GxS*^u!LcOmgQd{P^l zOEQCYK|bGZ-3`k1^A#OS78Cjc&rDIkxY)xbshX%4T z5~g;y>3&sMx(xSs6^mF#b+Zi$3xJ58uL=(Nch?-S`doZBNKtD$XWAG)jYC9QxHYC& z0Pr|;Z?egC7z2~-l&h~~zP8QrkJT$yxj56!{^<$GyA`>w;&6FU=2T*r-1N>fijn6) zgv|%*l#a|6fT$2t#0eS=*hKdfe87Bm!tUh)IR-0do+{zC#A>(&1w1L6b(!hi6=QjJ zMs@as#nsS%9L!BCSRhaXy4@TwQ81^Fk7LV7SiQXP=FtpVVrEbMr0JP3$R}t=+O90k zO9fW85}DIBs%;kheCC)Lb+8o5AixV8<=(9WScUb95sZY7rr`=mjt>G>`Zk$S#TY99Q3$Q83CAH2nX*?SQP?o zw2#1iPcW)Ss{3>}K}WdTMK5jfbxeO%VZ@b~+dlp}r5U!(-cHu;&-{1h3Ap~A11{0? zAc&y)04euU=SX}f*WugC{9U}{4!ABizpJrUw)%SB#3kX9{?Xji4)RKV8*1DXXj6_G zw9bP2BQL$J;kjT!>1d14{M&;SQBG%%+g7u9U98UYIB7e_$+x!;dTNY*1DLGVW4MRz zf35@C+TysNBsz@0Nfg6*OxGnf9e?Herby+t;{70VB_$b-5$7aWoD9(ZAls(0jFB?M zu{|P>BcqcBLV9|Z3HS8;FI`lsIUd-v_t`ZQfzv3I69oe?0t7kIMywG)9f_$0a_0_} zgkeNyvgOdi%uK3AcfNH}{Yb#j`1g8}oLxoH$l3rf;HvLSAkb^DrRmE&*eR z0++Hfe^~!~L@*p&E}Z%$vgtjf7vRir)@v^`C zjgkh@@Ii>xz%7J)NkEe;@iMp2Oce))X|+?upMSimWapSM7^-cVQ_>0aaO_SxE(Xdc z9S+nF!oBzfL=7NS48vm$-qX0+2*@sbhrsy}JjS+zyR3#eQgrAI5gm@VWz3O-N?l#1 zX5Jn}>wX$LoOSWxjCN1DaXSh#d4vEYOuK*=hm@d>OF&-QL7XE6H$H>$eOPqqn*@hC zrrn8rdv4DB-B}^E82PKoUWUvX{o)sjIrjs^$nPdQh_EJk^)Ew}UJ%EG+Nt&A^v~mw z*fRA`k4+$Q@|vE%mul>ekM^o+uWGU?D-ZGiGB)70`>Y@7xhp40d^msMy4YvnS>Abb zEbM__u>k9P7Zw8RySouUbNO7bQ!!YmFeo!47`DUkMQMs8{2uLF>3g4ouPZrm)0mV=YWUc*Eyt0TS4tFeWy zy6NSK7p&@Tr?CB;a`NmEaw*a zAMgwxurrOU8u$hKAC46_w@r(KpKNiY{cTfl4=-1Ako>MswfB4B`dz{6Y0Ek2~hFSL9(WPuZ1R?tiR zT%e=Bc+&Pqz|+_9ey$NEY4f{ZXUfaVtJzA&Cw2n%#bXRpz=$Iwh@=6{YD7Ap3N#1( zHX;Bn>a;l{MHa>*U=$)-2IgZKFiKO`T5SDjsk+W;ll8(^V(gZaet0#%zJL5~V(|e7 z-M4=Z9iOWFxlea)90y9=jzLVlT<_YnDk5}bK`#kc3+^*_GgxFWawjyIQvWpFOxU&Q zWN?K2gppRpjvGx2`N&z2p*2GO=#9hVWDJb^4p!?9wiz1GV)P**Biwqd4>cXI{^ZRB zfX+O*5+pN>Ck;8}*95!eNMRv<2D5pZ`bl53zaIEpwAB{1xcxn05sVB?1-U1%Zo>pX zWR)IeRo8N>v0kJ!52=w5cL&SZQ%sM91$Iis?QrB>v`YSntKG%v#@)yIcHNT^sXlhy zVH(T;ZzO?f6d$QRkJ;ukX>#h}e5!{(_U4~J?OIzc^MR39 zDm>h~rtWn|16LJ!0Lvx5`-SYd?+Z^{Po>b{J#4m6sb0O*1RIZb7 zR@sQkKPcV8ADCg{3YmfKg1|iVc^c#}UED!=&nkWf3wF8?qDyh#y+nJu4Re=xZFl!L z&UNq{HkUWuhU`9wouRVk8FTgCbR`U{6MiD|DQ+yxum*Hjd=}cZ&r99oEYP{+7rz~g zo1WxIr7vitk*}ps3Dd}Z=}CW>?&0Zxa(@tv{Ah~dk+cg7@{+E3ZP6KXc;%Gf$&Z3{ zPMcX|ZUU1u54s0L!wY`?pq^yn6ZGobt>_)&-pRrhVKG+snrjk-ckaFDd_^S2xH_$I zA%|~@5Bg{w2P7o04Pb*io|%wVTL?NQsnR#umkTLy)9T0BO2O`zOUglNmu7`8DoPfF z&2Ip&UGaa8@`MLaM+=T&j!vR7Y^YM<3k$eaG&9?RDVS>NSh_pOt0Y|4_RGlX&GD>5 z??^iDTlD*(z@LSui)19kAHTiArLG?Wqr%>>uR+6~IHy7K07g)5Dsiwcf*jVLFpH_T z?$t}f%3#55M+#LTgen8O40dYLwYCD4JKNhws{T+`S05JVU-dqJje7sXZ24op$9Eue zX(y%~+kaDZ6psV2dwqI3CyGv@yy?5pc-?OY!|64w^-okW6Ng;@!p2`V7|?Z7l)2EAttdKFG+z9%&*-IvA^xys;yo3 zM4{8J;!#YQdyJ)IZJW%+G@kaex3+<@YCfLqpT2Q`nhe4CdOc)6ZWzsxr?PiM3WAYu zXzXgH|NQYcpb|;poGF`fUv?VqQ=N8k>Hnf8+jr?&W|Mcn`onaUOXw+{{Jcc47`A3N#LvkSyOpITS5;F3R=pela)|pglVZ$I zGzVq$3Al=&uo-(dy<%fJtOkfwjp8go_^c6{M{^IjIDO1nr{7wfff)!`&CQxe+@!(A6u>aZ1p;S?gPaxNm6B{{EC+RHu?LKRyI2(*KaR?QV&A5 z8HLe5on3Cn);fS|&7*l|p8RqZ^|T#d(_O6+xpz4&VqvVv_RD!n)vg}NH@uw(`S+lZ z{EELIQ;tmo)5{@ZLo0CCd31bFlpM>S8A_g~DpNZuV7wEfH#VrO>mV;>yYpe!;)t_a zz3s)976dc=epw@1abR7JBOig{IA-SdDK6k&_~}I}i;Y%CdY8JBV}7qCgyx0z`-FJ= z1l=|bd*$K&!Q+rhG|N^>OD(hfe&$s@F+q+j;E_QjZVCA^btbQyTw}CCWC*uUh7tt! zPs8cXaDrDvi2CpK-fXGsxnVE27M;(OTbzA2ZETKG!6afuAX+}Tk=iqq2&f%;TaiAh zM0{zruf+7UVKHra#x5kxSX6DK`-Os7$%8v618zNM7h|rJwssfU&WLYFY;1N)xL)}K znlPGu>Rfm{5FM0bI=P;}031&ow@t@P^;#oF5qmZLt{fuiZiR(ys!VmBeQ(IWm-~>r zI9)_=stlENO+lf7!i`G2v?jJTv=z_ z-t=0L?q0h|!R6r}G)qQS4}SX`9$Au1q03)+@>zb?{fvysAt-xY6;X8pDa8U5*7T^o##|hkaeBYIv1qZ;GQM66#~bOV#Jz< zBiP1-aUclY*VQpkOe|2ZVZ1K^sdU$SGh_sa^pBCM|7omy4K?5#c1+MO@($cly&Jr% zc01>V83>ZGF<9CHkj!A50G}q;MXecH=>d$njmgAP2FGPBv7W5#Iy-cwVYKB{hd|wh zynQxTJ_*;pY4~V(rz&fJXM<@8?Kgoq%HV2G;7~FmWPqEVp_xupK>Cj!_;avTmTH-w z0~~|PUt6iF?7x6oj~yXDg{Ep*MGWd{1zR&*m{rBdi^tdbWBUj@VjUYtLgXdB*3E6b zHl4A0ecz?+o~nYvk5Ooc>fi;r_z(w$NL8j&Y~4Yt*724@QC?ST%*OQx1HRoU=n59| zD+XREK@YSy+CwwYGU&Hd@|-880QPZW-?@*!g|ydPPci&AVTC}KLr;?ctZ+;dzzW{L z*j@v1Tk4DlP%L=Q7@~jJBpkglXDr?!1sLEZeG!e10SCWbj;`{Kb`=D$+~&g5`mCZ$TaxGK0&!|- z`~qKi+C}Rmc?7qU7WI(reyk~wL?3|x&K%DMaeG0AI}QMhTr9&!ZY34ScnpB2OBWwx zTSOjeW3q8>>uDM*TkWXG)**!4w=}N9FPmwm`(6)BNS;;MazfAy@mXGw=FR$Y{ zUlK$J058}@WVodRXs`og3Q7iaq&RF>3}^RY#l7D=a(Kf$dt$>rF6Yrv4Sj3;!E<(6 zj}pga+(G~KyU@Fr&?9^Qo-@EboI$8w zTdgio(}`{^j_!=H`np&D04|QoJflG7$fzQTg6$a2Zb0XDom}o~$H97naRgPULn?#} zYo*^K2z-~Ag^v!nI*5lGooqXvw(gV`t`memkh`DKf27;Kb^KY9FsG&ly^rs&OI}wg z5cJvT+MH5vN!SDt02w^658N@Tk-FCe2p_Pu-W?FyZ%&`NC8kYn!hp2H(v;%W@L+wG z(t{T@9T z(6(OqdGD3fBuiE~ilYyXrvda2L;=dpjc`A6bOGw%dUxK!r0D!=)icM5Pai26ADhT> zYk8+~Upr?1qKcR~|DuxCA0U8>BJqCpY^dOq=7G3Az;A(6?&i&*sToA*5KRx>L+aFv zt^K}H+QW8NH{~gUQ)6b9{wT!V&3n6%kiF+>n(^gqP2n%IdlSriEm7R9)qE@x96@(T zA=gZHlkOrS@tOwd(&2m;M{bt=P?zYPG%`GJO#G_p-nEeal|!UO;s4Ev=3OW92}4)F zGwcUj6{<_7Jte<)T&HHfRsKBT{0ABLw0#Gr-~|vg!GkA#f`=S|6g_BmmmTFC*jb(|%BSmLY}$laq34 zS&t`<58AVY*|khKoKB;;9(aqX%zHJAJX^0QU*r+@MV6nxE3I`MGM&40rCE2x3b50i`{eDMe@ zlHwJ@G>~>Hbj=hVEL9siq+OUU!k6@f< z;pkbP;)BKovCdZaazmPq@Ng!~aSpX3u-n*eTvzB~R4It|0vyQ$>}@(1CEg?nvI-kY8^4u^E z6}i3ov%8V*@ccZW!~ywbQC1j8Isn(8S-*xbYECw9!tD09Z_z=9u2pBe{|m8aLBnlL z6`B}q>Nn!|F0Ll3T)0};LVel;nfQQIbub@z=j>Dk5LlY9D?_)lQY(|3(M?%ey8V^Fv`^TE5z#jwV zEJ6|$gY}#_8BlZVYlD;Yz=HK72&?u(eMebAl$(N|EYG9YN>6{MwuC_H7)mRqwv#2X z)~?3X9t9F0L6}FiOi?Zn$5a83;wc*%3eEci6K5CsX1EM{U_Wh1}$y$=ZIo?|ygI`uENkqSAQPmv{D|KBqB{46jsxQdA%nR5}D`d7!AN zOe4VjK`mUv?GMQ&Qv|T zWaf4tq=hnZ%}m9QS2=22=1D&_rW5|9~vT$b<8A10lE?@Y7zi zqO_n1*D(_KwS-_ z1&vs{kRfRL1rLZ>2a0#U;+PO4SPlT;+0D}1{D_`S3%2iPP1{B`|Dn`%-+3T^5i9M z4s`axg4_Ch(s9Z>Ky2kxp}pLoMj24xnwZ1;O!kZUzBp;5pL~%=u8)T%WdO8vE0d=PZ9^RQ4!z^90a0+ReulZ_SIkLh$_o&!ENR}*lGsNaZ` z#~_8f$&I7Odcl05p!7Ra0!{DF2B~2{9C6dfh6?hzLaJcX@1f_JzFtf@O*EJ#nIO`S z0e~S5su#SofP(1==4IvlVJJ|+P=`nSeh4aCNXH42O1-?kPz6yf3~dZLI0;?m9W-pJLH2CTo7~I zN98W@ym8}(=fU>%_1PuiW0cnbx_cz@oE2xP9D+@rhzCgmbPW$vBG#8`*;c2sv0PE) zu07zffeS43_c}i`Jy`Cn@h-sADU5bJnfp$)hqPsTgq!glnC*Yk$UhdoF zXh4?o7+G6)&^y7OeD?On@E#4N%G~63q07y(53PmRpQ9;uc)+Rg9noVJ0 zgBkPKbm(rhw2e3Y_7H1T#j{H^iSLuj-Thn?Kc!FaQtT76$RX&|V)!@O1-H3xFE?nQ zJPAq}1ID0Hf#$?2f=FM2(6k7OFMzcAUcYpjD(twg&J$@*b;Oq*d*$rc5ODd+VS-%5 zfm*>^h8FkCFW`ouu~q=Iv<)ZMd|Z#;77KZ^3L6!S3fy#RW*?3UWWGR+uK5N?xuI`| z3OZjg+8h`5L+_LIA%0 zJ#pNo9qg_cp!j1RyAG;1HvA{WGo7bVUX(=*@>sVUapbyhyqtP7D!h!q%+ zRd3Z9j@kAxV+Jw3(@Rf839mgw4SOx>3=z2<+;Y#KCuesU{@6#FK|cr3)Em$WxUJl{ zso)XElW$phFb6umA&B+HroB;?;+a`|N6jltG38rnIXTQkn_ce7p1H>-MIRoF;USwd zcl*JYGZ^5ad~fmx&u*UTOIu^^)d$z<0kBw=2074z6)PB+sqKsJ=0(?PnWGK&Y2rqd zNuTLzj|DTuH6YP+g?G;iV9#^uG-V=E!QPTHCYS2QhWaXlREez)`dECP~U zXJtdpT<9Nyn5tgZ(-I6z@H{i45A8;zs}G!d_-TcB=@xER>w$?5r*ww-@Fb^atH6N| z4CQ@``kX>9L4Xc53eyXw3ld>TS3MQFR*9R<uIfNc0%SAA2OQ`{CbGZW)EQx zIW{9M-j47QzLaM7mIEMXC=z;lLHzC8%M_ZKj^2*HCpOyABJP9p^%1Gw?shy8x%HKA zeLPG1K8y42**6{1yyEzZ!TUGHpqV*c5!azXL6UgdWvr3HpKcH91ZR*FMXG-naxoU} zI>e+JppNQ)=`;Q!fCwrvz46Mx5nfmmv<8;oc@;0H1AC(Ik0 zWtP~=QN}s!mFgsT5?s0W@qa=r*2W9&e)0a;+fMwCawjr_sU)m0 z7>dE)p(kk$JRn1`{)>)hKH7;ItaJ`M6fhpH=YBF1|*I5 zz|GGjQYPAPm^Wkj( zrV>2@NzJ58Ew51DlQRhc;6K{M~r3$n5Dpa|@bGiac4XQf)F<71geXx6(v zGZkI|xZ7v7V&2%SH=j7m&J-K->wrq9R zuF7^z!Q{5Ft3`@44=OAl8)qS>?TBDcoT1IjQ!{8*q91B2Xkclb1KyttRsif#f3zE# zZnK1EzQQ>8%aV}I=x7vYDcF}OE~ZM~as!7`_OV2s2*XU=^&$Q!NW0GerAUg5ZFuy= zPp_Md_}!<`;YzGvVBwUYABP(CK&A}(2L0L09$p57an*pWlKW;SO<*A4RD(k~^vgF0 z7s9fg1!PV+Ly=&ccKy1cuqTd5>2_xce(uVVTHzI%|pAA17FCcJmJv$xN+yK zTtYEFAcP67L&x*D(PrnC+^ zSFZnLZ)@Fv!@PN*CT&Ah-8kc*P_L)lgW}`7ji(sW?R(UpN{Vd${^xe!Hs7o$0J>Qg zkp8EI>`3}C8`$DLMHX(*>Sdu9tu~;@OO(ziT`dw`z-^Krw6T`@B%+KfqPk25eBPg{ ztF5e!<-D)Ye<|ciM@(^X9D8vg0->}H$c`|tKwg-%0J^4;O}+7w8!a+k7H@(KGf@VW(Ihoan!FDYa$Rdl9B~2h!ZBa zX9xv>K!KF$BqUP`xu}H^h1cWNgxzabw@%f?{gEFee?52^-9P8rqxO^uyg77wS?< zZbR6{B48Ata}$R7K%xJ?(77$ZY1ck*s6$0`D&shlpH-m8zeuba?j%I%U}77@%Is8Y zbEX+|`a@PYV(zlU22n(!U;g=FjzfmF@AY9Q847VZ0 z7Bh9I+ROW=XRSeevD!GAoeuua|j2i4*Rp6CWQk*`G*!c%!o2Fo1G*C3Z zz$QZe@2#o8ty7L<9Ce)(9M<}9>W;6Bqw-Q$S6VHS_{rq(&y?4O_e#4YEPiULzQ0N0 z*mO^g;1Grgi+(Vl3*#&}+&UP=%KtM3&ZPOGo=7`z;_U85RCFmsm3ht9zo}7eR8-sB zK}Kc#REd=br6$9^apoZ71pF1E++L6F#O;{Go?@kYX`Xthw#)t)uGGKa;t=Ly)w|uu z!}g)xfpU6wwY=)GiXFzl_45R}0KKU-6$MSv_Rar=*fJ3C`!?;8i<_?^<-#R+mQJns3tC!7VSb^pP4^I_WxHSk|s z-eh3mPoSM^xd^#ZKq;A#PeHd5mly7?tKOyghBFkP&klpF{-xetnwYl4UL`}TXZScR zH2ox)m>bJqw2{0AM#{@M_vqdvk2U?)e6S*S_iTgnMRh^Y4Z}L|y73ZNDhLdk-Jt6j z`XL&zjE<(+)6)Wvy~}E;ns9a8KPcI<>Wbucb$W22uz)R&HDG9@)fD_4L%Bc9+&v6v zaOuB`gMl3rgO!&gqhtSQ>X)CLKE+!juA*?Fr?fJM%eIfL@n~{J$ekn32-L&>yDJws zWB+K%B@C>E;3+zl*yjAd;zY~1LhbqQO-~}@XD`i(u)xAs!2mGccQUqKmR1CD~ zgn37-dO1}ww<5!kqC+MuyrvhQ#}qeKe|mrZ0;F>=@tGIkRIN4?t@Ph9ig4&}xS+n{ zdg}>bGlGe35b2ZorXD^XPcuo|#-mBungFff0G9+TbiTD3&s1LTg)b6bqn$wIeVYRU z+=P=?D5&K?io$uy#f-8=zQ9t|XM^Kr0qXf}>gL{Z$(7(Xij72$7jlg}MnX40aQ8PR z#jijZ2{hR=6E`&NYPAs zT~zHGZy7li1F-wblMe|d70-YEyn_y6=YO|rU}y(JaY8XC#8{uu3_M|K@wfSRv?fi5 z^#Zy_|CYTf%$*M z_@+>S8vT?vd?}kU19#=&;1@NIU?8{M9a32)S`O$|b} zr*r*GEN$idsgOLd$OBS6juy`zvPJLvjS1U(7})uJHL(pgDn&hjfkiRtmD}sa?4tv! zTILAd4U5qi$Y)QD(_$)(_qg7^VsUl1h<+1-Zj`pH!66T^K*}Kukj9ix3V|+q&uOaZ9CXIN0}ic-!d>W7*@BLC z*xrG~U*pmZm%4|Ocf5#}sy(LB)Ds*7t}3{>BoK%2ff#A3sxwqi+?ILiJw-dFQLL(y z^eWiYDbQ0Ux64zc=R>zt-{-w2uR`bF`a1hJf3o(yj-2`Ox7ABS{VMcZu>zxcjzoY9 zVy9{ktR4yr&K3-Z08y3-fHw7rm!MlH`T;SZP{C6-D{FySX6}j^RVU1VE|^TzPGXt} z<6Y7>F4e~gr;;bwZg6_co{`0={_4;cy?uKh3XDnF%&%0IDf|(x`U{cwC@6>* zd<-|*zkBL}&lFv8H750^HABaovS{dgftelCh^Rf9jF4DXURk4q=!hxxWEu$yKpp5^ zaKED}D?5WM7QZed@h9kjC)&tUXQ1V?b&^T=x!cdWWOXl1$0M@+B|v^^VPd;ixChVN zI|KBc!V8KA{3)>$etDbFwgtZjz#AZ~2`oJqSl^A?6bM({oA`^)(~&6@`iEX)|4MV} z`-Iz<71=X=yiN!7_m7vOvajpUpfbVa!Ws_D6UaHDK*q@noS#A;n*y_N0mVu{F&fjE z`w7e7Pm^MaRlwihAI;po%bx%_fJE3ix06OOjeUCGS$VXn6;$oa>dXrM!>`8Gsi%3t zHXv_}Y(#^TcuWVIBhWx(;!jH#FeQM)st`}b!=Lg9@Jm@jM{hjlmlf6woSVQJjC%xE z5x1dL^ikW-u@~R5_vF_`?XHkKTk-FB@I~Fz7k|{dmYx7}%d6M9-zs{GzCND$Q=JOLv&+b||$l*CE zUcnW^Go$_g7b*u$0WW0<$oFX`bb@gZ9&zLa{+T8`QsGbTZPNKx`yt0a@GRv$0BtDp z(Db%5QR(qMbR@S>x&v|8O5gK+=mv(ax_$id;P{n|Iy^aO@f6&O;1m+;a3d!lS87`5 zx>5)L2+j@Y?!UR`+|4I;w(89>*T)k=Tsbz;gl#zJLx7L_0V@-NmS5Qch4;Yb2RHt2 z(ff#L@&$+zt1+||XT}+Orp}XFw?2*wXOKCZj>`#Ub`T#DoxQ%d0;{?l&d~8c37Uql z4kvjp$0}QMAfo8LHp)leO4+b!fVZKoF?qF`X1!#zi zUgA(QTB}!L(NU#}p5antL`7Y@JQT%RD{X%^<#M{)^YQ88IF6%Se!ETNiR!)}usk~4 zp1d?%>2DA3L=~LP*BXz$2t`}a{p6}Txo3WqCZ!dYPv4VBxw0o2A&o$Sf)!~1q`}Q> zLW#>(R3w~{M?ioz%s`(V!E^(Gp;S+R6SWu&AbKDX5q^t#6ns^|$}1 z8EQr~TAWFx|(?(;F{Z+!9UqJnjpZJx&q z)3yi=@g|3EG1a;U`0on`<5U7FmFDt)Za-gwgWc0mZwZC0$wLaupR=ltAbjQe*!wTB z$_L?Kr@s)5?Wb%V&dRbQ&j=t?w{2A?s`e9y2^+KrEGt|__uf`YD+rP(mrlu_JbNT0m``ClplC(l++A$k&Sg|MB9K9AVUf%$>Cenx~QRE9vn>YRtY0gRdpc&Pv$Akr{c z@JR*l>tO&*7>lNO5+fnfQ?#@e5kS-f_f~^>n(91+lYg3B6BRe=nU`3aX9opQf%M$E z8N;T+9y+L?MMhID5$Kvyt>~K0mFOAsQ9lxXB^(uchZ$Hh?omG#T{Fe6m*2lwXKA7} z{^oPtQIRMU>9>iYirr1eCW^CAU(g}{7X;3KLckFLF*`7IX0QEaXQHDZtTT0(Hd{wE zkN8t|>F_#Ua9(d$klvDoakF#eUCC9yfp0cUr~V*g!GG00*5KVD?aPs0(M> z=QCY1nld#h4@_O$O3d;z7$@U-?E_nAWaBRslKR{GNdUAQ0qwV9fmGc8%UexCZtDqy z|Dq^{SG=}+6Qt<(fxOJH9yc6h9Kfm$snw`zw;nk^=>F*GPy4QSKipR3_uUK4dor?b zS|8z)h4f4l=;n}S3j#nofoah#9P3bk-%9(;lo(bC;CLLtT2Qzvw;q0&CPq%~`vzvo zj`xoatxlP}O_we)Df(EOc70=2=~Pcp$>M0r zV6S(eukwq7nYPlY$wm)v-@a)gm;AtkQN}6_-s3R^41qH)7Gh_r7TVdl2Sb{4sB~V( zcPOiSUsPb1MA=NhPURz~2iV)MyyOTm7hz)Qb?^?LAGv}l(Q!3s-f^%$Xg#^fPkD=b z>`Vf4>5oiHY)7>fZ$-KOO73WLb}+HNyhBbBv0+ZvT$}N0$45gHTMPoLNWtKI1<E+U&~74HwM%m>u@%KphHXQq{4p^pd~9;Hge6 z_+40E00!?8wtid_zte3)9iB6-V#w_g&(;sg`XpN=efMU70td<(tPUjj3{hawN_|IJYrzQoS-Bi^W~L+cvaM;6X;9uIv_KjA-tqau7s;7!@0J6G51KQz zg6&~Jpv%HQ0N8B83;@r5h8{^}kO<4|WVM*RBJ@ZwhT)zXze0jlYweOv=4 z4(C5Mf(ZdeN_NLT!w%b;Ka2}%b_|xsp6O>Ih;Q(?YPFT1Gbe8kVNJ44U2fbGC-vxl zIV$b0ZZb=4QeYC+%9S%vn~dR7XG>H7r5!yGrOg=uy85Krvko)@GVl(Fe=D@){ z3IJ5daS!m0QAmFg=kt+BhkwoiF zM2UF9T1*rRU)Hd_&i6Q8^YU@}gF_F;%nRe+UAKQB43+vOcsjI2@{gTX`OZ`%%+J7r z;ChDH29HkFqT8S-RJF@;73C3_f=YiFIlb9{E}3tXj$GAJcFQE2M!am|wqwsNUsv#e z_>V|E!*@*4AE1~bQ!r?D00P1bPNF3QeIt)mjdtIDG>?K6{$L#sV0vDOL(?cdRR)H6 z0bK=p+?8}NDRWN2YZbwBJ-siO_8ZJ^%!T1~0RXT$M{7k>JRX2u(TqyTLB;dAu&RA7 z6~b%Xzy%FiLSO&=aD@RGr7-ZDb3ds>uU!^!8L=u2W~7O|_AA<#qO8?BRWy_m!pb4_ zQVwP3tICHE9K8^*$C7~83MBNx2w1jn!F%Xy7Jd0tK=_RG_jWj@KJ6=@^cTXiw(fn^ zi;LEPE25WQaA?O;O}|7iIQ;OQU`?zQF!`jMy^?f+BJjLv&GpFE5&@g@l(|>&SJld0 z-kX#k%%<&N-+DFe9TWXQt>%ns~}(T)3ky9mhCGFKEh?2DCA%sP=S_ z1;Bddbve1Grh78QE}xZBe=7HD2lMTW#e<$yrMOk%HE_g35SSVn8>YQv)nZRDtYC1) zZ0it!4*ZJU)Xp_ z*ITlr-6m1~gDb830b;yOvCCKCX|f67hh!_f12}_RkG^_k56*x$`eCmm7QAq>J*=y$ zdB5l;x9xKXQLOPvm#qh~-=2N{Ue^ga5Wt<^GdHcU+(W$_$g68an&KXF+xj%9px#uK zP^EN&4;M!emE<)hf_=VM9((fTe+||kd>KeB6VGbX^ugj>AwV-cWgh9FV|V0@br z_a|JSqyR|W4YFB6dVaP*u3vp!bZ$*eo%(Toxj~bNIISDUnTzv2$ zxE)|SG%ENrY=ev(0NsNC$CaTPUfFXVCGQh;cyB)|MikplWb8ypkk}mY*O1G3lpF^- zl5rS&j4nftXPAC4Z$@RZ^;fGJE;hf&N)I#@_nh_bDy}%wpPcthTIexSWa_EDg9OC$ z8z?273NW2kai4J;dxZ5em%92|SHOkRG1c~%?jS$r>g!e`bN<_!47cv5bGwWrtLC~a z_n9@`$sq1o+7bhIqlQR&N~}{`@no1XPqIqT&b?vvINTpoh`oLKl%)x!0qI?I)!}Sl zkbl)Dgv7C52fy?mNZmriVe3uOi3#Z(0(on&|$)66W8aXm~T<%R=?(yWj-|+6YG`sDK7qPIP$5t?X9q0#dR>c~c zGy-iGSWf&2k~JjKOQyvK9lmNRtD1J3jJv1V#gDWqIwW6Hx)wv%vp13Ae8^=>+)j~A ze}pJD(3X)=JDw@00P0N$7Px)k2VijXhs`^Yp_0Q%HGIS!uA+|&i?Zj%_IQQ0tp2pm z%`^`Eq1!AR_-t2h`SPt~h&NaZ>rO9Y7%;7egO6kHQ75y>UZVaP7J(C9*S`Lk((3LB z9b*%@qNJp_Bc?bGFm=F*Xo%Dgf?0XBl|+Wn!e0o{U@iC?W@kB5NGuUVnmx$Ytz}Qa z{3)m8>Df$UW7fFYTyc?tidE9lFi5%)q*@z^fY>V>C9=L2*&o~v;JdFqI-CV?kJ6gA zREFzs7ofNftNz@nEqjQEU*v0YdGY7Z2xX|54QRD1+u(3q&;$CuDL!BlUE|3(nFV++ zRr;aj4K3Bfi)78rfmg9a4An@?tLuPO99)`w7o4N#E!kcMc0p11XE<0ieejdEmA+`TWlvR}BgB=J|w2pjT znZJu&H5C=L9$1FHG`O1c?hUcxt!sX5I&_RbxnuFHYzd+{lylb(!n_O2M+9Z~{;g=H zlnFzNPJ`m>D?QMnBuWjTEy!?U=#cNoVO_LufUc$RrHEvICl{+xarOeG?t)($XTj5n z0s+MMuDJJ}sRQ2yFwN_dX^Y2q2d)X2atftg&(3{9smS_bTETW&?lD*JPY7zh1)l2( z7{gm@pQ7*cYes(GL)wU5*0q`;h7L{ceMjYv&$+lJUtFuB)q%g4y8p4nOWo;j;)iY- zHmTMa^Ze$)4Ae0tY^gb90*-hK6^G};y3LfGy#nIZFs%IzRK%aI?{)bpGTj=cy7!)b z&N0rWxwzg0_1-EvNYbUrPH^ltvRRvBN`kAvxxgv{Ws9lrmYrLFAy&*u61ob7*t1mC zvdO2@WkwI&kG-*LS5etGD#UoTylJ3t-)j6c*8{e#nr#y7Um|r~V;pzbpg(-Ty$Qy8 z(yORXVcwI0`OGWS)f|EW2^VJ<9mQ1gJ=D-S_~&``gu4Ho{*U7~6IXAKh`);IlAIP2 zfv$M`u7d@l0m_goj;=7p++%Hy2p(*dj6XwFNMHsAkVlxh1F7``D~DC?sJ;$cxgWEN z92qrC5FM#}U11{_zjY8g%>{`}QS8^Y4aW&^99Q*DE#gI)frTCebZ>G#ary(t%D@lJ z75P@ncXicOkqAENOC2AC1-|sk-VKn+U|b^n;DGtSLvRAmAnY|bRjxyH5LJR=KqVBX z8#qne3+j?K)VrZ%6sKGAlwD1JUeKuleOaY&kv98o=z(F=ji(P%4UMiNdqH1_-;7Z^ z=KBmShT_)&g*obc)zu;=@;HVCDcBAk{I0^^^P~PZ;G28jL4Bh*znma(HJmjyOzmW- zL(N2Rb*!i2rmA@&8aN7r+b1>tpOues>Ni zYbZhZ{-T>CT zoE{Gb>1vNXv2)vaGhrfGbx2J!?9`KvtmJOq(cIF> z7W9Z*OIyO*HbQR)^E_oEB3K6-Jk+3dsxGp@OfNon-Y%iPvg`-c{^CO*PEA&@g!cT| z8@apVAFlNAtQlYnfItCX?%8R$A441>05*SXHTd+oJ}^-VjY2+&uE%?v`xLy?nDaudmGn( z)si_o%@gZcs-ogKYFiH7etQ1hl{ifc+ll9qJK2D~gZS^_q@K^7DJ|eHJ)-{y%q^(F z0!*-xK>yt(j+w(dg|dS?h&8j1c;YmBQ41SOuidvCp9p5?#ritm@D2WAEi!grx?-%s zS&9pgT{Q97uqcFCG%Vmifr({M^MhcdyPaNifS*^ct$!-CW|cYmeb!X@AqCDsq+45@ zbG@A|-g=BJ;F5PL$o@KlP+H1INmONY5aiU}<_FfZgz^rh2tam|p(XaH*uJ z?&yMOz2C^_6;l_7%1=id4EQ^D6e)bRO?mofk;q9TZJy$Ad`H1AeP3LD##yOuuJ zQv}Ox+}{|$$CkD-z*cJgk?}SH@_ybO+8Aw2^z+~@qT;UB56Zu*y=D5AEAVYQS zJw0((Hx3)lPk5*7Lj=D8a`rEgXZww2it^CrsUS2Dgn7Xf3$20T5?|ZKL2;@CsoZPuDTh zAJ#Ph&vcefbem>Lu5MigGlBVWUncTgNzrzdskT=YvOk-&^} z0jazXl1^UqL% zAqjM!0qtCma7+oYtR)=d3e%0MGe7KNhMQi0RkC>CdqdS!Y{Ae z%y@Ms@?4_;u4-|A@3v8`$ybFV^h{W4jE@05jPeK&ooZaiEiSu4OG0U;Se)s^P}8T1@cK!G=S=&d$?}pkslDYWUuI-uRGo)l%hV0EhU9 zh|eTZS;TkC8x}MH#~g~jgQqLltX!I2`+kykwx;jT2`5TTBeC57srBH^uY3Po`vhs; zAIrU5K9Zsvb#uH*R0gpR29RF+{s{QB;VD-5?^PEmtMPu&eN-oA=Y$DYO4a!#PRW|; z-=D4Y_shnekb5`}CWAwAhsTJNfQ#YfwotS6AFo}LuEY1QVwmy+ITekm6RA?saf`pZ zuRJ*R&TQc=`@^+sGY3MBmmujU{lEzENsl0bp1F2B&v>hJ@xT~5r1aVOtT2ldi_@|R zVMtw@Ki_~z{Doi@9fmw{BX!etD-J=^ihm9YXiqhe#REO-Kki zeWJsI?nGG826N_FOPQc*8+JChcN7(?>G462hK#n(^nJBnWuOum|7JXi?V7~p9Uo_R zxpFLSb<$(WvvdgfL=M#1TKNSZv%QO~vQ)FE8~kSSJtHT*>m-^x9$B|PKvxT-tmaXq zI<_9Za#L+@?~L$OB!%thYDgNYl&5J|IYbY*nM8vUTr za>aBIMDwM<>98twEyj6JljUa>>k@BlENh$l*#3>O#@mzJGm)Q;D*&SV|9V&cc~$x$ z`l(L+VKra_*21>dlM?;3lJGNQ>%} zgPkB@ZCRyg5)LDZzDva61(v-UY$zwqhsNx1=U>{jk_`v{)O@vSi8%1>m5kGM4n=1b z1dl7zXBgO#Rdc#Zei6ZO@e#U>$nBPzCicM8Bfphoe`5E*v76Qh1BBmmaVp+En51r^ zwLZm^0&hV}FA^z`r_IWX0=HoPZ{>G&jhtrWD1Uaj4ZL4Wz^d^h>cG$W4`|Qx4GI>=U2YHXLx!OzDK?2HZ3J1JX0O+L$5G!R>S9==8k3BWy+{f>?#V`46 zE2T~FT$-2Mr(a$Q>{{Yhmy$bGf^UOQHTIG5z5@2C$KTR=(8uF_yqR{3Nwy6gNa#Bhr=D!r?>njtA+j<^JX!dqu0i3azj4cTRiQ^L$l( zr0aHn_uWX!gK{>ukk4;E;}=s0?)!Skd~|5Mm-; zMkS6}B3+fp6c+0sQZDp@Va_Yu^JzBt zTXk-_F}vsHQfT2dVzMg!60^g6Y6JqRoH?*j0-WUp|^&PZK}Y10q- z8LhB*S_?$0k4O(HY%-pGm!fSZ&;1jNumv5V(?$a*ic?2;xdILpchP$q6CY}R_X%Pj zI(2SL`|GOj9`QbEUpbjXWWj0wAqE$2oi zoB#>!b8J~n+Lktf4C+9BK!<^(eb{^W3$30z(Jx2^CE}A&G*eaUW_SF+t6QnoJ9&4+ z{BNVkDgqEefZi9(LtjD0qbTPI(1`W$j962zz*R^8y7tV&;jKFI*+rppk{6Ok$H!~N z&gkz+F}1cjb7s}o>2aR<#5$b;4zR#g=Vuu>z)uwUf;`t@zhk4AjtsuUi{#0PSE>g* z)huIgrG&J(-%8fxAK}WiT5Q^d>{6r0+rxaOe<2JqTF`W_Wi&mG6$PF@OboQ!>m7r~ zrzqy=1T+eAgE$tpqT?EQzB@QBYHyuXeHpwz@=m#$lldWwvWAe;H&wC#`r~-``^^i4 zNig0~*Q}@U4NXc$)xH?S9xNIc@H^d5-LD(Zi(;H1x%z#+5zetu$(^jF^zMYO_^bs_ zm!2lL%k@A5y{TV=;@8bQS{!vLcO&yA7WbGHxK{Wp4fs}szI+y9qLlXdz~lGNQ;&Z- zL8wHNLeP{4Mk~_{SKttwZ2*no+cW7nI{rMB6_u54Sd{NdS*UWYv$htq`FY{Y+sqTY zzvn29?2kKgokRIA1X|;)R&}q}eLf?(JJ!Y%#?II8>GNbmgg)>HPxXfZnCD@e<_~90 z`pZ|#{*-v_c5wF|8`H>p$nr?+WE7-P75Az>i%a!0&tKr$mlD(V;7HgVS?C7tsXWie5vCN# zpz^`I{opG0(hC@8p=Lq717MGgU^VQa!E&ZEC6HlxVW>>yaAU6rS*k=?+4NccPGQz_ z{2R$_LHNLCZRHh;;W;CxSEOTFP!J`)7ri^ry%)YJt^a&aY451ZrpKEM9vGZ`!_Y9S zJWW0K-r452KIinaqS^4!-w;y-)Kj--OK?m@h(!!mVDhoj%E7$C6fgE9^VXtQUM21oq1H{cvfu$Gg%ydSZ)R zL4hJwn$|sXwJk?wLnBnI!e$dzjxS(9XU_EoG=c5{?|jEFach+8SJPy~>Yeb{OM0e^IB&xL&84Fbz9}alo-+dVyySY~M-JsfO3-a=zqnfvy z3;27qN@`8!6fR1#UVVtd-?%eq?yb=s_oP#j{mZg3$frFG#nhlEfnUPQk)zrqDE3Rd zAL}`GPvK~y(elcyL_5h_>_y^x+T$_rm(nu;(On@;Xc=_r_t67kzE{+}Yp}r2MzF6R z9GTLCa@3?(1&OErusv$|?0I2HXr4UJd>^5ACydl0_%K}j{;QeIql76mA062Vj2riv zr7a!0hLyJ0L#=Z)l0J=2R|$!pt9J8;$DbS2S9h4#%U<3b)i{lk{`bHS0+HV$jwy8% zY*Zhs7$~*^M?HEdLoaw}^dYWjNl&VOPn)DoTy?zovibM!e*2o2snqh0*5kg9uP3}Y zS@|A%F3AFSCtS`)=7IT`#tbrcroVj5EP6V@y2@SoPREWpkhUCQ3Y zO<+)1{?b1~zO?VM>~3lM7TE;>RWIA$2?r;rIS2DA%A6nchOH4m8>UrohRshjQ&L0D zYuoTs#yt+CVx&FBp6xZ-vU$Whp0$HedZ*^(37?hoUzFnmW)5lY>5!1Se8ei06n0W_ z6HqOJeKiJRud;5xxo&0-^LA16$UlGHP9>AQB1bKC%iB_K%R~IhCUQ@Qwf?Ar!zp0u zfC~)_j`4T881NGbAn2J9hFY_Tq-VrGiTF^6?*0c4AH^8-pGN8Y`|C#k>HQ*q9R9mM zEwZMk1i%v_f>HkNz0H{%lt%pP1xX)1fvB`R+q@4Ss@;zW$(|OuBG_b|B8*7e+sYvZ zzJKL9C zWYC;hFLXI5Tb}O6bB5$8n26Fp9Urd7iw{|PNlLJ%J>YPWO41q51tdcv)G6TR(YC=J z!UExzsVY?{aB}>yvBIS7YFv?D`^hNlg4?;R(0th{I*V!yV7Z?TNckWIwe2i2jQhNNsaVC4LuhEW!PkIvE zgce4VULJ#LqnjqYFH)kLfnjqM;Lgi?rKQpt5&81aPq#|@Gds^wN(_si{E%?ekZ|@@ zpyw07JkP;SX0Z&!umY@S+wv(Ge;Q>r1vM`LpdTswg|adTIQ$74M~#$K{EoV%UoI^c zK3ti z8#30Tq?1!w!rmO}e7u>Pn54tPfU9L_1 zh8%hNd}=-KP1pyJf807CS9|phb~t6?=9SLwFFcQ!Dxg>67uewn&`B0P=Ec--+znc{ z(ziQ6{z=LM`e6_Li<2kq8I7YIngJdIm}CZsTqp+0AJ+nbu$h#FsxnCH!0j^M))@MD`_AeYSZ82Wr|8CrhgO5C!7PNNK6 z(G*%vzF-!RA@R4 zoM%KPfDxg%v<;2+Mq*6?#c$T)!|O3~b-te;@8G)M7skdZG)fGshx>@6F#NA*weNHA zr*8x}`XD06*J?cxbAbXs^(A=N^pAf3ZhxwMqPr&|PVRE|av*|Dj$suygLYlp)W+|H zMqU&{3v$(L5IynlF{&VlG&*59gNyTGf)pudQ5F43VPlo{7&ZXaf)Tt}KO6eyZmRve zWH%0>#{-IT*Ea~+5TLiETm?PubDe9Wu`nmxfv<4JHT6pSXPCKGFTK;{$XQUp{Q3<- z^iiKA+m;gJ-51z}XXI>teZ#neQ>w@RN+&=eXqwhFAZhihA?Vj=`k_vybToqp{8&v- z_bfw;Y1)UqF@qLsEy$aS(d^~4v+H6-9Xyvjc{1%?rpRv7J+W51Dt1_9xcW?x_6mV> z9snS9jR&ZXFS0x!_C~e&hc=@T$2J}fW#=vsxFRl2+mQ{T)@N^>Ha&9AW&g!*kW(n; zF>!kisQ@#BXLX^s65O;XfCoGv_yC2_gdcrR>ej&|U{$AyD1W~8QHPpk>qm`v0XU<0 z)VN4@_wygtCcDCta>GZ4L8{>2l&Aw>TEM}V<|tC|$IeI~mV=&-J*TBzzJu-hgkdi) zfGP%b(IoTeMy@`E_n88EpOsZl8V#1sxH;dDcHgboF9MMC0Cv9;71a5~0H{mGH>_Wo zdpeFO)PdZ6?b_mo`nZ2AEI_~h>e8sSs?UYrrMaI?ph-n2<0oR)W*Ny8>4h&@G0sEZ zc>vab@fBbksGuxKsIgYt-GlWpe~dM z8n^>l0N0{;k+`MV$Fo$ooDCNF2mJnB&9zar7Ms1i;40m^ik178VSTk!Pj;NCOj&YisJ^6bR`T^6h+&vjGkxe^kF?2XH7~MM{#j?N7V%V zZCYql(?SfC8;D)_YAScX^is50_m>mC90x-P_q>j6ac;S+$sx07{}6$^QyLeUfp2in zOE&vJPUdtuhmEL)Lsaruo1-k%{D0B*=J8OzVc+n`o_*hqttcVcvSg@`{G_zlVw9a2 zWEmNAiR@bxDn$!XlI&T>HY7z^LyQ@UFwCGanK8fT?7r{!`8?0(z2En*=lMgQFs?c0 zx{mWWkL7!Ozv(JQQ68d{$Ty;MUFC@?aV=hEH8%3r&eI15*Q10Q&SZULMgxR92wl2N z5X;{>KTvcEvr`nWUPVP6cJq=YDEJ7T$QowoKMi?U>U8_rU4gf6tFx+d`g7=z4pRac8Imw@kooE1t!(hRCuWwKAOlIOW;mcX;vfOkoQx{MGZaG*_?0?cr}n83Zp zl5s^lITGGtjB|RTZe41$4nOPu;Ksve-uGWz8&CTv{7qGY0w;qEjdSa-J^U7$w!@F2 zL0{`xIn#4|Vf-K2l-^2YQn6xthI^jA^QW)hJ^s1(Phto@_dXZT{=72VEfigMhIMv{ zEClA4!251L-^KB2utU2b<4ZUTZdCwSNg^Z#-4M{5Us@|;?007JQHL{bc#rBkh}u)O ztVa4?X%5I~KhoC!qtJ8q(y>aS%yYgcc(%Pe)uap-PdPB((7ngFwRir|?v?z#w_n(t z3Bp(fIyY%J;kQl-TpS}&hR*5P#aFroPAfKFQy4E>)Z**0IpVl4@6bArN@L;dCWw8i z1L&72IYeY9Zl5y~R6r?&Q-FtXyK~{(7a)OR<>MEqFxXVcf4JhIukUgBJo^ZswUR(@ zW$|7EcXt^0iS$UgJRMMM{h|0dPc&pqp$y!eF46;DI)uf9sY>Mqc%%7D>qPNsYI(MA zar5eHUJMoN$DQedR}4MYCq)QGkGuKCO5U}hKhaFp@8t=Swe$0j6Vzv+fYkG5R{uo1 zHjqXW%7NDytwk~%gCxzBLH)@GN-0L*Mzy8|Om_M_1nb7i8q!jI&&-hwHA;b;s?1xj zP7uY|f%t2<2O;%~AZ)IH?Zk2GYek=FwX4!}9x<2u)wpxz=Btm;fcPsO@6Z^29+md# zbl4x@7%UkWs0)F+mNON82nUb~g286cGj!|4^vN^y0PWjBuLi3JIcma3ZJoJymwVYQ(isz zXmU)_+R^0|u%83ae8KcYS=bwJwo%SO2~tkcH`^wi^q4v$#kStpU7kGX$^#hB6Sma} z(EK3$j}@4h6sdl(!NEQ1bY< zj|#&zy6Y2Wg^gkJVvnz=_kk<|PE*(`V_q*2LHL(^1Id0#b_0FQap*)4evYSRETmvo z=Ap9au$UpH~q?7;Pe6=zqGW6aUjaRgWO)%H>S)Y8Gz{BZ#hphXnrmp^|+^Sdhqf ze<{MW$x_car6D=@2=zlvpTU7_JYcQ5TO&E!epbU13}ohf+$#fq=m27I7po0T9YpRK zPoL-BuKo+x=!Gbk0gWVT%?;eA`&7RamK-x=w9TG`lAtGiBB&>0h?VzgJ|gZ;_)?IkfAWcjc(Vk&}cxi`7o` z5u7<#?H2Nnoq7sxE!`hXqKA!ZZn^?FOZaV2Dv|~aWISyrs5A4p^!Z7kV)x{j>7Ll- zMvfyC+^GaMN1nr%;w?E8ts*0VB9PoKFk|MuR|2g8&ph*Lb>G^p!dwovuzN>UIUp8n zx!QWK8sQ5mviflKJJ~p&>a-fDcNMT4Hjlgw;n0$GFAc#IRg8C{4*O@`JZOFPY# z)VWXu;_txGVE{75XpwZ}(J$;NPIvT!vTUwQ=n!j0wi#!31|!2XQy0?2BYmSTvp1{6 zBN|KS|3QTPzw==Kgx9Ed*$2^X>}9e=7&YpF_59ZXlUf4!C!bv1 zd+_>>$^!DKD{p{4r-jm@69pd`c{A?j0f>9CzuC>z*0f;i*H7hA4wH^dYQ$e&V(beLv`L+HgNgq#{9NOnU5sN;!zZ1Hxru$Cj zyv^@X3r-A(^*AGeMVn!V0SFajYO+!wN?YA6Pl?>%4HR`p+u?VQirSZ_eLA@=ne-vs z`R5rg4*6sC7i!=2XvU$eVA#JkDq%!j^d20ze#-qiPB>rCYUxrGX- z%V2vJq^)igWEylNM5=GHX(zkT^#{2(taK;yJW100lBJ0GCFi@CUOl7ni1XaU-#bhh z0fz2yto{VN37+kF`-g8b7X@GR;7hT1(*q;Vu*dw9li*A`gQOb)^%(FG&S0!3S^8)j z(z(`!66sahu+ba8+XJVxte;%}lygKEFD7w)Ll$TvL3dzmLy!fK=|a#;7|yIg1PA7H zu@HTjVTVCblB*89{@!0kf6;fvByP~+)5qG>@_grK8l8qm&*l~jIML%6-mDJ@Nv2Ei zyiO9b0oj8R3u=0P%I`Tnw%;U<{GBLL8h*D&&otP|_c-_d(qgvig_|rskp;LU9ZWGa zF>Iif4n3;*v&yT}%n+)1NpRRJ53>!4vbV3oP{AQ@7W1<;u4k)RZa%ku{S?L-179P6 zKgA&So?^rn9QM&m&VOx)Gs0gH3)p>`j|K!bBYbyy`c;HK^o)VlHKUCF1mxU0T{?m~ znOHr{1d)KG-WB>-Ydc_GcaMm0?SJNg~?af;^x2UNTZv;fa>Xz#7F&^OoxiV$0JS6H5n*W6mVtZc_JFJrN2E@^whrkBdv2@5IYNCZ@nl;Qso%^|bly_hmum=*kyFuhs zQ@Hl)1@jQaX39)&UMHmklHUyeIc^~Q?ibCFS`hQH_)7Qd3cu*zMh$pshJ{xf_L?lJ@rJ_|E`?pv!gR2b{H?8~rkE zgF}dL)d-QtK2k6^nAD=e>FPUsaGS1Srg00{{c28e#0_x_XcRm(mu;+@ua@#J)~1xC zYH4d(mpu!tnVp)$7bbYy&WBeei>e0SNWLO397I>Io*Sm?fo4Vy8RuOx!`>6cG8(_< zK%wQ9uG|tx4?M;bu{!l8^0vVz-d2&bq=}iE75oIh47IhZus~)k2sk{!!JpNF?1Mx> zu-EfF0s*BI8N{5Sn3t36g@>PeFRi@|(auBOJ1#e7s9WN?E5_nk0uPrPYJ}FymiAZ~Hr{SNqnC4}c+55soo2hZ+)?=?fNs|ran|PC1 zLK?>YjeDSM_OtLEsmYzX`ns$Cthp>kUw4^GgFqIcHaSy539in>xDmm~yHJ-Vf*)*x z(Pt9|t*r{KnsWVVs9@J_;wRB0C0iM9YmF%xtK;+Mxi>)B&5I?Fzjq&es0u{=?Y-9_ zEG>ElaVNt-%UlyBY5#nxKSr|ti_*i}sdUpPc`2IkfeiUgo@?1S1+84&i@O2yi22P*XSm zs=*I2XgOayG|`Fub6O@r-T~kuJ&vj<2Co0e)w45r*HPdvDVwFuzBn& zP~-=(8kkBS6#6y_3dh?zvdn+>%GC+>WkWeAKgo)W4+*xs|W;^ECtMP$jFO&0EKNtIk^zsD8( z%DB7k-OzY<1q2g(Xbwv4;uox7F@)Zd=?&UBx&bMQ;(!OT0T7XpWN<&}V4P}ihrzp= z7H2~KP4Ugjq~phrcad!=3$%{Cd*-_I=ippZ*{i@RpetPq)C;s!d$5M@I$9zu3$KYE ztINMIaJ*9?hz_bH30d@he23{?2TKpYW#`n#44^E_4vnC94eRCTt&d*mEdLkw>$cOm z&!1<`AC5mRagnwUI7SRNBb2E^HA*mZsPNy-M2PBNO8*yT*M>f5l}FQ}tauW~e zY_8s$HD}ZAH>mbI!QFskP+F*o$JKiu(0IwWx{Zb*`VzsvLJSa{2;ctnm&ts0;pg z!VIKh87TUJTD0H!QI`C#TaI1OvjNGc8!O8YHvo$3GR$_|kXcTX7p`m;BbSKhhB;G@f8rf^(LA_EZ+oLu?DLCp@ zs#|mVwDyu&os{wMPWWR)1A-~X2mU_j-QH8?MOk2Aj@u4*Dz(d&%6qaLdLJ1VT8cjZ z0FI%N^((;-ffIDWuPK{l(@~>t|J~k3MfVwRF@#l^!SX zm8!2E2Y2jy_a^py=`{=s3>Tdb{I@iu2+ zDZPXy* zG=dgG|84La*EyVH&QC3UbiHpn5AA@Ld<3P{i-z#{NNyqu@ zMHk054!m!EpI`R=eeuZ+)tW&)-9=y<#SOdQMsrpDbtQ7Zmgg}5r*{I}^uGRHFpyU} zd4sH@c)UT*`-=hJ{p1Koj*7Z2j%UuoisCnXEP>Eo@L#fpgTQ>Rf@w$yS+@kd?|erV zWpqIh_Zl7TN@*;eEEcDRWOLjL=({N0oZ7vpRs#-C<#(5Jr9~2|{(xja*n;($upQR{ zTx_@FFwj79S~n19n+N+FL03m~R?`nmuGPI{4dU3c$m4IKYXh-LlLm+M!rrXp^~>M% zIP;X8bBO;;%}WW$Q4Oy9{qvUG@?H5koesX zAN|01WP*@V`WAt%1z2q_&x2+ocmIYITD7-u*Syk;GJGL#&~a3Ij+Z<$*teL=<8xfi zTk$ubhHA_9&K=Z~uhSh+p8z!up(KQ&IX5QyUF$fO8)EuW$}`C~4mwxA#Q=e>U~AI= zX&*=Hrf8ntaJDX$I&%%yIIec7b3+(UglB>9CmsW@bntHq{Z)L3nw4D=rpEs%<8Z&( zYdoK7h3C<;M~5t{f8Jy96R9Z~5dd@e8Td|pg4l}Wu5~44@N(g$5n@|pQ)`|~aHhi% zPcjsqL~MuW)rdHKUNX8TSA{Q!7Mz@m4N_Scp&=wdx^VMO4@G}jFAU>VWw z1&~79-MZY^l=SpLT|-7jv#8{e!S-{f5~{``SZ4}Lr8uraya1#51WfFYo2 z9lAkXpQ09EI25@^3@KZHi>=5u;Hzj&Ucu&N=&iOZUjhqtPwlS*`AFaE^cP7G7-i;! zVN(7(tNad8Y;bgC_}VOR+z?8fUP4^Tj;xkJx8lR9%9hf8S6rEWIy&|u;|wt^9p>Y{ zwsa))sV95ux`_t+khwDJ8=+RcTSwwMT8M-rU!V|rWPdz=;qE`DOvNYVe?Ppez}KS> zeO!?NT6N((s)-jdx_ea!A5>Pc8j2euyZ6!@j&5~w^L-zz*`ugegKHLk#a60oscG2h zVtK<;Qn7(~5Ef^)Qy=)mpDe@{#FI;R&Ac=mk)E(f@vtB`a zM**S=P+Z&sR)q!Dh(Y)w>6+y^0F;#2hZOluV5bkB98G>0_I9~oAi_Z4gv(;~9Kg}N zcC=OViR}9GO`e*Q9gLX&fSh`ySy9%y*Nh3%w^Mm|T~VseakbdV%-l~#-md{m{AwZZ z1oY4m8121$b1u3UEt0PFtfPpC;tA0GwQKOWNkFsIF3-+XzD`f_dYtn-+imx%dtX0C z-Gj!sA%P9LF<9XBk$J!*v4ea^QSTEtO!(MDJzc_cz-WW-~o|izLjIlB|W=5u%>ff z437l^BVC~CK0iqS6dY!4G2CPWh9|le{8$)!;9DmBro&fqS)6Lmh_||ZRzN?cPUdD91J)2;!H$-TbV8TO+xSN>Ah($APjfDVHxBBq zBlY|qwubx|(#?H5xp|MYK{xPX@ud)lB?GaFUqFr$q=5a{bNfi zf4?AIh;jw66e9rqZef_N;e>}WOX?6^PY@(TB1;)PbCK#>RP~KsXFIC+WyR>8`-#ji z$T=(ZhkR!)-r~x(`ZiywX9ItN9l(LgA`_B?_af_|#5Y)dCg&(ifqq3RobSoaKH0*| zZEz;>D5DHg8*ADNVoHOHL@u8!jGUvsT|%z>OouU1U{3?U(|kIz5n_siEiDN2nZ8Z- zOhXxwaDM;IWm#OJ&vP_3?J z4s=L^#_JOZSq~7s1d284TWmiLw4>?LD_%55=bT=82Gq(rYLM{EzLO zYK|M)J2UCef=%)SHcQ>A(P=veQ=JuZ1;M^A5+``P^y8a(X@hW zEbm3hV24XMfGMWViBJ*g8x6Cny^o-b?Kq=DF<}R~66|6bAIfZ1wa?#`kz>oxJ_8~j zPa(QND{KERZgLi7r?&*&!40B%L7=1hFHGEJ)ucb|w~(~|?%ixt+dSx?X)NEAPoC|5 z4i5?IHK-0|%z6prZSXpD6Hqw{W`sgrr?F;qZrba;45nT=dMHk(TbF*(opR$5{$){g zV6MA|hYj|g$FR8A>AoM&(K~Na2SqODyMIFD12%uY9HtbKl^OIe>=-H;@y8E2Nd_pk zYNn_k?I@VU7Mo=85-9yNbI?@H)F($@g(%p2;Vpe-rKeW|cZ`ghr1ETp9;Todf)Zsv zd`z~K4$ocxhO+{;CBZEC4`MF|Xug&p*r63Px1a97K!DFF+-6X|BA}SEV6WHLaPG0-&$OVK<)zz{lWifTnw6X};zzRf0zh8=8O{``V+jD;NO;Lq2meAD7z6qi!;z^x z3bY4ph)b!#ZFbSG=FpXA$JNHv8XhW|N-K}9{|r;hny@vrPJKJ&%PYr#nDRy7^aP!i zggYz%9&I{;Xqa3Jn%6?~Poq+qCj!b>rd5JyvhEXE29Hk~_y#djy^4)3KR)ef)uUVi97daGF0dUr z?Y|KpRVmND=ZM84>wfQmzNQjjLU{1oK@|VyI}<@s%qKFkT=~5Adz>hH6RKY+ZB?ooA#!=h|{OJD6l%*7e_Y54+{K?*#pqVR;uvC^s+yXDC=G&#_ zt?VAkm4|itWXW@}<)5>Pdj-OA@Ql0K;1oxlDI?bYQ^e{SZlInUT+TEDu~*NafVG|6 zp*F~-Jh0H@_viH|(Vohw0Xup}cxAuWSF4qZUKWwqsl)H>|7yVrqVvCBiYLHn$iDY5 zGG|7QrcQst(4@!W4W~hgCz}b6E}aT&Dj*-ZGV<|(O{62JrwAIumVoNskIIX{8@S-+ zTO?)k@mb|-Zk%i9&r$nhgQz+^36N$IzeO&`qlDvE*j2+TJvW~&I{hYtT8cPWd>xWv ziJIA>iTyL z_OL&Nto@^Z(S%Uf<-foYPJa3;Y7y-^9Z?QuaHD%2WCA_3Mt@i8NI<__-KNZ6#29=k z5bLV&IL*4D{R7iWkWc&Z{_rQN&E9d4m<%kffsmQ246MitgQV_s-{S5=#JfU0I6m7q z_=BwrCmi}7X+M*E7_8_WEjeK{GHjM9c3q`{%f@0uHY67TwhTxu=_&;BJ+PF66CHBQ zh`p?B1q9x;8oE=znk99Ka$~yeZAmEQv(95}Qt_t$Qy)h|eV_aeAI;XgxOZb+P>Cn}M;vpZM3`Mok-w09v)!4Wy+p<@T z6YcmPV@(brnGPGLt&uvPI)$1-Ve(F((E{8d1TOuA0j||m2~nK2RB6T3cXVY6+yN9V z?LK2kJEy53MGIPT=j=ZC-8?D~92YD8^-K}XBww@n+(cxL;wXz3xB#wRsLL6nz`$a( z0I7v72Ee-H(TL7sy;01CSF25PS|gE$?^(mG$v=(|6S_BD`0wdoV}Eza>Ij}Y?Q#~N z!<)V*()8xu(c@Ef@%x0iAa>2Z4!yrty1_pS;$o>_f&52Y4@VB(9O#6xJJJOPDP|SMQJJ4B z?fFWfOX3HP&zue2Pb@7yLdh;#@DfTka8{F*wf+?HrYx7|n5H)rtob-J9Dj*fZVdY! z^JF1uFH76X^lh5i;v+d-T7i#-xI`J}vp%GNf zP01S0!Prr^u6}{457smMQsEA{g+F1f++j+`-W};tf2FSA&;Dz_WLV7S^D1gFgk?K8K1oY=|QuRq&klk4lVrJC<0-+!K@9b@*`@71b> zJfRPlvFp?Y@#uio%ZUx|N1LXxZqlXGTY8G2p|>==M)P0`#&63VnhQ3&`3A^}vp$5p`LbJ2&6e))vFg}*5v`)00@<9=GU^)*Ee4ZUqGmqsv=EGNG9FLhl_#Eak&YI&2%6f{^X+jq}1s z$s&_{C0R1?KkKoA%*!L7l&XulR_bCKXg+>KZ2BK! zv6S`lxAjYvP-kTq&QB->S&sp{&t#?zWsp>GuPk7<;nSo5Dzdz^_Mzk{F+IK0y-S0? ze_clSRr9b$Oe&}+hI@c=&-9i1V!fjH*J=~CDxV{aV?_Q@Q`ywbjN{^j#R09hAQOj_ z`hkWg(T)j2BeT&CH*YGP&h(Qxf|k#{y2R8eNhsOA%mm&z(H}>D-Bd= zZJy%ywxTpO`u~X!djzA+5VD{W-Ze90^$Z{C`jdl25+gH%F+oO;(=_?-n0u_r-*hHW z#!Kop_QYEqnB3qTPOV_Z&cl;S_5;vn@`42&Dm~HA@TPF^T3pal=Z?l5)nm=ZYsb6r zrzj9{sYgcvbL=bT8VR1r63yPaP%HMSpoK$I8Wf~G&nzf>hDCaCK=eAw2I)rO1-eu( zHBie}c$VjRCCYCe_`yd_}B%wsLQW+!v{Zjw>?mb>I>!FNC(e5IfppV*m7OX^_>Aw%6 z`y+1?wl7B@m_iQJ7c5z+`RF|fckmkaQD#sdmWL@1W}Pvy^hDyGUsq$Hq3h+5wu?;<(u*RzVBrGsFhYkHP@t&LUuEgGI@Dyebd6bwjo<n5wz~fTx6SQgX(&Vx%V2^0Q4&6H8Y5^T~On4c1NSppU+yj zfPGImYB=mFXqwJ%Oc!0EBZDAH6DcI8%zxzS=S5c!H_bEB#)7tn8)Jz~WIim;$qF;G zpd*Z(An+P&2*Tx=&Z9ave#eE#StoaQi=NSLy{#4UfJ5|{V9+n#rSxg3_r(@anruN- zNIV-D-Y%ryrb3{4y*;7cHY?^-A5UTQSRR3H4Pb~g}@A(R$@VGcZlc(XwtWQ*S zBYe-iq_vks!$+1)&JWJ}`7px*$cJEC! z^Wv?EpMQ9-q#ynIRwCuFODPh15Dqi1XInc-`u>>~W)kyY=yMMu-@MJx;hR%u!1F(i z#YZP5k$gY3YeP~1b9(yiC(S#vtnnMz4<(|^%L^r3fhOEpZ?lHVh~(2#15uv~9_ntc zBYHcNW!7Qpj~~IdH-v*)kV)I7IFZ%pgL0;Fe#I;2lNzm`792lsJ0HOg87x5*pukkV zCzJ=xzXXPpHEBy-Vx1|^K&*MR0=^OZtMIi^p>7#0H#&}KDV03EwYK@^CpGv}^HwQi zrg=zrBHj=|tObY`hh4ZDh%W3W|Aom_LF@p>#f?=%rz7^$P4N!VamP&ic}JIWl&x*Q zFLOKGYNjD?!?&OfBAh{*XsaD%i6}Erzb?>?050wov0PO9;g3Jp?Q8Enc^uM(#E4kH zEiQV4MGqWFe{E7H&t~8D$Sue?M5!TQ?)X%mxRU-I{v;0Sz0pU1Ho6gP7!>35tUH}S zOP+{~<*{GP{nepxoB3?t3kh|8o(qrTEe5^9%+g`I3J_ocegRm3gD4Aj> zflV9?CMPRbhXv?)%ROObgtKzK3KGLG{u~1rXBb*(F>*IN-q<~QA*CC^zcRY)p+rx7 z3Cz)Oid_DF|8l4x&#uU21n~tHL~>s*eg&IR!|6W#6i5*h&6TkebLOMW0OCxXHILQ; zlqbE;rYtAIp9w$Z@d6yrpD9wTG%P?>V~pruIB=A&IFR}*zxo?hiffiqa2aiS3+_jP zuwoPB-+ZT(EyHdw3PV@)$d5?B0jJ))dt*}UO%!F=-bJ-OH#fo1a{rkVLEJPcFkvGC z-%EN6(Oh9P6HlU(iST2YqNui~MPTb<}R);2{UnR$wNO=7D z5b}^iJNy`DsI1P~AD#3!Wi6B?atz7+?DR3mfFlh!WAO!3=QpK-LZ_x*PqPMS2tlS^ zbY%dd9+{^9 z*Pk=LzC2pQ1<=*$c7T2lQGYH`PJ6In^!H*WoX~BlYR+JP>kwL;5PxDWeUPf1=Ce@3 zGrNqOzSxC&HT3{LTa=W0xnJ;3Vq)UmpdRc&)(~8fC5hJFf%>q@blifnL_O4UC$J{D z_~MQ#@?M_9;;~!o7K-^5%#duM>_&*AzC@s5t@c~oOV*xtLWXj}H=pU7n@J~Zv|qGG z=WqIT=1lc;n;)Xw!J1MD_>3y`3u6z9HEtOAd)+;G?$jAKwK|bj`LN8bQIJ<8U;hjG zu@R~UfLj{~fGfl-6nCu9QBN2ubZ%#-IMoFxJ>-rzx8d}6ho%#H4T74@^safKH0-ey zz<4s;`^U_fW~tHUa|bugeq-vfNLH&GYxgN{$$~@@uCJG_13e!^>F>I<)#36k)GlU6 z9bP!W@py^lQfxZugVg$tQKC9!JOz!7Ms8qYsjs#JgT?(P7$FeX_b@mnQSiKfLP3%QokExAobU5klCb0R$0@ z7YVsPz*V1Gp1i?XKlxcncle;Qa64UNig_aFU3EU>XD>y2(Fib3Ma0Zvc&tMBZqO^p zcZ?9Rq2VPKvMqDcVn@9_3Rzczhfkj(4OJ_)R%vGd+C}o)XIgJdzxIkct}3n_YlTZYWOq#81usA;pf`^2$x&{0j(tMz+jo?mq8DEElF>fe447F*Z*dLHky zP+zeMhu3yIV^~fz52t~%$Ee3h*1x0ZWAoU$`@yIWvrkf<8nYiai&#DbahIW^X>~w^ zyb9ajv#XxWwADn;+U&dfL|wSgJs~+t=Wg3(Xqd;-Pp|oOVS=TTg-{3!IQVT7HKC;D->#m(H%PXpNm^u1p>V|iccUb-I9tgihs?gA2x4E44yHcwHiwClZE3O)jI17O(|>V>vJnjIUPs5LXO3XdE6upCzD49^$${Q1N$d`l6LRCr1RY2XA~cAT3!8OK!)RGwVF~R_~`3z zHk3tYhDAO^A_l)?KczZMM8xUznq2NnsdDbW84%6)ADM52Du?VAJfVcgT#c#FN<}2s z@7{QcoS%Cdd0GH3HGJ5d`V@7eC%g#=&p;iQh3Nm2QFN`yl6W6FzqQ7S#* zd9ZhDebMX#tJ5Wb@C~2RRl->!vtn|3vWBl>{-~wWl|jG9C1sc`>bZW}V~5PS{SFdA zCSt3oBPHK-d~xqj@(_Oduq}E`-tH5yXE;o;M!#V6gs4uJDa>gb+I`E>vyN(8M;W%? zp~N4%Hy=lV1L7h|qnP6{Ket8aE$@q~L*_y*tP#oG792jy$)d!THQcND@|9RjvwbNu z-YaB{L26fwHyd={t2XCCqpr45)~iT%57yskFE%z?jrw}ls!CkD``#?^!`nLIBp%6z zxiacAj^V8?TMIpIcV}}-QRP;M6D)HcnqT1NRyd3&^i-!zZVldyo41#ZIT*fWYgj0z zpLx#qx>KHmW6;vZR1lIvD`A{RPxchcHQBiv?vM}m%_;5kmFe4e`%AE&k|2x}Z~h2O z_wLI{H~7}3LwsV>Tjy5)s@&N=f_kVcOjhc!(DktNA-em*szbUWh~=Zb$_D}<$*0)^ zD9vQ=8C7+)Fs3mxsGq40A`sE3DU3Vaav?bC=kAt$O-I}4Qrm;p_R7vqw&M6hsq>*` zJ^#=nrTWM=G;c~uN(X=cwUC`(k&@3U-M>o&scrpGzVG>aF@!Dkc;}K5JC3SO=6(LF zL@I)C$6RqFyDG*QMGkn@e90a%wit5GO|TeHRh1XC#YdM+w;=a8sz4=R6KrivJ0)Yj zq{nJ95&{iQI8z{6viYUv+;Fvdxykg^nYu9L(2tcvp>7+VOZ+@A z>r#!Ulp4j<=`pAW`LTw8n=$ze3cp1qEF+KG#GL)V=42??lKG0du%K(@R=zm@^Z%r# ziv5)C`XEoMR;H@sXRQgOf;!4zXGvXDNwwVzp6P%WEy*eua^=FVWMm^1>|rSAf4edS zE^Wg!gw{6xg`IjspdV@m1(3)^TzyVjTU;gPSixv7uFeXVOzxQC6L8li-tX;v3=?6K zYreYFVLAx_*|N;bDLuC^hRB+R(syNAw^?KJ;Ay@;l^FREEKW}86Ft+i3ZbP z_&MsG;jn0vQ~MOO=R=}AUmv^34&|9lf{?0DPc=#LO_+y0#g1f9-U~^udWcftU2 z)5!@nW)+IY4@|{w_Xe^_b1M}$Ofo$|G?|mxA+4xDJ^GkO;bnauZ=>|jZy&2RvSx+E z*-qhca$06`R6k34*qZ=8g<_o8UZIPTm*-=7o+z(Z2dr<7PiDU-Pms8w<1Tlp#nMin zkZcG)WI~5N-DFOWI`k0Tw0mPRY-O!*?Y8Ve-{s@dCmWg=%1jvwt&lR+q2mx7-cq*L zPI7&C@XwW*`>?k#wfpMP?XXz$cB%uhI_XPODkiqnrFg*bnY7pT)kW66S~G5jC8!xb zBsDYP^Hnh$fYa4A`mTeUiSXx^r!nG#fe}+VdBR; zzU$>l@`N4>w;0|sARBC{uT@hky8)}TiVi;)xeW9xODoemz;L?bWk%3-?pe-jj=;QuohQEANtk{H~vhP<#wG3|Nb`1{} z-w_+1G@_IfQ&P-D=*MQd=HxRD2@`IH_bfF(xZPsCb9_DuCOAIwpFKAFN78Sv#l|cM z<{fgnvTP}>@&vMiVVbDJ^ETZG9y>5ELH8jRwos9&Z*#qxTRQTr>zV`#?pK&vN98`j zMP8-7ogp=?%@B(=*aL3G71tA{%=y1;WKM^hn=x|K!#~VmSEV?M#^m|E$AhZBfguFh z;a&x|e%3+y=qpRw)DNDBwbIoaZa=cheCIj#^YCSBLQlV73_wgo=+Tj+^AY>gy>QK} z53Q|Z7aAnqKQNAexiizf#Gbhyu$B2i+^mt7>MpVh2!??o_GdjiP;7*I*SsHjkx#<+ z-rnpxcJ8X~%B=~%wO>SsY17Mzz2%(*asBo~*Qoq8D<|Vl9Nb#(lv{NFs_+`!r@O7N zjq`(Y_F=z18tx*iI#S?$MfR#^?~&m?qd!Je@}GlC)_%J4_`UDyTphyCfs^&6WxP7g ztqUo+Rv*Tl>YL-}G@1w1yTgn=7}<21d6j=T&)rU+={G{~bO;TQa55{p9)Tv1lh~sWrq`4py}q>Ud6C~-DO@}YistyIm%Qfx3^4+Lfv*oJY*hu_=}Gy`YON45!ZV?iFK>n6Xh_qr~D~ z#+@SjYJL@(YW%Ly{PFSncf!u+2qdK)=Rdq_kbz^$gPa9$`k3w&v2?+@xRsF{=t*LB zg9IV1?VPOKOsXu;&RG>GM>>-8UHCau+1=-D+@znCZ_x`RMAr;T1Rb|s^eZv|g^K7e zjEV18J0kv=pOQ7x9{U>>Q%?4wDgi}KW9*HkDX8*a*i)Ba#8GWQ8vL}t@GFT|3{$2t zoR1~)%UrHlp)f~iI#SViYw^zm_>Q&M^7FR-etxVecG3iS`vtVOo;Cl0HHzh)4Y+;9 zGcKHU$s4L2-`0Y~Y~T?vFb#r8T^PeEU_9-)x`HJX2kL(pMm&2^IH1sVA%_+O%B)Ez z7?(lm;3H@%Bt!t}aatf1+)6a3?Ed8XH*(AtFN;xhNw`=SS zTI?l6XT?K%4;reIAptW43715_hcA}MeY-xVHh`P`RKd*|zx?8rd(W%t3v?}NYd0io zX3~77crs4C?@yX$Q#(wZsH-Z$z-VDEsT)8V2`eL2}i zglTh)=08Y%0SjpdTk0zn#n6NSR5l|w3xjX*x!y|!-euYa99(c#;dGJzIyCG4g z2oVWG`;k0wT+g)1vnI4moNI@)^6ww~dE#c~?Xi7q>Q8soNywBks$Wlgbk9+tIpi-X zVXNEBU^&jcGfxHKw4-(DlQi^Kzbp$QG4}ywg%8cSh`jV?4F3QRuJ;f+b3o-ZwoI;NvohPdU|`{wBQTik0gFX%${bVqHb49Rd@TSq%{`tzp>N2}(}Pd7^Q zp7ML$dt5MVu(D?|IgG4*+?cYk(d*AVX!+&ySAXUUMmsBZZDvM#OP6e+SVLV-qN|g- z9hieV(C(qikC5NS6l%UH3oGPbe3^Q|?PW!h27@esdy zhM6*Y#7)0iPVKeinJMSYV~b?9YbkZ?7VzWCipLQOkbjoS(n8V)%9pP{tiPI_IPoUc zwZbdO_-&8Ys&XH&0 zS8r$kp&3j=-Ne1OxaIF;h++m&KjzHuFpH&n9d$$5V7iSbht)laU})WHO2aK)9Yd`4A@^XD_y1}O zZk|*H(t{1dtU(J&TyV}GBeER}rv+_?Pd@bTIJ%dJs39X7%#>AT;~pXElN*}VZ0kdY zCi|~$liH1FAPTvM9mB0|fy(+lgxuDLsv^=%a zv{n>mu+i-RpB?rL_Aa{I7TuuAB^umyC{yr>iC}&ZF%u=VCWXmUz9K7Fb+?Cm@5VM8 z0Xs<8w7}?sp04K>p!`q-CHWhb(i;fBxNxWQu5)%SUg|ZBe*z(EgI5e4Xn>8i|Bg+9g`Vc4j0Cn?cuOCUUwr%zto)T-L% zyP*GuPw!pUB^dXSPd2N_S61e~<{8tBeuupKrjIlzUn&(fKGaklATGU3KsG=a>|7M?{ra4_5G4 zfbD_&^3?MUnde_QrCof`z1_tw{cAd27pgr)F+HEDJC>#8-TCGqdShu^UaLt$qxJ ziK%c7sDk`BSW2Rn{!|Wsk<3!-OQB_!?Cc1^tdq;~0aWYwEPDvBUBnrV6!-@|aKk{uf7YPq*z5ZMrXzS_|kw)xS)# zU(BBgQKAYZ85}xgeOta{JAmCQ)aM8fyDH~D$6ly8Ow z#F2uv@zG`XV%~cz+=e}q&vp^t`-=3WJ2_HVHN;j0yY9lHgD(dIlkxKi^}VLLgU8|jOtdnZR4ZG&EL@neGhV+KBugm0kHiRQvxQOGcr|PT7*BqFk~! z47o&ZMhaO+sqBPg8_wAG{nE9Ll!O>$%`zA>s8D3dIy3gN%!Dy9+|Rk+|KR(>`Sm;= z=e*yq*ZcK;y`Iaq(7Mmdc^pd%4(vP`Mb5s-&d*s!fa*gzW9bn}e>2%?Jq`fEUP~dP zvamOeF=+0-;3B|%fbJ>U_3iL40*@GXw9b1P>BXT9(`2M)o}I3k;A1~QDQ~39ez>jl zTU1)NTJ`rIkm0bHe&O8&A_$8yCw86a{Mr#%qlqnV9=wVZq5H7vYQ+G-3r}CkoR07! zvZV05($tj%I5%1@B0PVDRz>5ml4!?zQ6eR%rLN)BgHy|inU z6%te4x*ev*lti=+LC}?SJjHf{LO~?#&tGJE(eI3>n_Lmvzfu4 z&8!!!!_Uz`y`uKfMqm>!IE)@e=tXxgcojV>9l;&qW zWMUWH4K&BR?o2zFo;LbzVu@Tja|~idqY%FWP(t{hD`UU)AVGNH$aAcl^oZtM9}vYv zCH5y61F2T>GxBYjbB|=KA|FI+wjAv{s&oU%kxI0E4BV@hWjMZv;2)$6Mby}SG#E3j zveq|NY`b1xz&MonDkmxcPm z%{6vabDB!#26<$lEx0!({#gbRqTuAh)(D_$k^D|Dd!;#l<_*3yzi`_v*tZ%^y#e_( z=|5sLDGtAfZ`|Iz^ooUKy2CF$AnS!&1b=&&Ir}5;lSGJRKO|nqb$vZ&x*c`WlgdTFxMIVEIq|YY6iPWQ_;yv!-lyz;X&b5aps^3a!+*9)Uu^|nZ6h6LD1~PTe^&aT; zC&CTi5sQcYfa4HY1GU=ulud_tosjd z89dlJ0dc=SKkaE_`9o8bM_H6#IrC$}`LFRtu!pBT?P*91Rv-w70ZJ}{6z#uD(eshu_Zj7 z_Bhg}0}1hcQjlr$^VV&3usPA!YnyCrSn`@R*e&4=0-E1pMHUaRIgBpXx_X3fW zplh>Nh@=nQDnRb)P~KnBJRIqN1hzNb#WSIc(!wFGeJC-6dzz`UC(S(=giw4LJkEHK z=vdjUJs@-D)emCEZDcH1&$ATi2!<_f8Wa;udFSZCXvtpB%k~tQQM&cy#VKb^F(~BG zst}tbwtL}ZNG)9hm5y&zVaffv#H-HSGu_-Ylx(gBR@;`J@6{*)$Y)PLJH1I1pzF0mI*Kvo|x-NBjbm~pT>%nU0 zs~ZyH7x=E;T7A1z8yvfA=$Fgcq?9|ffj;$Z6=__5aG-MP@mG+ZR$GqEf;L%tYRa}Po{0o#3D_jXMTPoRG z1R6n`2DY^ip8=sDWXy}RM>XT5Z2@q10AsJ4=~X9e?&o4ikiqp|F?z^IUY}`6{Z1gJZNEpCJwF;t%OtZ&?sV`%Yh;39bL$!ub3=<&sZ%!pMH&wC=MZFO~*V zk~T?^93NlB?CM;R>rAsEcLGZdTc^yVBhqzCatjYlu@Da|FAj}b#PitwAlyoAF7^pK zX;VC4EFtmBbfiw5ZKuBv5r=(05l0v7Es>-p^buRX>^UTf_6gp^^~gF@{e7AE?IC>W z*TUSQn8ptfZQsQoIjk=}G2BEOqq=-+I(P2ei)4}tztPindqor1vj8s{3TJ57UhhZo zz)yk!B}n;Gkfl{WYJ%^&?PlI`rLO4DUd<&pUn-2MBRtAPSTa4pT>)eLIquVS)}usv z60In_TwPu$@F?n){$2L4?Zc{G_JQR+GDx{?F-l>-+LDd{PGT zhMY?{c)Ip}ZOa{oIrBkPNFkv>i*&_}8l31>>{~IMe`QCnJkmbM!A=k9y`3Pij-SWq+5BHun|q6z7jEYgz5Fsb zEmy&s^W}wj2+e=8;1@m$@!kSUyV1Mf7x+&TUf@>4Z>_87ChJvi_xl2It+TTw{$B!r zg4#})PZZd+10(%!;^HBed86a%X2R$u^5FKzFs;B@^x@5j_;#PW3@fHa1nuzn3=e{@ zyy}fJpT!@1qqj5mfOw-%;g!GrNnBidvdNHNSPVexWL={2Y{!8R0&jd{0B_lUB3Ql|Ocy_}INsF@0bcwIVL$)hbY=@=hr|H{4z$+4 zke5IE@JxOf*cPmvp};l{OVJ_?6IVJ^sJAVsx1F^Kp8#3axJ*-_+HvWfFC~FrY{BMq zNFUH3Kawz|E-Zx+Z;WM;^BJqZXfmD`fT zf2{)9VZal=X^!rsQ0drX{TX|n!{)XQp4SOG3cj_$Zk&gCPf-g=?lrd@vrw53O?LWxioAF3dC_89EyzBjjc|tCUDP7NG75r;05)0b6biOJ zeIlj5Vydt_bMS(aifeXcfSC*|*I-0r>)J_sh5K^W0Tr%lOB_>KS=cwmx}ezJLKMJO z#?-mmRJFh?{b^s)_FUn^H>pnp+Dn@=od@r)31$_gYvXQ!sw}@U((Y|B7qCs$`E zY*~nnz~cR;ui?CAo40UIz3_VYj5$tWBHY{egF$J&gNIVIDM=Ohvczk{IS-|~Mowlj z_nw)hr+1yxTmh$~(Wt`ZpGC*QKXxtI9QXT<%&7`F6!9Qf+%(^tit2v#NT=MzII}a& z(%EZl93@EK_z!2n-4hYq3p9fIVBppbaYq|6Bi^j(S|nBVhpK;>H(9Rx=n$n;uk8qd zh{FpGWI@qrpx67Q-}eee^oKl}SjYLr{P&VQAmjaUMD~sPwbDFqpn=eEb{Chjz{QfJ zR}f+h==ZMTctjgITV%|Amt1lcqhaEAd%CySB=6M`bvFv+{fj6sWQ!mi5KCxm3pnY@ zbAU08SC8K8nS7q{V&_&pR4odJi0HFWa*S*Nmqk){-CZ)Y` ze1`%dY>KWNo6*QW-X(baHr;81mIvV*Zbrkc!2+5h5E4$(h35fUukMr>ai+fB^l3$0 zy3iCBB*X^pzNX+*yGkV6R@uD=Z=RBl^c|yYLEkXX}w+rJJF1vbHx|GMO zb*hC7g?9?EeP*4bMZP41fgl~T=Vlb->QWiF+GNw3e*BGO6V<3sIhX}i32J8*_*Oq4PWpXtEQ^oKv(3XSkQ?7_ni{0@ZTwcCq+5(D zW>%TW?9J^2tq-vzX)qEZ3NHvF6mshJM{UQph)lj8jCqjmKNc?+)gtD-B}Vu!KqU5w zX>6=Jw8i^h?)I&k92)0vRBm-+cm>P`em~`gr=P5KVBB&FjfAdOIN?jy0P!t-q8@2; z#BZf8t;_eRiRlO}YesmES=c)-MbFDVfF-N{gM0TnOUm4XbmwE;e@jXOLJ6Pkx3-K}Y9PK?y5|e?~bPO!i%n-R@ zN1)i}SuA+^yQd*uK=%v0G(+;iTN2};Q$gUxL z^xZode2Z95;GY!hW^`|&jKR7>RjTmtCP8)x)(y(XB(HT>D`gr=a_dz$&{Yy570@@B zszH<*69&PE&QAdpb(6H^-%59?5n=;&Mo35=v^5H%wJ}HACoi?= z!uh8`hbm6ILBr-HrT6C{O5z-k^7iz8eqa*GLpS;pB{%<*#|!dt!NiItuX|Q8G>DF_{+r$)D-{8x=2aeei9gh1>d_@+yF{E17?Cxn&X4 zSQ^l`leemKKE^-80eqq2jZwTeE@5JH{Xi6mMf0H>hQ}Pu{iL(qvbuIth08#z0BSxp z&79Ffk1RY&pyI!e^nNI=%+!ge9xG%#U7Pa420HJ6Vdh5I24VSUU!*A4pu@?q(;oJq zIeYdDa^rsic*20z4#jO~g29hx@`?8?I*BcW3mW9jc8)(9cSI$!F<45pGgecs9!cc47t|gaZUpvpxT}pqV`jGDTmDaYeSE+uIWM(cL(2X`%wQu`si*xQwSA%6v2&13w~w^<)f<+>}Wx&Eai-N z)G6=1g)>fS_9)fj2QM~y?#J_nP@-dIqpx4=7pDE}MTo-#4-E-)EI>s@QD}R$#jl!* zYj86nwwcA`h*2b~F~AJ+j6-AdnK^#@y#INs^yxZBX;$ii=6wYb?GMf9gVyAo zRI*FXXh#$ab@&i{7umq{>$V4Cl%F_itW+JYT86RST`K~{RQH6p?pZ>k*6?-B*fN4t zrz;(0*GN|m0Mx6i>_|xhI>Y{Agnb#-qD+Yn(Nj{8F+p=z^>9|AO(x4A_uMQo1XZa7 zcf)@n8Q7X|V=#niEQg~SWd-vHKgRRMzu(7s_-JTM>yRRCBb9K>c3Pn&v$`texzdp5 z#a$njJBLro->BczO0PbwZ`YnaZOHlTVPNKoV0B9X8r6v-jifA`>o`wU5k$+pAT&#z zZ7ncQNeViV`)C$w6sLE`03y|<5zNSCn&X(&PEB8rrI?0gfXKX&XfoVTBDTut^l0LG z376pW4c@=42Ni`k!8O&GOz>&+OBiJ$4Vq+?d?ahUDR+j)6U%UuQ6TlCf`Z}?Q6ZP) xi-L-I>(}nj6ZCrWrv2kB$F9RZpYi(oHpddNm$T5no>108 zvUQA*^g8B)KZ;w&d>I!)^O%18Yun%p;Qhl8Mae=3In<9T!4M~HsC>Du_mG*H+2ep# z1&-?ro%-QlzS-*Odu)~eg`)KFPwH1Cz2D`y))ZIS-e^r>c69X>(X^6Urip#BIqFvB-NF@X}Hab(vGwg&_x{1i0cLiTNM1%%tTUM}lAaSx^Q_tRA(jrTT6ZIOBTipAme<@@gBR)C*;`TQBA1S-5iD`QUvXWIl%DocT zVJ4#LWO^p~=L6bUDf)?@LS7@(keHk(;hj)nl<>ulj<=7SbHZ*EA3q#Bn{6%p{E1J7 z6NYc9;IuDObse+g*GwN*#?XnUnH^=J4L<~HIPzD+asnWRGk=9c{czefq5yXDg8Q(mIBszmB0l#mj5LSgs)uRY zy5`#ObYh+qo=7|=M3p-Hu_&V~L&&59+Ty|?A<8e-F}q&E{OgztE`4ZT<#&Fr*8f7l zhRTMD?^BBXm59x>uIp5n)2sT(q{ZJfX0qSg%kxV?UmKvv3fwztGVAE^#GKS43FQgbvd&Qiq()jYz=GcZd}{-9;Hsm# z>zKg1c7qAefhUTYo}l!JfZy%MFriDoc!D(8E(b7-%>kFIP!dZ z_G1e&!(m*+X+L}6ic>zKIJlk1on!`1jDHrEPK=9O89e}~jizVL;fcA`96EO)}c@p0x?$y-G) zzU`Mq%9Htb2}t279grKA<)?={?Ew~Dom9Dw`F_n=XkokT@q>ck!kQ9kG?q)jmtb^r zR>^ncbxhrICnWc|(%iwUtH@toi7+}kSh!0MH%502iM18>QQz~;<&z@;n52{sz?kff zb2eCAvHm7@#1iBCAw}H!8Wu2x_ zg86>Z7n{hU08f_>y}=PdGdkN$S(XT;+eAGPnjpmh?2h68Sa!WjTVsvlR@N~u+@@qP z$B)Qj+mcYPhm$lxvY$-1mHaL*jbFP}P zx+&wwOx=dEkUX&B3GB~@%W}|{oF~xymM0R_NeNWE7AQ-`whpy`O zuHFgRayFq!P(m|BUTWiYcNRu-h4gtHBLT-8Vqc2&olX-Ckf+AwJ*SrR#_}HZXu3%@ z#@R^?-Tz=IAVt|oj#@*hU>m8>&wI;bO=x#o9JDenh`FtC98dc)@2Y?K*5f7Dk&%9b z1_haJsclCDYHHNv4hm$1Z^4|Z`&!Vjiu@bj*~ir+Y664s?!XYzYaGYF3hpO$Laf;5)7 zk6{Ll(?lV9H|d@$Bv~6tPNLP%#gokU%0E?JtiPoY_hVag{uhrTWIK99ORyxCvijnvYIQowAj>9{FhJdlG$>0d--Mv4SmIztiN2as;~( z13f7?(m9c9J-ou5vkUZ1sje(FvA8Q~wE?mekk;W=!iJ=UDIM67;Uv4b3i*+TnvE}n zMB=hvd~@91XjVVWXYHODg;AtC3)vl!`(=a3o6TsoFOU5~H%G9WX7nfp3KQSec77&^ z+^|Tg@OY_MKD+(oN5^-uSIe({`S3{65)&m+@|{FK!R=Jj9oD-v^jlAM!C1jqx)uV$ zrr9p*RQ2fZ-GAOrGxNRS+B#+(v(Xp*j8`fr)1MNxdHIEK=1b$~I$FFNwetnzed#aV z5_0|P+EnHHDl#f}cBDIO^?nyM9$ksCC;`w(hA0I<(@Zz654^~E2n%vO{E3PK%@Aoa zJmyrtxnp5(m|=Zsn0e;pAL#nw52Cs{IY%^fURT;;Misex0VuPkcC#e9$Rgo0oG4zk zL0~|ur$vS8kBVa0*D{~VVI2-Dw%c9Vv&C0f@s;iAnMZE3@rZmMClQ`u*Py};?B(Nd z2^;4+qRFClctdTT8=P*`A272U{H>E0;9z^f@r8=Nl6aItXoLz>n_y$#+#%a zZ`zm>=L1WRsn1@=aH<9n#}WiLT0yWX@yp-Rf2lTi=2UI|ex~`@_DdV&viKYCWK$sp z^}UAN6nqC6k<#KM!Luv}RQL@`#u@ftTekv8@dx@TGl>l}n5E|krmQHg{0P<6Nu ze`Ew?Nb+kY-|u|-sLFYh_tQLe2s?ON4iWwm-qk^MSuuTLoJDjmg zHoCH0+~p3?g@D_N&N@c$46gyY%%#DGzVw`qF$ncUa{c}8L^`&y?(y`psDHi@>)8u>?-koCb+=8SNS!JfCZz37 zHKs-?{MuGIpL8W{dcf5W^=TA*y4?v!V@PrAz)e~)+Oq-kG7QHH%|d0MWBGM?qMjv3dn|pv+sQ)aNR=gT85=oA}qo!V19SR%EKxojB>1!W!)O$8zLZo6L(1au@z?oeN& ze(M;Y1uwT|hG!Zquq(FB{fa%~AB?1H6LwqRvk%uIm@{<~6tRbJ1(o>N|L zn_IljG6OS=#KpyRSQ2=EM|I-<$G28dso?R3)(c0urwhi~oLMFP1>F+_vc6AV7_*bv zEa+=8-9pJcIcEVO*}&w0Z=qBJCr;S|mZzLXZJUu`%K6V6fSd*{Rk12^;#c15>Q-7At#@3Coxa%a`16&k;tAXLTS(Nko&0}TFk}e6Z4*A~2$w#+j8lUGRV7ZcLRy|= zgvUFwXz@REmn)g0nX)Ikm#VVWcU>3f(-fU1TS4?DEX(Qu78Th{#|kvWp?XlO;tUq! za83T+b3C=KH+dv%2e$UeZdLYQp!eC9qFi{u9vE5r}XveUhIZ_71;Un{Z^P zmbNLpB&5Qr{nqy6!kxdB^Ng##Emz@g&j9soq1GeQ z5^Lg95FG??DUcNi!)%y?TyOsZmS00Vmb#&9W~4FSgmvR|cvzd-?qANddFs`n@+$)- z-*lQyEw5?Y?A_u1GZs=k>UWBN)jAOskVrrdwbulU7H7UUc0vugc0iCWL=c%M<2hg8 z9PYjeg8k4O3nXi;JtUbvM$*H(^*v?d3!dJ^Vd=|)Pw!Jgb_dU9N#9ZU7yCZxtk6o% z5m{%UZ@!2^8Y(vTHyCOO2Z&h5s1nr(H`6EJ?i+4Kit1U%G9F^LJ`;{ELRo5@uDD~U;$z`R@AV<>O_CD+%vmi@sgL- z4}Z*NN9^%^7#*cR7__Gd32)hRO#jRqd_KgwiR*%r)-g4!h_py&ix#Xq!(GRSEqJyx zJo9Tj4bkd^E?1YnET(*#4%~qzr{BA0@ROnvIZF&)u!I<7HqJG~>(MRYK-79F? z4$+2b!0bYK#YA0@Z>bVZkdq_kHq(7F3wpOJ78OPhoByoexZ&AQqM`Hi^ljzW3bnVr zeEA%MvdeznKt&?fAc9T8UkH=>8wMsQu#s0tG$E(}`8N<`(io(6rk8zEN507`*4(|o={Gd_(vryu z>r85|@((xb)>qvhOf?f+`rV<627JhZAH?#J=n7_*WtPovgNQg^{GAqY-a`vMSf3LD zN9vA$KeLh5ks4d{cBRlRmpUR4w{HnEG4AX`zv2Cu=nVADps5l6L1e+>@!vhMtx$q# z+@0W=f_o$RST>$>CE9q%pmT8f$C=?`{#%_ty*iq=dM}hH?_4V9TYKw{lSNg-Z_Eg_ zVWkjFHWdBoW}57G<6P^Px);ZG)xU{lIKmmeXs@%k%ig|Sf^+S1*#UZ#P?JB@EwLz zLfF-XHMJ^G9B6B})qJ&Zyn~-|T|FT$B5PVJKy+Vq)+NQR-`kI1Ah`k&;FA&{mMfH4 zAh{A8uLLSMtJC#qMs?3^{Bj!-&BdT+^COYt8v~LaGTa&WtUdmP@?)Zo@u9s$IoQjY zg=>=#gLPbU75CKM-uS8Wn?vXhql}%>M%wNhPGy;B@~5^ol>Kr-#X*)9-Ae!tA=;jx z3P%>BpYp{Ael%ZX>zx=9%xsTHuV{G_hd=J6X$CFg04;)+Uj&?{cdui~mnQ^A>^R`}=Hu zfxJyB3Zk6HeK%%}uVXfyTgP;jaWps}8I4{D@0saCB!6{bSts#{_)UR6Y`50#|b({3~8F&X+bG)($dJC#D?NEIf@d7rWBO z@BLls$V|$PnWHU3dV8MLXAygY4KDw*PT~)~&vf>7aZsfY{ab(&q>$;P!KvN@snAmv zbxI*CcZ)=H!=KAeBCmV1-_0Cg;|p=DASXm+7uOY{*pvPHq|gsgT}~|FKp3HAO9B3K zvlXL9qDr?p^rw|<{m+g!F+cRnbfPy=-_0drkT$TLc8H>YqnF;oA+lXFHc8FOTqXEd zmj#whZuj=>$X8`!!KXydKnooGFAHwJCOeeh3)p;kooss{TS554cl+g(qI9I~KUD%% zh~kRs5(kN!SUSWuNDA2{)|U`i+{D&OZN@^y^PV@QOtS4N_xRq`)-K#DY;5*un@*E< z0f?07&kI-~%ZRK9oy?qL8`HIxC=9YtIMiAoVeb7r+|%SY#FXJ4T3DNNYRsla)@R-l z@X$=oN|wml@HNBnDz2K0s0jj=+K#Km-MzAo@xw}lYqU>^H;+869ld!Q+eSOvn62p5 zGbxK+xN>v1ijSl{o#lwR4cwQ|Er_B|4s@c{OgpvGs8$>wSh<35uzv+F;mBZKuxG^; z7@{wHp{t|Pr!{*6q!s)W?eKez1uwS6PQ;S8K#b3LmJ4aky06O#F}&5Vj@d$Db|zwA z5X)}T=Wav%cyVNU79>LUm7vPl)ebg$my9nMQ|ll4-`v67+%gmtd}7K5vn$)+^{M;2 zE{d1SJ;HP?344e3Z8~-GE;uniB665i0_&TR0Cf5{Bo7Rf`jpwKOov@+b-Zpdc?cfjQ#nC*k66UXmphsL3C-=9cKA8SK5ErQkHhe%HcD}XBV$jT!Q_p%KF(&9WH9gl9ucuYw? zZ~po5Ab-X|(qtdJs}zy8;(TW3Aj*NP0?s}dqW4Hh5$s@#wPSClaiZUID$Gawy*%+Y z*Mcwn-H^4NsGZhQECGavf^z`755vdN%`RO2GsMpYB8Z}YIb7<+ zR&9tQ7*UfhjXE{GX?XMRnWHfevkmGA)|)0~_l16o!euSPHx&@M05rXZJqY~q8L;S1 zomCPl23RpV1RcdzIZ^bI3gpO_(_$5-gs{?e-fCDeIJh19$1bp}+|!p6?Rewbrj~U14 z<@|E!E%N^lMv-oCTNm6pEEUgD0DF%NxaG^VqAL<~%Hid%8{tXESW{0F4MmjoeFLsU z|B#2A17|M;_039qxa5Spe$bnLdlg$3+!cTuyTLgNCvhDvTVQis8{pgY+ivYQ=C2;r zh-|N}&Hmb2X({Y(Q!8!K3+-!M#~^ATJrTSyWH>&~6GWBgNAxG(Z_myrs9iGZ*OyL@ zNZF}ZXY1X*;adI*AMaE@KAy}9h7grVE_4>699+i)Uf^yej!Y@AMhaWDef~ir9H@YA zICA;@2dIaB*=9WZCZn0~kA$Fyjk99}sI~B;;*WXF&?VaVuP$JdOx{#kD>DG)AxeHf-RNys%vAe%1U2%#6VVtFt9Qq0??od`HJ2+whh8=vM@w@ z3WP$ORH?9Bowr%LsYIOa_U7SCW}n~Z37Y^V`CK`pkCi8P$9~Qlf0M(&$KYAvbBHXh z4U%2a4W0>#p*u;#_B3a!X;q1%thp~iK};v{DyMJ8IqE{!Gd6vQvffM@N`v<+qauN9 z8oJqu9vn9%!^<__T5LBBkrO&O%#g`j61C&=occ=>c3Yz4Xqs`enDYmnD=3=ILcc)8 zTRa-5vH}_~BAP;svSBjZLHr38&9V(1ckM#Oi0slOk;-thql2I#KdUvm0*Nem{u|2> zyX_w-m(`x!q}~|Ld}y@yONMQdMpyi`@nvJYj8|FXn54{N{D;`WO2lRsCGKs<{0#fUHQ=B@rwnwQH8Y- zMuvz-k$B)t>c_H=+*8f{<-~Iy%_Lgme>yyvLNb<2|GEt8k(v5{=9)mJ6g-y6InGLr z5r0z=e$M=O1)SF&3SCK{+Fc9rmwV(KTstr><&nduto-U_R$s2EUu^f&_WA5Hp>ld`}P2i2teXV^|rEnT--)ISHJy}^+Ip@Z6o z&-Pro>F4!LNOa4!`vJeTqi_$fFtl|Jkt#xk&x0EJDeT<>DWW>DYid6W8;1x@_c+PG z@$Fbif__91b$dy1H&fQh+^X;OedghNeCNGxTuISP$CWMob%F+D`U38Y8z++on1lW( z_skuxUbr-Ju6Kj0(tbqHvk2}7eH^~dT&)DkMx83Q(RAChE*mKfs zWM=^@JPnaca$`fp7cXIL8e=34WDhcSCG-2>qZ4l*H|^isI#Bzh+vRFWx4?`E=@mqe zhFI5EnMK^|fang^V!w=Y>R4T|zrC=kU+|SNW$+CLu0XEK@Z*%;Uj|8U+WX(DHa#=Z z9^Yg#v6Fwc05%y!H~%#hC3YAhXVnFIdP1#wV>VDHPLiO-mSP$nka2+_#^PJ=z;@SA zPPOIDSBD*&DU986aU*Ug%DD*w^y}E&+T41h*5j1!*wf++MgCp6dATqMwD1-ZEvqnPkBx9x5+Ch zg?z0~?Um^fY`$i;^=$Eri<;7Wo%@9B=fCIlitxEZI`2^Q+A(Ul~B9ZY-9csI9R!P3mx*Cfo8oImqV`zjEu-QNNK3=d#dqU4pAXo zgvo3XV2d4)BoY9Vrp;2k+KD@dpAChi9Qx*ZDcj$@s&m@*`2Ck}T3^0Q#XON<>Y-{7 zCmEb$ry49r;?s4ymMYI>HEx1D<8jOfMhgy}Vn;adM+LP~SjeQM#s8)w) zx(#7jN60B;=@x#7a*8MrUciXI=Q|h_0^M8NbLZuz;OR5z`?p262G z{`|`au-wsxC{e!Ge9+%sJGcdc{q&&=ak9SM&HXTkr7#k z)_MTu?K1}s(NL)ZjK{n@wStyVe6Q%igqNjxdi#Yt-rFOl4j&Os+|t}{qb|gLh;PQT z9I%jZc;O-{CdVt)4I~&tnvDesh?T`-mCzv1AKPmOju$8MaXZ9h6Fsi;rF5|#om$7l z_d=9XXeF*s2Cz;k3&gm(iyW|)2ChA^6I(B_H+8FzrX!mN z>Kwyn!eUl#%Q(JnZhpf)ayN9O{(fzS{Wk{2#YLdXA|@A21+XZ?vfQw+vDQ0meQ*L% z2^}FQo6nR z>w%f=?JOG=;deW}-Iu@AsuOsdtOT`Rrm-CK)<8Q0pfBt5I_A%{gILm#n-F&&3IHhx z!p}QI6<#sUm!#|k$Fy4Ps^T8Yd~r}XgI_jdmn72PqmsyOzQV6jv`ba2Nlg*Gx(94`mQcbNWu(c~loi#XObHCNZX7rAkLPf*}YqVdIx zmd#Ca=M{u)F6Y~FEk|gY z{;%mlA*!}=d8Ocq^MJ#8#^K#RAmV1^R&A&Si4Yry{~U&F87p3R1-o&wM=uTTk?x#xW<8K!O%d5tl5GP04$UwmoO|f% zp0soShQnIRqHHYd4+%C{M(>YJLx=&YvW}U3ORR<{@=zci-a%u=b9Xy}kYa>&q#@dT zKjXHiyJ274&+pl5ft=WD<{7pbPQk&N=~=IZ2W$+k+otmGU}&DO*&_7C;IG8pCw|eym|wy;jogiDWkg?+UA>Yjk<*{s?=Dc=MR@m z*wjrP_epq9Z;=2g*(Zqp;LJ{TP!6NQixJ<++%W1qbgur*Xh_=T`9S|t>mPEJTJqFK zHCu`Vnj((B-!W`<{F7`HvA!Q~f=+>I1pdnmx_OejZ;w9>q-PRp?_Y#Rwu|twZs**+ zYuu2p`77PW>U@Q^jdsM*?fVRpwj5O*yC}wjat?0hlZPz+_K%}U{-)2s@DLa|DO_=s zaD4stZ8NMo{X9@tcdP&QeCvTAm*Y~FyW8($_%(A;^)hZ4FBPIc$5u5kd%0WB5UJ$N zL`|Cu^ptG_3wVVBnLnX9YDz}^5e-t8g;fV+RNU{s3O$sp>hT~NQbpAPX8a-gP@q*; z<0Qlb38Hk^8Mf@Nh1GqI(;`lHbEP|8yve%PXlYR@`_xA(T-rJ&8)&|LH#cdJBno0Nfl|#jnZPDUKAWLA0R>APD;J+am=RRCDod$Bb zV`*a~RJJdfbn|;<<;}ogP|MA{F>U4-mTk1`d*fG0ZBbQ*9UE&rW@Kqr+#h7 z7B5faL)zzWohhxn9J;nAsnjVVBErBf$xg}h-Vv#KNdAw`W1N4oz!xPS0mm(W0q z!vE|OSNS;`cty}fCu#xbC=2Q!eQEqgA*%I0+g_Pc@@1@|svjv49+7_6Y`I(OtB<1M zS9w>_g9kGHGhT;$;8jBOB|Uii@=`qLbhl$`S}F<&4(!enMp6;vTefI{`Mk3D*vPC- z)8R_>B!0X$O?gA5i||H)jhZSpTrB|CF^J_*K<%2Di(kk1_L^D2dGyG2jEr~E0lm{~ zPJF4Cf86}*M4{|)=huEz>#efY0uu*!zmetp12!#0nl~-Su})y)a+jzVTRl!AlC{xO z?5F2stGmiCLlv5r;kJF{Rr1c)1rm-Ki=rlOtHE3wBB<5WC@>*Lp2S9U2)DspZ>Cqz zp@kcicX9gm4zuF=*I&AQ9O=yM?lGxFB+D}g0E<}xLp}@*m|Dm{jIBbjv(Q&OlON2e z*Z>WjBIwm*wuLPkD1CJsJUkMlWah@+@pRiUs7+C-HYimEDe<;7Q6Hrf8{X(Bc|%Br0=l_~)rpP21;vM40^zoK<7eHQJ&U_`eCm0# zwLaT@YJX_`n7s9@;<1qvMU)m3h{C|J?6EO8`8qH%dCM)XMH~L6D&!o1gY6NCC;pl9 zrb$ourbfI{tc7yb#~;|uLU(VM+;5Z%#qdrQlJ}F((qL2Y&@wKB_UF1SQRsoj(+bgq zBhvytEM`}`qf+ZY-^nJyLti>{D}GZH#6@I12p+ks z?)LRQyCDlnLdXE~N99c1?PH)xb<4rhFZrEfvE)Lv{c*nCtp;K5%v@6WxHe$uf(t4V z6f>gOI)tI%^l*NFA!(=j5nonJr>S@2ys2YJvFUww^yHB_uW^qy4eitB=`Zd}e|Y)) z)5pZ*cl_`((i9dp>}s>n1YL`0TjKbRp0?FN=yG`I58Bb66{;jz?8?0x6>X)uwyhthZ>} zINMz2fdnGP)8O&RhdxE7{KT0Gs4sDb=3T0U5rEZ*; z4LPf`c7Yv_NC9W=J+qFHTR5{H%q|GB8cq!VPMQxkbUZZI>iW$g@19C?#@_KBeQBpR zl{1_m9-z1Lv@cxuA;glqhxk!Vgr&-<306@mnTxx*v*O~Ldt0iDs*|$>GE*O}ISW<( zodEzL9!dsgGB6W~dtn9KpKuj+XQ zmS;|N7Fw11+hS+`C9{pthfAQ}qQ_MQZgNBzo@xO^6%&J-)rV>ocpyOM1_%;@4`RtQ z)&(pwsla=W;}(!;3kV-^bb628~Gg zkAcuJU`KOEkrkthm*art>0kbhbbkKtz2o0vSr@{PG;D1eFPfmUjwuT9L=?7=rqbZO zO{h&8)drjAy>adUI36i z1&Vq}s{z@g7w7pf?(#ZD6fnqKTS`ECl?GhdDYiMp-B6_6B_-(O$JYt(j`}5b?<;<; zzUSCMTvje`5C{htz}E@#`82Maxh1F*5Y$ETTd+4vyT*`B7DB*E+^tnq1O1 zzKn{A@*Y7~3aCvTv^t448WjxQhZ3&6bSw{#X)9yY4HpKwPP=9Q__pgJ)R;13SA1h@ z!jDZ$T;ZqggPA7zt#KT~1~}MXZBZms&5MU9m&o)OXakO3wV8pJ z4KJN0R?wXkVLP>f9;m(XcFuTF!Ld-^01w?W?t|N`-uk$T?tAw`FceW-$4Krc7y!^D z2aw^wtpG@C7_Jj4<_Qd3g#dmArCn(Vn7lLT^TN-xFA&EV#&I!dPreK<(?Vc;iWG0I z<*OO`>$0(=62R$v{uqbMj-=2EO~nC-emcAIKzcXd7t(tPu?X ztx+0r7&5&QCtHh6T`f?oON-Ap7^tlq_^s>NbyVqr>ElHG=DoGjj$7`{L}+gP{8J@P zNcf=8&%O9Yh~=X9=hR6K@CH?+HozW1BV~hEaP+R=SSh^Ko)Qq%in9ecJ=%E*=iv~6 z;B6a>wX5C+7AHqty*4UTLQF=XFCgT$DfrR@EFX~(1~t&JX-8{rs~SH`EmN#72C=8s zn761j);5$?^|)PhGuo(-hb>gFzDX0H83_8161`@hI`zS)G^VCio-Tm5=p zYi8aXf!*DThx4%h>dY%=qatvK3gGgc?!3EzAx;gDf2&lJT0fxgffcDvVQ0qYw|?lQURxs7>@Zwfu(2!>k|;KNpRU)xq7a5|4f$AgD+< z2GI-fD~}|EA$kpPEwdFE;1#2Vz>$m^T7xk!rrCEL^E_D&w!)8*5SbiwvjkU_*o|Wy zoAl*qvv3KQ;5)#E@7)SvvzPCQ7k9U4DU1{jL~JwqnbnXy9-AVdcVE^_b?MKIKMY_; zlr;Rt%HQMTBr%9$X>NDnSx=b8JjacQiJtZf7m-m3-q~!aqfJ-XbSFXU@*IYru9uT1 z%clt)2Agz(jnX*4@kO6Ph;Mn%DlJR{@KViQYX5R9lwLOtY|IHRw%J#Gnw+KblUHbS zNK#>|v%((-IUle*dNGKu3zinS5yauCt-P1ba$JW+vbdf(UT>V>SY*TaJ^b{|(7y#n z3&z>sjPBa~%vAXJv_Ve0cv}>to5g*b^UIfC8XB!almH5MK$pmZ;ABX%2RnxIo~@8# zu3vh2vqcrdgs?65-0=A`#80EgC!1Q6uWl(^9)GQ)A-TA!hsfj5O=&C*FuTZFZKdj^ znnN$c^*Yoxzs^CWNOJ!5v;fI*r&-=Js)Jj)vOA`l8apxyt(cU3(}VByFiLrv7ra3*=npC(TBD2chzyA%1RI0!WSI8=!EtvXQu%9( zjsr(?OjFBV9QC|?3_Y#*E>-3--uS{)b-!!t!d!^iOGH zO`JSliMb1Vm65~rCLy?Lc2Mb_(nv-OBX;;gG{<+}Y4aQJLzI^y6d!5u*H-aWMQ!7y zLG)g5+<|kASJbS_9-NM6UYM4^&QHk=%?wo_d(8i7JaKxml|uK|N?owol%ZX91XDi| z#J?ZI%l^BBU&^TXU!HxSAD)DERR9dyJ3?AnybgdHYv{bh9Wu+gZw%XktI=Z$J)iJj z+gO5Wa=bjU{QmZi1=06n-|U22jg%kazh3M%p{KP&|0g?q76ehsI540u31t+hr@*a_wUim@LnrqfdDWXaN@8z@Ic=tSo z(DX!}9$29ZM5%>dNi2P*n7e|EpazV{|BLzJ83S-0RXIcQ@zz}InywUyfvsOT-&ETF<; z0**(8#iq@AEzF)%*FKFhRwNzFx84aKI2k@|bg^6fcy#9DiezlR9=y{AkyhsH=Yca~ zk4&AG^k=+7Auy)MXnA~5>kh%z5Hbpet|jOR$<5eq$w0(!4mGU?qFPPWE_vQzj(*(II$uSFE$dtLK$M zUOX^me{orK(fEuiTZ^edSH(G3y$;vA65M?5qU^}er`r9E22a!P2*{~RT8mvlaBx2> zZ2Du4Gok}mF-S585}&@sODxu}KAW%zRgi4DGFeyk$k)Hi9y9Ovzou1CJVajsHEjEw zJAm4wA%FZ@#K_`LdGZo$n#v+ch`89~g9r5ovAOdQ>j!DOQfEu@B7n*n(-T1!-06^=B z1hU#_?K6P+9YCEXn7b!m3WR^XCE?zYUS7tMw6oY=l=tu+Jr#FhW^KJmxr#_+qhj%|2g_$ zvrBl;j)Py!+{@0+2>s~*-Yo{CM)Zbkx52RvuGoUYF42!kH-e{m3Ox@t91NKZ7att* z;g9K-GH`QN`vDvQ4t_GYt=G{fqx2;zz_^U_sDIK5=ag}l&iS!ZFRwj z&I%4aR)lUX1{Y)e;L_jG7J+hDZw9AK>|cw`2w%rcJ9ea#`we=?7OL8vL-uE5CkUjyk@p^cK#H`V z)K9kauT1Amit%~wYwlBZE)z@?YncqxnCCX&2oQ$V-|7-{ImnQ3`W9&DSff1F)w|^!AWuvq)K(V zE_&vw+b3K{O|x1`ICA=yyFaCGj4h}5gnC2oHRP^5vB98jW&asMaUn#)qtC#tN(ySV z60cZj2lQV)BQ{*e({Bx%f6={TV0TN;Gn0k}!nd7mr#~I4#QgG74fs{?f65MDU%+z0 z!P~-^daOKDY!yxeGWrP`x(AdG!}r+1TnBffWIytj?=%tU(lYGS zY{q+IzJ^P>iI?)PA6dIA-H=0Teay(25@~yvtUSSFWCM# zm_0S$4sIX~+p&W=F|aV08`=)+w9C^i$uRJS`{+$>mFfKGt4R@)!rIsAA8QQG@tuAC zq((aQZt%jkC}9MYR$|&wb$IR_nPumoilgM~4jH^Qw_{(h9|-xrv?q5fbbiD$nF$5% z88B|Y7_G1~e1~D5J7m$ts9P>^0M0pp^31QzU_oOp1;I^|DR_BY;G&vSXL;n0q|; z`*rz@R{zT4pPP%%Vk>obyx!)%<%K{|6u;gfoj;nSr9M=opSuwOd3H1?TTzS<5AJ?~ z7`ZB~14otj?9B9kZqdE`wf6pdj`N@PK=nT^d9y-_@AhMRNr)9-M8S8U_#{3|W|49v zb@?_L;trAN-4b9CEo_A@a-3Vs?xF20PQkP9w_;P1whtQCy&bBaEUwY?VR(<2P&A%f z=4ZinIc7WgbfA@fU=+Y_Qnuo*NWfb1nQW_R8Z(j+87y*{XzCzl=EzEKiCb9MR6Z4E zEd4%Mqn-aD>v{Px87;+kNIEYPmJRQ83i!htoCol(aON|A&piKC zM+NFqrI!0#L#F?qnVZ`^oCMrBN%)TtA_X`XXg#`w0D`gTG{RnRa>WJ_b&5YD1LhkX z@X`xP^`uI7@4U8veWqh=swM3e-@30F|8vSkv^`@(!wTb|AQAj^fp7l-4MYcjmqOw0 zO{Yz8jq_k2u;N=i$3~e;mAM8(x*PMJ_kBGoo5KF{s-5ekTy&C3_732D>0T0gD)q#kh8Ndm|K(&fHDDyZ z8j*=)`bb~$I!68`8HO}{nYYCY*vjtr%q0rX(83S6^4LYE>r*229RmA){QF)uIIHVp z)Nj(K<^KWx#6gI{hodJK)afn70{D%?Gpqnkex!tOc-c&LLD+@B!iakjJ7N@Hy{Wbf zQd8L5{n7WAk#(1Fk#6?ZgSec>zL3W-Vp9eV?Ba)h=mnE1)gpU(UL4Kn)2X@?c4_wC z3_Gn1M`9dAd08MyxW>T2x};XpX4to*F*Yt|I!RB`BRtKiuF)s{L_?D0(4L}1A0cV& zK7oDD_qSOW-zwcc+{^NPQN`GXIInU$x?4S4N zeVBHAj9**h`JyPa@e#pD`?rfDAA@@m!~{S=@y@YisTEH}s8e)16RJ1s@%x3J_HW69vCFNep_i95kDS+yfrk ze_bO$n#DH0;*bPC4IG2Z_HvGg_hcek9ozM}V4kSm@~? z2h6F52X8Jw0yNjU#xLG_HkzgAD!1AlANOCHEzR2e)vBX4WJCGb=_^_sS+De&2a?G-# zJjCHv97QUVE<7J7v5j*$iK@0Qm>H0vx=;FkyCce2e$6`!ED)@-%iv53daOBvs@ilU|r4@o%Q^>n6eJrP9fjkgRit~1CAN~;`@pm`jpbgl^|46*;v6VaSPHCCs} z>~`F7UM_yhgW|sLZTK$~v6Sfk8D7f;3E>5({U%nOY7(e>8E&E)PSehk$5ubsuR8LN zHy?X9)d{hbEgk>0wl0^cU_~8C`yh_?ZCtVhvkkVq`{3aPP>L7!-iaS=_JbB23{4wa z;Bk`Ir6F$Pkp=g)^55ohah^}P_WxXpH2r2Gb+Y~-aWaMRX%47H7oLD@mtlp2z{l(- zYSe_jb{8(&Iphud+-E$JE{Z#C+!9H9XrCeVKgUekXI9_YUF`jFK? z*sze_%0;^XtSj0I{0M{vRxKpvr;uy-yP}n*UbGpnUs@+?XMV6q&`tw@FNVS1M7@Uv znL}6%)UFjG6*7Sl(U3-6q9{bGY(i!JFREKQ4!Bmf=JXXB-E$Vmg4FL^8l)PW8hmVy z0eIeu99BwUCmxoC+Gm-2bA-QWcrHV=n|W1V)@8))2tRCFa^}3snWp4Vn#k4}qpzIO z+#N&+(_V#)gSqB9nuugKvKkYwiFpcDpTT#6F)GytJqfj^50W_CwUd`6$>-;;;Btf9 zw8gr=Qc`g47)S;mB)JL4!ttvxu{ z%Xql^__UF;R*iQM&zsZJGlt$09SMTAClAOO&7>Z@JrlnLsoGG%iSG}jsliFwYjIQu z_`TY`mBE6*T(i;YVIEAJ=$60kN9CHN({pS+hg0;&y&KSnvEPR0gqSF7UC=Om8VopJ zUc^M7?v2xvrd~B#P*Jk_BA<1MJd{RkkqCaQB@kjNV4kIOIQeMX%!Zkc-UB2WYTpk` zIRu8-)>-!~IoV{bn`*OX$zv9hqOC0ogj&gr+`_?kpVGI6>W;T0RND`4#GW*(=bMr= z{Xx;~Y89u5hU*yyLknu6ZUK+(6c@~%sRhpH?1+We$rI>=kx<1p;D6%L)461?s3%x= zkp3udwtJa(fHTsRe9URvgr6v^tPT?B`Eq^cQ0A4_(PP1HH%^H=>3(LL14h9|nWb1d3RP5^lC3c^~l5l5mYx);kltFnDfbWHHLRHpyUY z{Wg@&CMWQA#(t1r(Ue#4*f$B<=2-PA9b*-4Hd>nB(AfC$R&8zA$xY?>vsZ1TD&#{E zD_|z;3X1G746C5lb*OUSyN>I#Gd@FABDsZu_T6~t2ebOWe+OOB{Z3vnFEqd|v-tJg zdKr_^KQGHCBv>WyMY{oNV%aES1GN1Z%7EarFtGS(zyE@=JliRn6kL%9))YlK*ZW&W zNsKoIDx4cFZFZ_$u4H~Y1KqUiTypTCrvJJ;#vR^Zx}rpu4Mml2!df^6hcopo_~a$j z%`?^{-gS)$?Hpw?Cf$_5~(Y<7Mj^TblC3P3%zC7E7(}MH(zViazviEKzeC(_V zY5#Ai1QSLNY6nUGT{STVC|p>o#Ug8lJLkE+dP;1DZ=2p8Rn|oN+?^DhZTCnzcu`$e zgLE#m_rQxx#O45GUQo1} z7qf#EIvOL(RI^A4NGp8OrRt!}xlyI~0o@3;yKd~ty-()wsgDqhHy|^#gq5||L2AjW zFsiRWQ&6RU&@%+kJxp87%_8?XwIs{1q3_=%d$aWim9<8C4{UuBN84O6B!(F;{$0lm zXLz4xIsyIV0uxdD05PB?tmaJ@Le85pA~{T>&Dsmox^(kvwIj+koMpE> zKRyjT)e?3!MvmFzDI)cHl~e>~-h*L{B5ryEcWD52quC=zbD~+P$nNLg-LMDFZ&w0D z5|c4OT8ta5)+seRrxd>`u!WzMaeo6&^D`bI-#8;@S2d19dC3(UdiqOME>F%ee|{=@ zq4Yqkp>V2MMJVCn>DmLG$J&uK0A|Yzt5~ihsXE}peENKhStOI5Zhj!ti;_~QJ}TGv zoU@*$wfpxf;c3;m)Hi0cdf^fWMBP-y(`wgI{Yxz^-LoBbc4xN+l%B>O{ZgeX7z!bz zQY}3gp}?*+qgWVf%NX+AfVJF(b@PSvCSZ=AKnr}&dLH^Nr^Y-uZ^<(|t?Qtwj<8)y zl*Nbz*yv>s9%aL|i`2hCC@+=wL-)UJtqN-fI+Vne9(Ak?W?Y2mdf*BL_WB)(}a1R_c z4ocRaYjhJt?Z$poH&38ewY05U8J#f|A&KNya!|HoIulUXq=0dLO^=k@3jiT zS)C!mU(~8Mxora@ce}JF(v##*do^eByz!E%suxSJ3tEU4X;V`y`UXaU8vid{q5tas z{QqeRL70yyn&dI616GKh2;xl{NeaHV>~VF1U{{U|o;-89b4_Fi9bhcTIip^=7F})6 z$=#)n6YE1xW+(hkFLvR{FPJm(@$wNE@b+POGJ5`InhT(8R}O3nO}=)-%Dx`pZ3$qh6%Tsh>Wb1*R~O7e1xVu-2v;Kwp8K zsQrNM`Mr&y2{-K|!hX~nP`g>{iI5S9J-_K$r=tQTe(dMiTvxID=eP2*H<7`KYWK5PG^R z1r=W6vW@M#d!s{m+E3mCgzY}$-w1bDy9W@0_>t@};C%knUNo__ai zIZl|}a(U2Sb*x>S%ft|L6KUFN>maQ)iJ({l%rH-e7>da^V@{t54f%knO+}g&j4$gZ zdBE#-~ngguU9H+k_z;HX+4E4vz zdu{w5#BIS2)O`B^kqnlpJ z^cdT^uzhlJ_;}}Fl|QO_o_q!M$HcpLm?}GW4D=R_#vyKyC|rWtg&-mk2`kL6*HJGz z08)+vpgm7jd(x^x(Rwv0v2Xd>wIsIaxzS@kGUv>c*WYe^i)Br}`isf<@=>m`n%ne~ za0*04VfVwmkG+a%j_~0Hr0|WYu3J_06otrj?;I9|!EV=I$RF)iXd#&Y^0Q$Vdk7OEAicoQh&J1rGmld%)BSbzOzX7m)cwJBR7<-sSIFTIeaQ zV&I%bGu^Swh3dM3sa-F8&Gb?|OJN^!@B*@ZP=C=3HX>Jb<_Om`y|4)IT4>>!>(P0~ z>=tnSvZ2^#{z7uEs%oW*3+&PNKm4dR2^b*qOHPUD~4%RM`ZbuP8N ze*PA|HuHQ#Mcn$s)Q^;Q)$RGTD~vmBXdasPnkR@Z&A_&*YUUE0I~--2!8YGoMja5m z`X!`8|B%~`-Gg85+^01zEUDbhd(qP!rei$A|8*Vf1)<{b!o_?XbQXIj>T&Q*{MS52JaxgFA-S#%=!hqaFQ1jMpt=K)Ju4o7rz(eI0Dl^U zNl2ZsgYDaQ_+P4252(vm4amo^)>ND3F}+J$Ofc(EC0>6YX7Y`fX2MH$W~5NSNR?U{ z{*ORwFK`*}p(VDx0kpNl3wS!}I(kb1@DSJ0_?sAWxUdN;K-Hg^a(SSe_Pk*-hbjJfhObv18Kpf>SrzP&uy$9nC3KC@VmI=)US8oD1DUzn_~Zyl8JBk%5Pg?hZJ$B);g z(wsY|=qY+CWmCX`qAKe;AM0KjE6-~f`_0MAyi6YU(z`2uIPv_`lczs$i2=^faK{uQ z1U3O|Q96*Ammvf5UZ_m^6hDjOp^dS89SRs0?1tI(g=sK%3II{}W9m}t3JMY2bF5a9W zLvx4VIB4ORelN+(am>y_i_^i#?3dSNmk0tz*=zqLp$ex4gnGyaG{qh)&;u}3EsPNi zv}5Csp-Z?i@5B*@X);YzMR<{v6F~$uJ7gs)R!?+c*4)bcF}fQfr*;>qrb+tzXlZ`9 zvi0`WJ@x?BKAT(natS~469d!TvJ_XuEsHd1`zCjn-~EI5s5s4N0T>GdA*Nf+yF2Cd zLI(dHfpJw`EKmhxBB5n4BSKFaDtKIFli5E~_;PDi%Ev^4zAO(^!S~$8ecLpHn?G&H zlQgM(0Bbaae*>7k1JGN`YhdgraGvasqzSS#pva549667kkAPl9zyDqb2KWVH4ZZ>i znj+Q|2$bAGt1X;38LJ`bhZ`VHODaKC9_f;0>)-HZUQV%ms1t`%*|Vh$*8^iSB)g!G z8GNQ-XfI?BY)15kl|eNh;P3;0!05wC9e6IRHF?06@*{y}=|V3CUU4qtp>k^Pg?VN3 z0F_TEjp;!Pfuo06Ql#R$5e5ofD2f!SJ?XDWFHnDSYc0U0UicHm-~|WpJPUgo3;^)o z1Y-eZx9~CmE%Sk%j(7Kd@(-de7XoxmID!fB+m;980Ofn)7NN6bdCtDlJ5W`nFk(wv zK1NxU>Goks&(q00$p5$X`Gg|h#nE0b5>4oVp1FUaFD08Ch#oSsGdBv)si&924?7br zBs()N`c|hbXYynSXE!!URk4$7tjxZ>^1~!xAW0PQH*N|vEQEh=?#If}pXps*OyXY7 z44+X z0{wQ-6z{bkB(*4Xj@qm1u%}()#F%y1qV>be-0xz9*hdbW?3e%ZTT_dV!bYbT**Yw_0?T#T9)3vbYUFO&npO`L>rpWNdTG2*Y-l7WjdmMYB2ZvmQbkq) zHR0w3h`>>N4FU1XLTyKlp16JH(!AXWbbB*0Js{oh-9*O2R+h2F^aGc}mGYDimh59< zkqToHUYaLgn}?4=YxBU5P5_YNZ_9+-YJ51d+wKnTVWUh6vAfDPDq8JMq8V%6Oa8Z} zpDth67~S{A`R@K(@8>U}7jtPcEkH^qU@xGI{R96XE)r@GSRn5Fm%;NLhnDvfoM)s!8r4J8ca zeanCJD<)5O9wpXMA}}KmO^p5zLY{f?2R-q$8YA_F72BE?o+{1wIHxqJre{gYopN31 z$uIU0Rr7(0R%F;dWCp*?jsxSiSaRsUE~?vH z!0}99&w`S(Y7cO+H+5{>bv-rX^JNlkga`qV_Z1Xz*%A=fgZX)@`H=q8m^KTDq!z8g z={6j@q3r))%xO&Z_W5*O|C1fxA8ut!zr3IKDD?Ud@27jyYT?tUJ4ph8zf3>tA7pgr zD3QO%1Q!|c6yuCYAJs190&j{+uKnpk;CUDcu{`Mxv4Yw^;UT zZFd%QKYoU2kcYAtkzDg}W!l#&B7a#nviBbeyUHuM5GtqGMhrC$5`I*VoU&wC1T56i zqfet4DTWTe5s(_5{0O4uo3{a-HqgM)^;Z4=aAHR5XwvI@0TdigwVR|yok6?EbS@+HBGVIS zRS8&QKxc=dsCS}exZotJ3B21Sab^mzLav4!Z)vK1kr#E2Vy-0T<#J72N|?o0DQqvh zDz0AFwVKdm*T_E7;aSpDoGxi^?-tjm5r6KzV)@=IE1P^Kf)0b77yCcwg3#Z8&?E)H z8{;VuO;GLGjzANI7fcBxD3CDS7ItS`yJ~%t=D%Jy6i>3O$lyZzDh=z1ap3}g`h7qz zOJ_XU*>?{D{m%QyYTQ~C^$g=HhSOW0lK~K}2eu)xK#%Q>XBHDG;nCj^5$ReBZ&Q39 zetdc|$6!BRL(d0Pn4N*#AH$+aKu!1qAriVLiEGMXMkAfT6Gv*%H@820MlPm&`_B&- z>G;&9M-{kg z^S#^OpyY$*Q7jjR*1OZp%Y@L|z4;3WV+z_0*&lwP=< zaBa?u*-AS{oPvqG_qJ~QL4a?Vf0*s9bakmdiFYY|l6hXnZGnpq7z%gQHmjR{`ky(u zW$&5ge$75OTK*^nI{m6d>A#4nchkr`D=SnYIt+&^zn$sce7K-_?84H^uY?|Rfgt%D zGYzIc)?nwr-*z?pPa@+CXcn2hg|-UtrqE3_K#l}j|6zV1gMYQx@Kkm9mM1s!1(h?i zOWPgxvyOkp&reSJMqHNPx_cwd#)zXRHaa>ETD$sh1(|IT@KzWy*na>q-@qEuE3t-v zd#IdVZ`p2%49pk2Mp1~+@Q%*ZdMUirHMI6O&K_fTX<2!_;Z=#E?mlWjQJf&CmC- zdB7T(kVOH$MVHdsPS|+ze&*{OA@FU7z5Rd=;n+=|`yhf&z8d~HJQKm#-eb=rqC8Mj zQAFH&8W}`PRhV^7BJ{In{y<>Cz#1X{X}rx7E*TwXpHIHMvhSi7(;i&W5&i=rsQ>IY z^ndKP6JR_{g@_;^CmZ3u4l@qO<-w82smAH9)2(%+AR!Ni3MFk@^z^f7RSr8P1C|r- z6x&+I<3*g)?+}`R8HfX{%;yZuGJRlSZ7`|l?C$OFHWVit zBNxE+u0+Wx@hQ4#l59&Eww=&3qLg$KHlsD|vsPE}f*A9FpdejyI{<3h z2ik%-BETuQ?r_ixY5t(T*SqtO{-ISmaI(vI$h_ihplA2P$pB*(MM%a#>fZJD8#iA0 zWd2cLaDtqj1(?IIQz7F7;MYTrL+!>u69XB%fnj2~7NgC!H%WAqX~TrdXHk(<$)88e zQcr$beah98#T^MM<(eRINbtP=86v(15wpuq)cct!3x!_q1JimF&#Lc!9xRzQ)VK|R zay^@pYQ|_69f&2C>c`+cI^iFOhZPwqZyh}9PEOgsnq0iSfi6am`-{srB{+}GFN#K1_;!c#OK1AlN^a0fdG+{$ zaI=UZOH(nXgTWkMSB5J3%|1oTh`ms{%`71E z{y5X$KVXu14{&$^8Z`kU)QE-0AX)EKBIpazZo!;XH1?Q`KCDR*?aa|5->c3LalQ(D zu3^WOmUmPcX&8M@yUW)<@Mn_jg=t2TU7;AP5dcqyOl7*X;)A7YvtR$5%zWJFaLU~C zrLzBrU&gN&K2fnVSj$>`0my#PT{g}U7|%nP*}>e+iOsImVPO_{faf7iUw zSWV)cQCAE~o|%#BYIV}LSIR5dPi^2Z`DRJeQDATbOZo0Ia5ivYGYC`r`*$$#8K6SS zAlOw4Yt+msf&7HCH;oJj4Uhdzgc8|OV4lCC{HiqI#Pl+`Pd@lV zH_H>uy$P=T=ZrM05B-fkbI8ElK9m6DW*L_wMMg!*)+c^ll{q|qtFl(0p~Iy0-2TO{ zXjjnnq9MggSNY+*od@LYM`itAEpC$RWA2_`I~0_%y=2DW3c35E?~BEUuCak;N;!u7 zR}Gd217rcjd4}i&hUVLXCiRBm0rjWlj>~v6Kjq~7c}i|4uXJFr|1WyAX4>!YqBK{t z#hZ2aM9!y)3Ef6tV>Li_SOCJzesCaIB>4_+H}xmhrdWQWacMT9bl~wpH!qCcSY9CC z4<$+YROpz&R$9svG3T>RZ~rm$g;6~A3*!6y)tL@Sv5iKFoWru|M}V`J&b z+w)?01uzpdbx#IVi8ZY4&r-y~Fd+cUDZk5fVo{;Gma}GP=l6c1<>zh-=%cqy0 zGS02;$T7IyP5gcVD1USLn|1c%9%TB@w;jx)W|gqi-{cxIDsx*EDt}2Biqq&XHNJX* z=cY$38Lpw&oc<9O3SrGjZUgn$fyh#nQdqCB|TNomL#TZ@gt)SNE6=N8P74#KWBjfZ2X;lYmbfa5N;VwDEl-~E5rVIqK9BRl~;$Zoog3*bx_${%31yy z2&&$$^D;R3!TXg^RBK!p2ALTg;^eR`Fkjr5Na!Esg`}s{hc)7b`AuF9h#X{E@vwvy zUDqH@1|YsiYwkF? zFdDbijE6%tn;u{IfvGof>$LE@7b6&Cc;daRY!Tbi%%?%x^~p^nO=5vGEbI=ghuf4-QN~kyS-XIgPuN;&r=nwkJzL=VUc-?_N3}EKzSZeEP228uv1v; z1bnNKy_1=G{Dbcauyt4ZZlzt%xA(fG7cUemo>c&YWg_|M_Az!rORiABt)Y!6Wy>p; zewf#IvZq#6sXJ)q4u^}_-6`?&=E?IWnLj=!rf}Q7$Rx`D*m+;X{h#u&3Itzmw0-5yQ-YddXg>@B*;T6hcI1T;SEqQe^1TlWMEN?;@as5Ywd5L+b%T=qa zv73rQTI(fAQZVEPY!uj_b|pQw#X-5AOng^Te8|z-(ckRl1?1gmtVw~S^jW$6TdpF1 zA6$7cE#{PZNG^VI)k28L-e{ycm&a`)N9MD}&%0}PRr2a8t83%s@1Khm=D&LX{d)xB z!M@TFBZQXQ&H9M-4Ov?`f5o4x$!%lEMEkKb`c%OV8E2XeD!_VWBBT#otz9)KuW8 zz><{Mg1(=+VkjuFPx}^91Q|%+%2@9J&z6h@TiF>`1W^bG##9k53QwHfoYq~$txto8 zYXIC1n*?pgy76Xf@8bENb#`6XzU!z#i`5zZi9vr36}a^5G8yu}x}Car<G3UaznouvUsR6(&A(n9@gTZ-X?>UXRP{cgI!^guD1jiKd{{tcNJpY0WI!rrc z93u=!oN@3GOot@_T73Q&3lOHp;X0G54+hk#I{h?Tmb!7-OJO!LMo%0%7PPDGUcDh3 z`dHIQcz>bqj-6Ncx3eO#Z2sJzlMj3kRd|b!109T9vZE!e6wu=xYxkpD2{&CrQizN_G5#6lgYd*<*^KJ6!9kx zRsy(5B(aw!l)AZd1KgQq1_P%5r!-KH)(<{~<$e!Y77Q0z273y3Rn86wM!qUO#2R#NY%)2G;xJi56#)E)taByQd1rHf0+kr44&_1FmhIKIK%|sb1}ONRC^w zM2Rc%-`lI~-ZtZQMBypYJORC<04}dG3j90TG4DXx?Br(aI{iuFhMIJB0J;$?db0Ea zw6RtZl@bYom zy863%a`rFv8+vn=-apUDSJCr6`n6&xgb-q)m!NR+ zSdC*(kfa5Bx4MfwL^9_-WIiA`j2cRDvvaYhu_s(V)q91V)clOWodwRR6KXfo-wR!= zo8_n~G+%l_vpGeu9XN*v>(}P5RZ_l}`4smR_BrRzt?=EjHB-(igTASGFV%n7kxQ#P z^dM%D@xqPyj{^*LeLqOc+OGkYM^57r{Ue=3Zyxy3532;tY+qjC-hjht6)3$pj-w|( zh21JetTfgChbIJ6Kg9c$kQ1W_)$jAZtf8^HKzD$+=O@xm8BwwPgA$1 zt-z2fO&`PH01n*jAUp=#xF|r#E><(qt1e@bPeSb&)LjtkM?3~_1H&%DDY$k5`s&(c zc;Qa~)N$^CcikWR+Q(2yAAQGXDQI3hqiWY52DX$J6m-I>pixtO4Un>Re?h_P2z`FHL=m#KDu2ONa(rI0Y*9VHP+;++oYof8aO#ndNGc7o!a=G@DC_s?BD zM;Geq5^u;pH5xeK>X-iT&YfFKyGWYAs3qfOz#Vg@NxO z&jdLK3*+y97cHA2S2#5i_N@yvJ83*!usVUL{T%WFsALi_mAr&S28RLla$xWdj-Kj2 z%Y3ELo3s7(YOD05ziy#J)3~`FBD>7}3gKH##YhjITXpNlP?QlfYMw+GFsnm=cin=B==hkKK7`3bSfXKx2_G*bGL4W#&tLRNrr zF$bqw4&iH|Fcfim9^{6#g`M*v?yWNLGfJ3+f+0mTM#EDi)}DQ%OzmBLeTC%5+Ipzs zeT=@hvbmhGoR4G=(;rZv;R6iHE~uYIWL}17GDOk*y9tcf4%E)e&;nngg?AZ_o0Fs$ zV7&@@NyG6Y2F}5GZkJTwjHsyGY7G;I^k`2sDj(OHafAxnIE%}F!2F2%{|qVU-+(9> z0q2dc8D7Ve?MXktttcg=w8?~|P6um~tj-+y$=gA{UE{7WJ_nU;sM^NVCu&z;i>^O; zFzs`bafQ4SGHYHCMI43zDMM82!Bq)`&Qst*A(9%ey@!BFp64^TT-Qv^D;%4uq57_? z%URK^eOPlF;E?#)7~Bc;Cj4#CgGN3@d2A9oZOFY*+ZcqI)|*0tTV{ucx<`g3S2DY) z6^p6JwOc>VAeb{5PyTafGVd_*UoLV53TRffP*nm!X9Iz?F0_N2+Hwv)gRI9i5*!V< zTi;h1f9Ml`Md4ITASN5MOZi}8<|*vpNUa%9F$iD@g4h6)BW9cq+O{AeXUPO2B7-%d zr=p0-xx$ds9=b))>mV4grEsU9LzJtXXw+vRSZZ8#cTDYzBZ^`vbuMBfug znHv(^Kw&Y7gC4B&b}|rON?$jn>Yg#ex=Wo~IebgNDzWLAZ|Jn8+$p`u?H1}o+C56Z z+6YyEhM~WuF}dH=Zz7^1?KYz|+%!!c&D~W?*N)W3xSYRzC%)N3OzD{1DY-HTX$Lk> zq?5rhPqKIj@Obm&%TV0sK`7wxr_aj_?WUS1;$JHuaE;T%sbs;3^ENhO90&wx!6nfo zM}P<|g-qc;D6H2)p7VqscI1r&(I^ew5*6n2pKCO~Ag$9kL?ev@Zm?g`(Altkc^07Q z8f<9uP&>%B(SZCyyC)Yu##*=5K%wrWQjlG!1Mom+W<>XlzRtM9mvW6iYh~|pj(Hu1 z(gx064@+#$EAeeg6NlbOAe6S|89d<8!xR~N{QAS}SRF7I)k|_vU)>evp+sNcgM6LT6II?MK37B334Y6IAdW2p5Qs%W6+0&jcLlrw1^z(C7r z5B(hsF_X=@dg+9c#}VryM`hg_O!gmrF;jKQN<8-HY@P21wjXS?bBBQXKK}>tMzjqD zJAnA1YjQ1P?Ke1e1xoJ|@C9KqAY%-(f`=DX((th~cQ4g^sC;@#+)HEjf_X`BcdeLi ze$4G!Q-VR5p}OKL1oX-~qr!eAi2ExQdj9i(I{u6)%~2EI;rfImdACuyrOwch+W=6C1W)&j>ns@+YC6ou5Uke$ zRYlM3UIzDjurf=?7XS$HP6V_G#o&G{OMbV9c0DBv;PRF?usvkZVn3cr@GQ}7@IIV< zpD(I9Q6qlerDgHxjt$xo6u6HDR#F$}WpJPx`RzR*MFcuOYLdJN~aS-px4ZHnxDpz;vJ>vRahV5ppqe_JpZk`j1*Zdy1cqYe_~>2#XPXK zQz%y>z9KS6Ed7r2&~a98aMu@2REfb+NX(sfU3m9*b3Jb{SKK|bFkXYMU_<_q*lgne z-hC1jv-#T(2D)TKaxA5NDN#Zgtgy5FrWy%Bpp1wl+u>n_-dH@bV4iF_FB~$E!%Ru2 zjlg~r62ip2aaBuh-K*RrkF$On*u7DbOyCCG|4IkEz zpoo&x)=?yjLs-kj`3hT;g+YOD4u&qCZof`mgeIApa*VTEt&YDj(K&@I-bPa*C{+wj z5WKM7gQqDNM;PZv^`#k*8zwdPH(2rnDs_uy-g#E=rp-Lq-fan>pCT@&4t-VL*!Z-@ zM9-OA_(79Gfd^O)nL?7SP}BOngw-9BiRLpUEnYLgg`@V$=5|0!0a(#`_$i({L@s1f{{-bnBrTXUfZ~F~&;Fa?*k#!_j7Yep z+p2SSx$>mGVd1l>lRK^b+v_Z{|5_CI?lAE3PBq*fL!GwdAE6plosGQ26?(I$&|AXE zHjOOd0R7MAUQ(}|&}-TdO;{IHr75C2r9hyxvB$yB5E>u+#uXJC!ZGbO3Wsc>L+#sf z1BJ_#U&Y=zSx(WqvS+kEF)N0*%ZFruR$~AELp>c0_S6$Z2sFM$2D?ASZ-);oilsTy zb0K$Rw1^ht7(DCZ?^;6mdDa0keBCh-AmURwkgO`Nq?7z4Ld<5!FX;=%0v_bfF^#+J zn5hyHvd5Cs&hCashyYoutjzs`2Y*$&{7$X!u*Aor{%%fVW>yz@!7yIHVBZrs#^9oZ z@2hufRU6kxVsj7KH&LDCaL%oI@CMwW>lb{l{JdY#rKY@@*IHvAham39F~Ev&0axmQ z_uBj#2e^b@hfJYrS}PXHI4_ZfBdBhfm!e9JyivvZ2jBE_>R+qpGm#45J>;DFiT`X5 z33XTxJm?-DKv;d?kM6&P@xa>?-$25puH!Wa%9kofur2{EuFR%OpYGdttV*4TUvM_! zc#S~pYXufC9CCw*A0YB&40sB%jpcsECzKQf78C^F@Tjg}?rYQN3d$VLr?@3<${pSK zR9nQ?{E%l5cECr{`6%UMdvawDB^0{&1^tT&pcg?*)ebTxVjcsy#;^j=ZQKclb*R>G z$pUB=F{E1xvfeI-Fi+gmX!ku_Ro9pr3>Cy)96k{g8x~??WA$GD7N~gw;}!NDEObL( z5@0GQzcavdf=iw=OuooAS*Ue|s5Rj~G<|LRgBIEPc*SixPl?B1EeLe9PR7z4!5ih^ ziN%;&OW+%MLxzh1qSy@Dw}KR3%l|^N2Y|->+AmI2mD+6I+KGUD=O&;Ly^Omz;y(@u zD7@g$w?*;b7eF8Cy>-?NlR>XKAo_W-RRoSlkhRZXj_Wj&wl^u+h?%ebDBi;XlxZr=jX2Hdv+1}E@H ztPNm^*DIKeEMKOeeN?545z;=FjMopC zi>G-X?jl@!8SrQ#-D+yOQhBTIWQo@L=V?&y7s$g<5t0c*Q?4d=<71uU&lHWX|5mxSVXo~W->s11CloEE_TsHYMDxps z&;`xC;EQz%98T1>H!LsOmCH_JC2O~(EaD^s;|v`_7}Azkf|Vpyj(vyf>tdc3C5~SF zWq(17mFe0^vw@?pdB06Lz5FVBt~;iU>5>_4osGeMh79gKT?6OMk_0P|gBO#YsL@W4 zvei24PI_eW6pNW>p{hMZ(I@1KQ=OmnXPz9Xl|G=jQd*j~1<>rUWuko~;1Xf$2=*J& z^YMEH5j8^!Xue>-Arh7aMuTPJNAbkcOym>I7Q^$DGfywWo}Jq46*KyU@|A5W1@}!& zC)`}P4-VO!e7&(RRDbsU-)4cCW%RYrM}DNvj*Fh^Q#>n7+km?6=Srh~w(XJGM05jL zkwY&-G(NyAlCh8rD4HE`*0bLLJO4G@2pI+Op{L#(Lv<>A7z}5?F zk=qcTyYIzR&FuBs3~w%SdcsSwchLiBw+f{MbZ~b75CNE=>41gZczuio*Gq-CTe3OWxl8x=hf8gVe!Ft&UwiG*LSa$T)9|Phbh)$C&?cDO@YM z@oA??##PEI%Iul)UrQSO;C6%vt#l4k?pFdj00sesV=O6jUaS?8+w%|22Pom~C|-X_ zKnjA@8LP<}mh`0bh)kNr{xbTaTy{S71fox+tG`&_yD_Iy#4)bY+zI~>z}F&UG(oHqz)eU|i>ZMG2yxsB7SZC;&dKJ=@n)P-K0QftAD5uZ{NgVn zu9$2_(S*<+x$PY&qS1#dQXgL#4LaDwZzN?& zO-tiY5=3(&xOHrvtA%-nZG6+$BSpzv5Z! zYVOW1-!sUs+!V@N(~){Ply-OvR2p);{-Aev0a{bioXW$P=wxt_ET@WSdp>f(Nd-@n zH))>-MIbp#5NNzxX`;n%{?hNJ&dbRJfV(TU(7&>l`ao$>Z+Z*2xdF@_;MwTIJi&`9 zGgPro^tYQ>rXmAk9G=tur&fr3&cXD_)2Xv2y3^;)_8Xg$KV9X^dVYfSIkIPdT9SwX zmkamq}V=Scr*!3<(R>Z{Q+7+88sM&}R zwvaI-M1E9W2o7nn7hIf7=G-|x)@-nF25FNzY&JnNBDt4tY@UbY`pLbV(VeG!OX{ysi7(A@)|}%IURMNx0;7U{=@u{zFxlMm6Y{2;VohTyKe- zT1?4*YM=6PSa8B1zB!SO?vg@ z6^pyJW>d27Dn49mcabkYSF2wKtnCA!utasn1gcj13I2{{A}GrmWjw8(N3PZq_NSD? z0bK&~X8Vf+^zPd!SC|QeEG^xSC#E)qJB51Af$?c1*=HvU_`3{v>UGEv65_$TZc~Qi z)T*ye_~(^)Bssggx+<9_KYd{0E`KNXVm#u_qi>(twsHZdLJkhvpO)`FHtXPUvBKWo zzGGOv@&wc6tNUzY4kC2qC%1Kh*LR}Fg5qhmE`$3jEJrT*L5!@%nv=^HjV44eoWn7t=>weGf-}647=lwkIAKm)gnQN}=I?wZ1zQ=KVzpl1& z2QF9epjT4&_FNGUiV)7U?wLD>X5!IZS8irYNS#Y(h6AgEt!r?cV|~JAMklE0j%l zsHB{~h2fmmk>fk{QzdEXa%!(&u7|kgYj;^k(+EW64*U~PJ@U`!aJ99`f8@xgF|>F< zFp(hwQ9R7HMUpyp;P>TGT6CFG#Q}oJq36rfK0JtDoHTee^6T#FiQH+&FwafU1-)Ts zAZFbEKS=xpQ^0dso5@5jgATtP)L9}RSe!=Ra_MN%fGy~URod5G&!`66hzfZ=c}#TI zGfxXGl~>;q1x)X1z&BXZ@pzjV)OK7XwzCl3GO+5Vz2ahrJvkJh2)IuYC!gKzlaKz~ zsGH+?QTXl=zGMhtwj9QLF(VegF)>}38dQg6gPdRtR|g{bdwoRfUh?$~ToYeBWq9!2 zgg^tSwI2*nF>#Wk?r(7-gw__tTl?MK>3<(G^*8?bDbURMu3CdVC-}`?6(fprv>AZJ zZy!N6>@T?NX?$ zl}%VZjmxGoEl{vgRX-w}$vHKN^x`DR-h1#{>GF)-{>#EsK8;OO7dMemUu~5~?-DOx zjMm-0<tPXQl~x(>^;LOKvMtiW1pZ{Z)(X;zr=9I5FSTNtSnOt5Qc&gz&@ zg}#^+iQf>9yM8c`KO)dmaIG9zB7V~b%=d=`N8S$P9kpVO;a>_-p5nF>HuzHcYyaQ_ zW~JY-q~o}Iw1#l=7lMag2M(NntRAID0qRnn8M;Smln;b%<-BxviQ9XM)2AV`XQ}4*i=Ua2Aj)cHm!7Kgdics%|2{%|&L3F~?e4 z44F%b>~2WBy6@P($$>dwyUQzz^Gnl~$=55bcEqJxzm~|(+I&sV9_a-7 zez(@TG2oK=rWX&HH=AH)9XnJGjl$t4rVWD2<&S9o+K^QWaI9XG0vV<#1zGyz^a)F^MGobu2D>BL%!1;64%`Nq={Q7sT zL|(fjYVHW$Wi9yVli*wece?;^G}JUA(+M35LCjZwP5yW!wp&V3g{41|iPAYaP^PWo zyYxML*Oj#EuY`=eLoGy?UP0Az+%v#%gXN%1RuRV0bxwC6&x~|pRUGj)10RCVKNVL7 zN8xgLMMK}hTtf4WlusV*$kBe5qju^*{wf;QD&g+z^aM7$g7<8Z?3ocqu6_*xMH7X( zOsOaIr~>bEQS$!B3yw{63mp&9knB64#D915DIbB=z+@!iHwHEnp%40>^QYs!>}Y>k zBL4!JW}6ull$2IHlL>%__)&%CNKK7{~Hf~_IZq=d3P%cut@C~*?ID|mR;}}`_Bdpk<UPm|jT>|JS%O~v;~-ugLd6iJ3CiFldYYlR(0Qg)w1EiVuECpDjF9lY zFxRx*urF9w*|H&ggQc*;URkKj7^oMcfH`{n*BrqhyWNxo7tDgP zgMj^@K28zq`6?3QSf~`R>T-TjE8uHnfz8HemBzZ_vIGZn(P!LU%+Z;Q4xLnJyM9m{ zPQ+9mdf|_#ur?P-#-|kd=6=xX{ML~wsbZmEqyEGA5&z#Y56M|%+iUFKnZ=aXK(}i% zp-RkKs(XMt@`5=76Z#56b=Y zl0~he^O1pWM^#6bLvhk}hbN^vqi-mu(ICeN0oA>i{$^62Aqr4kHtfy=dyD`?Ys_k4 z&li(`#kHpAH#yNb?Y>RXr}Dru{TEu?U-Al-%_A$v)nqf?A9!+cpacEq*Z+;=*x+UU z#nWTwU;#wPlcNI*&RWHrNl(Iq9cak|hOzol4Qa(MK_lvo^4$&4z0E$oogv%LI;s_P zo4@w|rSZS~VYC^j)ZI2mHsUYBAj{ZT*fi9l#WfqkA4aFQsN+Bl-T!5WO1AyyxxvtP zN^j5iOlO{856bk#{ZbLyx0G%@MLy#X#5Gg}xe^N$puisVh2F<5WR}F1a}8fWV+(pd zW?G*D(eKmy9Ajm6=UmLoFqTT6leSXYX*R-tZ*&O27%cC9BLxvw3Jr7zL$h5EShr`E ziXQjSp=9H=WYa^V;MAV^yt?(o9DRT-Bylg-(2}yx#zx%wk!I$*6yEdG=IAz^#g|MS z`Wi!pV+rfgGFr(A>lsN9;K$i|b4zG}yj@CYwxpRyy3MXD>4!dl&9d5;*)88x^{N`R zH2<%HDhrgv)EUK04k9$mQH-RrJ78iFa2!^*ukOmMVHoh|FVX398ZJI(3nff{brkmt z%T)d}SM0NZjEStB>fPA&{U{;OkQfFA9f#Kyz?%f<0t5hEA2Rd2op%>T)YOe>g=~l9 zV22N37Bz#bo0}_}2AyXUB6EgiJ)K=Q#9ecY0aPI9q8cY!Jp;HD!DsUru-rZvN&F#L zkR#_it<0>8E495;vOrK~X^uoVrHPHHriosAlXfzB!jc|7d&o!<@LHNdG-eZVJcIof z0^~_EEGpZgiupWew(R=E(ZO2kHaOnpXTYlb%PjfRst(6*_xkSC*l*-UsW8{)=OkDV zfFq=YfTKKnSPv0Gvu?NJ0Pplae9<GYq%D#sD|S3>W9L`Cuj;)*mvFiN z>tzq|^CUmxguqRUK(ty!2Aqu^{BBl!WeD@RI(Bnp-wg7^S32ppPvD0S4_&5jJs$h8 zaq-oMHy7TQr#>#VDm*X+jTS|@K4p2jd&|itp@l2;8OZ=HxjIl2xl1qvfw&k z<_Yu12I_z~yCSITHwHA;uah2vfpZMghV&%_5lVQz00fk*+QJ8dOzn)#D$D%xR&clv z;(X$h5O%L-MXbo5D}wQ||Peha=*c?r4D3 zF^`eOBVAexRl!Sdi!5w!|CM%>K0&!q5JQ&h?A;ewDCuUrH&C_K;m1d3{Ie|Us@{PY z*HB-T$sj%Q$Nq6xqm}T|!pZA5hkZZeA@Is>tB97?vy)ghCh-CmvpWe!6+dy+)EenF zP=?&iB{NSQwxVEUHBGnrat@8b%0^jHzE6>!x&i#-k+p+a?T;!&(?m0+ZYYd;v4iiT z=(^+xK=0fCXB%OV%GQDX|KU42{HlbzF9_DvLDDrkp|$T_IBpEC6vsSHgc7Js8?-Rt zeSTpvIKCMpp&z{4q{n|(!YNK42K^k|AJaAZ>$>vx*2>5X96w#zu6@iS8j^Ed1p=1+ zKpcz%i0&N*F~2<_YqBh$>dt^w$X&~SyXDB@+)cy38CD&}x2LHWBBMH;&K35cx$UH}22-A7kox|*PI6iu39ONCEd4RBY`1>5Q7@AbQtI;nla zCr@;1^V`qfm6?@&jH=y30yu5}K<7Z=W(VNv9xV4ENP`YYUpqs}dP!oLXX8LVMXz1MbKM)$@PSXk{^q1NR~C{ zXT+{9n1tTgsrk){3*pT-M; zpC}mGi{=*9g9+BmAPK;_@N}LW--|3lUZ8g^f_;V8OxSdUmCX)^lZxP9t>Y_LuhG*k zvfGnb{^2Ud}#F0*`52MP3#3{q#>>x704lszzWE65_wg2*gq4jM-uc+{~tay zU|9&b{STiVyOwtdY5lwNNw7NTWA>(>Dgufwmte!-0Q#+CSSi~w|MlkaRJOgiZCUk6 zZ04MYphhgkTwPWW!=44lL>SEPVQ5c?(s{G>3~A=&6|BTVwYuhE)5B2NhZ2Jo(v)w9 zs*o${o=HfmD9aq(wSOf$G1ZLdw$=qL*&eP^BlSw=-Ac{FlUv2CoE_{l zM7n+|7pd&wTiQd4T?YQ;JE(s=e|YO)K`irZVtgm6ut85nX=V)-xn<>G`VR+cYl69Z8 z;mZ{6&J}SYL*sl|p9HhJY96%2fT6rhmT@ztbyh_(WhMG7)_Sz6Ly)Sk4!2E|KraB{-Q zv>a+Bgz5%!_o?t+=1~ME@gwoOB{=el>O+?+A5^9!KE0pgd~bu)NzVxJ{nxA>1BV7a zNdstE1sJlUp8f(Gz)2$yYhQ7Yb_CiA%uKBI7u+$t8Q=}LvawVtdA2)(Bu~3td$0a| ziHyn{QVqcfX@{!7dE{ujB3(EM!q%VCxNXk|W-Ds}(a4(^{jX`kqPef{@3%#t&D`I* z{xz1LtL^lAE69uWK)k2U2Cays3_<9J@zkx1|LK{#|AGGFnp@g=cfgYs605*`b<~Um zc@v-jiVyxF=Qzcgo{tFmI4a$kx_C;FPCwb)B(Z3IO3ac^)`*YY`tJjt_3MB+9~JLyFIMh|%M?Rf z&UbO}4?zmgybcCvP~GlY1Kb^tk*UTDIDfg+gXe-?!X~bjb&=Y;UozI`KmFGS zN%t@p(XetJcgG;mtDUvO(J_nF%-AT*c7G{<--ET~egpH;2bmjetssH-Hh0V{d2gm_ zkI)>6;~Yf91-!==-fe;=)Rx8rZJS&5+TAtX%rwS++ujm9GxV)4{^TD*a7EpNi{S#- z(vOy(&S5jcR0Zb!H0aB*fDvw)D^}0G1D8GKio=eat3M+ORapn0=E^DL7R|VMpGuzl z2B!n5CFf3IlR7bHfkV@aFB-h;37;QtXpsZw;vV#E3sI50C{hgloNsISR(@ud6px5T zHqrW%O%HP{zWgv$6nOJ#37XNyN|@S|Xct!IK3Gz85I;05(Gae*dumJu@RW;`=Cy`w zkc8Yp`-cz+az1>=)L@ev*(qJ;|KS@D=7C@;7KQ&?tX71B+T2Z8I$W`ZjuAzrd-Dg` z(+0yjAO4JJ@a*_SSnDY(0~rQ`%ttKbrs zNxM}2E*Wd=fyQe7T+>2dVzo3^-WRL@n2bUUevr{E(p-_Da)9UmNnN*LsV8Y0$ouT> z?la6ju*VI%uLEC_`GN1&tVLu713*}t(0`r5BSiMIsWZnL4_N&1JN)UkKehB_ysH}D zHnwExQ#Gstyd|n@f@5R8jeRS!6|df$yPs+db=YH=N0-;l?nBI}$ZWuoHN?O6L6SIw z0c_pfEmDK{5E^cW@A^p6>o3P{6)niw&AX0MFZ9M*m&Bj%vAGiJ=^3Wxe15yN!4sfL z$`{5%oMy0$28ii|h%yEse~jW!YK=~3*C*33RH7PA+NVWCQ8~c4&%X5+J%eLBVC!*> zb09`cBn+C&OF_-B55Tn@VbGWQmwcEMrJ=5QIsEg(1T>0g? z=WIBx777AUv+^k5enNyV(WlVefK$S%O?J7KAvIi9wpzBsF%R2NxZ#M{d*)p)Mqkx><`_a_V$6Qz;YCtDfT;8VDEan@|GAsvcrlj2=jz7vV95X0 zY|7ZhhgUX_DND6K=@Kq{oQW$^lxI>Cxbia#XrV}~MeShdTipQ@Hxq1S%;orTnW4WV zJCriYwk1wU}cT7|oeMJC-L{3ClU7ag&tm|fdJL$_ByI>IS zM*5L|ojNb4_hUW2;JTLWqMnt)YO|KHhNP2E`!d2-dS#H+G*G1n%Cu6#TlUJ4#Ky(4j z0%@isLq8+Y*r~N399gg&Qf{uV9%;T55$3DcMal7uo4A^ESGVN$bTq~ULt6^^qf7ph zfppqoL15XaK#gu3)k?QD-MVt5qAC8nRn*?C#UpvD9v3I< z^5BR3yPz9D!s`lPnJHrd+128lPr&U<#*j#wXa33;I)>QwW`O2Qqq{m%GWgZVy0}gS^3iz0~nYf!N1yc6h*Rc zf9eInR?KZoUG9Syk)@r6ZGGm0wJ4DVxvLQoYUiJAptC>jIcKF&rSH8}H8u0Gci5Rl z&>;?ZJq6gNV3lveB3L+&bD4X6uvzli{CL8b(E5g&hNi-mTgAzpLH-ie))J!C`y@}6 z(+x$VqZ9}Ph%!K2iS18=$g6|xnDwlmVg86RNR?xyq0hhD|KW>Y=pb^-NiRUZTn5{m z_Y81=eemugN^)&ybi}!mEW_K+h71b)&on3fPD$I@Iak})+wJ%CF_o8c^F| z7KG-UG)&cpHlwy1qas-hc>7XmAs4`K~xQ*azT z&VYLvOn-^X8s-7)fD-Z7lTsid{4v3pX7Ep&2rSRMgj&m7I%*5`XM#O4(imsynOgo( zqWW!Bu7mSvpOdHj?L&gUx^4b7r`88cuT~H~O{;htZV?L{>|787fJ0uTu=G%+fNal( zXpo6hte|gLXpD$-y_E-~Wg{BQDslr644YR{7ehD>gzicN>K$Ke#{$|=p|Bv-kfTi4sw9pbLGAqf=-1s-rwAZQOtgV$Da7B_rr^NF@z#Lexj9+}qNGB< zLUC&#qgo=6q0$cM5(-uiT6_S`#{`TGW`?kBVXypc4{p6MSSpPOzBX#uc;~yxg{UOm zl{;sIYWWXY(D<{7f&bkFCkv79>%fOy5Qm2}G)GC!2_YKvB1Hd{hsZV*IN}j4f3bNN zGi^{ixW^-!8Tz#BWW~qLbTB_wKZD=BZw%;qsL(k)q}Q%iBW|nE_90BSv8u{C{f9-q54A`OF7S?T>b=0B_LvUHR0FF(xxW-SfluyBRon%4mJnF$WudM8(bc@h#qw8mhk zZfKi3Qe?&-A7+9?fqHkXw zF9in`I0VQ#^ujMH919`B;Eh=6F>Qg}%I^jsQ|T&^y$5`gSfnjG82mJfBq2ht5P`rIgHV3w;uON`bF>B|@|ZHH=5ai(^h6NDiM^%MXZuPttrlVZwsN`}4CRpvC$(dT;) zITj4tzx#Se6z36l6xzR8kgGTg{MQ4fKhl%djXd%qb*v*KkOI%3VI>U6@M$DDatvr9 z5`p`l!TwzYv8rrqB9`Tyuiy06=Fg`i%l=!x$(U!Dm!LO5<-0yLAHV0ytlv& z`G>E;8+3@J?QY>C{d7S@z(L&H@HaP#O(sm_=P`&JZRC@g%z!r6vt(R7^|ub%CS+&$ zXn1%o+up`gt+=N6Y1Z_XweCHWSt|P@oR44Q=76`L>&;$iV7WgCnx&4DW+V#}c0Unb zFuZwuz4*hV6ja=rGZSySC(HS0;J!3fzxRCFkC*t+EdT-sY{XQ{`o-shTUW92;f^|XE^DG|rD~sswS$2sXtq2hu}$Oo zemy!1SPOaSZ17TaOdn~L%hxnhMj>O&ZhgsijpMOKi?ivz+y-2`| z!MY{DWzH@Gmq|j1`o_=zs1X1gV)TXq26Tk3n0lZ6yWMb4A!ZQBY~Hd^h>$ufs0&w9 z!VqMUEm-K(FzPl;d}x3=<+<0OR!xt#R^anOU+s(O&f0SR=N%;-9rUeKdp$R7M-3+f zPzQ6L3?B|c90HgN$UD0Pq_8%b8$p^cLbizk7jpw5?1CsRE$Wdv>=Yi6VCXZwla-fH zBgvd2J%jr`y6G;Qv{?##S}3daBd4e$ytEzIz#$tKapyy#@Fb!neTTlIGTCgSfH= z9Oq*!Z+9H%^m_qE1k=vEFp0dx4G4j@z0OUGt(?*-%WfNZenM7dm&Z8TBdWQEkNXwe zD~NS_T`}G|98Ebil2Z&p{gm(-g0Yrq1%*_e7aa;kwXlz^+nwYhxYHxSCRXc{RQ zVvu&%Zo`)p?s;8o{cUe-ZC))eo|oRn8x4)&<={Qn};H~X72|Aczc zWFI0&Z9M~t)_|)>Oa1INUOs4D^dR)LnDBD(-gFV3x-E+uUmlwB0efT+9(| zRn)!tNf5Zb|5qk36+;7v)XF^o+&UqB%c;=swFulNb?$ZzWa}FkEa%2*UQsQ1yo8+r zFU&BaIt0!0ESYm@wD^L>jve=W=Nhu+r`5F`M15W)@}UoCUYCdj#Jz*aRzXl-^2JVs z&;=Q491UckMYx{1+<_HygL642;TVHS!$v|$RIpm-q*!giv&_DrxhDOS;l@RYUyL$7 z-VGh|;6}Q2s?n{i9I^Tz(%hXY-~o29$Xv-KtN`8)R$=9xVD_h($_E|1e--aIm!|y- z`+z$Vz-IPnhP;lx^3B8ZvYA7}Mby`6azwK(mJ6Bxq-#TS7qL8Sfd6|k9b`FxpPjd} zpQ6K+^GAcMaTFAPx16CJ#voguFz{``Z((Zb{pH_elY7SFn`i@8hvk z(U3N=Mv$AAArF0-dI|kJoAZOpw59&BNmyh6`mez@%=(QQ(g=FI9}%VSUUIK+EF+t` zaMf7GF(wbIvPYRlR$lh=maF{9yVR_lz*B?R;Wh^2 ziy2#He}W;-Q4mzskYnbUqtSRPZ~CFDrkf1Dj*gD9&vXa-3&=NF&!+Ft^z*pXki6GI z828>Bl@HN=;r`gFCy*DDe*p6O;9-rGbL5wxll>41+4>#^_HmON%$R}3qu3Gfx5sc) z$2yd7g^9d<7WuivR_N-J+#8Scr)_=IGQCj2%8iHi_1eeAZGrj$C6I0eBD{O;wAvcu zF!RD8(d6A(3dfux&i1RF{Yg8}bxlZiRf3;$1jGpIP_Hb;o5}$eojre(qu5-^)>Y!& z>h#u;82#Qr|^PKF|}h93$~G7K2;sPDqHZ?(_E52;i^;?f~_j+ow;^Sv>J}Hz{ z=~|wZ;|~1wSrFwYsAfaJC)KNWq9uU(=6%+^GcJ*%rw7X*tr{XA8*y;~-EBP^4_Jw6 zzj$(5;+pO$o3EokK`JBr*Onr@3Nv8>Fk|9mKbi{ zOovJ?y@FlZA{q;FhRcTEXSb_{Omw`vp03*z)F!<7)3*vL6?ZoF*jl($Ho40k1RZY#wLiEVz;g*|r(q@&SkGvx-u9MS%Bm`iY03*XY zSZFo}#(D>Elwr>Cx&hO$;sxX9Wpo9Su)A;Wxw`O(VAqq&hdWL*e7UnP*7<(4pey7N zn#Z)`mw@=^$aN7NWGTE1FN2`Ly~I?QJh%S+4kEYE_%X%wljci>sdAzzvs&dE&3Ny= zNTrLTEDXJWB7xn*>IR~qaxfA2?YNMa{&IIl8{=A8x9B&jd*^KA{oYAle}9UPFMI>J z4p2VLBqW;|65f#|%|u`Yt%(aPs68#j;`0c!6r`;Z7Iw*0Oc1Y7+G5c|Y3(7=7eG1w z@Tno?LsG9<&*iwHoQS>cMYz#Zh?Gz za}_cLw$cDZaWfTTqRzu>bUXhxoLCLx@?fX2VWNR^P|h_2t301mu26UUIAT^_6MY!PE7w5ORkHL|2iXBC9M+lmH- zrYwj=28_@R48`@U<9G23CoXp2pO9bV^J%bvg5V)u1;%H2Y>hTb!QBP{_rCc;>CXhE z`J0UX@U~nEV<%4!-C_0mxJ{H z>~#aSc?EZ4;fR+DxyYVD?qR<*C+1dLjzPejtti*MCn~;P_NedKh}*L@006J zC4sm94h+fUa8ENEenV)k^TTtarfM^37lMOA*#0SQhm~yOE}Hdg5iPR808+J@vhA9mg+;E`>&_wJvUkKWn4)=Kkem}@%2IbCM84JoxE}6~N6u#WAr#6Y4WK29^F-E_V$8 zzjT~nr0of&8(_*=3E`pYNoSth5>LpS52miuk!9dGr)g!4!htOgEPCrTOfd} z2<;j%`%p;q;TS5qLnHVH%eekEuE{8y15Tj=3_NH2pd*;%y~)RW0v=KibCopiP4Dl- zK`FTE`c6!$@n{lW>pAR2#SNAjEQ~hgj;xkni5XM7SN*E$qQu96vpYSUmZu29V3R(g zR^S-EOcWjJgvK>yODGBvloo|yz(5&OT#_96^Qw+^`M@{z>$$)7^?ZB9|6mJW7T?Eh zn42#E)qFpK;^)OYh?d`|?I(l1n)TC@AApkFJ7!f4 zQFD*Pfn7IR=g7}`kzg`y33M@$7}+da_Ae$C0JkY@h-H|HS1*OP^&L-NXfrllzc(_P zTkYDTY$v*2H3vO78F!XHOJc|T z^{7$S_e)@HzJt>jGavEkXC1NPd*`55N9oR^Kr%94drnET+I&N5;j?Iye53=SWfS=b zYJ@%<({A}TRQ(<5QqY!1OGG+Rr0OnAJ(Aww7z;gkV3;l>AAM+6>6756;SXF7mUbzv z|Bjgk0KWHi-zts_TwnPxC9o-1BO+8)am!@U?UBY4-_7<%UOCIEZ^KH!ddI~cPT|_~ zM$TP!8gIE;x0ICqCGb+VIl4oqqkm$JJpwnd1kZy`b!Z%V*q!Y+zVt+G?ulA`av&}B z9V5Hy(AO==#+&T<_=GuFWNQPjne5L1#2sVI?wPP|#wL22Q~R`O%bNkb-Lr+p_oJUC zC(54vDwhfmNUu8n%$SyLbquGwRO<)2d1n=0jfp0NCwc{FViLw|8g1JeQvrYi7U=fxTaJ zd($zSxqDdK>K{+_ZRo9-zVLIfU*sMXC?jZb$aLpra*7tj9_CeWPm|o?hwi+W$gWAy zgLvJ|{S^Mn%@&WU0})p>qeUy#`S^e0#xuAQg0$r{km!HG&KkC2;+ZaebhJEPbM&{C zuto#-Md8Xaws6a_mTkj2%3s+#M_GBmB2W%Ov?ql!qa?iHYeG9DHL{F_nzF`!UWB7w zypOg1Qo1SYkh7|+py;!gJ7bT0;3EKxi}D;qmePz%FCmKkBH0_*<#3i_!wgWw_IAi~ z8!+HGxO}~FmEln+q@t%MQ{bYq&(bl`XfwH_7xBW-fcU1m6{C#S5AS_TWGJw7*dZ*Z zJEm&1{;huY^Wzz*kiJRU{yu|mukRj_5W1E;Sa~&fe2b%l&Bkcx`AvX>4zubdWo=96>y{%-s`FLEx*K0OS8#EG<$5`@W@t)`Q z*|ZA&8iD+Y;|)nyIH&MOdyFuwf0Gnm2Dr6@n1GD&Y`QuyRPt;$ID=K>%0k~(hdsmS z_rN45VJ#!*<;1+P=hHq3s46$I{aBcCqd;wPmvwJ4Hx0wYVkTHNXE@uq7il2g(I1BX zD3UKhG%!}8-p`0*USYSwDFBZ2gT?6R&58BCsz2hSv z2bRt&B`L=ZOKd!notm9>!@vm<0$JB4VP0(uQ0&Br;O%BG*6(88_{RmEr7R*4TdNhWYBvG5|Thn+`3lAV%#iG zP6I9HJO2F1E4Tz?Hl8_UWo7R}KB#6-9)Gbv@mhl>n9P5ST5JI{rl@E#ZBA$0dlt9& z5}4@wth^3PbfMuJ=2F9p1>?XJ?NSdus8GZ9<;jpO)YpM$q8DOyulfH0`CR-PUMckB zyV)*i#uvn#-Tv#{U|F*7>&bs|Zt)y=*|G&7Ej{xuj%6)1b8e*ui>QJfF4FnzTg<-DU%(C!=?}g_v zQVrpwXU4H%YQ3kVe1&eM?mAZSNFd92bg7K66MXa$rUHyya11B#m*MZMHQGomZA!j# zWZgaQu}frbm#xOcg8Bj>|qjDYt>DJ#vnj- z_mtF%OU#>=eZ%VRNWpDeSX*~3bnE-sf9ckBmerm=Pbb-*V|$NiG=#((tmonGqnY7W zi&31T$SBA@?Ftn8*r9O54SK;b(7GWOfVX<>v+nBU(xWNfwmvq#<*Zb}aQT(XDZeB| z%~dZ(L;H!$QFLt(Z|C<89WJD$LxvfUCB*c`FuVb7@)-yCAju#EltWInf)etW}5u}hFzn-EkG&|jdT)g>K9*fcH^yB z^$ZlV-hr7)z_2U|x)VY_T%Tf)3PsL8kB4m3>hB8Pk-nOMwWAD4ju`?n0>KtBt^{22D)Ndo3#or?OMm`3P;PpT=}T|3_4!X0)!wx4uFNZM z6AS1t`hPs+f1dF;l^I#TXvH~&3=#Rv#JFd9g@hfDU*Z00K(IRh875+xywWLLTYhw9 zy3m!nw3FGEic>kg8_wJmKe*(HfPMJC9u-8G(p>Rli0}apZXT?~)x2)>xO@j%T6+~E zhgWJG4aEw;DQD8W8)|Fn2K5(G!ZIg!m{_h|kBjKrw57ph69E$NgX*x1u-AT0axF!i z)A(aq^!bSV8<-j0xAId-UWdeK9)-g^PNez`@uC26)l)ya6S zmbSdrf5)xDuP$!BjbU#aCrd>qY*n1!wHf}&aWf21eT4}7LCjNFc#n4GOIKe*Du|Lk zAAY?INwi@Sj2|T{*a(W=mAsTWB#yGT<0AmLi}D0hM`fI$li)pGwVXrn=q!|I0N7^y zkxebbK{an7hpe@LsnXU;Tlx3zKNnQJ*<~$ypfHC8a22T{K(55`!P@{EQj8C-8X<#P zoYdG@~&%)epR(leu%&~sc%nDC4d#KepHQ4s~nV73`^JC4E&e7w|A=$}) zq!E$Bz}(b>OGHNk#mZwo1UbmvnkUqfLhn;|E7@$r9bV6{bzx1JUd9n|73tpSqzZk&UajJQ^_TA|)v^Z?W z0SVP{Gmor(D8$rur}c}}j=vd|*DfrZ@*fKU$`I&|ZU%UWP1|{O$U(x0I|hye0b@v) z|E&;u%DdynxWqtB#*<{(3ww&vW!xML&RW>>G2FrK*cb&KJq7`A$X38j4{hv<<5Wh0aSzPMPo3zRmo4_aoU4%%`9Q&00qYuj&0HyY87$UaBGWbl z-Xom`{$31iTBfpXS#vQ!44!^4jqV0UalPY6`b{P-A|kN1_Z5Q<=tOaEcdd%7mzRJ{ z4^nNTj?A#5K@%hjx!Nh(?4UoQ$Rs>q&iCU|PJS|w)l98lGjyux(X2Y!J8;<~ zRQR>EYn|YG|CQZ?pnOrL8Tj;V4ey27F4c;6BzdUK8y{X_B>3Q#1!(xAfWa(m#)Le5 zl+jLG9*>|cL#;aEk$r796WS-RDESN_-yQRM<6r{a{o{iEeic){o@;TgY0pYUWBD*y z;Y8*sGQ4N&>d?7!ptt0E=d9|$J|!rA{b_1;Ox8s{)b(pXr(y$m`9|EE@DmOeJj?+z zQG6skNLML^OF9VK(r@{n@yU^!Kbvda+$GF6>1ZyIJ^6KwR9rd-pmp@(DhtDQ02lDV z@8lIC`o3bm$X;gm9hLf##a(a2Q*PuttF83uY`SO7-z~e*W0myGCwteSNJyXqLwkm< zD+gv9)P|DGln7%4CFWsj(j`!Q7yb0(V1%uJ`{bnCG2bBh6Rn9wdahP)+ZxV__NJR3 z1Nm-;9{?lwmJm6q=<((hQGE)-1EAHdT1F>#P0 zkkrKj!D}k>8mS9H!x~PVI`V{%#2ONXxYfs9n~6kEt9)K13mgmHTo7EM${wkj_^@fX zW2LyQ?^=45W}K?k#dCVw_3-|G8yLT0>Vn#_<0V~GQOHYYY;Y!35ykvS4taJSlcE3C z@|VM#tZ~bD=VwuNN4MlSe@=ev;24~H#XSi4nbTIg{(<;nbGX*`Ejy)Y0YALD`3k&k zTKj2`3NDXYhj%!}xrBF0KRndu;-?i=C5K!V{T zRrDJ(FNJ%||Ea4TO-s+ddMLoV@qto&nE%_Okp)s#4nG`iT<^A~qJP{Z$1Q&mH&Wv@ zA|k5R(0eMg4sU#|NImV;BULnC=;{Q4t#)%~*fJ7hli`-%A$Y%D!~} zanqo7IeE1|@nBxWY1hL;Hu_>0EW9HE?NtMQHK%qyU`iehgp#=cj2A42D7K8+1^Em~ z!bjYO<(b)HmyjRE#woV0xgA})dz^;s`_{zR=Diqx7TI^^qT?<|MMY-A37wEA5@ zBT0-!ey22ArPWN_xLh})-xswIS;A+mTKPJ;EY{Ne;x-{!63xxAAXYi6gT`mHo7_wu z`{_*Jk5q8BR-vq4R7xlQsQe^&$=#U~NEpgTE!uI^K;pzM2N+D;(rS~=hREvy4sN@< z9Bl%7FKS*Q#`WpOPSQMRvw}klU$L-GsDm9i(ARlSRFV&RVNiklU+H3(RUV@{9@Aq}kNHmA#T7o2V z)2V;>o(eGq07B;mu9znYydq`kG>A;LW-|SPwCd3DPW#Y1y*ZZddblJo8;mD&rE%>G z;5})hndgUL*&FQmLDLvo5#ixx60d}pQ~z;+jjQEySaUZ_7pW_wU)8z#|KD#~yzT;8 zCOWo#E*Yanu$Q=3jV8cd%$T&?|8AB8AI>|~#eWpyM+QOMcoWU2eTs)z?lBw#1fZZI zn%w`x*qet#*}s3|L$<6T`!dRuvX-@wX|tuGBxD(7CnlAQOmkU6*(QWas1RitWh-Rt zLbC5+W>g}}Ol7)imiu|TKhN*`{Ep}Q{T;{eAIE)kA2V}Z@8vvS=Xt))*C|;NEHxb< z2`}1?l8l^_)W1B`Mb(aa-8Oh#efxNuz}AMN#nXTjq<;`its)rof2nJwkB^3SD{u#i zDH99WPPizMHjuRQk+Bs<(lg<_yxO1j>1cE~n)0xK9#OCw&axtW!iYm;AmvoXya|T^ zWH-=ZFFl33G{>Scb0L4IWLa)vzuSdRCNp|#>e>9+hf#+mnXkexIre3O=?Vw5xZ-B; zB3P$aKo=R!(guQ-{CJJ>;1S#nm52`0d7@5bsCH-5y7rY;tW5E!#MjXHso8l%2+wId zJ&GV(D(K)Yz`lzu;r82YzubW*(8G3E;S^_|YS0mnSP@)_9Z8nJP}DE|sJSfDEcva~ zB~4tHRGoxa0&^NSi11!^#9+@$!T`kMSLb^F(=owBqtef~A10b_#~!Fl-jgpwM7Azr zhS1=WoC!eJ)vx4Ikkj?;7V?clmzyG}?~Pop=Po%xUutxP0;Vz(+IhE#`gl?Nx4=%M zRcWr!e&uwiRR!`d{gRrM*k0ydCMx;!9pYBl|69f6!)w^RB)`5u^ zH3X$ZxUZ{WVJTl(k@@qvt9-iequ~2VHeYf+n>k62XkT^VZzml5*MLI*x8L;y$8-IR z&3v_B%ap9KX0!R|?|sUmQ@af?A9o=j4^TKK!MaI=X)^r;`OZAholPkaJe^*~R(v%! z-+cj{;AT91KU?wYvM7@`(ly;Dev~h($NBwSiy%7F6PV7p34jRYOgC1$y7^ZZK#ZuWnmA9i`Lkl{17)78V7C(k=&Ki_SK8$=hd3j%KLXtolX6}I#6=e`o$ zKrfHP02t|We+$tQM`~S0a3281=REzI$?8^c<+M0eD9$}{P(ch=Yy?|dp-5x3H7vHX z>>6Dv=EgN1;VNlOjc;^S1m82D*Q-9Q=Ox~nlexk};KBzG#^zsJuMRC>r>-&2atgSr zJx}2+XK%@!8*SbZ_$ zt8-vbkjT9Q7&zy|!Cxcaqov=_a7!-^+CU$jXP!GabjU}p6@R`>FMCEHhGwag#xq~c*r0IF@hr?U z6iJ%`zIQZQB-p#c$y|5mHP8_UW=ZUur=KgB#IzUG`hv0?V24#x_# zF+0x(>h$btO4L?3#_r1~sxvWR*kQ&d*ouEQmgD%TXEd?WD3CQf|zR>!*!3NFk1 zZ^^uW!OMPd9>JNUJl(X2?>O;ndzQ-#4gu$$t;(N2Jy`Q%_YX?c_w4;`KdW{$Wnmw- z;X6b{{jY4&OYe+9lo#{Aj#wZds#XOk7t+XQz&EW06oFRceww|maY!?CV9>QLg>tLt zC#Gi%0l5NO82XznXH9cvgFH@q{vkTCn|S*XG7%q4*y^Gq}; zd{t<6Xf6gt&#s@eMnneJV4RU#Q#W7uUpi8rHCl$8Akniq^|og2@&3aAlSKVVsy1Q?f(gk#Dqp9W47+hGyjd1%!mmMcA~1^Y=vwvu!oLt}5qb!td2U{X=}p zlCh?781TZam1h7bqkGuLd`O2fq`6XOKZ~v+=_$*F8Y3Bk3x|=}-vY?ZV{ZGy6BbOjOC1)U219pEKSqNmKL zP^W!#fe>h{mcx6e%;LP@cOEsfnjhsWbafS%wIU0%15bEZE26IMOz^vLRb8_05BMp` zmtY|r$oH3*)MJD z*)=KIv!1*4cP|~)&LG$=W9T;r*xP@2SV1%oB=G(c{Dr5#j*9?!|Fb>Hk!DV_;pfC# zykv!+q^ec<-Ysy-(5~p3FZ7X@ix`8+t!acdp*q)#*$Pt$iC!@w0RHYt{qjx3&|7 zraK|0opz<(FCaqs`@H16<0?GvLEDb71GfC?-lY1+ji5umuczh9jX;5VQ;k%Q^_a?y zfK|4wM9i!JS_RLiH5Vc>hlPfbk;dc1dSd+9UeW~6w%5J-W!j6VF z`RDR{uC%t_55Hvm^kfhFh4U(!t)QWt$ov8~?n|wch!+;|j)rPhaRZcqEq|Rd!+7H$ z-kdw^gVc3-_$D~Y7P^C%YdtiaUS|)S8{W`5mp}Jo?%ZA!5dJDpZ!pkO=arWdQk&>P zGQ@!iG~=cywHLtVpvI!x{OMV=c^v3}i-Ek!lS=KTtgrt$R{0e09@UTigBX1NL#mxrJkLgx2k6(+%be}70wZV;UJ%%@`fE!y@~vF_w=twFsK-i9%}_* z{ujOZ>UvI=jsd9qIS6i4v%J8+>x034fGhXiX()Vs`X@7Zuqha#evb_KmCTybh{f$l zk2Cc2ZA!0xPJN>#wY^#fQSIM@M+56ahg#*D^hlF%jtwV%hQ?qOe#di>V zE~`Xz>hxLf{481s=6DOKrDFm>3e^N^b=ti4t!VC6ZfEmOV9N@pZ)8IB-d~pfVbqx! ztZB$u{92NTO;*efMmu`yy<^_6HsV1Y&XEX}@)_oxz4|cHL$>{esx4eV~K( zFAV<8H$#tOxGV#)KMo2q^T6rL-(+`C9(r1E5?9#(%* zT+8++OWS-%afnQCRf#z)zM?DmQ56{1xCSzXXd%NmUqZdpp-@)I@6QMR5BdAR0jjkPQ@` z!3;gxyWcJAQwub4k}s{#!+dTkAYXSlp}_cs)S5mgQ-5FHGpD(4LSDDu@H_Rg!3*=3 zaLU0eeA3&>9j$BLOx4`O%h=yRO4U@z?0olz{xLVq{O@(XwY0f4yWj`b598jR`3>op zNi>K{EMWeB3F6f7{0(Or5N0rRe`X%*Fvx{d2UhZcgf>BkFk6F_#FZRvHYMa;Yl^&P zzlgDux4f+KA!R!y{~@vxSr3>_3rvvRj}2zaj&KihN61qAl@_@bv)x!~KNyV}y1tPU zQv7IK=Hl0`J8jmPJ7i9LsW;VvwuQ4wKzMQYA4JbvLMM``iHrLOVZ%Oj7#RxH{~qRX z{Nn6TpGeUih0_YhLN4nb znSS$Uox20zfLINQKUG+9z##@Xt>J>@lsD@T zfl#vPM7Lfl^a_&@ULb=@`t1Jw=IhzH>D(S>ynch*fO6DK0Hob|b;WMKc1J;3=@W^E zWIrUV9>JE@U>d+AETfMZ$hpThqY`DOrcZ_VR$5<+xMoyywQ|awRMW4P`tWM}sKE5c zx6>aaFf1>q)fBpfomrl+s^yT#{;ASk04Wh|)dNPBB@7KN{D~HH%HFDeD-m&D+gVN7 z$M4jX;PZ_G#YSgD54D%4_=eZqCEjiVDU( z+MI_il)f_s%){P}*ovYfUIeFb3dx`6@R9#Oh)aGiz$nj*LC_lxhGnfv0FAk;-VQzZ&Hsd4*0yW!C>nw35G;K z{KOD9`pr5Wr)s+%mcV2yS>&`^B(7rS!$s87BO#Bao}@okhLoL{B(@Ux-M!FD7$m8@ zoq%{k<+m*?jb}-FwtNB81q8duum-ZVVDkIfT&{3<(Ne=iJLl|Qp^NnX&R6+A9UGlR zV}}i|XZ9b-)=g=2zTqO!%EteH7_Swu4^fNkTx1KGg)~BQcAnsd1#Dq?Y(6(c7zF?K z6j=fENMSoLlNUhv1%$5+sP8PkuVzc}*Vs#xpYVO_wvpA7WO>#n{mUg+w@5$1Fm^Hn zwBv&}5D1>dt-HM)MPI7$Kl3K@{M*0{F{So~l=Bg~zL$nuk|D*SlLQ}!JG}RJNk<&H zLLfra93XrmGsPJN+-LhBXQ~(ia)H3~&*@~YkR(HdR+7vX1UB?F>?jzG4~{Bs-~Gay z6!_C^etA5sdcW*ISeSvS)s^@*O z>mekXavv!O`HN}UfMu^5bG?&` zbW_@^EAi=e|JN$(YdOENky0 z3GQPxKp`;zV-JBtO5!Ad&Q~TE8OJPTaF&C(8aG0;+C7zD$u-=qb7-xcQ~Ca-^&P%y z#cC0?CnU}B3l>gM=L-G#2Z1I_lj(ObTpLn91pim>oaKLd=Wt-w%sMl1)7yDTNJFjY zje=<`w2;_=7J)M<_IKFYBOc4rS#N5_b$0PvHibPl`1ny+s0wAfJy!DG>Lw!ipYXvs zJh0tW;+k`2O{h2Lx`%!1&`pNfD2`U$?XK%U&>+XRO-6qD-1P!QH03t9n6~I?XZ{k2Mp?hU{Te^)xuU=WWYn_&&YXNpo#*UnCVfkY zwz$m60tc12dD*d(xwL3_jHSoz@l>Ugl5!yX>|T{nrR1`QPZHcCV-2nu#_skq&N(b! zroR}x&z#8K?#LAZ(F@;SaJIcbZW_MGI)P(36?8Pz76LQk1v}fx{Wf+8du^9B_iWoA zGoFK9MGNv8D~jAx;78Xa{+#@sL|(BQME^P0oW)JjF`rgMnW|Wu2-D?3I=o~rlUBvkxhb=6i$%SV?gJTyEw~hsKs>FVPO1_Ac6+X&;cbEwhgBo?OU;>OOcs(k*Ve3xg4Au+1b3>W)ca7) z*n2nh2AK%a52IoA?lzK2AX=(Pj3tb_FyT75zVL2%EmV)Rc6RTA*ZZ$~*Hlef5!Cx> zrp}Ys9RxurgnSLHEV2UFyWhY{plty)+~iMn%YCmqe`4%ROBzjfR(YsLNjv>OpYCVh zT^@rOmAwQI;QwEc2V9>2V4nAQb(o)yp`csyG?+N}7=6TM5HI;G(Es!Z-oQ7&qEF@3 z>^lWFi;Gu3ejIpPxE<&!?VG*a_ZXz#v_vX*#FTNC~(Y+MgV<@DEf`7NgrFB{-pys zt?PxR=5>ubJ@kH)Z}Yp7_P*2BN_hF0X4T&=AUj;XE(afLBX2X``vND|ZK3jd?3agM zmz$2;iP4bk;HbVxhoW?w6yXN_4B_-xv4Ek&NJ&s;I8Z=8zW#UL|G}q!1)0Y=x5+-3 zNrTqE1<^n*1ru`t9c$Q+^kaK24+TDVMKwN9v(&dUXiAYeGF@C(`Bjt0PhR)>e4NUV>ypB<^SIo0(L;{OmdW7oRpiiEq5Y7_C} zvv&-+#9veWm4%g4t$Pp;zBX8TNxU+GHV0m27RYn=x^buGw?qEl*x{uhewK-#gCWlS z)a=K#CCnnzw7knXNr`KAp9fHv`%=GG8L#UYb2n=&HiUh6dl2s5U^_Dw0aJNdRc&m5 zXk=&9JnA>sgXRBJ)h?WN=dPmW%Tff-a5VajnclWWqFlzHGk?9%Ga}_RJ#G>)KasaPS*%DfdwDUp@c>GzbQG$8mEc zP<2xpYYH!QhcWH44Z0=NRVi8sJ#fO=+Cfh0xWuQQ=690!nAv(v_MoEC3lVGuS9+r} zTuoY@7X+*$W6){UIsW8lK(qXWxvY^=g}!XNkm@4qsPw+GHs|*!h0b=Sxd872v85Kc zVuyjA-gd&IrxKuxs}rWruutT+I*M7hHwYwC(~GrlChhxCsd3Nqv`?yjcOTEynX4VV z73^8|1-O=>wLtEu2wsru(o>v=lRLOTDStNj!sw#@sc-{TSK?tri}1deU`je-3rMWO zCgcUd4$v+_29ja1z^#fw?6A@R2Cks%Mg#KB)9T#IH`DUWV!-UKp`of0zAL@u`wpvZ zVX2QA-O{iX3imiWH))%fKnp)L`8_57W)ev**<8oGOw?l6$n0$FGaZ^#$1j9`CZ>0t ztwLqQ(V!HFQgO4p4U)#qJ2cTJ;+ph|h@t^k`?;wlN@aVVzX^Z_uc+FJyMA<8J@Z&I z<`4Ar`=g13@G`U%4(r@7vY}$JY?dp+Qc6e)4lMNQ(GD%rH?0zt-v(xbAh<)o6_bGf zKQWL1lBCEfcfx?@0Y>L~{Kg?Ims4O!7pgjLz{7&0o}n{5vwkE@$y)o4hLDJkCF>X;)5B0%u1!J)4Dqfww)@&C3nmVnj5_QK3&mr zX4jW5T{&3ok$nyb|MaOego3#+^Cbc%a<_s^7vB*WI8VN>oV)ly7IHb?coZK>(3qKR zWhl9)tG0#n;>Bi(t=zx%cv4>j|2pYfTcsQ!B`tYB$^- zC~^&gL4X_{DbaDOV15@gB)s1}c` z{oP?dePfY^zVl1Wf+oQ~J%tObi3%|WezKx6bT_8~8#Y&+V!O zLYJC!OuqFuz~}Mj3IQ+e%oPSh7hw%*@p`8lDRFwS?Y9rzK1p#)R7gm${*2|dkZYmC z-HYXSLj)#%N)9cYm49&^YSf}vb0u(a55v9bbSMnr1IGCsr^$UBoz^>2AHv&t=Dq=!tZJkkaJ0dC6vwvoYdo~LA=HgYPUf|?%VYt#Rs%{;^V zyl}A-v~7>zS|N_y7SM_8_s{~DcB7_V3i4PSrX~E# zm$|}7-KV7=UaGA|Y8iUsy+>tqcm;}`Vvs3ECP-jfYw+G z5)wQLS=$?{5TY6OC!hgf4ZGXWzYLeiunH2M=qR9HE$NcD3*=cLN(z{zXC6t9>-w9R z82-#K)ja5t``o!DY{biHH}{=yzL>OY-+g)U~SVG&ay`DQ+MU=1@{$GK8<<4^_L3^h~nbX4rqmD;wwthAxL zih8AJ_r-wjUv~e8f9JrkPWV+I5Br~?r@h-^=lyc>lz)`s!Ly7L@yfgh`N~wScakRb z813$3P5bH2U8=hVu@|OUr|Hd4ulZhymfrqq#^xJERbPI~Krw=_9e)lS)4c@bJl+}_ zUneMl&9l{RGNb;fRlwK`){=NDR>yAhzpm;zFnZ~ytF=;7Es7^SKX{%@$$;p=$Tsq% z2(xJz9gJ2XOeQl-9O?JZo}%8;u%=UnpVc&_E2@Vbl#3AXQjZWW*wg2;*0SYK9(a`BVRd!AT)J1yXuHfI8H*?Kq_^&}?WH&`RI0qPuMH*T5rW%XE9P%Ad_v6G zsjlv)LG`Ni%g}CjpP!?Ia8V0Q`nwN3SS-zMD=Izq*)HIy=m^A*;6IQFSPD|JKHCwF z0hTupPNuhad76$INr(e^#KDy&jzs6Z&AMcjTxeML%63C!Ms(kvA`{nG3XZ-AM zbaIlpErlqx#5JIYzKbjl`fK^ixxurf#J$EuHb?RR?rYNV9qD7E`c-PNJllCVIZ!KT zrvg|-4EUNVWHnToOHb@XOX}fwkl^6+ro`K9yO2O;X~phv$=+%=Y|3e(e3wbtUPQvO z#FrfQ;Rzb_*ES@Pt7r}2PBCGQ(9S);z;vunR#OE#NUFj5K^GS9fT^00<9N72meX40 z;)qvvpvxX%5wq%T2xU6|%~S;54frSq@EjU|b_BAyAPi{iB-8C@tvhcPB>!5!YkV^W zk_zt;ajko?YvV!n<5oqpTUq=KFZZ1HQ;LxFLrr{5yo086u^w)GpK1bEpI^-`WdGXd z5Yrh4*xf&Wj|fr+m2Mi1p)}?WN;58og8(2t8=_lbU}L)$4Di|0N53;m7!Y|? z>l#C;gV}Jd&O+ACUR-78%xTx5!nYa6Stq|zr-Yn(oE%(MFP$STi*XyDE}gw&bjWXb zYAQn0{z|FMM(ZKXSx6C0zk>Pq%F1pc0;JMKI1>O!S7N)WQPUv%kl4yh=IN1&&pWjO zoS*Id{FtZDV1M6!)E1UnEdw%1Sg({F?(2{>II;LSO{>Na`$TYeGl*|u7r?cU;)1Vi z^&%i~wj1lI20)w@6H{5DC~-Hxd$q2>@9q9u&*p-?4UhbmzobbW%gTJS%l?NJok&~k z7>FiI{jyJi1kHm>ao0LkmC1Iqau-H4_V)+t|MI%I&C2P1Y$7a;?N+^c^OiI6WQ-eTvw@rvtTGg#*)^l}$RNsH4XpEP-o+jNLQuVRmuZqdHl z4abqbU*n3M*}QiP|AqiOqN$ zQVNFMwp=%tr}iAehAVY|OE3TJCm2j?U}{ zaa3G*OY`IM;GJdqFL_(Muk?^Qh_I%{zk3gY2g*~lJl>ZP-1Rqs{Gyd1?J`oawz0MT zL5L2?FF|PAb(>*BC*dbNDI2Iq1@u$IP2T|t66P>lo5BG<2RM%n?}b&(p7rZXb%xm! za$(2JwM|XULwkE$VkJQA^n3sirPxXv^b(*+PMa6PU&c%;ViG)!>En+L4{jvC(+)B= zO}LNBzaCTZfqwbn?nlJ1#P?TI3I>{MbSJEyq#l)xt zq3Q9iY~6Z>sH%c%?=%#2>P67`VNZPtvR)=%Sefw+U3w$FdloJR={Tqp1MkGHG%f)2 zGL{3Fi0vJZn7NeIEhinH;*z3Tbyl07Wr94!5@Ff4;j60;uIfdi=2L_RIcxsE5D zo%yaUHlQ+-THmKE|D?n~XgNw9>p+T%!6~x%>t$Y6T z`KB>9of(19a!q7%lhQ?n0<-&(yaOBK$6O)ENUV zkwO+{FL(`*pa2O0c!6mNa9v-Q>VZ)s9l_Q)-70&Lz?@7h@ub~@-5J*t&7T>R-g-MN zpMlW6%ENL5q*i`64%TY;q2sx+)sTl>4({%nvZ_<&qvyGM=>4C*hXm8Kw)$MVon}_! z=lu;~KJfzr?o03x+Y2sWsB=YjF`QfH4g&o7V#z|Z;nfIUtHizk!Sz1pkyU>j?MT~N zw)2fcF6kLp_S18s@wXN*@%hHX%_#f&Bl?v)|?z%V5NTgeH~49&y& zG2|YxvQEp#?f$l!N8md==W(vU`fWXMuZxIN4tZ*zyF3I;(Qd` z*p~h_&0cz`xA#gf)|(Icv6%o|QY+p8jsX$V)?17b8mWyW;}JmEImlHBk~6g;U1~WM zxbNiFZTGIUYY5h#!E}s2EyXNP{9JCR_%&hTY+zV#Vj`83;VPf;@k68^h!{SDPJU=U z{6C>a;D17m+NNHr{nJ+3Lx}Y4f_vM%<9LNmk=7g3Z_V38cj=M1Kh8fvElc3?=88Q5 z8)a8ZjBT(^GZHe_ph+Y)q;g+TH*$^ZP0kO5sy%*6v3X|Aldx4RExl#0zBPkc>piKpKE=xZI-?6^p~Q zdAcE)hrh(`8?ZWW<>TY!Gz3_dmn442ALi$a=YIfE#EXV7;Ff{`tKb{9f<@lP4uXmF zs7=go^zHJmD0!n>npjPmyq-~+h+Vw3@>85fmWi%H#1|X2Z9L2f_HGNwe-KYgPW^*u z`SuT@t73B5l<;Efn-M+gn-t>_lvJX_Eqw{kV0bI(Qvs~+3~U;cQ%KNd^Fzu(5)fsG zg4-CSFZ4HzAF}bBX=N5U{zmOTXr-0fqPw!{j)1JLgC5c&(96I_=Yn-#l(t?jzfuvV z|3qU@D(EFlZoAy-+^HCbqVPl^(G`#(V8m^@!@ni$?FOv{yrd>L>8*o z4kPEAMl7+??(QdEDDL%-xXiyL!}bur!~YdGQT^H5>zVbO;&7ms-%aW6_`aU#pS--p?65d$Vn1stU z^G!J1d*#X%Yn7l2Y|t(mr`EimeTIQ)TPM!*Nry^@%n!htzZ0O# z0Bx*N3Z~RMnoXdgTCD7R(6sYib9KMa%as*tm8#d}Q081k^WCRcNijd{K_xb&k{G{QJL!=(qyt9|uF@ck^5lE(?a2^X~2 zVqS_yX8@fOGJBgeSNK&qcNQsXgh3K|@=xgtt@Q@`!hCabowq;4+!@wPkhFbN_?EuN|#z;u5#P-0yvuW=Fiw2Cl zN6(IY@UOqUAmAtV|!w37``6x`A8g*nF-rcgH`N1pyA~k zk9Obro5`iM(Z1hIt|JP5{Lz-7j^dr{QIkB6Qu{G`ORegx=C^4ZeGLDk)nB)c ztO7PCH5gg5^!L8^{rs)^TdmHD3yGF!wrsP!1(KS4;0K{t8L0KNxgPjwLm0y#E|{!z zlJICcIokY~;GOVP|4wGt^xkLnR|o^*`aTbPc+QE6yWHb3;@EYkS z;P+uQ3|m&C!*nApMve_ik~IE8jHhL2{e^@FJL+#Y#?Nb~Or{E4{eVMA-E?L~St25# zm1A`4j)H1H(57cHuQJk#{Ugg@LF)5(>g?Hddq3l!pIlvzg{XJA2<*9_56Cx?_q0Z} zC`kA~J6AZBQOMnjVMzv(rP2QVBB!C8&Srgv4?%UfHuV(WtANgmk~hKWTlTunU*?x_ z6BbF>5j_n7`r?1TkmIW|e*k4cU@-G-}y--!e3{oH^5l%eoY-k8+6M+2bW#mc_Y20#5)RZ0Cwdv#h{#XmZ8b zCiL!=SH(*=z{Ksm7eU8y_2=sCo)%qLB3}yhP8&bIzX$b|As6-8>aLoQ>(~N>;NBur zJ0Q9X_OEqf$G~QJhv~%11-nWEyK2gvfI#y26DV~7GVd+4-Jpd!+atvB!f?$gt8a0| zoR`y`YPC&XMb-ydJG2Y0*c& zds|fZJpC*&87Tr8R#p6@~=Tlp5`#Y^50KjvOUiqrE<=H#@ager!w~ z!G%az6BuN?bBc;v5Ga!3j+v&exX!U4{T4 z2vi38q=lAM5rgG;>F_5PTyWV2Vrx&s0JZBWH3jnD|E%TMe)SxLzaUZ?l-s6|^b0d( zQ{?=HWzX}!d0H|-OrkMa)6s)_5{nqNIdAcDoNMbj|_nu5*v~M za~Jm_yU=&gB9Je38~nT7(+rL)$5kI?2;SB?)&Ht)lvL7SgVGQB`XOHrhYJWHy zZ7aix8TvPGhy>Ta^*Y%bf|AMgWVY0A8fK-BzT0ds-np#3WNt~e{^uXt8%$^B?*zo; z4o}ONbi0cY`k!Mm(An#780Z^S;-*s>fbz-*ak8L5LkB?`kZJ|w2)<%YE&5Ucl6{)< z3>`EOFXt&x!w`rUk0}Z;U4F2k@Y!~lx@P|LUR5-20!sGM6-az5vi=;9BmRH-alF{) z=y~fzf;=v>Wjg$W!>8a2Vqb1c+vnZ+d6-dV%yWQ0=72&c7FeGypr8YrG;&3@&;j-H z6~zX9ab4j%W6AV(_Rx2NY8A^e^+%c4DZc2`Z|!`_TL#n=|E_bzeuGo33SRyMK@t45 zhX_dSAX$a%Hyca^jkl_w23{`TOiZrmC(oG7oZ@wxDecKPI-Ib=`vR3_z&QZ&RQ&~P zWl&k+1CTB@1s??dKE+it%Q<;u|9&@`M!W?o@UJ`bDttmZ%s59UEyuT{4|^j~z}T<; z=a~ejjKe&i(#q%sw;@V2hVDXy#T;Pq)GQbI)B+( zaZ!6Q-obVWGu64Br^01moEd?I7E-FxX;S zoymSnnf6R(WObbK`y--7-i5qJ1jsK=0F~$2f2W@lKaQYBo%7aQfgVQ#f+_SltfDRb zkC6^!=0^XTo-?X~y)anZapv(>b;rxcC~HcilxQ1H7ElCF#{T#7c5cT5%5LqL@ z`upJgDa2LD=QDxaJ>BnrD-W7LTd!w!I zI=OqyVXL&u`GBqpfKGbqd~9dDg%(op8yKmI6SB)wIVm!Wz(m&^KA0W1OVx& zRTq63GwF63CNT<`m9RfL78J5fl24CZ$A}00Ib|(-6>>Mvk+Ra4GB${Jf1J0;Tw*K$ zo2&%FPyU{~+Un$u+5s!J_3mu=Rd=iPlfQhVUdUUMmBsq!M_eVZ6!-%j#t zVeVP3G1NL+KtE4bX1h!@xq{HBS&42EPYM^jCD;~7wG%tOlwHWPGpg*4Xl74L&b?Jp zXr=RvT+ki?FnABhIopC?Jv2zBJVy%yWmYdv4%h$$&b!58Zw&*Cti!cU&KyG-XR7Do z!ZJ}mlr^8ImfJX?Ajewv+~(_}DU0jyUTY9H?s0Ahgj8k`Lp){y5(d3~tL%e1MHZ?x z!(T^A8nUFcCco6(k=@r$Pii}-aO;dcgMQ4-XBx6I{Eew7NeEh(xEr%JlSfi*DHn(L>rE#Ey=_LJS>d&aEjJEp@4TUw?ucaYX@WOG-gMwi8Xca1CQjf}>CkETGa?lAhlId_Ql9UPTn zuzH}p5rml(WS(cN)w0gvP_WxMkeIV`>o;gODYxW2mVMQ?HR|?~!VUb(&v|7T2zB(! ztR6fbE-KhU;KxlY_&ib=ooMAsjn>wUSPo=_b{3as{;}{IpK9!zX$YsQ)4pAa;l3ram_JY4 zj+b4@*j0V}J*@WHv}(FXO)v?34)o0)N5gydI>DcR5NRO88n7N+P@PcA5Ck-%IUu4; z!tdgFW4v4IfR5+fm(!wGLvL3tmzVFmDuh(V1CShtCsV30+nOSaaH^O||E>nbpoZJ$ z2g<)sk-v9a$_OtbwZ^A{{B|19{#eIUX|eobO;x9*$=q_Ldx08g<&3#3S} zQXiCCAwJ<);PUp-v{I&-(t`&_U+zBVG6;$6_xr@leo9~DiqC^%(E($+A6egNieOl8 z&X2rYD^v{T;f7HvEXIuFxA4**K8g29zWNz^R>+(M`c4kfy<0CNwqxVww!`6N9>)%i z$56|}jK4ofqQaTzY#uB;|P0_*jRjSpIYE4|D$M zi!B*{qUa_e^C=X6g_b3^Q63r+0JW*UE^_VsiZl=?@`JHazE~1&qSd6qGu^m&hYjBk zjrS_iyN;+UbSscPS381d7Y3_-$?)BYg;Nlalk0FzZCaOnG-D%Wz2mIaXUJ>P_rQDg z(y!!7?94nfw~3Pic_Z&awWo8b2Wn>ucwnn_=lWrdhJD4O>ARg2+Fm*JD71VOIPe|w zK^H`Fg1%5|(0aI5a-Sa2Q?WY1b(U6>NM%0g;mxqI)Z2OqL6Jo>jbtY_^aOv)*m+#QF0w*5ApyGt_@(-boeIWS`w zL^nLoczH5o>k&TcebQNR!8I8Kq>8N8V=IAhmH#UIlmSRzo&pnSZ_;+yw2O5iadmDF zBP6|Uq-N6LPsS<3I3N&v<)w&$j1ym#3@332la8TZD`CqnFe}0PUkXNwHw$rlwgsA6 z)zUbS#e(Z7-tC2}C4R`QT`xn&yZ)i#{%jkxLnLSu=781_ zDG4Z%w9Hr4C$W$CmbiZ#3^Mq0=2MUNi)qXQ6H0uf>{Q{=NsqP0q?B~_xBnL6s*(#q z?-zP?enAi3tHG9r8e5>+oDO@TselG%O?~!-&dR*~0lVH-H)M?4F7Rzpd_SbX7a5XU zw|3bt8l6?}-~2zJ4Qd4sNVi>|@n`M>G$%b~2InPL$^({accBmCuGtK3Z0uh)Im>x2 zlJiDVbT!_IckPmi!Nnuflbp_HP_*#os1aU>@*M-C zEfzxUTQzA9dq}EbHN!mh#f4eCwk?fHM_R5)^P^rTu=oE@*(GKs(2P3@-XBdriJhKj@`HdeiWaeo zq5riP^1$1T8oC(Ti!2|nEX2w_1$0lIngaeiWo) z#Njlc?s%tf(Gu0Kdapj*B*;}U19%vi>!Lt5z+>$u03QF)K@da%c`;vw!4F73Xc{p& zq(-a9uuL!Z=d|ZC6d3e%ZjN@#?KJLUBw{+lCi~~>y=qmlM|p%Mxev?I;Jqw!0&tGi z&I!7BY1TouC_N(DFu0G|+&FA5P0c-}IUe==cxnl*!uxoju(e{2dYH?ozwrLaK^{vP zh6Y4hT2{#z} z=>sw5_9cYH4Cwclzp zAe2Xu8{cW8w~7+mK4{NZ7WY15uX@;+tIZP$sx0{yMCnG0!?7yf%w)#pX&G9ev+SH= zP5tQfSZ7ncWZknLM_+pf+q}>|*(-iHeJf($z=}27G=jU;7o@31IGfRB|2K%8>{(6E zqyT}jXjPF!D!GMtyIFy5pBVBr!v!)#3Vk#F8*=}uS!jfd&d@f4{iu8T_Pe7^gEoh! zX#E7xR2U=y1my{xfXF|P(g3Y+7)ouRE4m=rZ=`eWRWZ&=3;&IA;C$Rb)LnIt;l0{2 zKbh*zT5u%(j)i|$F>vKPB4`KRh%ia-jCf%$khk-~@ek0JlRYYZX8e4w zocQm^Bl7p~g60Khu;ydH>E|KEH}W)&HZ@caH|x&yzXWotjotoAOB1iuds`$GYtrtB zc*%yJ>ER_mROF6qD$vo-+E1!8gE-NbKu2hxl80IMyeJ!beu0!|e!#FaT&U@y%4cLi ztPtgm>bX1gugAVZ7mx1MDE?RA=`dKWkl!4&#C9_5*M#I6@~o2SKEy=)krKjtvV!@s z(g5eGz`r>iB0EYjr-HtV4W)f3k2O&bUO@>=pB}afd&H?GWr2(XkkBOoA(oqj9%i#9 zC(&FmkU|oNgI0gdtS8nr>}t*fE#51gkB-Qt2bn|h&V27hBOTxT?!V;>f?wyai|p-j zT=57_0-Ev|Qxmed(n^DP;l6HF)X?fs69D}}Dd*=-XPHI!Vb$I3RF{nPUy0n?BKr6d zf}>7qwTCV2+6g-w=+MdpjTIumJhdBe-8}NvUZ|AiMn&CS81@wMJZxN)vFly?PAL%~ z2&J|ymFN!xSS45hW~<`E86sA6Y?1~4|Dx^9!=Zftzu}RsNVe?z7Fk;?6`7DNi3lO4 zl8}&)5xI)&TL`6?vP4-Xdm@v)NU~?0F{sGo8r8TmOZVyX{oeQSyYJukx&L^M=lNqC z9dk6-a-QdVf4^TZhAmu;d%hw#4IpKmV>)a+4ofe5%(?I?LSOs(jpyLLSN1KqRtIS zzG?mDae&=SURCX_59sv%KQ~NolSoZT3+fju0PvXS@iSgk0YKNz_hlvgToW3GS?Il4ieKGM*wi2 z%InOceeFRBx)lEld=i0d&Z;zi-+*`%QYMXI2o&YGROSmQ^j$=_bOeEe##WNh-E z5teK>kQ|9gn?qzk1?Cd4TD-k;Rk6Y)zizR(`de4bgwT<%3L;t?WdXvE1mB z)|$2^>(w8W5`Jm-p+h^XD-X%XY}aMG0#Ky4{qtD|nv3=kv^k+z11x!FIoj9uhK&xt z?|!v)N)hoWrN7|iv?-xmIW0~?K3o~)zdrq+1T3$NZ<{VO*~<_Dg)0rZ1~{jHCkBrs zkhBULiu{d5o2`BW7!_=m+Bj`|uR(-?MADo=x@|g}^C30RI{(`)2?L)ZBC2aWkQ9+& zi*(X?!MKM{lAep@n2`w?6Mw{s`FlZK^NM$2}F#mLG64DL-N+7 zA>@hnNkQ{{bIHFu4p!_}MN493fTwfvoW!{6v-3}K(hm(4Jb+)6CZU8eCqM>?B~9H0 zq>3|n{P%#PX=`Kcm=S-G@xzFt6(ufLa#`;6r!16tNYB+zi3>!CF!dQC%tSCq%s*Mz zwIOT3EI?nfiMQ=2LvP$!YKW65){)$AdLt`2_@0K%LdI$p zeP8rQ<7QYHWiR~@mX|E;X7+CLq~~GyR&HgZt^8?)yB^2M2inDg&-wTcnjDks;DiFV z!VSA!NNfihwi%rK-Z7{739tLc&JNL=?~h~}o^HUZHa^t-#%`$`$gZ?)yE=4CKHcBU z6sQ&tsQqA+PSqrm2M*dP!X398rDYzLA!RF~=pn9BY`4{X}d+kFDro5 zzA~FH{2)d0RWE*LeT9!bp9Zm|gx~l+&oTyWq6DnUu9=b)_}JM&4%u( zig&rrUpqHbxA{FZ5$w5KC?2cN^{4pU(tWu%(fc7lHUkIdHc=s;uGc+Hz$S(c8HGfI zRnOXlEg0hO%8F&#-mu64p6Lwvvfrsodrr6h=nu)?EM{#l2D_gJ*6VQeD5gHJlzu3> z2UBw+8zZx)t4oAaynm}>P84-9QNE^Kj@kD8ZmKGQ6f8GoHG>R{RO#%2RU z(mBUVZ)|L|U!noPJq#@4X5j;k~pPz5bC*6Cgu%;57;^OE5B7gm@iye_R zEp{A!`)1u{y1cR^L_exr3OmmOT88=MQ(c0bbos(9si)$+$q@e)!P(`_w32XVgKN;^ zl27lR4vlVbS}A>Z{>HvEh-l>{0PzJDoIv56C-MN%EPNUU_n&t6moXmul7CrnG443$ z1CsFDZ+y`1F(ykuT;%+ArF;IM(1oZZFocEOMyvG-d{{erVaXrsKf?1@ete(jN>R?X z>Z&uWbIbZ*t(^T{RhNxHf!ZHKAn@Y?aUiF6D^(^wF2WdB7Ns+mZBK|Fo04+Qb-y5< z`F1L<@1=cL;8gHqNmf}!JfZ;PasX4OU2v-yOb*r!-f5F_yOLcwRe-<4Q1L104OdKi zv0hbarF{MnXYDaE;V3}tA4HCPrLi*;0GA3*FuH%8VMHZlgkg3-=Nd8+e`9u-`)aQK z7N5fwYBJ;Z|-_?iy4f839`5IqRs)dv~E_ytT)#;zz;s%3{=Vl-DRm8+=Hj!Y2j zyfl6!@|B_i?j2OHct)pzms24;@T!_}bWzkLgby~LCBs<=J1qv;KaZC2)d4iAu_ zRCEo4*l6bc?mlt8q1Oj$U(V;;;fTdpL7im~DHp>0N<-h4LoWY9v&^ffX?e_IV-8ja zuFftI?Tb!*-9Sh%;v?PKWAIu(m!Nt2hn0L&ys~uNJF1za+ET%>Zq!X%5#$Y+9j<4! z<9_)O!~ljq7_|OQ0Z@QMh8REX1Q6AhcCv>CDK`g?4eDwWoi;9>x7*z@@C==#Zwhsd2fHB7dbv0`fv6C%1U;_WM_qH0 z@$at&K0Bo%81b{Xw&I8F^s#_u2@c6wW)ehN%>aW11I$t*J-RN`QBR$?qsK4i`3(2e zUsgT&)8rte^uol~wsrV`QS~%Pve(z}SZS)U|K5{~__KI`pUz2RW}rz~#{1xnPPvpR z?j$1@@@#ZJ1!(p(gr zqF&!VXYRV^(#ZBS>!n1o1PE7kQss6;b0H_}LY) z>#xF=xyRboTT9Oi3>X%1dA*f*Pw@oN8vm9r9smuSuz;4H|2za-;XSvHb|m9Vu58*R z>OS&_G#S%cs|>0dCil@khUID`7MLj~u2_Y6%g1xXY*%c>uw;XPTAl)4Xe?uE%Biyk zXx?>ORAih;A436IhAn9^@F?pb!b{EQXfqh1;Pd;wI>QNx0Sy|y)lPy_T)DHe&d0JW zx_8Wonjh-i`4id=1>ipSlc1DRn7aozwJ@J?SMQsG_2nU>(+d(?*$TtRvW(C~nw>R*!fu$OMxj zM$BP+hAzdFQgkvXn6LHYl8ch6xWtz>J+K(7z`s!-kRF5ZyAv^6D5Ip&Po9kuod`Xq zDba8+qXP+_GmbOk zS}{el*vL-VUG=pIZ^iO8O9tl0>zn9$EytCWtweWrIsMG>BIYx|3nnT;mk66jGZG1i z@=M^mp<-D(;{dvZ2eJUXAR0*2l-OckiTnf**ZqJCg>DSBt45>FMF{|8-}03_$?BpY z24q9_@JN^nbHB|~d_d>DWYGoF^53e5tBGLb6F$w4sDibOvMI+N?17pL+Gmx{4(pWG zj)fLDk^^&sK>MHY{oEZFvq-wvl`1rpP<O% zpZEeA3;tCS#OJqVQ@XM*_KL$FVi&@=v0?@KtLPZS|hbB!8LWn|c9kr=b@&?F{afW<~ zR<_-P7tDBj6G{)|dUQ_y2VK3YzQ?OYTycMkM0fN(aICxuq+H_549Wss-8Y|`;ziXS znUBta$FYO)#)`I;CO%>bVpDiaFuVDo^f^tdkT)0AukU*+*E1Hq46dM!Ub0^U zh%G&DAQzCE3w1-2r9j7yf&o)CwgZQFZY5x{G=39lhG6xbSeyAto!UsiglXxVg4LSC-BDz^j>Tt?EGK}~NO+>s|h zucF@?A~hRJj7(`f7pzpJuEjS^3v@-;zP>hzc^&f6Qg5S)A<)9w4jz!>Gc!KQm5iTa zr_V)j04MLX9jfMaZ=i|7a9~nV6dN-oL05LWE)nQAabRfQsd~2Kp*5Ve=O3diW8w zXKC=kt~rBy->1IDHt2{gi}rl^L3VWuFi|4pcsmu^uLiFN35oQ|2NP;Q4Fnu?4)aE4 zd8<+d)}6(3=V_GpsP3!@o)RtX$I=^8x#dBE(n?XK_Xi9Wul_tIzvOlG-hpG>;3T&H zK(9yOxCpBcW}xMsD-1h4sC=?`fm^}>&Q(k__#ekw#pZ*O2!Nf0*Bb_3bc#WfQKrE{#yw{<= z^h^Ll0&GpoGQLBSHBm(%WfN;ImxB8Zr?7U!U6^-$hB)fqZG!XLA}qvsv;fHFm(7aNr=zPHX|q%r?X@4_RV{ouZt; zML`G@BOUz#g4L}wQlFNL3Vc26(| z33eq8Ze@#JamNf$4NZ6i)=i(VEDxq2f$CTVlLZXWEH0JzvV7ZasF^rjM^o7&h9u6s~IGYBsDG}Ci4DD4kwQPxCyVo8T#B1~5SBvh+W=HSXI z35Y#+3iwAlaoq*&3(>`lfN+$GjWGo#dLh_~h$s?Qf!5)OgvwM_+3uE64kdeYziZ;nmKHZHE+ZA9YLqiNA|(0D3KtZh|B)bfDP)jc{PpiPa6($8o?&7@XOTpqp`S6TYk&6hr{QQ; z4C}wBXD!LUUw@#M#kL-#3bzBO?!PEWKfx7A{g!8TWh>*tLPB4re5Vdyyee?*-G?#` zSzi#Q;r|=C`fslu91PM{tFHL1g1V6g-GyQL=~w@Nsh@K_Q}XDMk6|@&kMG~MS_r+f zJ8ov?@ui7R5Nv}+^n++-;LMz74fWFb_ zj&jD%3{i}N6Omo(9vi}!Cx_c6291L@ZX%2C&K8Mh39-JENUDCP?QKYZC)@~$KC zo7jzvr&(Pxz_eIBt%f$AV?#QpbnZqS zy)bXL%Wv7Q%rEz}+gZ1sQ=c!SZByL2NK(HuSn2_?DH`2jpxCGbJavoFK;1M94M8;g z!u<|*hWUq3?X1$RE;%@5j#aZca|oC$3MQjyFIWoy8%O)bzc|{bT-u3KsCbl28dZfHFlimcdx1*uf1~NGEzTYdLI_+tZJJ0bnU| z5(3bZW!=O&ISOCNqrvUi3R{7dzPvqRcY77>L^`ba-%)LLUw>`{aiv}o%mn$6uxOBi zZO_83-$IL_SF?3kk0~0}|3NJRP^#e|L;r&$g@US$KuMuQTKK(d+B@Kf8)D>ob9Q5|EME3&#n z7j7M}9`aUkpm{%ZFrB*?FRhU(w9o$;*cq@p?I6~HEeP?LM%8$LngY~OOqwr2y49NW z-lDk((K$tp&w#F7c-@qxAOC0JWlpJgylpDmAA|d|-;NpY20qXLw4_-m3t-3g76VYE z-ZS|3S6Gb@=`{pf;*mXTlu0uG4n`Cc54;+G{+MX_mkD&y>F%k7#H&|}$0&Ip9%?4G zWqMt`XPKDg;_efck@XxE2faq_vxb^F1@ z>t)quuaVLlXZ{IGLLfHAZ>Svx#lGX7Mob(-SS(3r!u*ldM9}47QEGQ(-d?Q|KST3^ zcyr6JT^6M3x6;CxSP>@JjrJilY!Ngy!Qu%hX6Pbyba_|5Ir)hwuBNH5jacp9+cht@ zG__zcl>O&s`tE(vzsr@@mxa%{=;Fr+2M5R%_$_}?zo5&&i9oKm0%Oz9gQ8cE47xGN z;49cWs4FdyNVaIrYYhMC+nDE9c}gYrWM0<|{ozTa_`c~@VOw6)T|Mb8raU#=@DV~q z5a<^+T}#iG1MO{Oenf>o*;oUC?2tHezw;RGw7;M7NIm8FivH&P8$#9+n$)$Xv$9Jf zP~)1P&|XtcsGA>&%VY5ZJ8y9T*!%>IP8sMU!Kae^H_X3T?)ZJ$#n3S2hQ`;g#`VtrP)%37ph+Z9wr?;2|V%1$nL3A^Q@-RyVvFw=BsF`R867)doPK zHcb2>I;Rn-(X1Sy;@h$9Mp4Y!CN_5p0Xf3Y+U`UL`m@~}?HjZ$dK_JooA3;2Td;%A zc?wOm${1JpKbx5>5esk1#Pl9r9O~2!43bsN&`^dhADTQ~q4eMw`jI574Mxf`*)f;Q_RP6V$JH?_(WWVzxImt6sX+EA89dQF5i24Gc%DgF9D3 z@VPJN%V-B0?d>$9Rj}dg>!TW*VL-dM0tjXm=4~_y2k4+K=|~`by_j($iKp*(RlI5Q+hXe&dFcT%>DcX=orw%y4A?~KR?`=EFw#pDi(`A4;=Wy zeQ-1(vJ{N;bfASi{cb*c6bhf4oaru|F>*9l6c~D!{-STi#DR@N5YxGH#3oq7&ZzRo zs_X2uO44GMog-#WhXSd5G#Pxamuwmge6A5v-@y=bYP zw_jyo#!>US+ODFr3-`F#GTqft3P6E6fze08E~xz)O9=YbCQH?Sdcg<3Jf~bU5uk~` zl&obj7Ai&0i3xMy7)nWrjEFg;t{h{~v*>V`A34n3?WsmZGelDs?#e1tRaj5*J>U3y zKU4X=(5I#4+%ucIJyv+BM`GsgIW^5*z_kw29YozF8VB?81^IO)5!r>QL7FmxxP;HES#~Uu9Vv|p}YA_yB@96#^RYKX)K|pgq98`ECetg{;DxGRp5VZEX z^Hw=y!VWAiYy0zmh~7*I-ndv%Lap=DXGlUfhO?-7WDzc94Ee3$-R8_M?93B5XQ$q| z^Ik*p;do~rYQKU|P?FLE_CJ^LL^(tS(`}cu>BEpbOt-p{AKrEA=G@oLn_m+vqYQ(& zYK~Z!PQ9sqsMLP7y$<3CP#NgFe{!$ilCt+_TpUCe#enI0rbHZpC5_(&t#=TvKqMSs zY=AN_Tm;e`B7pS*QHfZz(zn;>F8-9H@|BIA?R;Zp@jt356s6=ISHSvF%X|E%cgmtm z&N%w8m#Yej<@jECF`o%aJFy3Wa!WI0@f48xxBIb_AwsP6C`z(1= zi>)=WjTUzLwxxG8zUc3N^|f~^2pl6K+O`G1#&bc?`#Wbe_{6(B`zUJ|vA%t1*VDC} zn8M=gv9a&ClbIkMiuHy_F(6Cjv&v0(LGozz%v#35XNZ;poYR|7Z@1IDa&jg7NM>{H z4#;z{-^VD^L^}GIeWgvy;Xh~8G}#bf0XXkcU6PHwz(3rXC1F8gSP&^Xy-rw2NY=d2 zJc^f%P-JZ5~tfT;C-dQ|w&paCVbnPw#(q{~!o|ggM_9V0)dO+N4Ua^i=$#Y;bhuUkEm7P*Wq{z;ewotPl{ ziD|>e((b_lc|F9~BQBkTJm%HyA=OA5y+`s?Nuwy-l;0h26))=tl>|HzL^%Nl)&-A` zPh^*Y9eItd=vt#YFvMUpB#ZH_2D|OYrk4C1deUKetx*55c|nY6VlqZ5xhR#y!#IXt z?q%)TijMb0Ps+g#LE=$|nNJ|EG!0v-McI6sq6fPR9PqI$pll{W=irRis*9MTC2#pf zFW-KR)M9NII;*;fXPEYbRr(0r?|n@mFOV}o$U1A!z1GB#q?7Txv7M~#DR7VjH7H5v zx6PxFYFCrs@;Re5nTEE8NDcvw!^)P@r%QZemyVk|p->uYn}1?i+b8L}unVjm?|~^_ zKtkwBQ8rZX_&T!Ou4&Ji?=CmboopPMnVTDogp>ty4`*KVj}^EV_tWSN-SNiIFXlgN zDOhfXNDAw;H{AxBBp~1cPlo3(Ok8Mm@TkCxz3&=5Or!;Yj0|c-W(g8Fg90pL|Newx z_}T=c&LF*HL(KN5JxEYBHqWW8**)?o+H`p7b-thbkD|&e)?T0OxgkUITV-$pi*Fh% z;b7F&nml_Lw<<;+II3__A_^1Y^8|QlVS0O?T;9CxUyp|U4H#mFcH%y_5i{(PJf#8n zOdKjY>NLiJ0etnC61g8$raK4Rn%R*F~~?ZsmcEI2ic1Xg(^udJ_YnI8V} zC0(M}>pImaGRtc6qBkT|nI(?a{FiIoG$eROoj}Whry^;lt0l7eVa~LkxiS z;H1%{5jjK?NN37Tuz4&&hVXEkJmO_H>UosACs+2s(64X5{T<`ez5FAvP`3N~`rFi( z3La-Q`tDph&kr)K@t*2bOEBJqN7oIk#Tfx(G5}iGZE9XWI@OAm^mqLtXvcnEzW%z! zkMx8k?~5e@R(Q%0LOvc*0`Mt%;2fJFKvi%GBLUPk+B3wvlyL;k=o}PqM{>VAyrSwq zdeL9r@2_rjV_&D2V3?9d-{%im0msd6^uM&i0jiEWlOhm*9yksSAbdf`K%{J3J$G`H z1lf?ON-K zRwiy0vj0f!jqs9oG8Ekl6%jYbZpfO7%*7ArR9c8^3^LqVl0 zgq?b>dPntndu<|nPsd@C}atAJ?UIIKK@Yz981m8n|o%yHCT&NjI4Sa4t z_|w|-U%VfPISaIt(P2v!Jpn7EJ5v+DX1FM9qTPY)!_x*KCj^M=GokBkbH;zeVd;l?*Glb|1 zvv?kY$OFvSbfeSc!1S4x4%srcO%dCBni`0!bjRXO4wu$spKi(j;$e)1bgGM?1KgSS ztR5|7p|#f22X`Km%R+N)MG)*!9&l4CLl{Ipa%YEI&3wX3SEig6re`JxrN6z`bm%d& zSa~b3RbAkNZvJKzMFZB+E8GlG-fp5?tLoFJ%ZOHKlo4E#|G~kZb$~!k&n$oTFhJs` z%=ma8^-H0}j@m<`5T~CEd#m;9XlZn_{qr{K{6I-86VzL&Wkzp}bpXk^)8Yl}18X4%^Qt2)kHk0(` z@Dsm$!*H_oC%!cRr^^pb+w2QTmZUI0Fuf3HM zM0&fzHyeu$?kHK~K~A91PxSCo8oOaum%8E;4gl*+;@SMtYRTHsOjq;Ay_{Ipg8OMF zspu4n9>wM@OCr6~eoUfLD_>3ZamzcFTeH^7gE>2|omIP7QBXWx4tSJ9A$BToj@rNr zzKs41z2wa37>tE#T{56~{#Vr2Bk^Sw*utLOk&8Cg^5#t$A9g0EURFsTKJ|&~Fut3J zB)YJ8kDeY!o&Cq1j@x&lwpq8fnSBf&Y4t=_Sf+VVb76R=P^yK z291#2*{(C^g5s~06;u=Use(Bp3pEq~cKr7T0+bnHQn|GwUt7(0UWj|l{)r^)=oD~q zueNSVxq|6O+Q;(ZW_3VvjW7}5sd4WG;&y?xH_BncSRM(XHIGNhm94yv9rT)qc1^o$ z2aL>fe+SRSvMz!2=H;hL%YV7EJb7+a4d`v5);Ij(5=E1|$|4DX+z#Fu2FMM8Pb|GaZ^!EDSPJj0ju3?&?a~zZr~xd2Z0$iT%9UoxBddno|$K?caC8UZ^Sat zUco9ou^bj(oeF7FhbO=Ko$e7<5Ilq8vVI?kKFUCYpMxE1_o?T*+3OKB-PA2c?pxnG zkp9%qN99dVFW*vMk$|g|_yu49-r={xtJf^BUvL>igy?}}Lu549is()57J|~Jd=jSZM=1h*p*l^z z1J6$eZ~*M>Sk>H0(!BejfoNH%%aj1S@o!p*rT#^&4S>q9IDRw(IiyMpM!uIq?XQTn z8FBbFQGecY#{S_bWh3ZvV`<5!Rvh1(l4m6C8yv5~(lUt3Axrt2+Z){NF-kHNcrw)R(fLMt4PoPB&DUoOsjxpR<_UcY zBk=m?wMkI5GeH-kTi8tQ>$td8zTM*^J>9Z<1#hdZynPPRLju0+G=M%0^5QWoP`d@e z7ZgJLjJ}9Bde%aYMS|&zlVs)H)YC^dF-D2|r3hF$n zs8yK7XO5hvs`dgn224CrAWDT)WoiG3AyVc)(WWt0RrS7UnDpw?Z(b`&pKop6T~v$W z3eiL2&Pu@Y#O8qWb{qk*2WKYIiN_Lwet76bF7PBe!jxHPw)t#>l$j4lbg1pn#LXj* zLp26aF8@qIC_(LBELn!>P-{SC^rYKHQh@FoQ^OP-|M$j5yid7BRc^d}rbfVpd)G{4 z%%gju_5olZT%f7%G&SaZ7ROX{zp-*``{qQCyZ?ZV8&&VVsm+@s8fu}FZG!<{xSWKJ z(j-2MLeru{X^Rs-4M~`7AQ%fztshJv@JAhtto)uouzDlkIj7A1q0j4r=_@_StzZmx z4nmA9NqOB(%uKYmoNPBtKa*_%Tjaf1j@BM6YSayShNxN3$}&y$eAFD_LKVVVb<*_ zM!iLht8RFwxV|{H(AZf2P`m8(deVb)lQ3EZ%|#~f}%759O@uzhpOI2 zmmJp>NnOvq2<|xmUN{ohoJd?AcJ+gM!^Djq6p-#~gr0vW!6C%9`3Ph!p?_xZ$q5XZ z2(Uq6%m(xYokTf4v`@yQ2H6n1*U0RtEq;ZiJaWOZ?Lo=9d+Eyq8tUEcA$PKZKXOK+ z|0S_(e^H0GQE|*ab%1k3WRnx9q$Y+IlA_uRwjU=!h6A6lcH42zEDQNB2ijJK6R&1n z2>f`+Wj`RXkiO(2(cU|shJyTX@Nu1KG&zs4o13*G10JE3AY%a+3)RL+g!*yik>b4~ zU5zdqqMp$s8*V0n%HGmk*SW>hZ&P9EP}pVh@D|OQQ#)yq!&+g>9{}1s zL+@&?FA)y#p+JtnZy0ZCLq^~Nyz!nW zmAeCaRsOv;X@gI$}{r&)6xk2f5M}$xNQ6k4hjknq~%>xt~n8>oNm6e zy5xKeKZoEH;Q<;sT*umOSriS>WVgt2n3=7DT4f!0VPX!zAt-Bq6vTT)NcxTd=C^a! z<5jyBsi+jLxTz=X>pXdw@@R85;Lo!(qg*E16cI04-55jc z*~(O`EvR?%O;4w2Y0GXi5ql(Th?{1LA|Xa68R9ioiV{RpDjCX1GikuA zvrPs5&B%2dPkAx@psc=NmxP4fm+kN5V|H+WVWKl&lQesz{B)YPimdTH4~6=-zy%7F3)G}1G!8BZ?q!A0bJ{K z)b^BlVyqtD`EO`}Uw~miUw^4P9T#vd%PwTEw6gD;vm5d|v)EXJW5@Fl!3l%>q)ex8l)M81kK;4vUkY_s7q-6_~D5ascj3|R6BN7pzQ(>piK=- z78;JgbPyA9f@snMS+R*IULbhR^)RkGgvzt^zOSqw=?b-n|;)x!C(nCauDE zKzm98fhvG!h{m$AI2X|LpFRMLPh$he;yeErLd85}jmp8N(UU&CZs8OsGL~Up1saKE z8VYd9DoGA&_Ax7mFL%E^CQ!Ruy%N=TkN*VAQDdvT6|1)8iNYDH52@i~)np^7a=%Tv z) zR%h-rBK{h{*Re64eCAAxR=fhi!D0+i$N_YT3(_O8jWg#?t@B3u`|G7^brqEFR0X`x z2e2ZW;XwStnGZl;pA^U_=-JVrRM-~Zp%n}owqKWlo^5u&_;;0?W7*4Y=L840NsC!# zjfXihJUw0HV<4P=zu}uBKRnR(4Ou%#nZ<=W?p5@xTywg!C}6@Nd!LQY1iA<$C)@o9 zK?GozKaUf<<)-S$43_aw0~nD0?(A0Bz~LYMZ1BpE_Pcx)d#~WV(7W^3&ItH#tPt`* zZU(^ocX!jg(8x8W#>a=ZBg-udxTpNj9#1EUB0gM(F?8xmUnR7^x?8)W77U2=9xU94G9UMK`S z(*#gfMvnH+LqT2cxD+C1+W~j1fqiyIpjPeRTJpy`jmJ%4pHNXv@fWFf^*g+NpXfA3pGXmWFG@68m8ha^2R7bMiDv|ES zH~~p@7GZ;pEK9H%j&Y)!vuR!WgZ>Z3oQf1WQ&TJ0KSJUYKf^GuAovmv2sQy8fyCN$ zfk(mK;q~@3NJ8AxYZk34*ZbYtF?kLij`00>#)n;^$Kn{eIDc!7p9@V|0SU_q(4{FE zJrOO$yboPkOF4+;w@GOiol_odD3IMgwPBzCh5Un!xBf+6jpB45+c6cKGOV$_0uUNx zfI}2BIOSm;3I!3w`j@P#mTWm6TDE@_kC2X?YkLJ=>eE*IG6GmtJ^7H{TciO~G|vi= zXI1x&A;v$1=DHTJ7POc@gSOl6;QQeDSAM{G7;V(fvEf+SP_Lt$b?EGZxyr+&x2Rnu z0EfO99b|`St^w@EE%cvJBpm1;P7WkIpv0FTWV|Qd_QCrK2MEtgi12b^wS^zwg+!0id7QXFXB;eH0Smk>Fk*lxkG9vSCgb5>?0(qgTz zMN8Y$?&vTZY~=Oj`03ysUK#g|-BHqzu`V0a;xDM}mjO5S%-Yl-P|NTm@Pj)ZW_a#-fS@7(dyz!XAMR5NP$^<*35ErUkCvF$y zi%EIC5%ZfDy%Q+{H4{-!neiV1eJ16Dm-Iqz+^eCpT&AAvv_!p6T@*lvm;XB;&sIC+ z2W`tG%!9gd#i=8f$+}WnCx#v$bR$lc4CTt~n&7zPEXGhH#9)3|MuB5@_jeOm1`ns* zY)iyOIf@7DPvc_dM!C|@F#N^FS%R?s{p>k+-uZ#3VAb;(DxhJ>4=uvjg=o=*fR(Ke;#tY^%fjOAsegv}W}#6{ug8{GLchlgRwnmV zf8>q(*!>O<<`w$Tf2^S{h^`P$+>993Sf|rK(I=aLfctu@dVn&|-Ct}>uIMfd9D(k@ zOoh-2;XKeVPR*n39W|C&F4P&jI9$nays$7*eXDRY{46EQ=0cf!Q||hKr(KHx1*=HS^VI74Xd6cRg64={f^+!}yK+V~E9cTGZ_zCMmG znK4rDl1W|{o2-9nu}6;vYQkuSj0Sx_3;GN-ipJ9S0c6e{jmS}G!IAlBeo)BQQz6CK zy~^a_N$2bmx3G#%=?91Ii?MZZIXPUi`*OJrtU`>$UsNB4;1fhwck^MTKH*uTeM zUD8B%Q#0=BzB%ijtH0nlvViN;N}CRUKK*#-LN#~k2PpF}W^bbuwlR{u~1NwQ01Mo9KwNAagcBv45b(EKO-E-}jHXMyurCu^Z&Mn9L-~$ z{te?<{mPh4zfp}p1y3OURnQ;|_LH-anfDhJwD#G*Tdv_$3P~d)ejLq9sa`}9=DmbH zx}SD)2zltBbV`xx`n#nN~$t9CumX3YiH8p8ph-@yFDKGW{2I@mUQZ4xC+z z76s>D3!HyY+lakSxQs*85-30xYK@3BsTeLJNAl4lHW}Odi@)z*X?(q!`_e8rIKZ&m z>q-s1jNoAqKXWE{``6Q#=hlz@5j#L<5W$^nFd=3Dw>NsIZ3ObP$&|c!@qzMw{QvN z3JI_W8qu}{f(2Hja3o3v{@FvUdy3SM*F*zHH5sbmH*@u!8wK;ysuy2pP0dFQ9$d7_ z)xT(vdx4w8v>j3h>#%-bM%Byy>pP&+ge4F`F^Ss*Q`@7dK3G{jY(qqa@e9d4;4#AI1wsIsKo`_$S zZ*7b9f(FywbJPqlKT{%Ap0iJT7e!wtn3Zoce6AcnZhavmvhq?9guU|?Pp+UXfIDVJ z6M>}@Cj|0~Mg<}j)a)KlacXWRx&J|Bz5h|qNIS&B_DyHy4f~$BDHjBsRxehLD`52XVN~`r~|)POnJ^9&h{n;zi6CJ)3(ko~S)$ zdy9Yif=}bbi0;t?MDw9gBtVlQZxs*_u&_gVe>qS2i$kE4kNU=GcT6r82S)%(Qu;}T zavNexH)d!fX_ZK1J8RF-+HX05J!v6YHH7A5?C_z|Oz}W-%manBfvrfR{UN3?h~h{! zAmh3SlObfCjNci8SoY)ab$>Uj@xVRJtoHW*qE4Ig^J0d<|9)qZqF$Cze z=utg)M^Z-296!|dS{O5XZido49=MQzQd22wePEG#{4}?cwiDOcd;H}ZcbR2qA51)< zp=p&#UYo2V)A9pb4HvX_-8y)Bhjf6lqIt)l*n`TVv&5^3-BvEUr8VXNW>wqD;WjX7_v2S@&5-8W=ce2y=zyAcvZ%eZ(a5oD5$phHJ(U25 z0l4>Zu|jcXE{iKl+{%&PpPUwj(fx|Xic5Z#I|(9pFM^-kox*+pgbZjY)<;M{Db+uk zlj^~WdtSeDKx)AM8eG@&;f0(2-e!09>n{&|dG&$`(z!8dKzD;!0QR(-YFZdfg06bn zr5KAK7Z!BHk*X>{7EjU<(t7C?;KNzs?e#i&`^5fs+309@)i=>$(PKFHq#dFrw`!U_ z#~LNn+NBw(KAYCnKqsmW4}8qR>udtYuwE-Qw{-ehyY?zI^%M8qyhkS6jO?~FhO89O zR1_Y%DM01si3E-_9HK6w;PMkfS>33FvH z-~_OZGTPtWl0;l6d&X_|%+FiMuktH|h?{;fmKS;OE%)`w zgKGZXuKB~z;>^46%@5RHJemGkC+frsN0XAF^*sK`epWwM0+ zoHnJzYb(e<46iDGN< zizw|UKjgsB8El_{XoAjkI_P6*Hwe|4{^f0=9rFW@L_P5iQP7_i$nZP-US%&(mi8VURP&kNHl=q9xg{8E2%LG}S1$JM$ zKZs>NvEsrI@@Mgy(D#EInGr4$M52c!xjQQ|;_oqZ;=kA>8Vi4(xaa$_cXtz{=pXSk zW`CGer&ODe%Dp46@6#)R-0=M~V}^&;ljNqsyqgW4xo<_6jfgC;_|xGV4;PNQ*dib( zq_0S@ck7403NFw#S9vszDxEC$hLWS{>I7usW$QsR&$@;TTH^*`P3 zPol@{qu)jAGxHkw<27VeCPM#wg+02AHIFPcR_WCX&CE>q^?P3p@CcZXIGq?8ES>Bs z)$a=BB%u03PcDL;sph{t=J5{1xYPh{*WMfMCaMkO-L zP~+At@8|OQeV*m`KELDn}P<1H6j+zxI9q#B05Of1v^I`}t&y{fBSJI<;n14e< z$i9!Bhon?^E-`b$JPXTG2d3WV>o2YuOMI9_X1lk77Nv>>CNZ(CAgPzNyg_hkIgi0zGck8a(q>2Cz`4~v%okT@ zx@Lkwq0_}b;c|-v$?Xlg_cZwZ9#4wv3ltqx!yM3fG_Jp#qP4#pz#8$Y^u9#4_c=}; zm5AJPdMUE)EL0aFn{xP8k`0-Xo?aC53)ZBg*jdJUoP=u9?~sX&PX#=#Ah?}@Xe5>| zTNykfJpO�Zx^fp~X#{4&|-kQQeF6X?n)D9_)T-5*m8?Zi2Y>x|M{sn;d>2!1?2` z5E2?qS;TBw1oqDUU7RCu?61E;umN5?`2$`U8kXHUE+*-Nvi6?AU$D#gwpQa$E+Fx{ z!2pNS*Fmbg4Ld?qFlj9xT`}Dkh$~=vyt#It%JPpoMV>0Z=U{1ykyn+U^fEtV%TAe~ z>9G~S@KkezagF<4A4J0eVEpv#CRpc(08!*$gmCl+vI&^>d7m6Rm+Hi@t~lLiT;it` zd)fTRyg?4C9ejI@U2P9NzH{k=g*z5GwN2=!?R&__dI!%sn2uU;MU?4&T;UAX&QOqC zCVELCU*W`wk3O1K^oGZN#y_52*_o(hl#`b$bl+Aj*TY>{@u;x+atat)lBoIwa6V4R z5&^96kBtC#M){}kQuRe@OK4JkbJLqlA(!j-EzIV=_bzI<2)H~yx|?Q%W9iW~mo}D-1%F3eHMB=#DUfyrB@E$B%GLWeVE)GLXZa`0Fbv|rZ zNH%qL*TMA6=(eS9xOy9Wu}*iT>iGF%r_K4|c#a%dTSbD%djkm6 z744g-!W}9GWPxq@8>d=R#|Jc90B7;toTQ`axqoC;1Cn_wR6W-F!MX{W@jU}|1hd1# zu0*K!fu1O`O0p%lredV{{0^1aT#WL2gUw2F)RZk-CX{)DbDI>5`O=RCl(ZeLE+f!k zQqs++f^wtOm2ekw%ytT{=Bw99xAbf0J|JY8Dr;rVcB6WUaAW~nk>_88{V@iXEBhH* zd(nWVkpA`(K@B7@&h0)l9nz$KjG0I2ZHp>Qd5xy0IBZ_U46Y-l7jlXk;DhEUvMpOG zjcE%O$^{T2lM3d*>`maITn2=fhg)cl#L$Ucs|tH+;q*g^Ipp zeNjdK$)9b_DELU@+R7?d)QLg6v{HJ{ikcQZCv`L|Sz9OhR8joh_CHUMi|ccJY*`jp z*nknBU3ami$Mda|2MP>FOt0F^ zmE;<#<=uX#nt4a%533q1%hV;WbV40=*(YRU{ji6M;>7dU=^q^WcTN6O^GU>C;F;xH zGis`Uo*>7OaIz>@Xb3DX0XmAq1;zTj+aF{nHVrFoX2LcY!XE^hR&9k;j3}Jip?BRU z+~KuUM-=b%fOFG3{5P=;}LC1=6S0D1Gr`pUx<~j^SNhdI z&~*jNi}XOIBXC1rot$UJ(03CO+3Is?lA)!G?)I@IyNhi7KIXn#9)?WQ9ZgyeN!c4r z{dd&#|7YUN!?^)&IN9!MP-_E&u2IRKO@yjZg4c zF9Vwv=rWrh9^}PAP7ESjdYy_)PCpP`ultsgymIMG+0;{?CPPNWrbMgvnST+|^05mt z9NGVJm}KVAbeGwwN9_>(B>GEPAG)C_b3XGMe9dv+lSOf(eDcf6gKJf^9SP@)fA~+& zdHbKY7g7^P;5NXn#e@C-WfAqtrlEw}57lA29zbS$xS!38P6^&=l{{a)i)^p!zHi0N z?0iexIeFDruKCveNGVPxD0aSLq;ZjO9+K7sKIb#se2Y;>&6qx@gcdk6EB!Pm8!^!M zGAF@PQX}V!g+%7GA=Mpouo4JX0y0G$KAOAQr3kY20&~6U%XEZM~GkexC#9= z%u(aa=I`W@dtB^L74qU$a-O~G?Y~5G1-vjF9oMD^V=#2NE`m_-O@V|`a&=?g)1MD< zk&XG6%RBpa9Nb?TWU@b&Kg8ol0gVJ!{YenNO`&rHa+W(M4+l^H$whm|S`BV7(K`SI zaJXt_1oOJ@laP_zDbtrpN#xg%a+-?PZvay0AtueLn8^j96u3a(Q3uyW+gjbP0n!R>j#~sr+}x(mQikQOy$~!J%}~UXiC9NBJBi3sDqf8EXs(+#6T9R zJ2XRO?7E-PeLv|OCZS%Z=2EVFlUSUy?rB5-xG^w`ri`J#^fBMknZ*n_<`Mc#9v!0Y z?`so1COc{|$FWbMI|j>7g>;Jv#_nHORnvI2<&YJsHwk38m#rC7-kiszU_As-Qt46L zF%{VO_yVML|0J&Ez6MJB^E&}|Ys%UY)!Q7hM>}wW%>oWuodmio;mf-77+bT1+`#~c z0@o#9;jx70iMNW0k$07g_uab_0=wn3M&O@I4f-LjvS9T`%&(R#;$F*Xjltu=5-#x%ebna!J2mb z7x8(OLHHLTDvIQKOaM$sN8Lt26u?c-#;(tNwLg?`$TQva<>@<4QKycdDf;kY$M$xF zvB%eYJnOb>X)IUxAZsVou^%T`32Q-2PkK=?lPEDG{AX4g&RjC7c46z{%6f<80GovbN7;#vuTQ>ipTUwiFb(NIo zyR+$yKO_^sJW74~l;4_j3>0~0&^aP(c#SKZ!{Wn#z8Oe6NS{5`|-#oQr>|=nB4*&-4XkbBu;JPNp4Q>G%?J85RD8~K! z%wLQEr(q^?&pdbEt%c3^X!OV8=0R1Jr?KEHGe^eKg8)=)qX&Y$EkK9*1qbx4q~R>6j_;q2USO!_G8|=C_dnao$akxO@(R_=>b=0# zgS7r00x>}~Tn~)C+DttiqtvaC>Gjvdbkz1IR0mF*|2%XXsAoMvMuB9Z@Ez1e84M`h z03&mHhW*dy;?V}-m*>Z zDg$squQ|nL0x;A~-be$W$eBUNejFF^q67DLzg^YcWA`&>b|>2V`6=01q4Pa=ym+q) zH3$8#Y@H2J>St>=*wRMK9L`-n65Vxm$7$HL6OS4X`!ocjZ8!IUvCc5KO>^|3W}6mB*kmNv&7&CVU_cuMS71?Q_^Zq%>o^~5 zkDj*LJKKOXN0IL98UkmLVSS?$>tmpX-B?C%>69C`g3}lub;N8NzJErE4)}F^@T0D} zGuS0L{hlaf>aL<=O-2*z*=^TXq(kfY58iN8ta22wA!M8F2KRpf>Uc8FRUg$^K~tWo zsK3h53xGVe!%QQjdoFdnJ6NK8@V(nl;R_l(>naVXEsHPDaW!z&V84UD(%7X_%WMoa zdU-PN>{+X)KR>!{$DeARrbJyh_6aZfS?~|2y!iHQNw6gmcw3$3Q7ND|ib~IBJ2CVv zWQyvj%7I=Wn>r8cUblx3JYXDVEjm! zM~XzJb*F0rBjau3VO1CZC=H)}AETk~?ZDsNEZ|}J)JDL@RsG#63|_E6q^}Q%i~JF} z16mcb3T?8#_5|g<^Avd>+N56Gd-BkT0ms!D;Mm$x@ZlB^g6PP3&7?McsP7rP(JKXC zbM~Hsw!>51vP}=&re7cnnD^D|(?6zu_h0M3cs|znjjRpFa0`Hy&MAfh2jfW%9vzSb zcMw9g3I|ZJLebCL$)U!_{s!!J>c|<*^vh9pvHRwta#T{;E`Trcy0hT7iT~$W7X9C| z>`PF5Qyu`-$k{KT!)g5tjzdqzcap0IYJiwK%ex1!^)_|g^;Q7c<;V zplWkiO5M^6Ao-J(iKd@kh4%<9Pcp3jf%O2+ZIq5pYIS{TSF=Pm7@N-0DXC2q;(8qr zkJE2{XDT7Ii-;cp{2&s%XMq;VFoclt*g66&0kkD#;O8r3?=&ZAOiM&j^gN>3Vwdh3 z&%n^#*pml@Q9TOp={yr}j|TTMnLYM=QGA(T3f&2~hCJV)#;9dK)-f(Li>;e!0u~%2 zD#BS_^qB_v0Dm>z`l)AvAKupAi#zu^y=Ff}!ovUT&y|c#&>@rvf;zxb{u#CP&Jz-n zfIQf*Kl!PM%_*)hD1!JJIXP%b9C^eF#Aa;xLZ>D>cBl(eN=>xyyj zZQz$PzBkWJ8Yh@8hpRQ=NEhU|Z;ONwr?C3~wHzIim5#2#u%%k48#Eodpl9OepYP);(?WqP zNf0mV&QlTsO#xEQ`}Al;mK8veNudJ>WOZgdL$ezx)<@Zgb1NI14}*MTT%BK$Qkk8f ztBgFh4_6^&I&_40^Y&KV7IpAvgESupv~&H?lrpp=I*gd|@EM8L%+Tv(-h|R4#Pys$ zjn0<3Ms-Z9cnYHI4tA4P{{ zrI=}rG!Sn<&XMXv?7D+r)5U=(+| zGv^il7#=kA=#F|)Y!}XB)3Y-wj6IdaFG1>OZli}UR=-S7)6S>me0OTNTzl13=Pc&_Kt42@0WMkQO)>p1!3b5uBA_Jds(2 z6C>Lc&(XYlyzWepAKPrM9{>DybYR#~!tmX@m-dz)d8Qiv;bk_0gB22FXT3qTk}Jy@ z+5#r(D!R_rMIz)A1!#jmhxJM#A&LBfF=Lm-hTP z{&6bjGTbWJ4|Ez514lguV*$1~)_SOs;ENu_&b`tQ00eur9NUL)!;$#8!*utF# zjB+$nhu3Z5_XA&LOY+Cb>FEOvh4Kci>7RE!kozIx9qeL-{IOB+Zw%05as<6;608=U z3Y{HIuwB>`u*t)B5un>^=xW7ZC)%a`)T+<)93bYGRz{wc65{#pTj zPX)~B;txWU@<^5aqsurshV7i}vy!fICzZS9v(cH%@zBb%->kK>)AsS5eBY2Nc)2vE zx#sR)TksUKxnI%s060X;W9^Jy1;M#~qDRN7*TW3KxG1Zr^7iGl{k01FFjMhEyL&#S z`_6|hgqoe^jkDuLRbA(S!3yD11>X&ZTOnJTo-4!Xi?ierD&dYBPO@w!OEjoRg!J(D zl95NFhoSzXf+C^T^7+KMFMMJH%dO>>s3NdiK+Xl)8i47sK_jqkz?FbZe-FPudVkoV zzAPDL!*h7hrRYPL8Dp27!|`)X_Z+3QL)r^VrEEhb;QWeb)iS8MSP0F7}MfkNgpGcj657hi!CDe&kmbhujzDf*P^V@llU8P4JCd^Vzomcx-9~ zelUb5=CZcqCrOf~%1@*9s5T9A*zGXXjz35mstQ{M!k^S}sdEhv`0g6_wb^5}KlWv( zCb#(jR1{bS7toY)fR_j+@WIRp<`0-aJqS!hgBCuw2G3X9dW`55=XJYSAFKXI+nM@q z`B3bK%kMsXs6wFY$p3LVnTJ4JptmFk=8@K9rqZcJI`j` zkwWrK{8T{C$vL4D^ z*-5ymSpa`~Im^7tDQ+_Y29pZE^*}WTv|l1nka#r7qGcUufS!P6wzXHgfCv)dI^t7vg`r1UbQ!cerJf5F2S12-|=VyQo zBd|2X$JlxDFTUydnbX+POKuMpN`j^Zl(&zYGKmXaAkaebdU)eSQe#-d2mG*^Vj88V zu~~U7nE)s^s@)~mH0FDf2kvI50faA)k#tpp>~ihGck|s z!U|Ql-VFHBu)QT9G!&6(^3SNwe5=I%%=ZgngSZQA2Z%$!S>(-M0F>g2hB3?>mEJu+ z{W}XO-A$8D6!}%^Rk^RvQb;wk+hf740H3L&{S~P5zQ+InxwBv*1Yi^Ar}SXWq-7)eI@73@(b}eACNlgZb>U3@C}hEGNlUAE zxe$;pU5hrPny1>UZTogNH)h~JJQ40bGvG=Y3{hU7c5<~tXXOk?QzU9p{54_-@;X^F zzV%i`u(40vWBxeu@JvLe{E%h2y!Msm$*&KR1({9QG|(uLV=D!-z~1h#Lx=tfCoeEh z!Gt-xm4=w@vnLt4(0^);;jwCGTd~G9E57IEzY0IJ%V=fWT||Bght--C3 zHn0^%Pagk;Ab|^>`xmhjZtL&z-Srt4LuJt}40xL~4=K+B*)55IJ6akQiF{{s(|^ud zfbSgx2ho2KQ+;zla}me=NCE>Rpu!FFcH*VzZ4eWHLodGCEps1tc&3p^q=1s9?k#f0@>|VJ1+$Z$Aih~(7y;PQb!hBTMtNLW3jcs zp%u-R+@PY^Qiqrq8R!J=A-3g61W6%!^-PlUIPLJBz*i&rUo}$Y=kkVBTt0k%@ZcZ+ z1l~Bz8s~o()k%>g%G4mduZfnzNaI4n0W9n2zWMMTErQZ7EZ^_Sez1an;kEO0_b%pg@BU6c{QCm;nh}b z@V%!EaG70e>Qla5U4At+HIQQ=uPM4Sr2Gj$or$mjD;~tC zw#knC5glu?tVwy^m}x5_tfBkk2w#!O1gN|oLv+k6tY)yW4Bi#rImRqo+Jsurd+G|2 z?djv_%!q6*@fK_x4kw1`gryr6w+BB>(NR;H*0SrF5)ygAwO)eg7qD!Z3*rt`aUf>U z0en3FDI6zLu`<}>YrVqckA!n!chaSrj|}MFocN;#G7E@7Zvh`Wkfsj4@B^pp50Tsg zfO~~=H~t)9o0z4|(L_5Ya=&(IM~N#rnXY{jvv$~YkdZlg)&5u|^W^_USl>i)Mf+H8 z>;O(GusZ@OSLmSC$qk<*$bo^+2zc%l^tW`pGrUIWtNTE&{gC(fbr+KYe{GQ@x80|= zmGb`jptVICNCM1|j+3NCbw~wGUk$~w?}F4t4_sjH0FE=9(zZkFQvgX5M9RYNVlZz% zhuutVpM8Y~-3czE+AG{K{yL2)aBjV!opo1ox7FSNvgZZ@iYC#*!Lq-?omJeRsm&v$ zxC1CTymH8l*jPjJ>Thq7%r6f^kGVa5TDFHzc{x1Fc~Ee*QGV}5-OI*c%1>e|8nMFJ z5}YQyGA^1^j@O0-F?3>VtE(T?1_YZ8xrwHxNTY+(-c$S!$2II`>?Uq~%YG88-%;>a zeaoW`+62Ij_+~I`n{5(oCrkx)0C@63JBoa$2aROw1W-gVk`6fJ%QK4}1rtq19@$2O zD`C6rHA3=#HJT>yKS=hgCw}`5COE*t9SVq$`SzdM_t{*I4{x{-5H`NGng1blJ@Pt^4rIZ4z+%DG1?+ZH_*#dGA{}yhmi?bSh50Q{6%oUf{BCig9LdfBplYj_HOax5p2l_Dn>Ts zjSb<-v!9W>ajcHu7n3t5a`-DNLO{0B-C-L{MhQ<{C zE(dpc<>_)YBOCenUA&ulVC$=K(nul1I6@Pq2Q2{T+Tzqu$4OjxMTjid=I-upSMu~% z`@_<<+7>xOB@wC>W`#hR0|?m&g04G|ZstNFzZi185W-BKy?unOIgi;MRMxE+wvZT@ z>twj<=vhrG&9pG7&zaVkkj%F5zn;zMLam}t0a=F^;E8zSdr=LTUM##9-HF;B%}YV2 zyu1V=4{Fi)Eo(a1#k(ggAozyZhV*-l;!`qyHl{gufp~%p=fw~F8k^#8)U{j@ZLWFELbIhxxZB6!GFAY7N=1SLo^UX7 zvY_jsE@8f9Z1}npwxd8!+~8%2I#T0ff&QzY?F!z|C50#frC0yN;^$9cvr+UjThz=x zH2p*txl?Zv#e_J7!{JiD6m4?&SJMoza)bKbE6pjpn^MQZzPTUCs9jmeKU-|`^%H{x z4nmX-0B1Ue@5RXA`%vP~hsW2{E3C_Amn5f;waxv|KDky~nfJ~;;Our`zANUaYfuLT z3x>6QNU|s}l$k3GrSx@GTXkQ7m8cqCvnfx9=hrXPzha4xH9xj7i*m~i+6dz--<^rj zfDX>mfg>F>w3Fe%m7YMets!rS?h5n)F>%m2aHPP*LLe$7f&=MltJM|8%|3^MxgTj& zRQb737B8fPy>ZFm&Ta3jy{8Kx9xFr?@&g*qXkr4)W{ElM_AlbLaRFEcPk=@#<Ad0%B08EHgc};b`lzvgp-b!-*o`@WF|y!nviVT?@3)o3sdg(J2nlkfD|rn!?y`HMPHgnbvHzaPJ`Elljgi)Xdw zyZ|7b{gVi0XuAU0%1ZMj7#w41gmZukH-xThOq=^;+w&%7v`B2+RLOKzWSZtjMmWTmpkbB$b4P_#coy^Ie_U}zHk*Lt5#5m*PE!U+s~4W2TTuoRzKyq@qU!NZ zPX)a>p}+s_bVnY$LDziR^0C<3WkC&(VYeTIMl|i%9P&#G^9*B<<~sjLisdql6QA2` zaQuWmF|{y#r~UNjW<)@0dQt4%_;d;69FqkxW8n4YzMq~iq4-Tb#h7dLuH9VaX0gmN ze$4w(32XctsG|^78%0wn-70!oPqD0j%?d@lJ=F0~uWO1X%km!6CX)}wTsu~pV;<*J zt-d4S3r#SnclnY+Qw#_bOgKi-3P?moeDq}L7+2)!b< zqA78+^eYSCp;wYrfLgsi?lLU?4EFA3J`{WSBfdP^J=pg{sGaZLh0JrN>5HntCHp=0 zx4%-edyim)23p(*x0leM39KCUC-q&KN8iS4r0VSeS~0CobF4}yUB~-+zu#K9G&eWA z&a{iy63A#*KWgh?!;=@g1}IOf%>A6owqy8FGk(|uK=9;J_5M_2y7Qltg|DZa&2Tx& zS8KQSK}P*}hVtZO+=~hg1{Sp4=q_m3WP`f8+`|e67&2xk=MgDjRxIQVs%sXA;$Hn1 z;m#=RdH1GuwCbfsQ%mmQ#ymIGb-#f0opt+HO`?`hYBA4(#5rOH9f3*n8wogdmNoZ1 zUZRnHsj{*spxOsE{;|rk`MTRg_Ov_IPbn|beeaj)C*=Ve2N2wZa^fe@iCqB79^T;g zEkf|ZuqT~BX~^1=llpbkz;ic4Y}mHUEBL*-rTLnV|BZ(heb#PRs?HiW4DevWNR1mD zMZr!YTWAy_!5jp>1?m9OdhJ=fU!$|SbD!sxd;esboY;_vaDbbP z=3#rm&8*|m^K&POCDE2!Y{tLhiPbO2kFH$9H=P<_*yNs@y%??C9;S3pI+q`^IDD#L z^D;OZ%PbOV(osSy=uk}JBI8%@LYJ=?Rxx;x>J#my0Q&4rLWBF#ueS5N`~B%gA*d{5 zp$=DwLJv+ua_l^Ho}|Jls7yYhd<62 z869n>oaF&h@gS;poCn-AkTf5+iN5Ak;`f~ny^BMSUTC8?^b+R!os1prze5KeC`D9V z$$uZGK4=-6ehsS!YVRp9UX^Ap(8Jj~yqSCGM!aM6BxLGtqWg2VXXimsB(45h5M zJIW{RcU`WvS$elrRSJf<{{?StYer^~cH)(}p8&}XoYJVhEdr0`Rz-fUFBunpTrc(W z5Ar@=pcYlb^N*DNNB;2WmL9G&49&$@^{JRLYzm)3L2sN&3%p*|RL=B;g$8ch7ntwq zucmH|$o*)J@PEklfha)O@rdODI381#vLwNXJ>GT*+7cSr`Xj^E!d>K~phMA<^VHtAbPRQ38Gua+4|U^}MpXvg z>YP+Ohy3N$a!=>wo}P4g1NaPMdH1p{h%3)Yh1d)N{gy{xbeq<%&-ejQW$VR1qaVK+ zUzS+Li1{?jd_3MCTRSa!<4(|uMOa$)?YNbgmSn*$6y2i0GacY@0p-!;5!0Si>cJI> z(d_Hji+}Td;|>lxr(-y4G5)(>wmJ1iwd(1WUEzBVW{f3op4nz40BpbhPH=@m%n-cH zcIObGFakffpHyG4+&5RPYrjk~(zr985Df`pvB=IQNIUN9^NDT9+=RD{?r4 zRcMatXiEY&C5D zErgW{^$ih_h()0}Yuu%O5l@d&iEPc65S+>Z&4hRe?3(jgMs+TzBGeFXC*;rHLJz-l#VXgCGj!&4JHAi?lj1;(M~c z+CPhS?b>)VP^Nq@WZmFx71A`$;DYmj+jbsR=}cQM1XtcPn57LsK-<~a7tx}<1RflO zEnE7lTWpY7;b)Ze(r9Y>_;-!;IUt3T)k(5MQ^p1-k<5Z|ppVkV1CJm(Vs1`#&LS@w zJQ|e!bu+Vw`T2BTR0;OW)XPQ7)f<|HJ9Y)UQm$!`&x&G{u~lYS?xeIoHUN*Rj5E!$ zFupVUrRJB>#F|g_%k}TV50APjI^Bn+-Q4U45Hf!-Y)M=0_G~yF#O9rxhU3$gwDFCZ!*A;Ai`Ifx#;G=Hib5J+7T;YL`6p>Q7Ih!6v@y#N%W2C7 z=nvx%HL`^6+ASRo_0>&=6zv%M%{jVqwyi0%lRRUosM_HfkTP0wF6jes#M_2f1bVv~ z_>oyz+>KJ)AsC_~d){O=ylI;zo8K;s4?+d4P0UtcN(wDp(SA_}fHM{H{dQj!+f-q6leUN3kf8#`od7xU}n0ywt&E=oV!U{r%`J#h-KzZ4E2x)AjLy!Ug~JZQnB5<7~tckOcvO)2fE$ zD44ibNouHtz_XHQN2i)mRZp4jYdxVmuj+o+FjTO6r=c=7KCu8)!uq+Qm1R(eE-3&? z0!MB+oQ4??UG3>%*~V8Jj34+y_Uuuv-;naz#Qab&ZGp;j)U7fGK4K+lch}~*>S3HF z($CLkG7#nDGr%JH$@|y!X&dNu#_53w?|dEjB4d}`f?1ATbY{5dubUU%qLXmgQeo46 z|1s6LnHw4nR~-#n`XWSM(iI%_m8bZ!{^b29#_TwSeT1T82Vp(m8jyt{GhsN2UPtMH z3P5czzjybjJ4WIJW3lle4)eVcSKz5~;p%}S&O92qml0_DUsQmIKjIBa!VcCcs6&OT zkCTVRDQQGW`03w%`?obi7acY?PECf&dekM4=Dh8eU8{Z4f0XAR*|W&`1-8u;D;yen zg4_QuqCLYAIJHfzGy&fVCWB4_$EZr?K6(@ln!^gt1WCe~3Uh5ktIB7{Gu|hd8$y|7 z3t68sm>}#1p^!c{%2J|TVN|(V`;zv!!3>ZpR?KekoR0 zG|4Guk!dSp2TSiqO-Zd>aCVDQ)sS_DibM7y1OPIack@*NPwqBv=r;|ltW~hG(%ImP z_#VMu=LnN!l!EpQy&m8FqpfI_x7D-o^I(_WnO@HeiSlz2#w{Owg+*mMUnC$7UL<0yE7yH$G+VRU`6Je@?P^a{h2Xv-%ezMr^Tn#41%U^Li{2MOW(x*3@S8({IFIab zIteM_4nH~pO|%BjRl4MUU1T2I^W;KyZ@OcBXoR%!9#4TabD=L>96*Boj~K?K0B#z& zq!ZP+uv|e2CzIN5igOE|MbBGiooX0~IQIP3dC#2r^LG-vN)TO2v4XruREmM#4#?X9 z;ma+I4a!0%lu1y+?_(brGu?}x8&nBVQLdxs{iqhq{+s?Vido*`pTf}0&xjJakziRD z?xxW@2VE|hJI3QzKtIMK0ig>?uo;T$D7X9v1k%A|^zWx1NW#BZpu~%tTEfz8f9XyB zvH|FLPH^dTw9lVWGo{;YUpZyWS7lckxC6VEb!Y!P+Wz(RY!BWG3WGW_3BUJ&4gwe6 z0PDn=A9EX8E1l>;vJ?lo&u#|bWj(;ENdhk?4glb~-h>(Jag|81(#$X{ubqbXvm}<} z_3*tZxr-Ml87+67o@qWmwbnf`zYhe^z7zi<-aVrtS*-b4u8FydM}fo>9vNsP4<`y< z^}bPXN>1{?3o&{~Y{}j}`vzXO+pS^ZgUkYEKYm$A(dtE=&m&BmddJ%F#IPqv@1kSC z#O$LSTcsK}-Z;^boNRF|V0iQJJA`T2+=3DCPmBS*<;XTUx=s^Y*0|5{&0X4tIOTrW z$uiE%$~x)V!hQKay`WqPv-xii_&oVLDZHD!#N|+~I5=gKzE zTIzITRnzw%MB90H@Qk@R-Qt=@k^U@da^ph2s>_Ht=A=W{itax_Ql#3zZTt_l2I9ov zl)4Z2-owv06{goTX*_fACETTm&8*Y~`mN}y;8m#au^+cT+w0P7;SaU=J!`Xu^S+?Z z#dm!iqDB$*Y{d!NV!Nq}En*8IJ&?prP>3u{+U4DXm zPl8SKxUSywvn*NDW5A2+UFiA+QX^q1i7Si*9|<;PTxN8E4FjO2SVukKveTc#{2N9Z zLSrtloSG$$ihsA)NvF5I6WZ-=*&%w=*{BhR1(!m4TNB zHy~9)C)`(lcEt5VViTWszO&;giSR#{-5v)T!kF3Kp#ql1sHGQMjYtEs zSa=fsByl^w8?OfBW=d~H-bQr_olO+&SGGL*Pm&9t@P)=Ng@JrXy=6Gt?|Tpa$p3CR z!-geX=R8&32L3|VWIh0Y<9k#4xvz8lO>;)IcJ4Ja z*%5WvH?8y1iEcm64QV7h*DnkU`q=276DO8zbbr>L{C&@h_tVd_4DZYGqsKO zyRR^AJdjsEMdIZd0(E8889y2#s{)jJ?o&!jWKXXq z4cr5NANWlGcQnKA>j5=Bf%8<(fpI)vD@r%#S&+w@$N4oiotv}%sTQI}&crxd)$_4t zP{Y$U{{NhKj(}$%Z>EE@9BNaD_GVT+RCe?kRj++@_cT4S!tdhlY}zS}^{J{W+w7DI zW4~R);;arENj&Rmt<<=>*Z=y(M8RyJ`-m`Zvjo?$34Krk=R_<^vkCRr(D*O&6@K3t zoIwK|_Hr8Mf7cEx{5x))-&ZKK)-N~?yK}muh{@S{;;5rBs{NRj#?>T;p;>ix!qIiB-L&uZW zJWO=}g;Y6Y_kYFT61yQ91eXLd(xRVC?q>F`4YHj| zLoIZnF*yaP<$X!ype-B2V|`rAA<6{ju2L4~$R`M!@`uyQotmYeM128^-(XH$s{_@D zZU>5~lR;6vt9E(Bm5VQ{HTJvRi89RX(Jv9Zly26$zckLxZj-=O1Gg0?4}1gfpMmB+ zFi`&@wD)d9!UmyS=PN*!h_vOmSH`gY)QMUm)i7&`0^7@hG+-3_XcO_z8#l zJv-fbUqldr5I`(-Z|rFU&9mfcG$k4P`?^;jSE(@cUqnt2?{D8!I`qYrD}rSK8x?y~ z^~&geCK`SpP7RIOPWKLB-aS1S5Eas6rFHCWr^Zvm_heRNjXUC-;znu{(B1gPW4gm* zkknMbmeFMH1Q#6~^jEf;b&h?pi9Q17f2R&zpNcV~{$+mKmAjU9%m2o4t}QV3087Ifz3G^={2eF_#ajWa)o?wL7D-ZrC|Pd?*@^?Rjn?T{ zx>Jnez#z5%=Y&?VJvEUbd^IWXVIwr9J0zO>0C9u3Af^lFtrZJAF~<3idSk|Hwap6u1_i#Z?EnG0t%@QWlad z{VX^|YC72%A$*wr?Va{O?edgSn0^}0U+Cilp36M2+m@$d3WPoYs)Cj9O?}`A%o$h6l7=)!e^-9pzkq2uO(?ApAHC8FUfYt8VB9sV zJy^v9f^3y_K)QCNTXEkri-FifC#mEwe2XiZQ~`A;RB=V40ncB1cTwK8r;Ux61T`$k zJRvDe)WIjlexfz92Y-Z_d72nqYm7x_I&fb@lu_)aCAtd(yjaoz#0Dc5vAM}<013+& z`uu;^ZpW}Ks9J;B<&j$~16Y5aX=;pQYaM#hZo1P{e}`xFeP;<(-#gdO;=?At0{1@@}%9z%gJL%;rtoyP8yqld3N$t#8`sp-r z!&MillQR;T-Ag6zb~UWwleztubyB8s)YdMhC2&=>)~D>Z5)qN+7+x2R@sx@wHm!@& z%uKgv5(}Tz{o2;jb|w%Ujh2$wN{c|l%AZ4kmTdt#rQ+A}%>}&VG)|`l#?G;_;`D<) zQ8tEk1`{z!t)y3+w6kltEWK}%%e~tesXzZKKfgHz)+Zt(10WmocM3aLNN z`bdJ~G?^eGluT|xOXEzygM1=RynP3>gQc-M0F329)xU@(kLN&=0Xh#sq7Wq!C4iFy z59rcXMV_O$(Ib9o-jT7rX!w_}cEfS$JufCs`KBfNoy*#-doDfs{`VY*e3wboJraWt zLkkA3Np#w{XLsOGEKA&ecrlo=su&HKW4A9|31#MD%DygM8N7B)*g|4<=c2*KNzMH~ z=U472#X{(N#@t>2X2F16(2im8gg}3z+lcQ$$ris09jem>(dy`{3JZ$L+$)8`qxTTvpN}d7@h@u3o+U@tEx!*vhdK>SbRAZljS$;;G?4z_ zm3x5T_)O&xCk+bI6UL9BL29vj3TF4m=Ydy$eGW20|+_t3xrNbO9WM9DP? zK}0SBeJKd++^zB`eiHhQ!ioR_d95`#85CNQ9+0_%Xi5W#?%*WH6fR3d-U>%H}WqQF$FR-V7&K&u?XLyx(ef*TE2&^`8Sv9-k zB+~g{?=@gW>+>J`^P8Lvwt^(fRdg}&0V@oLHk=8Kc|2H*72HQCh!i-K9=P*D+Y(qP zl3<~{z6mTwqUH(7P;hwKEK0i-OMUcJ#7Ww!flhmz?dk3Lkzs$Re?fdzEyzzaogeW> zGZwi38jc-cm!IG~gO>l}yM>9$`vwU@-(G@VWG`fb;_P+U>L8bp0ErI3v_ddbCVhnZ zrsZ{(+fT3USwPcR4!>)(K3q1qNqX9ADeCU~W8^*!-cZYcoCiES_Y6l0i?ifd2ceF9 zloVSMU{7Zb0X|YFpqGXCN(>tWtwv;+bUyU?=&|7_Ig?|n(RKb(g!Ae61dZ5}{0Im# z-x3wZgNW2IhF*&&r9n(7`Ye}FrL`$dT+!l+*s{c7BY;R`$-^VeV~mNE)$8798L#%R zKcR?cG+o+?dUI2DrI@jBaeD2Q=BqpkpYU@3?{#bimaz@=p2`jcGk{=lab-fk{@N2i z9VNl4S!@YiEUf-#HRJza?Y*O#>bibW1e8vscY=VT6j6#OAi+YD2V|&!H z;B&S?LJAFx*-@lPG_3bT?mi-z!Qh_vxFk>D9vv$ZycusimuRjrN=2-IPD@E6z-mp%8Bn6oqNtk-}4u35$BSx(Njn|tj0 zE@?~v7bB`Dnfwk3gEm2BSvW~uqSn#``4wd)WxgD})vnZ_K(UJ9kQ2D~vES86)jdVA zS2J&CHMJ^4M<2~H=VCX5;oSr3@$(ojU6!^y8p$b!@U<`)bzTsqW2{W^EG6PJQG`>5 z@%Eq2KF*LXq#m|B>}4_3tWkpHt(SkWx5EFc=>j(-rSfX4!hLV|E!vptySYd`82E5F za=HSzboZs@A@^#1IjN*fGPQ1G^x{^d~H96- zE@H+1F;`}9c3ak=wyiisXj_XAW0SFyO5e<^Adz@i{-SZ6f(WxEEYPDnQq&pl`!vT0>-N9{EQWu&)NIt0Wfs_0tDeg|78y%LOwuXc{Xbv;j0oA#1-T1d#>s z`OWnkSMJl^?ET`pggf{r%ksz30|y}cK5aD&k+VT+VA^_H;a2}I= zp%o#wAyYmQPzv8n6J`>x?X4BEStuBDx9J{>J&`;R=^VWx`_YMu0ID%Q@Y?U3H#>)y z9YM-Bl3s?(s^x6v9-(CF$9`KxaD-9PfvHJo$7URGM=`G4d-N>s#S9$(!DhMg>cuRd z$KJ8WZk?#csvPR|^cW;zwoL=hVu$@M`yV|R#VFsZuD&lUnXOa=pmB+P=4zbzYL*mI=8TT?N>qj^Sx zSz7~JF=m(-;hhyQ3rqKFDK5ru7{vyC|1IJm)08z!yXU83ndM{!WxabJc}Ke_vcUrm z00-Mcuqpv6b=UyGfK2-S1m=%)B<3m^oGmk7nH?i!>fnGJGI9Vy8+HOc;aq>wZ)W@E z5>8c*d}#12iM2^y71ycxVVV84|EPV=*pM7l0Kv{^G0e9JpsDX;1Zc0g;lkEJfO%Yb zNv7kil7W?WVF3x-L;*R!N)~z?O zZMtv}kaY*jStX3WLAPCat^u_7Yqo48fKdQOo1n0+g{FVfpyiPY!nv`vW}yK393)|s zpu0+d)y5u=8Z6m2rM9?e>^sS5X@6o4RjmnMOfC5phyz71-5V$-XfY|A`#6a?1*q#I5P(6TyDDkOy2Q_ z&c!sIT(J!O3um?CfH?691cIn*Sa_GwN(ajl8ONb8KwYDQc?)n@vBCh++Eho+;gm!L z;U7Al8ChFd`>oLbbK13gSf@GDt2K;rqgh_x2n7V^=_& zR3v_LS;g|!$@7f9*4rna;STu1%>#m3dgH6Pt03Xm_;Mmcv-dRR@_ z4(q89l~=h}u3+A_Lv6djnNap&o{ueYb^)NdyoY;MkCDyW}z#!~z z5hK1QE4|RJFm;=M$Q@8Q<+ENMsPtk&re4M!n@@(c8nqYGPgVOnOn-QNam%k2$FY*! zi4zSCQFYBXS9>Qv@Sm}8Dm66m(KVP!IhZJ-Et>4a$l`y5n%7v+SUC%ZW)LG;JCKHv zCF;Wot_-6jL%uiThPG)IgZ)~!RHdwk%mu5SJXd}GJn}ggNKHh3aY{jl*dL5B-GLJ0 z0%4-fbeD2(y6kFV-W>IMsj{cP~G zPtyCC>sR_>amyeO)5O$JdRF`q!KyMg>J41=f)v{s>_*Yf5vna(ydXjCMB70%ws-H9}%c!e9(Gm`9sS! zEEpY^U-7WfG+hk(t6pZi`YB5Rg`&){?Vk7UaGUcn<-oTu!cP~->1aB3Es7#@{;?Ptj3+tPr7-;|>{GUMD)fFwi(F0aPsja>kk?7; zy*kS@%kaBHD{`J`RwFcPXlt6XKfw+V%&!`K^Z>|A2%u z;Ut*Qv(R^WsD=-bdlUFK#FdHP0J9r|18taTjeivWF1W%P%=^I8*{jn`mr?OGd8OGT zv=iqu?sy>g^f9GR$9ep(Tf8zczrje7x-6g)I>__u%qAOr0c7m1th2}vu&<#$Co5kA zPBb19!6?|0sEJ49>FmwYe&kmL=4?YOc)<`Y#*DCVGCh*xHxNt00)(u1q@UqQ1l z#8LOHfczV?U`BqjwBeYfL+pu=Vp(&)RY|s@K1))EfZ_GgO$ zY8M`)RPdgXZs3o3mZ<}$Eky8j1DVp`Lg~;zu+e~T2>zzTq_0a{*QoV~OKba{L4b^V zE41QyS$$jD(@b^~(+teo=8`{$=prKp{B%j^WnLFruy`WT?{fn~Cc~7k%zOG|A@9kZ zx%iZ(`@zRgC3f!aYw{3tn*xAs&}?d3B-dp0;Dia6U<;NB>_sD%mxSuAwP!D_=ZnN5 ze>oZ_&)RYisO^$JT9x)(HpdNme~$Y-c0P}-4JAUq%)rYSfWs%5^i2a1<$$O0BTi## zQJthX@VUz)bOjt7^TfMF`F_WA_dV(xO>lzf>~*Kv#^zM0q!sPc%WCnB7wk)zX)pp! zhmI16T1Nhrh5-)-U->kX|@7!sx zl$u^ctrb_>`bLR}#JNQyqcYyqr~dhwACl*7cL!gPixeid|0C_fZ)m5F1@N+m3Xavu zUxuMozWBwt(2k<2#+=KZrrVFG#Ei-#s+;2eFEO1sc-O?ry#>@%Opi;%_v#VDJsR}f zsu1aIq^G^he%E5Sr0zbGV$!R%NWKU~>k|OUXt2fln49gVS8UCccEGZ|c>g%#N(NAn z0c@wglI%Vds)>q?G7NUUbkVcCe*GNu(;VG3E43lT0zzrPi^_pArEfnE9`^rawmw%LM5|hfyW;Lx(GL zs}$((|A)Up|A(JJ|KI-k|MhmL8 zN;55QM|tRG6(ei44-sPt&0MHH_IN5ORX?>fn035fyhXqEY5X=NJgAwdSwM$LBZ+_k z*vth$PD0s_M7zkYUNw{YCbJZH?}&p?shic^N#X4mqaI@U`qi{>Ew^?h6@~g+)za4! zeobmGU-cUnGO_h;w-UDIey*7g$b(zCTmS?K+VorD`8k7#LNJGs!33pt^zqPvE($LJ zikPo(v>*S~FTH+dL{|Mv_u?g`&J&-cGj0)zZ^yT(h-Ks68nLxA5)1z=ZY(V!qWd}N zEhflE91shUB`5%~kiGd|lXVPjMXphfOZXKa3O2%79UzrC>1d)R&^u>2c*7ji=a@I& z$t?v{E=sW8qTVlMK$78`dXja#Jh~Qdc=kfgg_>J?X zuMQXWR$B=C9J7DG{n;tfQ5cdnI|!x}H1t7E#%-pv1UO#7xJeBfDQVYW@U}W-6*?x; z@bD$;(X(5!XydDMvsq3LPrUrLl$E7w+_;<13!I^%o81)%=&w6uMCTHDAh@m4*%hwu zkych@=#;zH3@#eG&35`6uBm$4;Or{cy0H6Oda<*8!Z3y&lCfgQSU(3Q>`5P86H4Cb zByOieu{CemCXZZZWp!3|HAP*1ohP5g+-I-oWYGBpFDpD72lHFP@= z6hFjCYf*tEItq9gp1TT@*cL#eu>eIL(jFcj4xN3HY+LF$@cM@R8QJyy>QyuH*57~0 z)MLopid!H|CJ5|@`k0S6kJzAi+ZqKwq<`%!lrd}wLUUt1i(y;qczTEbv+1`UegUda ztZJlvJa2kkZ6Z9%k}?N(Ljx;vjIZ_5iEf~DUm@Igf154>ua-w9^-}X`^2F}ylJuL#AK3a2Gx{DKm0Vc8NL3%l|HDtyIU zEXqRj=3C01`7nzoQ3q&?UwD)G^A364P6jwjj2;tk(dhn+a?$*$A<>vRqqN$tMHX#%{2+U#G}qMuRrOb5 zwRW5-RGYJ~$v62QN!2Difmx3T@UViAanP#?2KdC}g@v0Hfk00sFQ1u2|A}iB>koc9 z>H7WFRVMvK)($=Xf>jT>>~PCCF>C7udxY>ClY*1{{qV$s(^$(+{JK#{VDp5L{*r`# zpPKmlW!uYHHtJA8(Znxsw)2*NDX}9?D{lqc={j*g(u)ym7QoMw63Ozr#%sw!@FllK ze0YlNwO_MlbJGg+vmEjH{`-fLs)w|zUbLK#wq9VP0cb6=kZ$jWKKptYB>YrRvM4LN zI?u_uPt5PFGX3jamyL~D{pHLX2R(N>-T23`5aI>j2cof0Mg~}YZI-`DQyaMC#;%?i zQwyrjv9n`rOjwXtG``O<^2kB)%iW9W0e7@sD;_YHg=c`I0n33nM3y5{pa0FJv}&df zZfAqV=KYi9*WyhNalP5sQQ{*{EDrSd@evO2JAOP(&-$N4(MGL_2$JHq5;PsMCCzkD zGgcNVkWmWyZ>}q3DYgl{GKt#MFAFNBX#0ZIJThW)=rsL~94#yWm0fCbK zIxmEGyv_(qce*UN*b6xyvt-z+@bzG-zpl8ktce4-8mpKVJ>c3nL=%zd%$IKBoxb)| z^wIED?K#({R=Yms8l}w{rHy3{O*sLhjpa4Aauv&(5Jb=D#KZu)O`Ms=Vxw0~Ee17* z&(BRL8Tix_3@Hs{D-P>(+cfq;T6ngH zlMUQf^86Yup0tC&2Kw*32n3XWs*=s3q!KJHhmjkzlA6yWxwzj*ApEnNH4JVWnOchG3C6?& zCHLCdL*OblX<89>! z1;B;VnsVqN$OD(GtQAGD4G#38>284~&Ws0@!y1{-lAZQeJi2`0^n~X1Z&Qx;o|-0r zqX>X~0CW{dgWx?fZT;b$>_q zU@|~)4^>iGiW5dRGhL2x8*fKEi?R$-CwVS#eNN;+q%Ud+%4H;2c2k4`Wtenzm#YoJ z<1^S6G$f{&WmSRK{#KDHeOmo;RIm6ROz0^-6*4^(sBBR8>3Y$jqR22bJ?_0ea7MG| zY{pt6%vEnP^%3UfdIx!CtEu#y5~Sb_Zo{A4FFfcn08D*{pSNuLq<3_=DDq~`z#-E7 zJ90G&HbR{oH6X#NP^+!onRdJnHi7i%N2TiS8MMQjpsUK)-@R*H7v`q|T+f~VV%7YM zT?1Kg-N2xcy*^|vGah(mp}VRGTj~<3(Qcee4)(Y@f4L$3!?2cauKMy}g-SDn zpg*g9yhkx>S88wSWGi$K$Q$cL~B$wFW&uZor6b38K~3u)nzqv2hsK!VaJZ>{*~!#%K3^Iv$+DycifT(>x42dP1Q2 zKSkE6dmRiWwB0^N4cQ;Lc#BKGR1i7l&9hMmMsM?sZ6c$|)N*$f#(JCobV%#8qVg_! z!9(d+o+Q|Js-mLgRhE6i@)}Q5rfS1S#RL2o`DNueZlo_i5#f}PS{`8I={*(~+dV{7 zaydG2RPA$;$i_aqS)=U&J?D9F+`~$FQB3ar5b+q;d%ZY1?r*MXT|~f>kpfUnuQ1Cb z>ftTyBNecZDYhasyGyzv6ZBk7)6@=d@jp^}Z}lbPKfU)ysBLNE z+Ebv{mN#EX(uwW!!L+gtUTE}AOt+D68gm|UmDPS?{TAZ6trWN(M?45@)P%qYz2nq? z+!4(ZB$#ljz!d|(nPOgd{<$A>-ejMjx#M5vR`E8`W?Q`9*RA$i^Ih5-)*t?6(=lY* z5E$aTe}))^FPpUU)ADiYELUVmUQ*2e@>tzJhOM3Y0(JBSKQ)>4{9m+bHkh9nD27Z0 z$I3KF$HrI}XOLb>wSN?0KX2oKIKNUjUz2&!S23&@X5RPoi3)me}2>d)s zXiUf7ZCvDVNxNO3>Z#Q!F^Xu2I;nTF;6)V)# zrgtmP&Tq9p-6w9%C2C`^g*^|CASiVZ?@E1vm!+XD4}5k$KKsc>UkRI{?;%9l{4cJy z7eDn#^fTjrLe7p{En^=ouj?motRwv33W(1DnZ(pXD}Npn0ZqFwil7mOU$`$cKRe2IJmY)TUs@@8s z7V7`C6K;VBfQv)_D*^;FW!Pw?5?^t)8RtQ;_lsS4C&LU$i^K0a{|ad8f}TlQUW>Y` zb{nOxcIh$cD}Xyx17yYkTXG!$`5jP`X6R`Y4HH!nq$~MNd3OVyoHTI~`UNzh!tt&# zA>86F>RBeQe=3OEkF}~_D6GN%&_{%uMuHW9Pp47JD(R|5ycQ)rzh?Y~t(bkm{-SS; zyrWj{n-+q4dq1WcM{7a9Km2PqYzpiBSwjTNSl4aAo6|T^W~#)g14Kb()h2pe+LP}a z$3BMH93Gopd6T-$a#79s+G&bnjswpF$H5%B3|mR4#1|;*Z|_@(sJrK3_&%{k>E1%L z>T*>NO-Ij05kxqtdl}}>ZK$XCX#X`z|HMh?zXRn)6xhYHP#gHMHwM-ur4hpEYRt#< z1937*ru&O_y?F-5bR4L_`{~@v*ScAk#TR0BAw(3!J@xJT^w~8ND>$~?&Jte~>na;J zmfL*^k8bas56=|)F>X0paqm>Ud8gKPh?mzOIFAIK|nf{I8iv9uu;0eNM@Y0o> zXhZDuG0-QD_NyHb2YZ2>&}^v00_@Of_m5pB#iNV-dRFqSU;B8a{hq3bTTv-_fA@p7 z$UU%ZGkh9Bk;oP|?haQFcxH=D$v2q5N?OG3=j3vX8qmQZ#+6vP3ns~kBj{{^i^v#% z47CQ50a2k6C(xqmgRaqts;R(qL&CpBj8-aGpp6LVBE;BaN3PDvw&mPA7!Q%iBmi{-_50LbGkyz4% zBLM9yZ4U#(#o3bO9F{X%fK!ib35A$BoCgHFv~t6|znIIP+>z25!2R#YNk^Ecw$_a-ML6Nne1`Er^15p22)bKLD))48WHL zb7hhYTdE!rQ4`;TrrV&T_6YCmnQzJLA+zI_)@PUy0dAt7T^MN76(+cI9yB`^HR z#`~QNA%Y%YMT7di4Fgeg(UL*rXq*y>;0OeWD3cL&EOYv{d-!m(3?9E9F7%J*TQdzF zymljtywKUE;gHz3?s2$oPls1^qkQD-u&rnE|SJ0RZmQfvff;uRLX>(9+&~jjqMx zeFz%5@x{qZNJ8OmXWjRb-NCjo>bPc+W|dT#EB0Hjuh4}q`Mfk!dlnb_K=YVk_#aZR zR{h{PgP-rd*eJ%Sa61D_@jx7qHM^|K&R^?6v(PmZj;b#QXzD9+?n2Z;OqJ2H8&tkH z2i%#VD_DM59ME>^(kTva=t%C%zChut&X>GCtq9w+9@OCd%A?$4A<5;~L#DfsXZR3- z$E@ANcegi-6a#v2VJh2@=66Ll2%Qk{ecz&ddVMg@)0*+|A8+O3>K#z(E9%}+ zCTMH0#(@mhIby%^zClwpx>Y+->2gDDE^if1VCRbq{T?CQxj*)*e%xY?>z~WA94;E_ z1rcQB9|y#}n9rxyrpuA;HxRJJ3UcAv_`*U^=3o#{yIjE8`S{bz7t%%pPW0yO3$?Ws z*vGeJ)U5QymPMo82TdbGNo5fAE<|@DO#&IgHkw~}EBaMkG0hIuCBLf$DqO9g_a*B3&Tzq5>ye&Rlu%yM38bW7-% z)v&w+->VHh{_^7yl^@7^Rh4_6avol^z9XGEdyd;$3O@XQ7#B;3x>)$<2&Ie65(xDN zn}1x2odlG=K?%mI%c+N`Sx_w+RDx2WCWt@+y%IvjcUX>Xr6duyLXT&@H>U8}pCj+@ zFz7STw#rqy?3eo5({8EhJ>)$Ao5Uajwk#9&RfbHPZWY0XQPUznD*tY&vt{ex)x9Vqs$|L?DHA1N2X2mOH4Q80#^nM`%gJ{`tOvL}Q(thv|4Z}*SoXOl|t4*)Hbl#etc+|1?#RC^Lg)0 z4AZ&>$I&D|LX8NaU{lZl60iu15Gn3yC#N6C?9(gCb5s@|et*48Mdm{O{r#s$R~!y1 za)*+}P;nS3q9XeS?DvFl0k&AsLT`sJb>0ec+}Kt7tpCfA5Py`)yT-!7nCPPI<|@eu z|5A42q$&g=_g1jwqO*y8;DTf=%4%U^#fp~lT6yR@9#hR#3v zJbQ0Q%1T>kw|^8wGsu84sM0SCk>z{7SHn{^=x-QG@VKt~bRWHCI*dY;%~u=j`St zmlK1&O6tyl_56|3Um3PYpDAU-XQZ6z9x2FXaNdBU0DLt#3nZ49@uMJmBtrBDa7SmO z`X114;vpO!T3*u$YRWo!^-C`82kKXxTk#N2?3~nH<3`_KxY>E7yGQT)n@IsvIm4M z>dLY`j?Xr{Q}M#R03q}lhVHO7xdvv9K4AsNv9NH$xz`O0rSJ9hwYX*-yvTTY+16># z{N2U5>f1ZzAlu&6=R!}SoyV{YK+!%vAuTQR{84gw25caMocjC>nzERHejS8Ld47v;f))BP(+h}b4Oj`TNH2tDA7Po` zb3t2^?ejeJ;*OvpCFO!{*UQ>}ip9~=vA%D~# zGu>1Q)1GYs(Ap-+r?G?&Fe;gYsqrkNGzUwHNZXN`ih5NM@9djUy@3+L(0z(M0pA%+?fE( zr?z>1#ZxxN5qc9=$X2cgu=xX<#GK2j9UzXmbP)lOXgz~7;JVRAE>C z$$F|zxMeaV#Ju@dq6^_V*di@tYBp{wvUx!V2^z3aNd-n3=A-U!o9(F)JlKX|L%zTG!ktUV zaV3dnpT%PZcFhjG>h4|H$OCLiF31h{66hK;K#(2ekRk7fx?n)~8I%};4XJx+{yWHk z#vN|vKb+L6d$-4LO{1$Lx=X5E>(1^Mm%WGnkl?);Y)Rmi76AOfZRv$jI~MptEN__Y zh1;jwU7M6pN9kkFibEc`^ws0f8>}Coi5<|(T)nbGD97IO3UU+rf8l9dVJZSP`%Uno zCzc8Lr*mM~+jLAVp4f?&U1Q)K$-*@~@ollm7M&^5B^b5p9e;YQXA{rHUW&tz$*5*O zDwO2TJJhm+pvTx%_yP`e9Z_i+to7S9?^)Yilf%=ceX>wXZO0k@1ztzGQQkg0k394z zq#HxG0G+S347PI_{IwhNQnw>~#nq`Lcm3>}yd$SZ2GsIGc35-es?)i?fNQM`MBp=P zJJN0ZrG(}`g=1czCqxA+T^ZA{M-)xFSEiodmab6px+zlqeeA)skbP&0 z>sHUh>m3km>a}T0$kQ{8iTaHZVxOmX%GW)XdQq^0p%M4%*UOQPmpJ43ir)g4o$?M{ zarzkbVV@SAm2Lh}25Ls7VYta&{xfLmA2^Ufp8uLafT$JTuDG4?CERXPA6K-rCAuw- zEWcX}<16F++K!p;gy>d<8&(M&Kw%gVR_s^F)rBAh>+ol74}v(V@lA_BPF!~iuxL01 zvd2&fh%(y{4u7PdlO{dhVf`}7c7H3COE3J@{?(pB=zTvHL4iCV`qZSaqx1o?P|HF0 zF5xxBklZ=35#GR3b%E_RW#w@my&s=XZe}YlG1NxrN5a~)1`us~`Zw1jK}5ThhoucZ z;S#8Lp$p)$7l;cF#8Y0)J*D~abr~y_HkrvW<{tWag+V?;t2(hV5=FhK=MH{U3fZFt z)x@~tS=XVqA>iucKk3L!9S4~|S`MuoI?`z-=fRh_iYR-JJJuNo)3tZL46D>tnGKNU z%DE(3)g*f_wyIANN~8R5Fm@U^I0 z^G7)d#}aadzKxOA@*@RHXeiV-ckMxvsV*%-dFB*oRygaBmUVYH?|2$O0K4~Qbb`Hn^98B z6y!O-a`=L45;J!0(~VBeNWJTadd|9M{7X`c0P?b}n=r2LL|`dr^CqMN0owY87_lXH z4Thvhu-`n6=R0#fk6Cx2I;APDfW{lYEBNwJTwry}*bYU~yShU9Eg3c>p7mu>E38F= z9awLpZXfoAo_puazLP=+W>3}eosGmQ@(h!W$o~(|)BC@8o-UD?I_!)L^E-Hen{Wss z&|i)KSOhjL2qg_sHV82khB;-0x-Q>FHzQU1}u@A?(Ai zK-KCC5Wto&NZdqPK>Y5zg#jFMMR&{8(_Is|E_AB_?s&?hefgA3e5o5`b<$61A_K6=Z>inOM@WUKyDiT zWtfGZJ$vp!^G4dQ{ubKj_!B*EejHcyi8~UhAC2_^FMNz;01b79$^bgak}hI`+n+bR z0MyFX?zhdE(v3O~@BdBW3LUVSxJrA_rZbmO+7&E*Ldxl}_he+N-UF|L7s8XrNnf5K z!Vln_1(yFzAuXfro9U%e-8d3a0{db%6k4%k+8FW&A2#N%Y$iAEbSk-LBzm)f@1QJ_ z11LTH6nO6ddwU7f9*pYa(2RDpGI1v|0wvKyy3Nql@>>{o?0{M4#l{O^tE$!bu+<9| zxrd4ij^`Q)J2=~qo2%|aG97=Kfa01&u!W;Sz-r`$@${Pu-NrGY4jt6lf>&PiKW|Vj zr<{5{R_nB@QS8@G7cKAY^=?&>CZBj3GY}CWPDWoOM6DxAd1U6`1bqHr{y0OVS;~|v z(+k#R2@u~a%W(2rP&(a(RLYib&*nZv;LqsWzSQqO{Qjrewy&9=b9#m3^L9vW*UXqd zC|;!ZKx{A>iN@eT7aX%N{um?$H`&^7TsR<&w~#0dwP!8K%xvrt47N1sEQ9zsV7^}-gQ9#l)}GJ?rJIpRsrHZVgjlcb zoeD}f1-X_GeZae*a1%TjXWbasHfScsh3*wsD`K%~fD?3wC4EUyQTs>E1GieXo$;Pt zhak!;?<+UA&VYqa&4=hdUO*F8m-&jmzT(P2F-ymgoo=CKxH|aKajXz@vBSS?kFx#f z@Xk;Ddfi51GQGuhx!RAO5*q+K>>ir#go6)wQ6V5Z&5V1+DK)@NOW4Ccw5gic%Y?Jq zw@}Qky2{(o^WvHmiH{vjAMt&sldG&wC0u-YGy5|J1;ST4PP8I;NudyQ71}Io%V+gW zdhefp^%A}$Rr9k|QBE!0Yjmg6!@M2mdsNOG**!T8g?@Vk&Nylj8k|}{*7rSFZXw>d zDvczzT>za0sz~8qtS$8Q&T`^U-T1gP(v*r16xaD~?^Ef|PXhE;FLcF}&-Z*rv5(3D zr@CRkO=->8TyU|}m%mYtpiJS+g*9HN&4?{AKfni*PmO33u#A%;n9c)rjym$b(&Z&{ zf9<$~YALvUx@Dwox7MHNN2p~`eDQ-Mo41BjK&Iy6s((>LiQ5R+yq*xVJ)`L*5Dlw% zPs(HL%lRI3rtpooiei>A7X+7WuW9#0k}Z4-Yy6jgKVA$m-doS0wB?561~$vw%yZve zlkT+BQnPn)nEM64O0*t-8>zz|K7d;o=yp>>_#d;}&-+lsew(To?z`DfBZ>gv9S=8HVkAUT5|VvM8eZ&G^qN0cP8iMfz-*!nz;jv%cSl@$OqI2fv?&2WXV` z!UUsJlkB7L{rQz`_`7{G9WRznxzs!t722IM`f@-{iVYI-wla(aaYrbmj|e%_`&tnx zM($IOr=eu*#B%fL;^|qvw&zDg6H47W6nhU|&)aUH`t^42{?V@)AS1lfq8`fX0XImv z)O@4VW?ZT7%>spSznWR@@gJWH4x0@+&%FKASrY5Ex)L>71zJ?sy`dV7R*b|VbAvA) zIT(sLMQ*xnh4;KfbA_C*+#9esZX>jx$=z}pEEuq8+GJY->%-l`fuDCahwSdXC}a~Q1!eVpjI?GwTGr}p~m5_C%j zLISlsi*r_^V!Z0Ll$<34%fb;F8Q# z2u2LM)uf8=EnE!|82gcu{n%^od*d>DG;h8V&T!#k$BaSL!Hmg8r88f&{9zqT{K_TQ z1hE;nP*-f2$EhCv6;5T-T~UGQ1{FH6N(BxZx_~UZ3=%iqK{%7Zm7J~`;d!KHrSttiCIw-uCE@q)71*fX7`6fU ztjpp0f zgNi4b5KxTDTA6sZ0wcRUzvbz5**1@7p~=%UAMUa%(}|zcCB?5F6f zn1D2}jxCb6gXM@flK1MyihFMaHOBQ>`(4%kZF@i5)%)kVTCyzYd~oTGH)k-N;OvF9 zSR7Q75QV4$jT_?QM@x;kWbLAn-)?(9KQ7p&ulevr^N*rv0GRF{_NIm?pEw|<>0BuTz&u%flPoSOm|s}hMUH873`*C*^)DMuK+t9F9NtYYX!beES5SyTEIQ*-2xYq8Bhz!eta_*&NNWA{F_3eq5(>~o!5^K5e zNv}$PU`5atuuQX$OHF`{y4$El%8Lu_h+!qD$ML9E==S)cX%;@UT3FYdA+ zhOKaVYDqMzbQ`%&o{KH@5-b*lW+|m#dcFWqYj+0knPb0&yyZ#m zNUpW=A0z}N3|2lUP{?!}*TWd7b#C9N!Ci2WI@%7g&GRGrESQ|b%e8U^<8I_zyR!J5 zNXI^>ehM&0KgH2UF|RNjMXM-?URe!%?rX1JAo+Y7@2ud_95!aIqv&@KNe>K}s z^YD^>)jKpNJv|SaZnyG9c+1xp5aC~dVr&Op0w#QI2|XK6eau9m;hHzt_QvqpR~ALH zAE%Z?Ob7QIOnX@2H&qq+m-+9vk5IFAfy`KV>-b@>c% z+x|&9*f!;EKGU3vhkfd|;vQGO&VHL?CEB!L+4k)mh!Y*V0mt|cU=4vvo40y1=_sST z;zgs(4iQe}wT8G$jmDSEla&Od9uMz|sOp2hcBof%nV!+;J!bsUuh{8^!jH4a4q?u1 z%=ZH=An|Iq#exo3e*_PBfC@Hvm{?FQ6~-cb^nsM$LLzFNEeD&i+kM9Kv@?ALM4rDD zfA@4P?uHXjcNpnQWhAB+2dksl5?HDXTYQaaSp%!l6?$I5ZwxDr{3dUQhn=DL{u>+C z!x*#OSyt)%u>qEeiTytwX@BUt%PM%xvPM2k+p#y?%NT2IHzyoM3>*(kDDSDUQO&dw zPJQ-QIXAz`4}KD$v>2^3qu`k$a1PevYBQ2R3o_Yw^j1R0Lz6BFJ=$I9i=MZ;V?3=X z)ph@ueXp9VA6=JAl47$K<)JvF0x~OqPO%FN4%D{|3=G&4wlgzHd$}j~qK$e1KxS}O zfKi`+hkuelfJO&7(O9{q&eH<;1JoED<(`^Gzq>N?up;$$+b_~PfG>1p%iGg+nG8;a zsRhdcz;T`jL-7-4`|?)iGzKZ{l$lclWmRT=qk4xw=?K34ku4@P$Fr(wf8q(^_BU6a z*ebUk^c$@8crf)CYA+xNDx=`?M{xN9c7b+te2U?B|ayoj%`(U;=3gi-`P_Z38uQt3r|UY;$ltzOXF{ zZE_g?^)^@`H>cvo<4|^~Wc?0_06`OA4Z+#7N%^3!F%b~H&O+cKHicXZr4V&PLl8lX z2R)PET-s62Up)GOTa!X!}W zX8@p8*$gfRbHEM11)K?GaALf%AnwFFB!!iouc@q~+@DCWAyB9v`8}!D)qkggm7$Fr;B<2#*odY{xHci-6sm{hwfRMZnpz6UKqMtA-p4G znFPM%_!t4r=%ab9>@!1!c68(}997nCv95ekYiCfiE;eBssVvV- zPvz4IKL`H9Yz2U|5-!t8PL#o8>O(qEF(Yu#X@9{ff_nY9AHnR}{pHL>iN*Xo;-)Vc=IweOmU4BUgk*N}uqrs9q2dV~ZeC)RfuZFTL~WdP4mE1cKi1|GP_bwhv?`o& zxNEGlB95Y7ZL7Z`019bw1HqK?y#0$n@Gp^4gI(Fl_`yPi$KY^=>QB&BOWX-`GJ~b# zo-g^A^!ptkU42jK`cL&~SIKc-0) z=i@l^@#y;!;l~R*ZTLzDq~LpuCPX+3=3Bs~0BC&*k)JqYJQ2m-Q@W@d@B|7y`m=Sx zF+z1$8*5*4I9w1qR?q8&+qY|QO>J9dGkB`#&COtS9{x#x_BU78M(muzhyh39^hk}( zHRp6I6=mF*bvEX~NJY0Pi_P3?qQ)H(A@$+T6<+bly9r>9b$d+rDe87+E3nizV>X1W!c zfe22O5=j!?ks9Qg=-TE{w_>hBxVi_+t|0glrpd(u*ff)_YmVt0yO|I@qCbjIr-|)qW$+Vw> zzrd7zdFRuyqO-HsdD^0v9SU}Tx)7=z1G*|+o9jsea`R6h*vwO8z72v_)2-jRWhJwF z4oqwW4`YQNi5HbW)|^yHN1d{OxTQEdp|N^{cNu zH#CS5;IIE)^$gK`Ik{6*K<*K0MsrhMX9t;Dg`@iwE(fx%A>$#qT8CoR4HvdF0|ljA z9b(%)TRLzX*HQlQXp*2?`(9nYX>QfyR`wrbpWNP_JNvV+Z4EX+Gga=>3Zl@-pxIK9 zk4+7ttT@@N#|0f0>>gfoNZhC6C)ro_en>wf??(40uc&+&g^2oZ_8=6YHZP(s9Nx8= zk7}WW-M?=@2`^SxooOQ#C+|b}{mD|Ms`u1GRD!i_NO7hbv;m1wiOz9w?dVPpDfZp4 z{}|$Ot=PlF!my^dE$eu&N;&ryUT|0SPyi02pPm2wFz!Q{$2@3J!4S5-y1wx2DDQO_ z^Qawh@;>)Dom=l_S%1oQg8oM~27{cN?20ebHRWWY@IfARI@(NyjK-^UPT zCbKxkQl48@-i>;yet8LLnj2Fv%xuKYv()s+cOd{ioABJYzHPgLrxlcb_ZF1^I4ACpVB6A z(324(bDh9@iZ_o%n4k{Es!=XpU6DvFJYGaF!M7zNSj1rwE zS;{JJTI}=0KMzMn-^VK+d9|vG{_sCTaxlR$s~5=ZVD~w#`xtwY(btnrsVRB@nE#N$ zRJuz8-uYH5$lVw5H2z2+^UII}A??K0#%5iPj>O#6Z-p6NUUlbUZzcHoTFPD0my7^O z)vNywAAm&wEHV!uVk8DMp#*1oTZQR)Ix>v$6CgBHu3`G)Tl;Gf)Ew!Xm9?)Yp^|HZDuPkCT5A5-iSWW z)V}=tywU)H_*)IQ@-c|`CX5F$e*_EzmfIc6p{${Tx!{>eRTlAE!1k)Iq4nh6oH*+J zURQ}s5i4lOaEpMf3=O!F5+mOD?w8901;8!J3vdRM!}{^Hd+bVA!2dYf50>AR5`fF2 z2sG*GS?lea^X^)SP5sOBePR)FE`={XM%Cvle?B-MtC{=@0~8K-9$XgZ;&sxH2o{bZfm#> zfK^1C00&V-hXeeSU(=IB{VWGMVzr( z^tJcvm-{5+HZ7Z^R&zJK-5>zc@c@r%XY``^Ft~pKbYh_!GbKl|YAAAN$dg?&?mn>n zU^bp+lI=D3*3jS0T_Lj5h3lH>L5YkE^qsz^@7UahzFeeQ zgO0s8Sn)A%ef0@5c*+{RFmxZ)n$d^lN$)*9k;@v{VK)@Lj+2R?^cbn(wgk2`1!T2V z)wrD5X3Is1F$0Wm=DO%!X7G&nSlC_|x>ZEkPv-0Ix!PE83tFBFn`ev?K;wEk1cd#t zH~I;NnC-|n`TM-tqV?lDk+zxh2W-9SB>vgltdy^gHf&p9q&eA2aD0*qyEYqEZQTRG zj+Ka_T6Z#X7%cybE^tiSHYCg%$qqfG?s-2N9%gSPm}YK0q}L2cmwgho;IcaXm-ANr zw|zD&e@nB%cbJ8AsNi0)0hx}&7*7In#Oz7fF`5OvLR*eE%3m$^q^{M`CE4aL@h zT0z=sLxAJS`cbcSx7LPM&sQod4S?%{DRxHkoovOo`(nlx>rA_!02G}`KMZ85|C!MO zo%V~Alk^X7=&2YdC{_#Vq#x;cD3mi%Qrb(RV{SA}d~az;@@c`&TuCv61XALwi|?4! zoMV2AX9)^n@PHC^#J9Um^rY&Nu&EJCcvXAECAX|>R%VEnakgYXcm z>auK~Hhn1=wJs-jwi5cj#E{ub+r9$8PGnFr@TA$+|DqHoHzAtv5e6rgdI^xe!3Nk! ze0}fw8p%tob6x}4D7gpGnxGN5Omjp-L` zA55(an4}*Ryklo}>duLd$Dhs6Po-3V4fOzop;Dq`t0Gdf+Sh!`ank6Y1Ld#BYd?gT zdU}p04#bg8-jw@imV3a~vc9U1J)XV_{*o((_!e6Ws!}YJD`dwK>J1%yNl;CjlzhS! zJk)*qbgEr_jUZVp^;<)Pdi2qQ&pvv*Z96)>Ux8r!gXQlicz`d^u!CT8!W5Y1b0Fb? zR&@!Y16-=kMs#1ndYk4)MXOVRoHInrHM2%xar_~ z|Dl1dT|{QkFG?CcJB+QL%-aFV;CvOg$5l2TI2cT@k2>6nXRFUqb$Pn^(LPdDkNayp z|Y!e^J+O&=#l>OR}25=$?H0K zo|;*VTG?KyFU-rCN2FPn)S@x)J)mIW)Jx#2#jyn?P%e&XtK2T}fcV=(7F9g$Mn6^L z97Jaavug5uyWvc_kv>AGNRG}G#JAyjLNUaU(%Sb~2E!`K&bb;g&UbIy45A(~Tc6pf zT;2lZK1uLh`^sQ!*LjKVuygbxWo@T?8xQ_ZZ-eZw!#q)*Y$Yg+Hm^p()9c;uo!JxH z+1Ybr{s5K)fDV8nM#9E1uvz7l8?k(7JtaFZy2YIQnzQ@@Bs4eOS!|LwW7}QykwT&D zoX{3%>G^wY@Yr#qrTf)esm`%1e+pYb{o9s$9I2rU(IXhg!Ui2#C)R&&xb@5n;IEFV zX`tpW`K6V;pll0f^n|#bFZ~PfOtJX{V$;8<-=PbCpF%CbHw_FhKo&BOMijSt@W{~x z+QD~2OMtnq$bG3tN|J0k8C92qHI2J%>a|f`?kcjue5|15oytH=5@v48z}aTJ%ZQ1# zf`vc{2)3gtO_hyO_$Q0_FZUgZbQl>hV(R&J_{SlseCb0{8M`1(k$YzK=V8M z&)ettURYWNyI#8-VZ>+vSi9~@M5}?ZpGJlvxT7);8c!+(|izfhY*K{%^>7(U?+CUdP%C_ zw%?)`$2?=pM%+_4`amUaQ|$^<^1=yv{-E-iJ5Q4KSC;OLf-sCAj|I@-WpGf5glWwl zE#6N2RQy`+L%vD37sh1Q$E7yCp2d1f!Notb4!((PJ7Z<#C}zXl#tfDJRAhnv-UBOl z8ES7Nz?Nn_P`f^ZKlBE&MLll@?{ws%$nzPCpPU>f>rBq3+5BGns@mEwU~ei{J1W|t zs%yFLAAFWOPTIdFGlgPR7KrHe>ERgrT6}2Msh4gU<}k=HwYWiT8`o z9Zl@PrIq0S_$tCY-hH6m>|)(BBCNMUaEFL_c=#rIS^^^W()iR!K$zT(#Qyk!smP*8 z_i*9r>35eGJnG@XRJ8g;j2QZ5${_c0+(08qIIwcmq&w6}np z8~I7;UsT!}C7r?j8oa+YV}ek%wf=(K4|#l$8tt&#=?U?4IA;QP)f9Ap&ftEj^^|$> zTb?VFyi*?1XfB$P%6t{;!N5RW<_oX`w5x-o^*Dm&%{P}Y5Ky&O;9+ZM8t~0V!q`RZ z6M|>yhjaF1OsAyTmE*|B=MK)J{vlo_8x^TTUN~{xG+i&TOR2}$6PLJNECnl|XvWBM zaM1?AgIq(|!T3Lniy=5mUhEbYhrILTg~Mcr>pp1P{XC&`+b!1Qev(i$kC>jM(i#Sl z#?d5zt=@MWQ2;-c;UgPIsU|d@bkg@IhSbpMA|F6*ak{3z#E7!pR{uqz_MPNZmXB*2 zQ%E{?BvBAZFbFdJO$1K#68P082=2g|h^%K3m+2Y5?H~Dd0o80R)6@BCQPU5im##+# zWeP|Ki#hQ;VLR64?8Yv~4md4lpaVnqI^Tq$L6 zUIZ_C3~ILs`_DCyV+<|4>+PPykb0lAN_nOSJzOxWfh)4!8W#_;r0mp~6x&@H*o3aK z`fOaXG>`WB+=d=054!-hPhenUK-0+JI7-4rK|&a3+F+6lCM9*1ml06QQeen z^|TVPkIX^30-O_ygW9(=g z$59IsM~vxA8~c}M*4wNQ5y&#CHp(Z@r0_iCjdm0Tf&9_B-3)Ltfh4OR!P^W`gf@hh z2vg@MU&vSSOU4htewr@MarPX@pIU9O^4vF<_%f51`x*a5^{Y5x%apf$PoXqsXIs8Y zOn>)IsTjD1z@re$2B}`O%Vc!>!c^;0FGHM+PT2bMNv`W5PpO;WN&R3Sg|@h~#RvYW z+Mk{tH!#pU`*r~~8$wR^GB_rKsV+3F@mAHaPgy!l@Jh^Lv!s(XS@px)DHB${>8t6_ z4&cr;H?L2>;&R^B6&i=2_Dclfk4i)d5P^Gy8o;=9D!J1P@h7Qq1yxWljA&~XAsEB4 zs?0zE`{&E=2{oD4X&1$OnQq(+e|y}xoXP~X1HK(=`{WrB(pEG>hvqlRkVQa6ocloLqB`^sltVo`IuIX(vgOX+_>*M%t)HH6UPM2d#y6QH;0QOfw}KfvA>Y zeHdOja+k0aXF^BzQ1m^J(CmI%Z2r(<^6~TPp_KQHCtjXjc6JoivEEMj|6DJ#P!JkK z90?np?gfKkBs_9hBqmh<)lc+6Z8;9llijRkDr(+>j>YV8VaE1%v;^bJ!-#p9<7hR zE|FK2y?Ifz`2;cUDHLSz)#{#iUmxc4k^h%huKd5ea&3|(jtAe*no6&Nh&j8~-D(L7 zbwiP_;hdc$r_tXVY8~lw`sbHVOtoxI3e>8S-!|eo^5QxKd|q0wONIf9-Cn)`+oJa_ zs!M*TDRP~9drb?9(;zUFN9WyMQhIc9x+CPhn$!cT`UIugkbUzbYgz1DqtN5>-k5Yq zfiuU3?rTou=P zw}pn1csOL2L8C$K9|`b945Dbb;YkfSvf%umK{6GX=o3Ko{k|B@bIgSqG1 z5R9WJAYzXSg8)Au%SNq4+`23z6RLYp{rT^|uj}3S-v~JMT;!v)*qU!>cUbPi=2tj;rZo{nYcjDLmL8(ma2k{0Oy_bjER;7b)CV( z_UPT?IFJewe_aL|AWignuvqp4;y>tGz!Vcgu_t#=Zdm8p|Im>}q861rziKP13V7z6 zhjfi{+JjbtJPY>T&2b`=y}H0IsJRQX&&zqV4uj_IJZV4cF?@68tGz^m>+?*^TwURc z;jPO?M)HLdG`s&sy5XUR$(=M)#F~VUJL&%VQ_DpXRmHSy0()H7?Xkl4&Mog5f>_(< zQ;!|?p+Z@JSD1#{wZK$gy9R;yvX_L}P>{?iO3-##@+1g5{s>5mlQ1wzIH%-XldZ4d z=wg3{C4KA4n5mp@8%Nig=79lkmOxse^69NDsuZ8mM)+ zoaE{G{#6U7`n>O)+|^YH>2@oLNM=2;A#}|Hpv+15Ut_#o53m7;!nz&Vv3#wR)S>Yp)N?9jg^&Pew3=UM8oZ3ORI{gX|J|qCWNm5z^;+%Wq!T)KJCNp@R zfd?rv<}vl(H@rw{)6Eop#97<_7K5W6o-fUF->K3IE*ix7_-oB-PRm~kdU@6~V4 z%Y0MlED9d>+M)?}{QRlpN{_5VE~OoN?lSqHDlR?oQ^M=}S}$3!z+c0DAm4TaeeiG$ zVq-0BoJ0q`)8u%elLDt|=5J?H;%jtxi4RJ5Vr}184y*GF-qG zjNoC1bDr-j4%6$G-Wx4j1sz7*NUzdRzLxc??NtvC@dXo;YMJP|z@Q!{xZp*@K>4If zq=PVpUF2zSDP9ZXgj)ja%IBe+=05k2$#PFr`Z=6ua!)m!5vvTU9M}TGn4B@x3p5Zu zM?%`{zz7{d?1|@j4UrhKvGRk2J0HEh)H$1(#u^h&i_dSDlUON0Bc z2_$Z3qvxVp?p5-FV&hg{AAg3Qc>2m~qw$MxV#PEKMoe}~(wW-*Av7=evc6U>a$qt< zj3Y?W&J!WPG|`O=*EV;$6xy}`1vM|%j+I4+*aK67{3c5E$4ewmv{`r}OO;#OEf7O; zjS7l}!^xp&Foo(~0BxT~K_JnAoxewhI`~r|nmH2XMS66>l&zPzKJC9*o9XQ(-Ga9q zEr_1`)&q5C8f|>{zYut`Qw9}vhVhTm3j_<}NWAUb%(w$IVQ`%Y``xWY5(9k-5$wf= z(T#8KoiDj(pW(NBS$4M|wts*A8D@eaknyfWv@sMt`U}7bco0lMsjV+95~QXPtVOz6 z6ImsxlKbhwxl7%Ljh3=cQq4P6ml9`u*+7CwVUDLo(*c|pMg!UA0{RlIdnk&){CBiOfk`Zbfr4VdId|9Y#=U(1oeVR z#Y(6{fs}ahI%AOBnh2FViuklbRe7zQ_dWgAIXR1;oG+Q)JO9EKX4u;d|4!yxg+v_D zWVS4?N4Fo{qwLgMrzls=UifLb^x&4|__LH75&Pu<*oH-EXN$oV2;&mUw!r5}%^^|>3xe*kn=Q3P`@(3bHnJ#-WMSI!^j5BfDC=A;_idoJ*HJTc`B z@nH<8@-lcD=;@&NfuGL0K+n`jeynpC2`NEJ5-X$Y&R@5P-{ph`w+*vAIt01-eVRRA zRuGxm&Z5w%;P?dcg5v|KqwUVx=8K8APVuPR7Zr%f#Vq|X)q8U?grsw#q|`j$(|y91 z9^83DdKTGN`SG9InTwVU4RN`Kw~AvMfwB)uE1U64M=5ORUldd8xn3vUUfu{tesY?7 zqMUolB7uAQCnWo&zv1Hv&e(;)x$095VqY77C}j-;K*`sTPPM8XzHWczshE7?1gh_68(&+*e$dlg9HJc(fnyfyK{CiT$LO6_t%N5X>ssKe z&xt>1PM$jHCyn#o1>N3w?mbrF)t8egD?bx+>%�zYv(Z+wfy>fR@5O4tf@F1<p%YAlM(IlJfcLn9Xm(Gd*F(r5-47?DH8@Sx?8txc6rFrO*VTPH8F#I)3Vk!s z;U4MJBlnHiK+e5{K@qP&9 z^#m``B%_a)w;8K`Ii21QX$p%&$aw+MdvxOD*RwHxE~d{BO;cyta=b;S(U{p5me zi%E#b#OWV3T{cYeE~#TDmtBm8*a#0ngz^jz`(gisPq{#UgCS*ez#r&)D_yKCd~ug45xU)9tIpis`~*cJ>N)Z2-VB?l~j9JRXB1dS*C12F5DD=)zXW@o(=CQ6+?f2|XPS=TOnj}~e*5FZ{Q-}u{d~TXJ zy0hECPI~^#2o003zAJqM?V-2t?d1TsBTj5RKC9SfDH%X}R+)_F5iukeJ$Zly#?!H` zp-IcTxxg^cEm+v%F5vOMTrVn@IG8fP3yv*piUoS2G_k=w<8>W0`JiItKdX{vqD z;=L=y|3x|UD9My($YkXA;F)suInAr}Mk*UBOS-9@f_IsF;@=8c>Y~6u*Y-gJdqqB_ zZ7k&%I0y{@Yo7eIlTt{=t`kVfQ`BxWpSQM9@nAtvq{C>PQJL4`xOD1LQ{AQPlS9jW zg{RY_RLew;v1qDL0Uo%01zr8U2cP69ftZugl)Gfy;|DpAg~?HXXDQQr#FM)nTHTV; zEgEo9#_~?%G0}PnQE4z3q;3r^4Qzu$REyTZHDnEIfLo);ylvA%Ph0F4t9DLHi4GUq z2gJ8?%FP*T$6w<;&U!=aPM{P@46)DRR$vi<1_@5Q)4zyY8zkx}#F|oyaF@*9MBKVF z%~XnJRHl!Xe@ilq4ha_g9kh~UG$UqlEyYM0Z~h-nZDv9i8qscOZ&d{X#U*wkNgBLd z9(;GFi#k4>TreFktnsgJ%9eQ1sqWKfFzNhDZk9Wd%bhYXFp{DNG!?^l8PT8$ zX~q^rFhzo?20_$oaBCi0OsU7bNXERDq$pn+uj1hmk%mInZ?W$|$dC2B zVY{IG+-bJpl)od|0%b*R18>Q+?&ldgesD5PYL0sBCqrpe9mvmhGc-rko69eV)~VQG z`0Zxvo^cV4kH;DmTEv^&)bQ2@|r!)r19(c&Pl zl)l%I&N(@KgLu<`vw0>_DcYs6e7b8gs!Yq=jYrTq3uFm^vt9$04T&WzO{pA%jHAMq zhp}hCBUK5+LIU%DIA3eDv9-8%P6@i^`gFo-t~^XBZiMj0giFTaZ17o#c{_lka#dnN`9;V_YR(IOyx8Cft#;vN;-sOGuCFJ2hqLr?X#aFd((29{(V z_Z=qGa?n819|1!7_D&3J0!@9V5XK2^L-Cj@o-FcUR~SdQd-g({hQq3_?GjHS+u9pe6c z@`iIbFNw3rj-CcyT`^Q>NrHeVE_7AKPP7V*Il~M*eD#-U>%>U^r-BdPdMe)x?rR=k z+l#_0B>8eaRvGEaNJu>a2GOs9#=qW(ffu(pav$b!b^=h%a>jr7dx2jfzFVlm&;+%% zy8lHnV_deTUPGhA-q_!YNgw=(FSw6Qh-#cuZpR&dmFy>Vt!u|vjp=4yha$p6Vn#yC7Lt2?I|VScFLm&w8P`#h70<72;%wIy-ipoeRN z06~f|i{0OHiU@Iei1!qzbtMWLm8myt=DHbe-AVll$ub+=u)5>ZW?{xms2hp}Gqq;x zMdXC8PXLZds5#QPAmpjWabdS1mL}r#$alDiBii=0+p%+8#DF)5KUFSl0K-l*LnpQh zgnh*FYO9oWAM6&cBn|ER`gKnBOlNVXXhz%ZcCmKJyDVRRR7Qd&s7AwB+*A?c08=Gm zPl9@rLYc~ug`QT$8m+V-H~k#1*6xC>Ooism;2T9l>Sp2wud26~oobJ5Aukwm5IqO_ z(HRPCOq74ozbNrnyv3m8X2ymz1EhojL8Oxe{D64=>in_c%B@ken&8B`GHbE4!2LX5 z$M_qEShTYlmrJQvk!BD#;^=YO8X%Bl?L@=Zxv>^alRCmdF&8mY(+{SNHO1d-w`BI3 zritvcx_HurW%J{${G5;PRsMoq zX^(tJi5JfEY?*v1^YV|W@$u9c^PSgDQGP!?a?I%IjYHu-jh3xRYENm97`W##Ix1r{ z7BWl6ZggD5-G-4Vk6oQ|SIUl+^@(a@narc8yK^=|$J+Sx*a%g;h|JJlun7|5h!mOv znHQkZT`-SRLX}!zm!JLRp*nS=D#Yk{rD)u-%~PJ@7di|K*-^)x&$35^y&?~RX%IBc zjBL4(#86-iV7Z}C{R$7Ra&XC8_l<6DZeD)<`uX8Z;XoOo8_qBHvAhDT!eQVeIrmPI zIQS0$NpPDnV+W>a+@%W1DflTeygGq9Wk0Q_HF&=B3Hv>tp0f4tQI_&4`yPQ;WPcb0 zqUj?vA*wamwFa!osL2G4|8;$G@vLo=UZ`vXwEgXPqlEKUVyV{=)?2@@qYE8dQ+^Z| z1YI$-m>fZ(*m5W<-cQ zJwGtI1d{s$K?5opbH9JISd#`F^u265VooH)&dS^>O?&@&I5#336RG$*GuUdsoLk^6 zs5thdO>Q{P3ZVgy$OAln%XOj}a#(GhPQ)Jy@}(M1sN*ABC2Pmk%dd_T-`D1-BbJMq znoOzVkM|j|@<%#YrBd0#IvvsD<^CJ&loN=#(<7(*S;E%!5KnC%fSie*G)k&sep<%F zwO=}1|KSIept)_>8FEpyjs&7+gFtahUbf2W4lE|`$NWQ9`Ja20-%2dVydWqnV{Ti; zqVUQ=S)Pnz3G9iN86P<>#*(^r-w~ zxhJ3c(s}X=utt}Yk+Yo?J8)L;VI$Ry0g(5wYwN?*Zo3M+MsL(XYp6AeB6*t_Z@ekZ z%fWm2QpAyGxWBgMJs0fBSc@sh(ajnr{%~ti4RR@;7ERW0 z>w`uuKWxJV3|9BM=b)IEw6&Y4Hb?a*Cu8e6p_hxosVd18{u$9$)^|T?Sl((_* z2VCXDQDX9wJ0WD~I>n2g6Q(sk?SYkOWkQsCG{r+rC7&>k!F7eNDSchSHq(omiU&|B z#}5d_k<7j(P}E>_oN|i`5t=$k3QH_qZQ#_&dAq8E8ZF=Sa=|vs`}Xd4zCHCULXpLj z;xVFu?VM9R$Y9ON};67|&YR=ae_}~h| zI>R&@>yrZQL$b)GZ(MJ%M)~I2P8+rNTs`&l3;U*|&|eTAumW2V83%|hu!xdbw+fEv z21U1udbYBXqhA!}7+xLGcv0JIdi4`W+#BONy2Zmahb^m9+9HC5mhlafG|4((Z@aN| zn0VkiI73;;hF$0I`xcA~kYtBZ|1)J?q;Z+lrN$cl?2{wGs?Fk81>fIqwB%_+m(^M? z8Ls~YyO|r-ixqCw2*Q!@U4*Hu?s;h6u*N6jdt2J}{$=ftc>cM`m)6f^9&?xW>2dt> z+v-~q-lJ4Cz#we>obi=V2awTUoha^Pw+&nJn&tZ%xFK>~VCh0{%k<*(n zcFR=EI&*D=x!b@9^?N`GXmb7!vt8K;5tGrd!@?BQ$>3z3KME5-GQsSr!$pRouzNiow(PlY4847%sfeAef&(g{Z?b!XC;UNdgqZ%kTW#HOL+xXt`9Fl z1P~x#c$dL^vv5}4+HO)Q_n&5u>qgZx_F5N3odx>ZS+-e(=E6--yq`g;&H_Ae6}Sv9 z5yRfBdC@VMVGXq>T8|2IN(Wuny!?FjF-1pR z`K}X}4W^_N8XMd!Lcm)>Y2Bge4(4kJp_QqW5KvC?8Adul_Nc9E<~<#%{TMA9oeyA zUsQ3%X93J z`&;*iIc0%b0r|J@*Xa3i7CWbDT3C)hX63uDS6Yj*hM!Yi0Dsj2%?gAM@6s#w2&^75 z*4OstjM3nhgQPk#u*DD%o0o!UKoUdDtHCATz|AvHj_5T4QBdArl$o)rDp{j zT`Xq8X9dG~N9>e7>tMj7+B;~y@Ig1d5a@20G1GLkCSTA~v|ScoIH=DdHSW8`uf)zm*mf-!@UU@;@}=WEEd!d*LKx<(NFCZ zawvHk-GvE7Yg(S|w+}YDG$a;P2ix-T#XAT33VOSqm1o)J&1$Vwlk40@OA z04TxQf^2ARD8RjTl*=!|CFJ^2BRTDRYHs$L~JcBHqGmo4&_1<;A?TzBp zouFQt@I}Uc@IkEV$PU%Ks24MHjxt<>NG#z6R!dd$-Pn_BAIY!H+wv#$#$1cP@LA}R z%jjJh179%qui_8USHgzS+q!#%Vs`?bgSqt7$wjrjT$&ZY|03P}&62`|s$rl&r8Bd5 z_08_b!@X^OG&^{s!!wO-V7k^SMSetye;}XPQ1U?K3?U9H)XEk%xWH+aq@xDt=@uSq zOTIS5c}x>x{cRplIW=5uZa*D7b?mUsYsDe?aL~x8eBJ5`BJg&w|DYYRDvxJ@;2*0H zE$qWWd+vD{(}iPQB?z=IEz(cf!IbKvUg*tQw&icXX-B)h$0I=n7tEJjWI%He5XY43 zhi*Lr=foeJcI(xw8uup9nDrz+6%pR9z22_YwXs(gind}4jo2?8{~I5Uufwe?_fl`T z2ktHaIbc2HR4?V>%{8_Hf8v(FjEDJj-(aJK!Q-!T{C(_)K;JPki|ZnjJOmlx;TsEx zEJWjDSv1^8T~v$z7gf8pvE~M2dZkp&%YtG$tO)$IrYCT!ZfOST7N>XMep&5Jv42YT zZYt zoCqjkaG=Q@qa^KIG=a(!Mc$9R?lpcsqPzLv7eM`evO7bCTXvhNW63iN#CQ{gQCJ>tg{(?=o)+*W=fOd1!uLXO~qWQ ztA=$V3*P=PargVK(qVL^^8U#3-%6$C%Oh?p?wo;@m;YNhj8__BoM((L)QGHC7TS0M z#oCPnL_!QDyKIB!x!)36V8*?~U$$b!=;LNy-=UOKk9DOUgM9@vtNR{+FSPN18osP% z;{Y;{n9=>|W2sOJAMNVM)N+9IvEqZvX-24!$uGkuhsb4L`YMw0Qpc8$pWlD}2h(^@ z*zm$O4$yO>s!!bj428y?HE(xY(+fR!O(&wtZF>FPLcC#znJdkEie2xHZ|;}p-_k|5 zIXo%_9`z?Q;L(Z#LI3!ker4fW6J4AB8sM$LjN4&Q*Xg#>0X58s;+as*v2?o^8?CG{ zi<2?8OnEL$d0NK*WY5;HxG!uV&vVZD3m!JuLgWZEzDXi_wPOttxPplM>)KMI0w7T2bN@O{7D_s|zh=CsjLbM|65}rPY*?sD+0q=vb z0B{Z1ZKxD3@Ctdfz!*w8l2eZKwznS7GBYuSuEQD|Wtf=+0r`M!2Zc{910(tZ`!l{M z338O#7`zB&@4&z=px}0{vsGzSTdzFyKGI!b`}HT^tiz9{q?*El545|a?0Kt*ZiO57 zIjR$fp*AOg*9_;)AP~QS%=<#`I&d?03N!vi6@SYJ6Sk)GQui`eVE>)K{EK4NdSgCg z04>s@;FEvXQP|!ewgkn4+;bt7Jh4tTsa_`euZz+mWS>4Q&|Q+|hYe|Z7+`zAd|m!O z$Q-ahrof?Rj4`p(#54g9_AjIW0?siPM^|4Rr>Z&ffWz@_V(>-pt>Wp1vrk$_TW(U02av)jA1Or z(jQ9~_oP}>_5o&zEE+#RE^3k!C{3-%C#pSTHUMJL_Frsf9fLG-m}eG+6AQR)2b*&A zUyaG8cJ|#UJ?Uj|B0aG?4#l|TN&P(nEeipY?g*Ukt9Y97UN>Pz1O4&(geK$VieJ>BF_it~WDCKV7f3M^gXg zm8FFuPy?xw1@S$Q-|)z@TWd^MI8H97L@d+QlFJO`2CBwC5l;q>1^6l%+C$LuR|OkW zRLb!u83{$x1d+AqYRLfK)Um}D6hAMRy>x+$NrK9h zy_XVu%(IUcSTn^)r+zvD=0+7O*Ey*df>stt$!*CqCM3W&WEgm`-!m`Btv5hq(odHA z^Pu3#nkw^t>HCT-f49I1n*ey;!wWow^(Qva>Y7buZ#-iVynQJezR3F(fDgZikSYaf zA^5H^NwQhg;^|CB71(Q*Zyp*dN|&QY<8%Q0k6$_3asX$bRu`V8yC4O19RwSE6}@{l zIgx;<4})vspo=~g3XX1uqK**sb%L<_%)=kC#m*FUsUZr<>(+okj4gq2pfVVKt)v!} zX?J1DUEK27`r*-KYj=U|v;Wjcq@Jqnv6^+5OYolc+k0<5$P6t18R*A~4#+k}fzAHr zj|v4_z;+Jmze&#n0mV&ti}4I={re&oxYl-Mh?sy^J+6m4bTBKQ7qIOcKI z@wedM5N?IKgZFfV+E0a=FVe4fWgqDwB<$|XVSuQqBBsz&+;hgA+?;76>%Q)s@|s!5 zr1>;z@ZhHqV5b0qOd~xK1At;u3jnIK}P}7s%SMcoM)1*^336H^1D8a&}xVT)k(N;9=9KE%b*E4&wIv}&jD)|jff3!js zcf>C$Npa-P=Pnpf?xS|H_k0kZ_I{{WhB7f)5#O=g5EA@3@;+!p_}0}R74kW{P4 z+5(%73gWn64-duFHa?)^E;qCG`r~9b9-LYBnf2x24!hz3!7f0t@Xn6;`<5Dd9z%Vq zQ+< z^DbO0|HNgcXxQ*B;%woKsg-}mt2$&4ofN)K)g*UP;t+P~sUSTa>9Ap}xV8$>fy2J8 zF4>0c-OPr+_p3_bj=~KXch^@!6Oe#VB?SBb1 zK{a~XiHKkS2F67O!{iVQd1BI(%-)$S`a`eoZ0oErd;LDQ_@8ITNL=?)Kc+=)zf)rA z2E@V|R1X7%fDO3I! ziiwsH_65D|s13fxDWCudVpv$`L+xtNH{fV%8t7SOpxd&#GD7gh;2=<=Hls=8KvXM) zV22QZmlLZ&PAxiIreQtNyec|J?h5m}*IY@EJ2|s(QT|IDXK=HLmjvUQ2UNbK3*;M;1S2YJ+WV3`J6ybgnD97NG<1MLob z-mI}2c~B{)mhgK+*1>M>;(+qah5X)=`KS8cJFGQU>AJELC66%np99OlPKCjjLC13Dj_+`v>u(k_ke=i*Zqe~Y5Q5MnD zk0kU#j>4{^3C1q+XzX5OL!w%@8E?=~r*+z(WFdj{=#IqN*<8(|vSnXtWRA6!pkO_= zrmU(SCfW4ka@QJ)HCa9~LH`-R{4WNQeH++6_Iqp7D-#CC7zmr!cDj*wW76a z?wevub%|wisc9NV_7}{}Mztw-{W(lHRl0i^xT_fOZF6cffs`%!`x#0rJ!Mm!m){P- zU#zX2R5!J^Ot_k5oc(z}@j|7NSB`@ITxvv_D1Tx>D^kkFH9NNph}*yLxPFr zCqJ)jD^~nD+T!PV%mMi^Ysw6#SOFgHjD8Q#;jD}6P18XH6Rd)r0~K3x+)_wX^RmLC|SRvPcLJg)n zpwRQMR4S=%JbR{J^L5W>C7gC|V?A!W!K=T=Cw<}r^M6Hy7-ulVrJTT|(s&^S71mb%zI!M^~7exiJefxz_ zCxE!`2R=9{s7McatZbL~traRN?<^cDxcTNp+ZTSNJC?H2q)BQQ--$kUFxy}JQ(PyH zt^CcxC(l67lb<~Oy^b2oq#C*SyR=&Y3<2QXXvV`qL-90oAL?;N&-D4;IJIRUK6~Dq zYEc)`dtz?R(B-wV-@uziW5JtOg=UmlM^FZop-)(@Jyt+O2^ZH2$=sc0@4|B-m`<&I z^Ld9J_Oxm{cm>>3>HqtUuWRO;v>S)kOJA+YHFnyb{b0R>?0~J3q5l)cdlwiNP)>O; z2;@e^;#YbW^d=m<A7ptRftpe^;^^1IUx}Uh+)Dn5$#qvJyx61@T;F~djj{G5+ zCVql?#c?WV5VDN+lIW6>tsDN9a4o~GhrMEY+pdIU;jmsEvZMT`=EAChGHlWaA`t8zZeG!$o7k^Ic`gs_|;;M?Cw8_CApJ& zEzJ=V8ns-ak6sz5$nnY&y#uGkEN#Ov=>v2RaBd;>x;ft=21j*8P}rz#b9@Jtbht!e z<7psQ)vUI4fR4QuOWgqO(VGbC5GOH>ZZN5s4>r)Y*>kmg12hItuPAfJ(~{fdZZVDxImbZ0Nlq^i=bpm6}@8X!FgdHk}VCBhv1CH^2eme-U?8J zt*WEVi+g(7QP)~B0m_amJWLWC7DxXLKfvT|2~C?_BlFY*LO$qTtk87t`EQ+<*I32t zJ|6Gl6u4k0=+l!hY%ZjWVsbQu*&QcnYUI)OTv>RcCJ+d=IUXHdscEf#f449~|NVuE zq$ek<&VwQYJrb-KsGGHyLNH{3PCz$IcB7M^8c9O5HkPCo8+KwW)Y>x`THLQbwO%-; zy3&;W*{JYdb=6 zOp|+Ac>|JFf33a2V`*vPB(EseCWItl$1t=*Ao^w1pqF5X#ZpShBk}^pbX@(>#hY+_ zhH6YnkiE?Z6KTrOsn4qU?^EjqM4#R5W6zMz)uj~!OrJ4Cti}E|(gq6~MNbC8B)VwO zi}%d@P!R;hF!n!&eScvmyekW%O&?B(;Z((~lJ@&I|9BlMEdRu4*!;%*x35qrW`gb- zm=u>kvVLk2v;y~Pazmu8^+*8TwD+7QM3+|{8L#71Dl&M zkE?#`oIpb779Rn>@Tfw7B=LZ|xaErs!|fBK$mT%dyL zyZ6_|&-);YIs^~wGH}}M1WdgqP0*fF1lWC@sIrbf9?lMtJRh;#gy$*=5HZ{3NXThpA=}!;<50N3bZQ!~dewxn#0TrX3Y&U{v6`Kpvc ze}%)TFNC*hjy+gwguFIYXfh?ntZ<+Cl7C%FsS?i~JnDzO|DR+q&e9_1&GQzPZGy9s{3=a7={c#cPD|t?%Rh_Nx^Y6@~8J-rfpo^6yS4 z?Hovnn}7`%9bYk zZmh{(B$A90l0*ujnhM!MDH+k+mh4*yMYbYEOpB$E$-Y&}nypzVg<+;L?##^nelG9N z=l2}XaeTkeb3A|KXq%aPuIs$c^SsXUb-rGo$WsY_HFot+(bT=PZz}w{IX;4ASP|j9 zUMz&03yS*|Xl^w&5DnW#>6Jxcvx5%xPaPX88>t+vJtwGq8kPJ)EJiDo4sk!B_EC7( z(f@_|{|osi(q<9iH?@l(U*Smcvu>B18w*}JH$`(T3k+Th7g0$ODDl3kb@9f-okCTn zPAHTR@_F&^5HP7Ou)Rba10?yA{D_b+lb4mj&7EbCQ=@h>g-_?>44y1L_kZ=X(ddn# zhxs*=!Me0<&r+`RcPJ}7L2=AJ8^6t8r}1b)gca~U%j*a@l1mfR`mx&XqN^47l!Zv1 z(67gG%7$|_ygv>tygX8t)C>BE_r^bzYyklU18WbaB3eHHVz8?mZ{MyY)jT3*`8qz?6ve0MV@oyJ<;10U(MBkQwFA&{o=M;Y==$LK+6*QV*Vcpr9>PC#j{96K!xoAuA_(r zewJ0mt*t)*nEzKD`3I;D==OpGobc~{JDOg$d(zc-?l&I!08lX~gg~N{ zc`P^59H$0nZC=pyE-fQ;L~HTKnp>lYZral)$CHG2tH)cr@IH6q-&93t2mhB!obsYd z)l;DlmsT8GQfaP9s?9H7`&C$uf2v0HACz@f{@tq{P85ze-wMPnbY=Ti^x z$md%h;q8FdBf0m8E33E(B6ay`)wvIV36gSahi747vM>-D_7(NLWd5I{UrT%5Dg~=z z6L4Ew_A$OHS`)=PCMAmt@`I#0Ok*XI_MIteZt#^g9X#n-w=shZ3UDgmo5g_5qyZ4< zJYC}noaxy-qOS&jqZJ;RfBVM&?D;sz<#nt;+-`OCD?JK+ZzV-fp(KQmsl~sq!6B>y z-w;IB6N9PP#`(^b4ARR#1X*U{?N(KMvlH`-^v#5~Tkc=IaP`UA57&(EW*sA&-2El# zE|GhECs{|z8pR2K9f*MhX&6v`is60Tn}_S=MQ(nibtB!>vVuwP`v5Ng2}-bW-oTEe zrEo!ji~AZja|QWK=H(cM6@vaXrO4(7Nag)Pl#hpOl8tJilj}Cmq*1&~iArFkgYp>- zYX{deXWJXF3Uy>b*rZ+Ubjn$%vjN%BFrx9YF>@`=t0B43UM)zV`i_aIZhz@>E~q5O z-6v5%nhzW(M-q~Dfg_letP`Vksr1NpDe)g`h}4*Y*n5YdJ+0xBULXehoDSN3N04v3 zqcZ5|jI!rJhZ3HFD7_Al8%{RhAzgpm-oEUK{dc`o{Mgrt8mr*m+wVRIW-j0R@_6q9 zx1JKFpaDlJ6G`R=*$RUMnqahQ7SuszSYeVHO28DM^%_S^iaoodpx9vKI+K7XdrIwf zTJSoZd0OiA%kANcPLCOlpU1nXxoIehYS-HdN%fi@mG%BanZ%75J;Zhd7u^o-4vv~G;l?wk%Ww{O!toN2=F zO>iKgNo>y~;G8rre0vC4>GpV#@76djqC32=DMxpC?iD695LD(UcMF!^PzPbH`6cF4 ze58Bx^VcEi#WH}QNhcxsy-@&^#~4uhrcB4re9nKQ^YTHg^{wA*kerrd8K(?j{}9;) z4C)V%j(~gG%@W>Tz4dT$%|fGbxT08VdZ58R!t;7?e818WjZi^8xg{VlM`i>7dagoh zK7EQYFl$0L2%=^R;O*ZL42C(55s(*6##n2);(1fbcby~DrjB?Eq;{l$@ zt|e?XauD9b&kBFr%*?3PooskpHnEUn_oV+r#O1hCT3gE-ywi{Ex|4oqt6j{mG=lK> zYBKvWxLUMoazGXoi-vU&!A!;g?PF2IEH+`cdq9C}nL@bV`MR-{n(QC-k7*MzIJ&d; z(FxhZ(-KCKoPhj{LqOUy2x?0=Lz=IZ5r)r6zaFcd8`fRsV8Z+>4-CjC7yFE5`#T%n zsBUW8^ol2Md$*v9G^RD|Z#)PHAD}jQtRDu}kZuo2KDPtNp3?Qq-I@EHhQ=JX`is}R z_i(8#WPKK|?^k?jxBG2Y%9D0&&G+v2PyS*3C&}Cx0@#_ewJ8KbtUw}yWt;b5Z>CX%A12X>-x|^wZx`N-y2exSy>4hq(BWl;{X+ zIkttQkU^6v1f2q)7>0OpC(!Q%#F1l_QF&%*o*wp|sZV#rURTp*wgf+`EssT?1FFm5 z=#=a1b&0VdI^96A2R% zc5f7qX!o1N9%KVeGH}<9eLzlEE*MMC7=5Lom*lYo`IXsTcXyNf$ZC+|NcV12X!^GK#;Ol zK<9B^gss4$`>e=9?IyMO#U4@oO%TZYt(Qvzg7g7z>Y44TxoJX$j3cf=uP^UH!bnsW znc?QRex#>skfU3>;UedOi|lI^;<&<~h6M3_VGvUg1z6l^?6SEibBCf=*R`L_h?0p0 z%=zZ%>E1`*e|6^A$VRJJ1z+B&4(Ws0P=LJy5sd*PGc#GgxS1_Mm4uP62XeY@mUMuN zn^mPhL4Cx;qAWbbTyLM}eLF7Grq@+@?{Mt#E!a~!K>u;;YgmuzNhL4YGu5bbAE?*9dRLH`z_EV>Xa5XSR6Ch^IVQPq-6kDd-d3DPQ9H(smXq7kl9 zv0N4wrq}7?d*4$}PM0@e>(*D|uPK=W5Ugt3<_M+ygHpg!s>!H2;_a_|BgPXg`rhjkvYSdtD=M_{F`!4$pio#2?N5lF+o_N-=HPI;*5z>H| z9yYNqSzfQ1mw(xG9WnK}Xr{voL6Qd?X59=lM?RBL$(n}4h(C)la2E5{ zpBnKq%q_W>de;Nadf@C1e7K{3SiJE|d*a^~lJUf*=jl0r)1NTb~%%t&ZYXF93rz5ak)>#ULFESn+`I@)BguMoED4KvjitVGkEuv7gnV z4~!*OTKPOxUlX?}Z}^h_e#fy*7o9&+T}^yYoZDcNrl7X7*q>?`X)NRAn}1UlKnffV z7N&{-fRc$z$xOCA_pyZvMxM-(ud5s37|;}UU1izrA_&ynv>Wdm=YQI^7EOF6)8l80k(Bgb0@aceOc^DfKpz zCQO8P`gRFNgw)Qx&L%rD%^MhpKX#?`NO-Fs5o~`LT?K~ibk=9Vu4YH=VGedfux5S_ z?lyL7)B*h(nxh0qnW>ah>hIK>vXJtQ7mk4xt8QcF;9_H0LiQZgmo8S)fR5)M~(+~vB%eLDO7(k;nD zRaGfuhMz6S0ltD-U+i;#vJ7W8INrb8TZI`7RjoV6XH%t`uFXu^i_`_r#kAV?f@wa< zDftsg3^Z(NO93#&I6y}glJ15P=LkOWdkz!3-GDbssBB3UUAP(4dAlo2IM(G8^)BU` zJ0yF0i!*976o+M8L31RBDJxwEglX3ZQ>~`VLw-%MXY2t(bl@wTe?j&(CE+C51bD)Z zpU-g4@a6NZ*q2kXli&O0^HF=Ubhev$r;RhtG6E^%T?UiVw7f)yBc&v!9xVn}cQ(bk z`xWfCzwK}BAoi!>huI<)0zPL94^?%1ir%>?;4fetP-MGt zPI41Tl>21RcErZoIs&vJ?ojlPoPZmxQM(+jMuzy$%k=R~?%o$BlB=8g#yzz1XTgS^ z!wg{HBGLsxtWSie82R8CLOAxcAp4H7?4OfLxzhKq-OyW=_q^|_{?YrY1Z23&df?63 zD%m7w9?r{>f3t|s$yf$6Gc~ICtKq-?WR-}BXn~!0fq=xQK9lakpAhygK->4o zAeb}wMu$;zm(Vqjypr&cTa^>{E*2GHeY5wse~{nO(fKUi#E$frq5M?)T`XD|DE@L} z4>GK1qvP~8CICWbCBE}WI4Gk!bT|vT#G}MzG z_^u$nGLCc5zuf<(66vZ`ufa0CqA4;I|s`% z2j1OGOo)QqsYM5u>OY`QrFxGI>!UBae7;*wz@2=1ku zO<+d`38a!|X_-o}Z%?aE9phtjDNH_ku@e-lM}XZw@B zZuevkD8$BYijstDK#nE!jbRz3$JkL;LyT_;tArCfLno~7{h84pqnQ*h`vGMldB0g`_K~ffoi+vC^>BvBVC7GX7N{q*v3_b5#s7?G>}> z3Rmes^?&KuWNG0Hyd32Xa33|i9!oap{TNwG+O+LdM?3sHteO^rWmy#E8=tI3H0$V@#(7Dzb0Q{DwM@55?o zEwiB4?U-Vrf%Jm;WCa*w7sw6l#B4MEU}F2iPEzE<2ur+6;=+S?=a2n{ojk=y0AM>B z9!7*gxCgPUF?H2Z2DphfOAGa%Ljg^<$3uWxTZX5-y_Z_vFKU9J*v@g&Q%*qS=O-=< zYO9BE40j0LN$LV3t>JMbA95yC2b73a@U}GLgzBKD9{6FW>~#kVdp&&?BA;q|>Aj(J zqv-Q*it3l2`lRPDd%BB-nc*EkC8`9?xkas}%b)pdEch%eUF~FpU$AcN#n5ak-C$m| zYAxq-@9lwq91)R2|3-WaN6MEv-vt?L1~-Kf^L-Ryk>Is%kLlgeBY&qEzk>LpH@P2C z{=qvAAC|rG_ly3QnY+B}V;jUczYLIV6TmX>AY;(;%6HiK5;WxFm@E$Oou%2ldRi3T zIcc@$=7U{7+xw4D8XsV$pJj#~i??BkR&5a)(g7()2c1NzE%}p`nOXuv?f@?VI*WS1 zn>7GaC(dFgep4#DiGncNX(URAQgqEDu4{6%+>%n5 zoTc}U?PW8Bs({ASFU_cgL!Zq!L~v-Z1OcE<1i~&~@!OGBZJ7VWCsih?OVO&cRZ{Nm zInM>TV@<*L`Akw0zn?CT^iPrEy}3CnG$V~%1&WxzZh=ov0H6K?JkAMyAis{2;qWu& zSudH6(JlKZSj++<23qflE$FHb7yKFug8s0er+ZSK9cvU@G6dRf_7V#65;%g@9Vld9+i_9!cC;|@4VLVPB}7Fl~X@D z8$^_^ioSt+rOA_-Lzf*W+>STATe&g`!Yr-hCL^kjm?q|VW{iru8uaCyx z3wE`=EUEqD%XJ?gSIn1HNC5kiyitGvY9r#5l~}MA=7LY~`CVqLw|r*D(;r*6x=(n% zzcCk`l=Z0h*KUENv?Z5QSzK~s!dYHnp2Pnkl>~>DUym?NuH#=40q@W^LeM09uuw}q z43Gm+?mpBQqCo1IpLyfE&f4j+DwS<5l!abyzY%iDNkPZpw`2Yt9Iqtw2MgXL_+S55 zhzKf1YPN&aTaL-d&)l;jicdMP*I@Tj-D^FfSY*S&-!ijCYWtjS?3ll-YJ{F+MPn;V z;5{LTkQ@69G6M;=?uydoR$FLH<3Bnk6eJ{HEyd@bzLb9|?|>uwFraYS7ecm`ch^eG zdV9FGA|+`Qdwvnis7AIXz#2@iHc)#(4n$o8+qf!%Z9iu`Vs7F?)eBwLyR$I#6EuF> zfA{uH{}exaD8X{Cw0=ygttxT3CO?L?DHsCbI!YMW#lVJL;IW9nEGWHy@!%*G_9z)L z>v9Ile+(^b=^xs^KQz^RXj))SB_?ZAsk5E>1D*m7a3o1IWGBE6nf*a?_5thuC2|lf z3b>Z{c>&F;C3hx2a@XCB3Mh-NU>L{9Q)|?Djzka30*uW2GG;r*p(g9r=RWl#b*aq! zm#r}kjsjOZlQ8?wcwK3Gw@-2<_iqlkFr-8MvgW|(y;?zuq{3xA2yn5>5GEi-M8lJ z4Q@@<{w!CylOL{zNEC;5sbu>WPdcdy$Di)IXaXr?-y1B6g6A1#fae)Z#QvEN!IhI3 zHyb$>ChxGUkaaht#JC^D0yo__n#GP_&={i>nW1FoF{0lAlz0p28esmN9Hf%y!)#p` zsJy~<{!mCMe*CCOo@-ON>i*OlcIRX--q~d1e0uI!tW{92x+t!_0`dZo@JbF;hM{P3 zqwBF;bDR_Z^r?ZuB)VRC)4e-cD&fjEcz2Yz7_a>00b@PmQJ_#p{ zWWKyo*m>p&U(vg$Ob~ow3$^Z0C~#~yT#X3+d~8)wKWF7!=m^KF=L?3kM5K!yD5!pY zCfC)6k0Qoz%mh=Fz-+W?m@C{=B_#kA^EUqb$`Wu|0qP1wIEZ1(+PGC+$M0t$pQ6N{ z;v2?kxAF=NI?_ej>e$zC?%W1U-5b}v3wj; zFA*^EUh>!V5lb3C5^aHRTiu>V67b_?gFnZf25$RGD{xnB*4PVV1;Y4O6^6)K>^Imw zC>-1AAhtK~3tcsK(w<~EX;~G_6Ki<2-t&r@q9A9S8N$Jc&{84EfGc=_KKWgF*6-<) zV@(G>K9!yQtMVpqWZ$o1zoE~g>RNR#t9S9;k%=kj$S?i^=%q&hzPuSEi3~DFVw}~` zz|CycCcK}2CkK|fj-A{h9a+9O_;l*)hrkfy(f8tc32HoSbNwG*e7xkfw2uaNg2x7x z;kI!zO^!I3^$wIyyi0E5UN*tYzuWBI2jV;bQV zPv3sDf8V70wTw4JGMMPfv}a}}Twjs)%icc`(6IfhdrOi_ftHh*7}}~nQIaF*wsxOw z4)viJCfwvnUnJYsQZYUPh$OS^@VqPw?t>^F_HoWhD1*ttQi`?(IiAdPjC)^v_}B%} z$XB~h4D#+XcPkz5#W8ROf2{z)Irwrch%2v%A~=J-3lk?Q#qt>juUyxe67*c6V>{h* z`+dLZZq7o@%A&H0zqihS0 zPJro}upbyWR1?%d=H$4*>f)-PZ~PDl`%4;B5*K^g?UyE?$_wXg??HsWQ309hvDRG- z{Z|ZU+$NxKx62mVdmnJ)vG*74oAGjYy{6!OL|xrb^2e#zh@Kz*E}9w#IXZtEz}u=} z=a>ZG-6xFd!Ku0(#?^4zwAp2DC*~mT9>nlmflW-;+Ci<#4wtshzx0zrnxMBpv_-D0 zar&hLR;Po^hPwM*vs9*3yR`Cb{L1^|c@F&wwImKt=HEHYBLzjGVG9h67iZbDp-OVt z-pi0%Mn_bS4(Ue7r^I>lT=oTQpVMy3gG<%yBVgM}+y^c4YzK~h2Gbln8ziK7@>r%* z+(5DE9Omlyt4#t2G7L?L)NQ0XAd6ecDH~u#fNn%s;6JFcK1SCT+qqi+bxyHN7>@+xL-uT+;13yoS0|YBLn8-b^qaN*172I+QbeGO{8OIy z^{c!2ti+{#+1nzMV$y150bdOL3aRgGzUUDi31|_T0u^e_VaPXgGO-)83E6^lf5wxn z85yrEvJ8@p2POL|?XQK0s99fa*xZmWnk<=Xclo*OyWS^|7hsd>IS#L~kHJ=y#vbgK zj~GEe3+*wLAre*jQ;YkKN5(bpm(`;0$xZ6@di!j(^5%V@;6^o}0~fE^#VkbnNl6gt z&-_&v=py;AdB|-t>*O8`*YuzzX2h2xjYYOeGl0)s>rvVcYf+&XLzvy8q%oAx8155M z)llN&RUzveyKn2FLV{f}UVYQ9*TV8r7w(EMOgOgEOj{&tycDALg2LsUz&?;DtdjoP zHFgl$DpHvBn`1gm0GVhC`2t0=S|(K(K5lSf$G=n zLd*?OM>KXMoSH7L4w@F^bF2E-vfbJ-jLSqYWk<2);sZ<8p^@qpAyE1}ztvgdQ0OUO z3L9tlN9nE2n_Zfs33cF)mk)mC|Ay0n+AaY8t#iD=>y(5Uhs= zM&Fp3Utcn!6UHeop8)qph#czxV$^2;yG?~kabRUM@QZuTVMj-?2{xoDRyv#Cw% zu0Z*rk2lg9wp_XTd%U9rKEQ@fjzmS2`~QOi8N>NhJ%f&%1<@UiPyGvGUNXo!mOv-s=fmz+3{a#pL4?6WZtjC|;A_9&-is1s zR@=fp;j_2GX~|-+{GmL`gy%p&a)@!~n$gm=ZC`3tCu3yq&+$U2Ef@ZRwG!=&Iu1O> zbvy*-tjx)rI2SBsIpnV)=}RWk{QmHzR9z$z0qH~E!BCa$9y?rXv9T_3lt~=nI_1+KSw#bAfzn< zMJ(_Z>i9rJ=mT@WlUo>djcwi-ZSEfUsIS`3%Wes;#sgPdec7Pq)Jv@rI{PGtH@XTz_Y zt)VIh)wZ9h-rjs8hk@b9f^(|{tfI*sq?kVjj>05FXb}9BBj}DUk?=iW&@}uy4_l25 zsD2j5l=i`7YRQ%;JU6L^vzw~tcFG>mo5-P4=SKAF{@jst&zq_75`*GHvb+ z%nYR0LX2CBNdFk03@)VkX}H&Yutw#5To&{YPO?)lb@vL}W_q4y1=o?DzpD()L!r2G zzlK40s6g>V@BRZq+yfg5TQ2kP1ImuOMErlj4)`jr01Mi%lN$KkGc|(AA5tI z;}&wt)3-X?$IW#m6%`*7lN8{&7Y!+5X{MRUfi)7gR5pqHUr1%qF zpGki*PY}OlM`K@@L9hy#B5!QNFx(Apf7Kt?LnP1Ob-I;!M)WQ)ZK4fCW}d|Urf>O( z*B4T`W7=@fFxS;4ZPnEk8=7B6=81fA%x*a z3zI<;b)=VG9p%@aOpA7#Xj(LsDVUS5;v;gm#4AP68youZgSYLD; z=A~O(x-fFyEEHPy?_kBC$=tOK>W|pVTTLhlzTOl)T7XF9h^5>m#K2-(AeeZTlcfJj7~&L>!c~Tb&{C^i5yWP zavO(%s1aDup~Y3!Iq@eW-Qa<-6Gp6U+>&zceV|7l@0xc`s>Mk0lA@jU^yHhnlGno3 zq_g7I!40#y_zTrfXD~Z^I;*d8kgj*hie^zXj;vD*nsDW@Q80Ao*`+|N^vHc)ZM^As(g|BVJw08$i_f1oWC$pydnL#?Y1>4f*N%CI+KSRX zx)W6k(Ma)#EIdvveoF=S4cDAVs?N*~JjXVu_BLC7Vrj?8-B7O6PW2bcJ7s&j155Ln z6`Le{QJkW2=n}Y$8NM-Y?tIOy=2%O_E?6?%&%w!5tJiZjedqg~ZYbZe*>&VQ=^#Y8 z4LEZsx4#NEGfAlsCELQRj(MpeRv404D0sFO8j$C@Y^31V_^C_Vvsp_c>xNOSu-I!% zFFmguu`0r_d)S6!2 zkv)CBdZ%(M5Ah>-!vEcq&0}C7+9;O)@9g7wfG}mOEEO@n2APFyKPwn^E70w&N&cEp zDc|jn(Cn4uBf8tq`y}r(?fEa@d?Zz4PSy|&%2cYefbz!5t2X;~JnL$}A7HxhFn`78FH;w;( zBjeci8_%cui*Mg_`&w+wY+RaP3nCvHp>u}J;uobHbJ&Nmb_iz{eqwX+eP1Op)6d(* z-oJO=lg?|WoAIRTBL!nc>_fxmYsd*JY6x3j&K4;&4nVD=5moS+u}aWq7|03PERr~J zxD$kaOoQ~&AUiZlEp@hWsf-C7V^+pjkczwN_4e4w9}E-RbYw?@kKOq@TOivBRUL`! zeJCweBS^5G3obZmz(57LmDt7EHuCzmMoUcV$bUBoPp0><`H=Tm#tCfftEes9rwHb8 z%g53Q^`ZcI4#_Y&^KRKLn4-Yd4+UGARp z?sn^;q!coc%;Z$8umAw)Wpx$EeTKmeC8Q1dm)2R}FWZuH>JtEUX>l9===_5cgY%v2 z6}PpoSdHqsE_{PCM2$zEUDbKh{=|3lOs<>mJIsqfm=`cD+JX4)(#VahL}GVt>H;ik zQ5c424{0;!BoHt#vHwH_;`U|JnbKA?-}F7J&rc70P+w6ImQ#ItO-1d}hY4rsjbc!g zG05pa)wy5P;sEqT`Y3Qzqwy%hIqcFbFsD;1`?xhNr}q8OZ?VhdnvDO%aJENMh#XP* zcdgP)aw>_ErL_A6(`5iRH31Kv8QIa4?^b7H$?(jIlsMTH@J(?%KwvI=pT#A=BaU;b zBpkR#^_ToT8j?v`iVh$_>rtNo=>T>Eh6}EzTgr*_BHBRa)c#pVjw(i6-#Z18@+;on zGazrg5^oU!dNB(ZEr4{`MwWeVHm!>W(P8Sr?K1};a>>ZCQSW5XGdN7`^=q>Cge;D{N{_C=}#RG+(AVO5KAQt`+h8Ter z2FPBYGRP2X-|!qb#U24o=V}_S3_Cwi~qhaVo-usp?=d#Xx817X^$LFfSk?M=}Qyq=KdAV-ji@0EBjsTUiIm1UBs` zZd}jEQjGtW+dr8vIIkXlr*r7x8<$IIyR6hNG^~JbqG;~g8b=O9HmxyOc9m&>JTj|6 z5x||I?J@$$-Wm&5=q8Wxj}W17rVWNa%6{a!M)r{wlWR{jM!BUyS&sNTDd_iwR_7ECv0Dk+mRn8*&ToNF&3II z*puk+h%4nS8}BYX;EmDIpNDRoP#@_k$h;0W^ev11rJZFjZ)wIYf^{QroNTrZb^nM=P;}IwR zeh6evuWN|29Q?bAFocmIhy}1Lc8-f~YsAJI&kvW8impM0_b;a3s9P35Yi?%6PF&7e za{<9@mLZOa0ON{1Oc=zFe|Tcy3h zX3qH_FMlO?&36t4{50cO!UohL2oa*v3eh4zZ!~?5Yj0X@85dos*?nl}AbLVAJ3>_r z+DcKe-fyRUWN>J^P5% zSW5e%ZCg%`)z`aukR;aUud#Uw>yPvihckfvCA7pPKp2i%6Tv03n}o^bpU(xAoOCseQFTkAW2>6 zm^MW2<$mvx`qVqIb^;m~(m7nh+adrTthNs(a5kSr6h-J$4aKzY1`tMmXB7PINYdv6lNn=<_ zKKV~b38-2z3O{522I1ZtOZ@+SBIy6?iB=OiN@d_sH#WB>49!#AdIC>>%XPMY$}xI$ z_qKTKlDNwy<*@yy9&F<=cl$BEX4>uF&Iu`R5(oB|8nLT#Qgc3ihu|QCGNBG6g z=Ps`s?mPPl2dVR0_s(4jgnfw+9&A)7a5t6+*s`|0P=E+8D;?IM>AZFx`n8ke@X&Ve z7vr}k@3#cnor_-I?(Kpwq$l?Vft#let@&N1)t} zGE`9h-uzwVpZc4lO})d^l$CA@kxRGXW}$Wn9d#2np#6i2Db{Okff|Nd{HeFB4vj2d z|0x=*+St;N{na+P$o+7P`o)UeXk~BFF31Ei04^LXIOSnOvF@lI5F~eqk|ce4IC9BB zitRSC8fo8Lcu-|~pSkGIVRgfoL3(ln+K@K16> zA|1z^ZF-hI`l`XWn8T!CC) z=k_ZSI66guAA^GSufRGco#c2ATe=F6UhHVyM2b7Y_?ktnxiNb@LHJtG*7s-6hy7Of z%}Tz}*UgV|;$MxohPGh<{XjXcv9cYT|t^L8|9zfZ)C&yPn92R7}$vtQ&rzI0nk3yC$# z%{Mn;UncY`9RRbTXK8_SX-6hG@zT(;anRm95jDb;XDPLN8)Q%An)deg=VUENIPw3M zg&+`5K?>4#Y__%FfB`EFmNb6Z-~Kf)J<>hHHWRC3D54kpLPo=C7k^rDJqYOX;M8oP zaeRQ)xFVm1cFnK6#-HR{jn257%xzG7OSWPK4WFJjrJxgpkS#}BVlhn-4; zH43k|aKk{2L+e=+m00v8x)2kk1|%0YOk|B z511Su3Dygm6+Ni>G$?D?SNYZc-I}FJE^H7<+CZ8a0>$F$R$>r!*oyU>IrD*@m_)zp zu~@vbnN;^W!&cNz;<=-mdQ$IFxhNT24@=SX>=ce5Feubr>*(~b)&CO{S!aygEgdtW!QjpouA$Z_`0<<-_Z@nStZQk>PnbI3r?hl zZ2x&qjVbx@QDDU51+($+nC6kSHQ>^sUbI?BUwT%X}{&g%47z=r4A~6o%Xj- zA2N5SoetRbVbkQG*X5vp#*|UyZSb^nk81#F5Zsy{`u=_ZlX%ZZR6^ zBDBnHcw1_B$)icU?-54pD~kt7274lf*~yo7ncXk>eLhbr4TH^tn$*ExO?NWM=Kx$CPOWj91t_s|@QO2oxi@EezyIFxXLtH!MK;NntKU|O-f|Te zi)Bnfq=rPUBDo9AF<8YQ2P`o&K9cZ8%-I3HMG7*ufqII&Me-wNSnP!>t`gyDO5S~% z4`WAtt>$E9e9#`575Z3ss6%&Ox%nIQCqi~zge@TFZycxh(Qe&b>ue&^6Pae7Gd9ci z)o)Jrm@pIbZ1m@}k4hd?XO!}MLP0xu*=58wBB1}~$a%7lBjeaclXPqVZo-Ono1q|Q zkh#h~7Iq8FiqPpbqY006#Lb=Ri|+o^Yqh+7>C0kEnaHd6&;y7vf}UDrwJ`xoWqp8! zXExG#Xdz;-eK$dNq++ZN&~b_fpzM7E)fx8U$Co}mzZ`oiy}xriz6Szhhi#E0%0qGk zIT-|)YOFLz5luIk6_K|E_|xU61Jt*g9XGk%ML~gHAu0mq&EZbVc9(oFzmhVaMWJr# zG2D=?CEN!6R-GTa37?_5@%8}>FXX3&FNQV;+{_)TVI3T(coyz)-RUneI%e+H)8+{G zy?=wanx2+izB*!937OZ(8LN3{4q4fhN%@JXH!ia-g$9nA9iH>Tx z_7`i+*>v{@z9B<$Nw7-M z9pK&*r$wSvvNOn&(iAOON{Q$qj`NiGxdb)=a^DoV8kkic+z9kEJz3fCFqIzJZ3e9* zVmi!rq*Ed)^N%$StG}*qDxgIK*HCkpS7DKYV{0884AA zJPyqfMj&{V&;_>d0n`{m_hFu7b;0Ruy(l-BOvG2W{f^qx#5}dPSYyPewP4)2ab(Yc zHU9Da`?EF)KRZhIpMI#YsU`X}#C=Pme1s-vX)E-rvn8n@F(=b8jBgw@8l5oj+1SWL z%Amlf9%s%T9d_guwikSuC1o3wA#3N%<7DH3{t^htpf@^8$Y`q|T**E`7{ZFs7*PD` z;;oA;3~E6xeylP~uP$}ZuF~e`3F@v{5=MC!s~@OnW_|>%nQ@?r`iQjzKVsjCIz3uX zoGj^P2|mWJoEq9QgSBMpjg|NZ`ud0KZtuR+rdryr+}C|N+wME4Qni6{2*9;AhPP)< z6;i{Ku45)2^+=O1GiwVU<4y-68cid5`a!|9Ua@|22@$gLJLmaj`T0bFZyx+XoXGie@)ODQXw-A;ozM;^kx9j(I&Q$gd6Ebvx;7J%*wQ- zC==QN&CX)wHzPr{&-CZhHE{aWFFI*DI<>NpyRB(2_At~ofomM3L)1)IE0XZBl@I39 zLbd`;C<}?<+UrF|9da#`grCb4_epL|Ye;Mpm>wVthW|+>5?VZA0jyVPeaqj+r z8cFmH>f3<9PBSb$o@u7jEc|3S$Qx3`^@<%n-+#2F!jo^Cfb!ZF(kmjYiRpG!Bz(p^ zmu`bHG2$zqgly*6scx69WD!s`n@ z?s<_-lgzf`J84vx-Mxh+q}DE#^XFXSS_P{v}GhZy2@GR63P*2JLWXu1^5`HFfkhW;=ek%Olh;&%-TtuK29}xP!aegZ^LBQ&5L)K3aRTkmw0f!S@M{Gf4So z%gvG;XJiV;IEk4WpT*L>K)n=$uV&RUM{7oqoy{?Nd$8g&UpzcJw$@0T=;9O8R;^Oa z*mB&^PSf}Ym{SJ3sU=PSz*s|sE-;*hF#tSMP*jqPsQb0bk6$Nkhco7#>YImGRwxax z)1imOHfN_4aX~n7l8|=-IgSpJgvu96sS!gh#uKiz}y?Hd$ z{r@*UqU`&=4pGWlmJ~7~TP{h3kfxGsA+m;fix5Hxp-e&)vTs?&SSw^F#4J+TGLxF# zHOqCsT%Y^<`Fzj)y?^KY&i%*bbewC(dwIT|>*M)&th=Uk*D;@a=3003RCZq#60tg$ zz0~zEZ=Af#Gw*2#n~=UBQ2_KF_)&Xc)d5Jj!`6>Dn$#sct-pEoM{WH?h#$Ass{o$j zFmAC^?_tO5M>*MSzN+K$Y75xKj-EtOyr7LB3bF@X3u^CXi zuAi}yR=kBC*}9J+f~$oOED};fZEFy$5fsM{CXc&i`1lR|G*#!0_U1;N!->bfWSmV< z%{k}Wm@0N^gO_Ve=qdXj-x`ouvjjk`9T$dy=4n`p6FM0u{sTSN(hr7tNzqZzo*Ggo zx>tsOpoDLX*Za!l&49EO>l)czY4P_JPSlJeVI_zmFG@Fs@FyYvO(U8%v#N_D!sv7x zwdhG0P)O^NpLbu_S2q2mtbc>L)E^msEaFof=bXx4RaurV&ssd{KsEhfV;o}~{u@HX z`LYO*q=}OE-w3`AWjZq2QU)Z+R$kZUGvaIfHK#uk^|Eb6O1%wpuObk$v{_1O?`G+-_U-65S)DVj&S$!f%Dt${Fn&fP1k4UL;Rag%Mhx9%_JQAlL-WEJr)Rt^vyrFT0#i*zRLidwV%><9fV=<9a1KP+Lg1$?#l$Dthzwn)UseigX*?-wnu-@*~-9%?6ju6#%|Kii$uf6%3 zT0?sRIGi9L9E~w;duGEV!l(memwmaB%jeU?g&$=_UEXCa{s#x5oIXgF zIw8mp31(d+IEj9g^$yiA*xNwDCSmq}JJ2Xx|OFy?# z)w~s6F~X~3r2c3Tk$L_(Jj5-=J`Mp!Hv11zvOr#F6sH6dyV}U3e5*y$*xo-6 zi)1H1b+$Hj`sHg084Fb&wKTg4FXSj@4nP2&KuW~bANu;T3E;t)bT(}oS=S-f8WHj8KdMZ^~pa+8}9 zbpo_4(xR2T6l(UA6W=^fwQZj)&AcMq=&U=UajP`RD8AmX<>bL_1~>gIg#U_d#v}m= zgN9zvSwL>gQFh#tV!Hjr@d-Mdw5_AQZ z^td15D5%-kq$v0~=GzTJM_9<2XIa+Y4bKj~;zWJUlA*S|M+LH9gXV7U3=WfXneLd5$JB{UGU>4PyJyJVS}AhVCf>}hPIjuv{6 zIz31$IV)AU2IGC(*H^T)y+nH6%Dy`1r+%3P;S~`(Y|NZi5Z)NNK_>tfjWj@>h2mzt zUph$DzKc;_{bNB$Y=U^b8`cgGJjORPwDvx7E0I!hdFs(bZIhQ0&CVFbaRDPSh60gt z5jB5NoRd%x2N*-3LeH5Lj= z$Ht*4z3Aparu0@nieWIH_cubSW^Mal1SWA*$sH)zl`#fHSs=yaC3pBNyZiqb<JB1# zUm2swxU|<^$%WkoTCWM7MHcO292y@}HlC?`!%P1dFf$0m#++?4=eLb_oub#)Dn0KSSV1fB^*Fxyc?@FW(464MN(| z(RwYE!4!r*`JkSlgsac}!PBiB)_Wh@d&@7KOfD%xWtq9i27kNBj&1zP{yQDm%o*He z!3D`)h;nX0dtP}E-LRF+&?RZ_iSRm`>N0KYP%>J6-EGK4G46mfPnLsWOSb~yFsHRI8GYf#OCjjTfdx*b;l5{te{zPO6Lax3}SSkw-@>B0M z-tgBZrs(+yH!C_RneIzIdI5DM12&GQ;o)N7wp2-hn>O^L4E;R((t+9PA5nABY|qyu z3Z7K8=x7`oP7ANF7k&9=%~j{6%dMod#}G~6D5hh2P)*qW8fdKuIffWK?Jh;bIo++top z)B@HTIB5SA=C}su0l_)opH6_nJR1H@%Gf|ldfz_&d_ZF`z%PhQ^4t<1Q3$UKO{POh##$J6t;lozKWWQ(K(N-`v^$bkBc zy@zE)$s0u7y={Y z#-w)%l+(zC)P6uNY#(2AFpuUz%}p(ZwFjI`RHzEo zIHm;;4<8+{{-fM-vXEVj4PvF?Jhw86WSK8_kZ^3Q36|h^P2;Be;F8GXPo`n-S#DaY zcfjUENV?tqp-%JIuq>UUdTfh8f!CQ~OjYeC%=wX>d!Rk$Vg-XK#%xuqhUddY?k)Qj z7D^Z2(^U`;fBlIjHw%G*Bfp6rc=TbIB}kBgeiC)HbXBX zNkNR6VWI;0_}#wi6lMu5DX)2qTRtvcrM~Zl%>7RdLWr7=I9Su3A#X#sX5m5eSoTEhsjxAWp#`acxEa&VD4b7nZ8YuAUgEA0bAz?kY zNK(V>k0^i33E46SH49@~waNsQJ#4W@RlJVwZ$7~O1>^#1<=r?Sz&FYe&0aR>*1qfd zS2$}w6&)K`rDuE4OE~$Pw8{C8A9n@aDf;A0ZF|AOqgoU)D?a)dCT-lJTBtTXAw6;K zc8yO6RS}ZDqWE1<@j;aHFUSg5#t|&VyJbU^ehEP1dHAa_6OagI7o6R*Bs>-^7*o1P zfU!1(_K^YKiuR-&^?2e~_WSD9eoM*wMSg#5K^3e{L9+z+Gw(sZf$1iOunpZTS`yZ+ z2B~Vn{)F)2wGq+~w|0Pkz)I_O$s)(8(YlAo$9&f=FQ+iS)8>2QUqiNyu*vHLc?7x8{N|a$X zqrQ$}4nm}^DI7ckx!@=Pvm5=?5*o-g<4!U3V0|j+QfWVh3y$d@KecXMVCvbN@|_j+ z=&Y&922^zMMVsM^H^W@F(~dXUx((njwax$xzUn@#U}x#BWDF5TH2q z!=>ECX^A!y-rOpDbcLkTx5A6)xBkfX=BZ%uKMevkls^{UkJ|i6@dKl`zY(z=4)p!- zx9^_?#oXWLy%x+HD`vj{1%WXIC318#hlH9p?`H`zAEB;qB|0r}jW^|2>7il!kWR}j z$NVdSnxCI+>T_K=`>l$sf3EJr!ALHx+cANOF`snFLp_?y zTOuxSusR{>btv|9LiJVR0Xl(4hv6c!u8=6 z^6&&Ocp%)lw3q69Hx!wai$S^(_D)!6M<>JCXZF{|?`5Q+MeuZr{j0~uI!AB_PGyW(d z5fJ4ONTGiR0(8cT_-~M{?D-D-hrf}rC&&#_V2kC}uEUH2lg7HsfEX+xWqlI%yymX# z&^X1v;A#^nw{5GZQ_+yWFKuylCEWsA(sRHhXP4O2I?{4xT(mO}b}n9!MJH9DH$d z)Mxj`xZU9*&$GO0=7G8iZ$9Fkm5(9cYl7ae9GExn?&$0)`mU_KkbS1v=DC-fAH?O| zb2MIj_rguswwXJrgJfkR_%4#d@E5UjqhHOmdGz2Gx(M1TNZBf!?)5I_L81H~&q zE$c-lFvRVNxcdxIy45Ppei&drV%fEZb=|&Ab{@HVFt#Sc$eH6JP^v7FXqRptvta zcNTYfR4{cDeWrl+@#51*_2SM9A*$y)B7`si+h@_{(nSbc^I4^n0dcZZ%T*r3a4y75oy;m1Om^B6?sS7q1CF5!@{E3i{0xD0=BQ1sTUW7@_V2G|G@Y zGloldUafxrux@Blext!{$?a5`O3KO0=IRF(C?kffMxzUvP8$ z9>wAVkn-$)coaa>-h;~bwD9b5cz?}rwH*_vOZ){I6jq&3Ag=DqhB#f3)qRP33ZsB| zgJjpM;aGL;da50IVdHvHWWZ-0VON`PDp}4vZ9HY~UVWb5^LaFr5e1B20K$XBV>I>k zmq92ETNjW}8Bx=h2l>wG+s1gDt&_}?HESFZuO1GHOiO!DGB=V4(*Umkl|2fpYyn;h z8sr1aEtV3*AXrxGbZ}JTc%XN3I4>++ z)d}}UF6XzwQ_LigrYgO?nZ1B{tDux<(SCnWpN6gJ;95wOv52BMotOf0)WFn=SW z4^r)W?PVEyW6>@YKc|VB4%L`ayf$bm?ZtXT9gns6wDH?)H{hc~_zxU93@{by_n9TQ zPfO8}K)vTWu!+*FSucmBORhdMSzyFDb@K#=zsmM7>8zeZ*04PS!L9P+sUT6I3!IPm zP}GV(8UbziwgLQl=pv?&HHmD}W=IcJV8%=ib;5b{Ba7=xN_pru|3FuedsXd!5*6C< zyS`g9@Tzr^$C1+>7H5M^zv%x!Glb94lEF9nHxcI3fAs>Qm92!s5!&SIWyKv%lW@Td zQ%k!#eorp1L09V|&ThUTg&EIF;cT&Ne}?iPU2G{oybFQ+@_aXCeWEuONB}ArFST+Cw zW5DgbonreAAl;yv-qA$_3*!GJi+Is_W1D^q=#w%*Gsg$XhVw!v&WGP+C>Ng`UoFsj zlvaOy*kvvsIDuhXz4YM?8Pp3$|@u$f%K05f zkAU7%qpyR(I#f;8W*+yF9wd zO_%q+{VO%}4wCk&L87|iLg&KX1m{dmt-V$;Pj{_^%kgF3v10<{M>r5QTA76+2U|rRjeqK#M^cYFUIyWL3~@4A$E?%^OMdG z{;rk=0|h9A(Ns0^=vk^>1~Z$)6zE=B6Wq0DF=VWGw<6z>$h6q^H{0eHFrbThuLC&N zv^k)tHbfR=34xrJ8p4KR$WMYdXh%I3MHYF>KhsB|R_vpqcWy~;yGBQ+!E;W8j2knv#gy#mN)6hJn! z!ASt6s>cR?6$bXU+vvcTerO{RmMWeWFdOM)h{d z*Unx-i$sJH3p7s~l_MaHUwc2>sD^r8RBot`EpwJqjQkq`s&T_`7B@RZmpsr(Npgc* z$eIDadgtTkF8N|4DMJLG?Y|x^DYZKBs#SW1Yri}7Smcm3=hsdM_-INvC*yfkKbp4B z8~Mjl{Z}eY4H!`h0K|eK4m-Fr+yFF~$FC0c%T%{_WkCMqPVPpyiTixUkBDnwO3xLu z>L?cuva2HK=uKdL95`ki0I&W@Os>LwfEvOwmu158g*Q1Y+`BJL^qGBJ^WncVs$vWP6q%ya>an+69h3&5ZjY zkvH~=9SMDGZf>q`PJjP?ac`mux0r*XqXj#gsT_h+>M7@c(Wx?k@#Ej%z83~&Ks4ke z?#H+XT5$HK5lG;G<{$WT3x6X7S(CQ|6Ch$al7~z!@2fU2yReeo>(uhy{eo*QK}<5k zBS>&$TjFH)Ewwsj?*NPs^D$UaL;=Z;+r7(!cLPlnSR)1n3hk2jGz+a}jL(((X_vcv zOy~8DKLTb8kAm-l0_BBezLM4mGmJa+6-q3{0 zH5$V#H&L~?@@y^7!R_mjc~{={5qAQJUKR-g*BvcIbmN=OaXhY1gcl&z-fdti2GH0F z6`VdU8wh@lV46Pb2&Y9eyP#{M^MSPUNwi7Wk(zx9OCr!hT5jjIL>c0l*(+MBqz~YVYj(Wv~o+5WYG5CqvcKjz-W2op*5;9S-s*Miv|k%Q8?}(kYkbl;=la z9*?7eoq{eg&?Cat0xDKIdWM0dYcgy*3onjcarxG$Ewq|;C0vg)Uhi<*{`2Qz`!TD^ z$^}H!jT-pqI*Z$n&d-RPMgc#P3V3dRrkVHskx`sgq7>bozb_&B`16d5LtNEY1VQSp zUH1_womjzkzu(=!8ke9?YZxwQ&SVPpqt$se9O|}*4UJgbqI5IHRnXSJqxcxERDrY? z5xEQ(pFbHX(R#Ea+(ztmo~>c(LSby|219MLDtwvKKr0WU3*qHi^3jr%orP**b(dHz z^+f_*4PUCol<2OtECZm{oz<_hr`HD*Sjo()E3sQ3i( zt_S6ZpL})avjY})9H8lX4-chY^@E&`$2d!Y;j%HLSpPo+384vGHFBPl#RWVrM;`Nq ziPkzOWR775WF+dyIH8ZcX@QRM&Qmuqb4SLWJEu&1J}{hizhU#^hV5Yu8$)FMFNQF9 zdj53jX!izqn|_4##t@kl?OyISB=DrV>jf0E@7cR{_h)YH$!87JbGF4ed4`|H4s&r% z{cs)v`+?dJtbyGC$X$-kuCYe#`Es)3VpknmB{GD`X*!wqb8X?3)x1hF*Wtbk;(Ub2 zJM2e~{(e5m5&*-MxFd`1{l3w+mVqSBuE=^==1g+-{W?N75XUV5FZvy7Ma0#fnX4%a z$TEJ)uw1}6gxQ4i9)(q(gILT#qpA)70x_r`D3Yjl*NDL0`k%6ow#8qJyYgDf-;pa) z%VkQcqrP6{o}8HXFYRSchTb4U8Z@CT!LlraoeF9%4R7{J@Ry|iMtuH?`vxB1MZIAa@_c*PE3Sz-{AJ+u?o?CX<@h&R2iSg`^C&oYYvojRdb``v zXEw$6$S#hzCU0GhmA5z?;Qu79tViTFf{gvo!p1*~8yPA=X%j~s)&|8!o93a>Vd&R~ zr`!ihYQWy_j?z-L!v`Sv`jUsy4<0FK8RJ_`#K6Cr8D3*3@0vZBvEXPF6@x9=pHXES z^Q^hdC&p9Mnv#%w{a*jx#(K|qA$+YRKp8TMW@!oFPQhm%9s`R&wA0nlk7MfJvwB&4 zAyhKKY`z?XSnH@OYK3=O;6SxEae4yx#v{2I%bCUfg-LoVjAjZr_&ra2aVzjvEvXJ}ELS-mYD(7KP z`_(V<>I49A0w>iMN<dqXq7h!borVqFBuLh+2LI>V^u^-K{ROuHvpXm>+jg^K)SMm zB=ALcO;P%nqcz|l5-N_s8*M;I!VB317Dw2dHjMv>c%`ai9F|+FB2;M<+_(3DMx<)N z0^Plw!5I7-5tD+*O0n<^<*VX4&ZXA~NGJ8bBq}%CZz^@} z$%Ou*a)LrBIaZmT`^PStsg>$sOU_W{oW=KJonT^qM zC*Vn2vOZ=L-?vUZwh9V(t7gYHp2ek+)h|not8U>CvVJN^a>Ai;|K4FG{6sTNyhArQ zS$r{6+GzrOP%w#Fq)S;sv+Qdi0QyV>^91cUi1+|46zvAI2G9znaQt$Hu=mK*Il`Qe zm3@8fM3Gxm^40Hr_4)fFH_0AKH`E4-&j$o|Bg=MDbLcRda+|;?T_DXeAEXL0kNb(k zRP@XsL(-hZzd`wjw0~ep2%aPTjmXepm{4(vjm6L0(MvvF(3yJN#NCEbIZHMT%VSwS zpRVlf;(7D#-OU8dZjfLiqak48N+3G1WFw*9Cb4QezxY3ts^{$c!PXygV{sW&tLa#9 z6dz7>xYb;M(I+M!e!ZkpM6?Z8D9sabPkn3J)9ROB8g|d$y{0KF_V8~X@NGgSF!=f` z3vh9aBcQAHJqDrsul%y*zvq`dr&u705)(lIkLCc;&sdpEpMeJr(mOnsBcEnmm?765 zamt)`%R6K)-j-#?rSryRD9bhs$nPMRy%^Gx&m&SnURe{po7q|nxTLrAISWrvSWr9h ztv+wH$sYW7g_VHi?YJ!bg{N1G!zFIiAsE@f8yy5&&lgc%+H?&WBrB&3@@s;7!5zK_ zW@D+R4K(-qDA-L0&@*m9%k>hYrlzA&#ok?#G1`Wur>i*{PR)wsC39Ur99#BC>-6F~ z$gAQX*SF*To;{j%dM9zJsb`mA969&cvpkdYFF&j|0gA&x?YwS3cJY>sE_OXOh!c33 zcqHS}d`_3O24(SZ*}Z{GNFEa25zF@(0*I<6zxh)>-D?h>!+1 zfV0fap#tVEEbA9Vd?XdZKUF|7YoHrQ8O$WmX#fD1 ze#-H@jL_7y@QV~y3n_e>?{?GFgk89XuI8}Cl`JbQ<;$leF3Sc?pAq~bny0DHt_c0i zm?N>t6uIRow8+eaX8n4Jaz^=>UF#>gP6l)l4PV0XKtO<4K1Sp4@g_%ZDPNL@4 zcOq!q*{>jV@J9|9zE64v>%5ys?~cgwi3U?%_0Ynhw${HYijMevCMY?L29ITrJKX$#=#sO+>tHPL3VNq!A_vWd z)PdM?qy|0g51j_C@(ZKHoHWSHvqgga_n(Y`F}D8_U`BT$DIVdB3}j;lGcH;kl)a&@ z3`wb)r40CRJIg9#-in1c5eEnBbu|Jq%gK8p%XS5Frw2v+>L>8vuGZW~ih)UL*OQuv zK!AnUw-Y^+t{2wS-ZB}mKk>{fo;nv{H$JYTpTH&|A4Z)D#2>=RX|@BbBUJuxHC5{d z+i2fE&1E6mAD0$SeYWw+wc1yjxd?=90H@F60@gYJ>D`_GHHe$@Afvl+yuK2-1=utb z2)S-301k>Smr~L90l$bPvy@>Wz1TWORRmGAA3~admVcZOIPK^TC33ZWPdJKVKbmY% z?{oX^OxC4U6NRj`nXt5X(nT@4~Kei$obqP7D7 zi0g~KKq>D-%KOT^$|rW7=2z`0h#AJUO>D51_Q!L zEFMh?_>lqTTII`N=DYGL8VuTdV~oyY&$Lp@9_F51J4^5he;s`Uq6O7w>n65t%TexS zKCbmjL+1jA84uc_K*tB#+3D+2ipId|^ur(ma3)}s+|1vn-M^dA+nqE(^fAk87pMM$1ojdsWH^3F|0del;E21(%1R-`8Tf?O^|hSLaq>w zzZQOQf+K|mee2>~FQ`do3BdM!uQy4pT~pDzS8he#e8-mSqB*`;i(uEV=|dPe_QhtUQ_V8#E`p(cIV24Tq7VW9yJPSV+cskp!f)<8{RM^} zjH095BG%8O;=>h0u-t7p;c|7_U??+#!$>V)_+{a@1pvK)c?rW!Kq%IpA zM5W;UKqr1* z?hwf%G1dI!&0MM$2Z#4&R+9z9(!R{YH)to4XN_%1+A?zx{G!8TpZcwrt_!Z0@EQ^&Qd!HACuVfcx2EL; zuL#GY9?Y}ufXrpIW?p?cba2BB=#3e4z}YV@dZPvOMo{M|TKS=hxtb!Mzb!cDu2lEN z*QsR9Nb^B!zmjeimp*9u^owx}jMuk7HTe~80Q2R$0P~7&0fVYhhDbRL>Sty{*U%aB z*iBsu16Do*zxDBX9^uhDPlKCyJsaZB-uV_#%=4FUN8U>zxi>?iUES5PM>)4|QPs)L z8C^T-z8ByZRC}OB_v_wIrkU-cm&?>QzJ|q*)(Ztz9#h#1Po)6C4f-a8uP5xBXsw+S z%{Tx~BM_YSm4Te{mfbDCoT6TO;&1Rx-}bOqkWRmvT;hU?_q(7rYvTRK6dX`Lx&u+1 zv58n9euOaC*tg6Krz@Yyn({M|8!PLm?d16&dP0ZKMwU+~Mo>{MaYO8rUxgeRX!c+Z#z&B4dt4BJ>EE`B7uLH@2Pp|*bn%xGnCohP1MPH%Jn3@*5-+;ugCT13)q7Q_UIN-qmCr4 z?!CF{a=y67ED@?#it%o8wZWhX7ghXQX0`V$h2SI8To^KTaqeOt+4_0PNu`oZdB+^+ zVH_7nNX=Xze9VKz1)j-n3Ai2nw4nw9Rwhnb@GNF;ge}<;TlUdsRso=;XzoYc3XDuIx>_l^dqyctQn=J;46ck0lVme1YCLvQ%GHGJWkw(K=(Ow{j@7##Zk3RSemDg=TnI9LU@d1q%Q{RABqtpu}^j35~} zk9aA&+$AEK(-_f7rn55b#bSVQ5G*cBVO4W*y?Wy$@`Hh`>4Smq~A~9a~yQ3+~ zhv$XL0sqPbgy1X>V*-5K6}pU^KSu*^J`VD`&ADk#q>@I$2ThLU^*I&Ex89ldts}I;?{LI9pnD{k zNbJZvy!zU|&gpq5>j9En}-^ei)v~Yh8B~TZ5yMQ=p2&r|$~Au_`&}>3J0}Y*B74Bm0qa+vM~l zY7MbGU0;wNuI#-Xcj;@vQvQ**)z>dAHyq@sRNnQf>RorG5EOwU($Hx*2|arr1_!KE z60v=zk%CRpYDcz4Vp z#<{UBskd|y(0rh-yT-YVkWrG}4K6;fZ+PVuk*}Yt6F;GfXt=Cac^JWZ2jSv28nHL@8=)V14S&3Kbr#^0LRRLvJIY{t2@_uLIUC0y&LQP?*%Hpb}e z4|BRQYYwnDT7qY_=Wg^#2wp5_+?pi=bzRR^a5fV=PPd>}=ffWB8_8NE9<1v;b;7u= zG@l@)Uia`>&OZumJI!Ly&svp7N&Q)rPDO^Ue%e)nyc5N4pmigZv9F zTTZ}UoD5OlG#zLj{xy?VH+Rvd<%Tm~ZvUaveUV~D6@1Wn_TO-W8&-dvg(5AYBqEM?D5GHKb!Uc*Gce0thWIId-<=-LK3mvL2eM}_pd<#|QtpL1C5|N7R`ahAuC8i!>T43rQWlkGajuhKC3-&peq=5->lECbq2zd-OUCpx+fhyg7)z6k z1~9tRz&^C3ZxM>Rm8{(av~6$&bm!?Nquc`W_s`DSDid$|8rpRwG)~*>$DPCxr%-7~ zaf|}E&+A?C3~h>6dJ{hc=4&RkGaDTq&w9209I51QPOiA*mVD-uL^@e4Bq#~}=P4i$VW%P;Vzkc`jQ`}9?fvfVdhyZ6ntZI13u0X&@U1H^Kdvb`U(&71dS?d`2Q6BUY}855 z;r9xktOpZL-d~B=QICy&+fqHoeuE7H)PKUuv|eVp297wRodG!PuE68f@({^!>z*Du z>GklWMxLPpRv_oO{q7E(L#o&8Lk;rU_v?F}RLgu;vpxR8*EPMN;-l}Me#Wox@N>qq z&dxgdDg~DN@)sXv4TDMb+x@t5ZIB}}Kv#p-3wr_ow820#Q@T{Y-u6m@FnbwZg)TE8 znw;m`uf=F;{aEMqi;5x(-ShIe_Wq4Q9#|Y zekJKadzQS%-QecwO>KSN!siJO7kY;BP72GvzqyCF=d>y7?pH+37r;Oq0FP6h2FcF& zn%E;IhEs|ZPw-~6^}$;c=smLFR|tQLuK4mm;F_nI8T3s>i)_k~jUMaA0CLs^Q0r$b zSkXqPUOj|)UfKoNB4)W#o8L#$RiU96zP%rNHqORtJ1Yo*)H}M{_TWA@X@rn9=XNCe zgB3&e1hs@*K8S^tlX|w~qCIx9`Qz86oBR)l+MPw~gf%rzd9}2pVJkOM^z24*l`L(H zXQ6G-f)FqVcMP(l2}2VNki{wi2m4bQcvmrIS{Itn@bQSj& zq?E0*>*uGQ6O>iX={Zrs zMg(bk)QtN#1e3L1fv&a)Y+#j3KaGovl=1$Q;^=_Q=~nC3foj#dg)G9+6q_1Y$%-Ma zw!Ks1#`hD1P}efVVaZ+=Kb4kEjtTF(q(8Az*O)SC-I`)Btikh7$DBP;Z#N+{mJ&R;ahPt3AwZPBs;x*#8`Gq(O*Ntse*L+ZRAmGUoR-hE zG{>zSyWA|b*M@r3^L{4oisQa_m5@GQfB!#AJPc^f;vW!XSn4p2WHpGp({+&gxws<2 z9uGA2wxU&w7l*no%B%UC(=RMVO0S4GH6;vR-Vl)PNz5`Myl#T&;HZKP61*PH5BtHE zWH2C@f^G`z^p^|3S;p$H8qGKyRpYYk-qb?ZN#qaIC@KW+kAAZ!+RgA|1y`$aFBh zso}TxCk2YyKU+1FDABu785wjv@Y3?oo8Fhq=le9cWm~iz_1Ng+d3CaEqR{Mg&i{lz zA|UY}JA;5gB*g~x+ahpd8ssXXJ;*=T8*7TyzkmcE%Xrp8^9A#JZjvBV*VTFK;ZCOj z;k>;7N|jzRUFjs!7oYpE6Lnnugz(9mW->%BNpllKSEY#a+L6 z^64od&^`{yIvv5fTL}}vr_)2^Z|xlIiRjR5xA9jSGxR>EStrfVfbrCOG6RcF=RfZG zvG2kjX^t1t#4*lhu=0zD-$AduQjmfZN+ROPR;k8H=f%5gM`H)RhhqEEN^9T6U@!QLQ+Z21Q zd17DNp|auB1PdT)(*8*FP8J*D{--GM=tA&RXU0NDK?4M@2o~AoMSE}SFBL}_^bTe7 zm!-AaY81Ll*QRhEJ!8{N+!X~FutsA*!!iKEdj$kuZv)sABHg<*r$9Y~BA(6l@NvTK z2U&<^l`rX#-;Y2RKX5YfvA7!mPu(jxan!Os&%O?}d1(<9Bj2ERWJ_}~ zFZUJE-;9p*R+0Gd;XOa%wl+iDfF6v(=4(gmf>+NSZA{HABiXvC z69gC5&Ml3UFOK+WZe$JT?{U6gn`V06u5$a3SD*;=uze1So|GD25@e>Z-WQP4<4iw%I7#_)Mu!_ zn9zrnH+Bq_lWg7;Z4_Ob)$Kjtgm{>BP&re!M7eU;i9H<|#69t^B1)EP;*E_+Ph_?n zrnI9S%X_`9T)8o6XPkEPj)O4U^{clLIsnD|eg|aw8Q=i~H^<5L@cv&6Q2?mgj3kb7 zx~{(-P14-h{E^B#_s&!+P#aQeNO3vDeJ$;ah?tay(5sdAR@<3?W7H4A`#@hCBCNmw zftdt;fo?-lLejxCq_Vh@sF=y#Kl+P5YVDsc+KE4DKQ)dQX!6vkuRUXZGRPrg4|};C~U7U<|EOoK+?3%YZ%kH{(i;O#euA}^mHX1uqbFJuXM!!xYhdSzUhC#C=ZIf^2yJl~ql@(j&9bz1t z7#p1%M5kI{auqmYDOMb8&y6M!*ik-1qx=!QJ!m0@M|s5AwRc0kv_Cg!=X@n^&i6*i z)jq$__kGXdTD8?qjFZ|rbN?(=7JNh*Py0l7K+NWoJagcKA2Nucz5CBUK-? z2HYaRD%kD(#lX^!7Z(=oXtSwOj|Xz!OX!&8g=n@pW_Hcjyv4!B4JT#+9is}=oC<;Z zCj5*KBpnh}ecI12ETl2^!`hi-Pc^PIz6J}0&)ozyt>A7U!MFBssvC>f7GR$Wdr)xT z1`1ALf&o)}1d7-KfmUb#sNS>j7ew7-nr*(e&OOjs3*tNHWID7 z`o6kjoSpAK$kr}kjL%yL$3!PYu@*5p1;BOz-lc4GQm=!FU7yw@jtVD zRPR~;=Xg)Dt;_wR<7AS_MUN%ka=X|L0kvl`> z;Cq2XoKy%sgJj6)10g&lL%7#0CeX6IL2}7(=37J5D^C@|;#_zi)l- zh0bPzvO~Jgg*ugG1m=tL_jvHZu&;)ncRc+4+0Z9emhCMKQyqfET~9*w+f~9(y|yc;<@9txFpi>mn4Hu;&d@3Bv)Os%Gzy>}qDsbx##IK>z;YO`rut>5@j9ZLi&Be4osTIASYO z777RijhO*t2Cflz=LgfgAI-k;swPyHDWCfuCDhaGELm6nu)=A{RPnIH89>q*aAJ}1 z+XLT;7nyp`Mgl`h)L zCEpl-ctfaP>JVGb-w14ealyP-OKU<&))94;fKr{>ql)okvtPmWzXetX@D%FArQ*`B zv*AvEz!}kMI<~o)Kk=TOv}q}-c2mw-oN$-=*`&@$AwH3E7fyl$D;cfUi8N9z* zm=sj?bv7(>P3C!F=;dEdn^JADt6}%{-LI9Mgn|vfNlRNwztN|Rq@{g(5hYOgd~PV{ zR*+hMU?7L*lCwvGqr;GhHA28fkBzw*tx4^GL^0A4x5y`5Z@HA;3z0o;<1gd>na%j% z0kaRX&07OEb<^Wo{F0cDe$bh!v>R|#nQybMo4Z>C(kA8eJb7=H#ooC-X=!^fVB<25 z6I5^gZ-W^XTpoUf{TNi;!~lT{zhx-Uit;AhVW}GaFW%lf9?Ji38y+FM3E9_B%39XS zGD%tT^_^(3o65ddl2JH?C~K&MmuZE?b<@t!C zNi;Dfa@Z+G2@sVy3DLw^lHdSxs8A#r1?#ebNQE(6tSXZ*62l$oIzGNO=u}tpqzxJ7 zAGW_T+me@C!t&l5i-Ojg?9>UG6@;0AintplvY|MCwg|WRe0PT)o$@KsoH%)gdynxtt;| z*}bJ8(>UrKcTldk#TIzo^mze*?gk__9PT+xF-i6guqEyx6S8Hsc1TCbvP#z5^yZL! z*x$X|P@;}L1BG;5tebuhg^SlaUi-xF^`!7buatQB*EUgU2tZ0c@wib5tNZ5Gk#N3c z>j-|;#QRl$xMy_8mrnvIuXL-^ISM~piF4ejlfpMBcQsGlGs$+ z=c;lFdgK$ej`WhYnOcG)eo&#c!P7N{`hD<3r4IlzwX4TjUddgqq?6q?e8V> z49ZwnscMmA#YGRAyxOO$H=>fafdZlt=Fo%|iu5XGFHL?tEkdH;HZ{~P@dMlTrc=oN zl(VCMm8Dxwc!~Bo?K<-(NRGXEg9+OXF6wCTkO2ZsNBg{i%o*G~o&J1kvYR*ri9 z2-s;ubOjZRJ^6nzSPtf)=hE!@?nfx(Mn;eT?d<8*C*~&9>e}C@vP2pRB(TyC<;wDO zPY5@&e*Y(lmLT*WBwEO2S+)=uYI~@Q^=0=*gK*`X%FYl-&CzTa@Y$<6sQ(9S~dF?0p$T`Lmn5>dOGNEC{RPen20TAU`}~e;$sV zg0Ik+SA?~rGJC;A#ysGi*RRMVJ}=%746X_U(?97OEY6*_hBcCu7^vuLOi}DcPpXmVckup_8et`YE=JvHhSN)*y%$v-`gy%BPsRydCsNNtum)-c_+zd| zutiA}WX~$rZg5>56bTUWC1>!#*JqD~YdIGk$xpiPeIw2DuBv53#0A+QbIqTni_`xG zZ-}_9!?fJFMKFC-1WMds8CF#QphJB70uW94MywlT?#0o)hf)3jsqdWk`{bDI^pZ|G4zFVaY3~>zEnO z;I(TSwkE%KUYhSWkNI4=^jMA>b%PLzBgj@EKepgh!sn?jGIRHc~Xp41pspR2yO5lcbrw3Us_uUoaiTa@viohRqT>#ss5kl4SFh%wLqJD=INfVpIJ=l_| z4Ll!wW?DKl?S+@=%%5MqGNfjN)+3!XU%yV5HDqFE)II95%unt>SofNNxkrsjKZ+xwq-hLkmG zYs&ZGgRFcyK~jH6M$4aqi-RGV}D8qU4qa6X1 z`Ji8RG1X9)1V}5_VYPms$bIB1_)tfO+;kLbC`$fSdt)8-v(~N&+Z^-96dUh!%clAR zLOHVDkG#^J=<#vv8ybbA{=KHqYcpf8R2cB^jm!?h{g6GJL{p>l!p8XJ>Glz~rIyb% z)H%E6mbToEjjRFH{RJ71WOwR+{1;KVI19j2T%L*b~#(QjFYh&+%9O!cJ<(1F91+B zgjTX@VXJstzW;sbvZ9<8zfln>Tev+ zu`?&I8&o|$1hj!V&lUVVXycm}mWL@R%XZje0Fd{EF$mi5e}(I=#1q#5(D9pgtl48= zA(Gg9U!?9ZjxYmMOf3tF!6SRb&wzN?d^X?#POwtJLRa5vx*sJ?JuuSaAxpy~YW1!X z6PTXCT#h+1k$!VVU;c;=$YpBQs@nYyvAps1ne|;ln#&GMBwihwKeVHERR@hh8-JdE z;aSA5Mu1Qc)Ub{c0PYIegQ**$D)yk{zQG=wmErDTU33YpOWI$cM|oVg^UHFljku-m z&b-;f*Hs4U%tY82V)GFg=K;aqdZ)ArH%+W_LP4#=Rm<3e7JWB)QfuTvjfINqw=+Mz zu8TNTf6dI&!+iQL0=@rUpLYVu6bDl8MGTM%DzCR0kRf!2#H=z+F*(X!q_vis6JK@d zF|ItuL7fn})I-f~m{wcc+usg$>C} z>@(!LEd3hmMGTTX4so^_F)W$8XsO4+q?L(tslX@6^LM39dcM9}dIyp8udCjpwHy{q zPc7ahbaRXfSOzr!+VUtY@(;p8W`T{{RA;;eVE;!u48lAaP?v6pSd`O5hYajXCiD!- zPWk1lmbMIB@Z@r#$kaP8GvqL51QUP;<=+=VH10K~&ki`(2 z`d8J8Vg_avK_wx>M5r3hJ`CUd-PdU7?_cIIf9mEL`h=eN;}fnv)bie&YhOPIh3-rg zizl&pR~ZMR?5TsD88ajF3$^e!N`>tq9rBl)E*Fn#wtV8HeD|5NH=I(t%=r^)cm)vr z+013INkIBEudP^@!+N~K^KfRk;~}_`H>i0%XjUEI;WbWx{@sxiiWnGR$H9i!{mH-3 zU}#ed1tdE|$$wTiZ6hyDt4oynk#@2#fp{z-rU(9Du`B|5DHr~!wgh)6d z0!1oZWjMqP*ZRj8!uE5~>eE^RCsY1PI*XPp`F_Ap>rz~hl~Jx}9p&U6WxdcN95#4L zEg^zHYQg-fXj%cA*xdxwOUg=E2t_Iz0~?~)4hotXQY=Pi%Hq)=&Dh6SW+ldv#sYu;*=N3r~Jj&NM116Uhc5?XsESla0<4T*J;FF0fU6^}7&8=|n1VFIJ$?M~@o!9l;zc0}cJ;Ay zaojQeg=;F^Qdt=7lfs`?nx|p*g}`-a9X6O?pR1zkEmJTo%WxOO4t_K4BUr*K5a}LN zH?evwlqyLdW~FMz9E0b&Z~3|YDWXbbRwyF*ZoX^8YZobtM>y2(%~0A^Qj>`}7R3R+ z4kqln8g4oE0flk}Y4C4*mnh>=<#Cc>%rPh-M()myV@9z}KDN$$aE*;PG<}!DJe)o4~tC zOag5+WGF@cbP$28PV>NB(qHD26OnZ@X#ZNPthaiS%wJ9Yc){W@OBepeFky1DO4^hQ z5n%5Nz^?r+dn5eA6*4HRyNULV&ZB#3@0D7fCFvn1-@V6^H9DbRCO-*u2@q%vRbU{Z z0U~M{90!7xVZ#n?(9y0KhTz_4?iW8>P)YhJ7$gkYHr2eYZL-!}soLj6iBsjc zAhH($s75$CcU1y@XaJN`I70Jf^SuYR!G<^IS^=+_Ea3|Ju7VCt`^6s2GJ8LDH|}<1 zSk0~je_q}#pqH1Y79uzsAm<%=d@eTY?hgHz3=+)h)~y2~Q+oq4E^Ysi3%WL;^$)+IWv7GbnF{~bwC&X zp8s@Po?UC`JpAfs5cMY*?9gGU_>wSenRYcG zXnMvG*vH-5&0w_1PwV`N_uJ9~ae>c?NH4XJ3no8Z5zBXA_TzHzj4Kk7fiKQ zL%#vB>jb$M^i4AU+pl(h1GdqB`S}0%r=kDXwfs*Wd;;*tUIg%I9K9LrmTIhUKdI>+ zQ>6$i(0R7aNG2yyU*NvBaA_nYm%6AqV$HeNt-tXwLP%ldDJdpqM*|EbVEnLakH=A( zWj+bd>o*~jUR8JP2>DXwToUHz8266Cx4Ib6lCi}^&{Wgu&(GMBW$=mHrhP*lZAN{v69WCrl{EMtD#kKi@x*(1t1E zP@Tp;?a?Otx!s#j_ewl$-mP(3Rzpy?iI(zg8Hg%{51_=bp3W^J4O(XN+L}5mAu@-m zURnxfsk{s%yFfA8j!9ZQ|~Y#p>!1RWOlfP#^U9W26r# zUVP6!p_mHMxR5K~fOh80Tn2N$CdkO=V_rgCh9E#ML~#;GPcXF|*+2}0}pW)l*s z4s>~#`)1tWn1g=duhTq1w-fBk`<`5|9*yde2!(FpL99i3!`24$eBg&nAQuqjNXif; z{@gAp?ViEcE*%R*kv~guIGeq7?g?uu{ zNcohDkMXjvx&SgFj#nvuxD|aDBOk-dyg8!XfR9s@VrtB37)$p^Hi5dUP-BsJ8&nd( zb+ajgwBGt(L&kX|Y5qgvu9(IfyY9Rjj}!~i%@5nxC%?qzoY0R_E=i~Z1#ZuXPTy=cGpR|T|pztrwNJ_o(7a=>dr2*OSAP>3m7ZQRrfdIcOz3BI{u z9khW#Vi(ZutxuKp(;cLIe1pp>`w`^x*_{SVFh6m{#FIhvN?@9EGK`=hIt;$ZXNzJV!@L1i ztpP^@1d3*pZwJ|Y8EQ;X%GQatcyf@8bewa^=v0X38?pTEvif4u8ST2OVT|^#!85-m z=7>y-{lMXYkhgdVJ_oF%4ift19rFn40-Q#)&^Q$7J|txT?Z)EH4u>p6i+%_!Fr<;P zYy;D-e3m9_CWy&p8uu*^J({vs%EY{b#1RmNKn1l5ST%G-2OM~bngm46V?JQ&wVHZY z9ea&XQ`@+Vpaxghd&*5IUU6L#!&djLud80M32QzsOmK}dXVntC6cai$Td0z~m}xw@ zDKUIzpLE+$l=)nN*@rd6&)R)hpImn%6%D~Vk1knpIg*l?($aHCiHvDaXbv4~b$=7N za)7$OfZ<0Bf@dma$m|FhoJD5D=+jEmndq`Wm%b88 z%HpQHrgK1yDme#SEZi+fK3fz&eaGFlX;}{l%(q?Hn>B z?X)C$Xjg5;8PB+jm+tTva0FeF;512wt#TGwoFK2g3X#$~z{&1_-B^LEOROLbg15qM zY@%;n(0|KFCU~y%*R*@UWBQii$~$wvk~nAO8z_z9)Jk4e#~A`OGBh>H71Te^>FaPU zDX5LA4b>Cwx_I~H7k|R82K#GH7!}^g?E8Dv_t)Il6Wop3VRud_5TAp$w+5nG&Qw7G zpGTptP^LkQS4$dq8sQ$(OtpD{W4?N(Hp0j}_HVQWrb@99#S{&sE5mc(aecx}N>VtD z5`YYa?CJF7ZV#nkM^CJW>%sj~Wo>nbEzjE=(2Xp;Y@^C}s{0Q@GT>vXyi_8OQv5pu z28d<3F))FK0l{p%4d4r34Ll3Mx%k0IN?qMVQ*gqEVdAWIg~SOeQoVrv z2Y44Mqi3RnpSHf^TuzPOnEH4;J-dQ=VkEe_%3I#Sy7Zo0II2}Dv{t##8DG_Qh|+3mY3nz+id3F?H@~-#<0uzUpuh)CFC>Sz5=7yAr)x&P)YXaF?9{lMP^1}*6x;-Oyj>LIF2+dF%$}E8R0c-eJYIj% zC_{91y5Vm3ll}ZBvUK02+SlqPU+V;}F@pkRfTS6wOs?C!?Bh`p)X=z@^P&YuqMRuy zc1LG<>3a7cXzAd(fTx$d!b;hmS~KK`*40d>gwv>NswsYhj|R`QVe- zBd<%>r9|E-{o(~>m5e^X(`D6-;-AGXQ&6cl@W`KeH!g1qdAg1&*EcjcbXnMFR+h!q zab#Jk3UB_vP-%*=+7?UEs%wyWti|yk2%+i-I8Wr}X|9x*otMdtmF6?XR$k6QQ&<8v z?KD@nfVyMjA(ruhItAg6K^Z*zV?^{|cJt(dY93PvT!CJ%hXQQGxWE(`i;{iyZLv_B zs+a-9ol7w`F9S{n+Bw>_+pI5Fs z6&4r%h(%QZ7#M&W?tuW`P!~fqDr5rXA38Ni?|lu=Q>ygP?q&n_BjFX&FQ`jyl_~qY zYVuBacwM}pD{|Lynr|LV_;sBIU;$`KgTBRjWk8(9*}w}FD5(XPj9c~CKfF7>~=1X58>1$2b1*=Kcoktb30O2Sx)x`BV)a4mvKu&`sd$#Ys%&*2;xTQFctm5VO z5v*4VoyNT!Bu>;SwN$+Qh&#`%H7{|aBd$1L-f!)wvrhNt0wMRj? zIkU|OqEtz94ZE5f4%WNY>mIfdMXT#&CpsjXFCnM)p5-AA{--Ae#_vBoX%V230?#4? zS2Oq)42S{)n{3)PR+&YL7kuo=rH&XBHeKsN43JkN@_d8-&}WSsAMxWtYrT*F|93- z5{O)>qqv7~H5wjw#+KZ+etthW>bHelhKhGaj+E>P;G82t-VI`8V5+5A48@pHs2)m2 z`5||+zpyo_f*u;9UZbJRW2K8%XS?LTS9(NxX88(pWgUT{OZVBz$z9?lc%fju3A!mO z59OC37!?6@1;r?X3Nx^)Lxz1lb@q9Qg*Vrh{b8(v&A#jN4kz&So;Sbn^~Q971{PS3 zuFC`fz^p?tL3r*#Z2}Y{*x#;Hbef5d-;Gh4;eCELR-G+Tv%*(X^ffxm-{Vx~)$Wg( zf0LkrItm2G@z6s+$8eQKV2eJ6nAZu{%AO&CxGUtn4wCJfh?R9P7I2$%AjzZt?#SJZw@s08~7Qaq$QLm6KwvR z7g772!*xu}w>yN%$2^1JoN6Mu@>8k39f656IrDz#}UJW-gLIdW7MCg6t&c z*54B0g9jeb276-o5|*~r?ru~;e?hO^tu(J_ojhOjCQ@G62eBLCFlq@=TWcxPoBX?5g`rg?|`QytMMjI-mF4MTKs`I`r*%&IrUWh%Q_o4Q? zZ#I(wzAX?1!cvg#cX^HYc zSZ_=N8$>HsL?918#fk^S(V)a~@5TT`6b7R#(fkOopYfkt)|VTiJfU3J&; zmbmrBMbT~Shev4+hEtD>9b?0APdh*}uIn}Ems);voI@sV?4-MqUzpK9miHjxll6n5 zesOayf9^2UrL+s(It-b~u7w)E2id|`Xc;L{APeAr1rL>Cb3h9IJoLqqOn7<*_v+k- z$A-kI$61kw6IHDgb5%1Mt{NH3dF8>br^g#Igq9^bl&JKIFi*(yYW9@J)W@L#t}@6% zqv#d4(;i->?;@b=Yn0e_>;;VK&Vv%73@$ac+|H4a%!|mXz z&wwQlN0G9L(ohg2!n^@P_rd@256A2dWFPmLXc*)TC{!AeT#aZFJL5Zg#dSI1k!Ykv z4Px(a3Agrl?Z2y&H-lAlEMRnBxP?9~%vw~iU1XacNfL}<3Rl-zpj|R~Zx5bc|&U^1yvOREU z37ld5JG_nyZS*ka{q^5ZCVZbzj%3S)=oV?BLOTvbxx(kDj@=k)F4GGxAg}ZJ{UB@g zRZxNh1O!42?r13H8<(%`a}{{{RPhX3H0J{!mm7oxngiGjK!DB8C?YMe<-TbEYE6xa zAbe1R@xgHU<7*lzx9>f+R*O5asC@WC?Q0T9<c92z$ilqBDH{6 z2ZUV{t1Jq{5BM9)Pyx45aUH57tArY5+QsLU&RxRzfR{E!^@nb2Udu@S3PP#Q* za&gEr2sVx2i<)APBUWBfjk*2YBJrx$)0FYW8-=G z{r5vdR@ZhFzI7@Jy&%pC1J3%LB=aPQWwFB!vHi_}K^j zJ^*`yv7K;M%zyjB|M2Ib|J^6}|2aPN|8d8bU}1OHLtQsQbx3v1tmMn}>CP8aTi(Dq z05IrdFVoPlIv zwF_Uw&Ig7bbUb z0b;IG%Pp9m=v~Iama3VLv@hG{|9t86{jB_{ntmlgPG!UMko!Xb2l@uS4y}O&01lCA zKRBhw<~Jg*&qxAk;mA^^@s}6{|Lyq2=^GvUMjBl4d)zVpTQ?QXW2UzBv|3Iz32D4P z#FZ(dY_^4s#KYQKOxYbnYRGy4KnI0}V269*@8&^07{MruLM^-LnVxB%9J7q$=5j)T zx&e}B|KOJ5D>K(1E<;|2!%$HGkYZ7SG2n_ypsoOBAWh7YS_*JcKDNl+Kby7_t}YcG z8Z(K0?rvVvi5@iW_K(*5wJN<&<)jnos;<>$8g=yp{pdb5_~E&E=+_TmNm9W1UPE0q-~nA21Yyp}4h^OkE0-;} zxDzKR(ggaZaVh%WLi&62{(eGFvTw^ZDlzlCZ7AGj@pZc>|XQNWQ&bAiap(+z1Wb)Qs%CX>tewpMSY zMSv*6gF?A`}af^+E@Rza@H=*%ekMW8apMSA33Y8 zV6i~~?23s2!xG#5H>^VyL&NvUjuQvod|_UG2@jqgDH_$94>ho9>}XHMdbu;?rOzZ; zPX0PD@ptIb8emNT9XJpYP=g>a4bllw&l2t`N<;p*_*+mn!~>*^8gnV%Y+X8NKH24_ zGyW3~zU@yv?Brpqmak_icKdF+{_OWU^D z*Y6R6Fq6yVSsYEw9KK(=I2<4o<*{$1z527XNAdCpH`T>?@>A~8b>kmfMcbPki=a%PRv1{t-&DrKIe zj^oPoZz-nf56b+^i*jkG>9#VO!iN?W@^tajS{_taihn;_YqVJk8iM*D#2JJQGuZt3 zRNM~idIAu;+ftLza41H-O>4F^l2WB{KKpFexEFTiQWCl~wRR; zbnHBjy62S-7vc}fIumY#q~N;|sIA94L8Ruqj%MHDCB>TdmpR;Hqu0={h|Dw8WplsO zPU+%3#P|D7an*XSfMyGTpkg-&XS!@b{{`6D^;9T;FeOWl z0q=Y_w1@o((shM%?1mSHLiRNzj#bUuom^7XKb!zr*IGJVzIUm_ay=_X7JUB|f$FL# zg=JV#o6=C;Ge|yv$?b%*OCATt%#{2b4Y$1nJ59TLcU%Csw6Uk9XDY`mp<4YQCl1Ut zM!?=wd^bv{c=YZv_I5Yj#nXegt>#!9SIWnk?53L^=hUS8>~8#vI~X*5p(9|$p^G@! z(3s7S2O-=+-(GYou}-p!g`zmdcPL{A9B3i9DrYSI%<$~{P1*O=Uat#^5p4JeI`G|owg4VX1RMN= z$OTa>vkt##%jSK>;AaAp6n70w;6-sV4n;Z94&YwF*;cO5%xZw07J6!R4P;L(w{Kj5 zZYk^jiSa>1x5gqU9Too%oDU@ocIrX~yiaG9%z)UVvtq1Df~O*nKikNj#DIjBV3ynjg7VYFneVK|8GP8)@iD-{pUrj32H}`^bj_lF2_pfkYc}|)ug!>HkV^nx{pi- z9Dj-9wBq`m6R9|W{2e4XZ+=vBuV_U<(bpWZDTikc)s*A>|Ea%DLkk9k6f-XBLOQs` zq~o6(mBzx44oshFEHn)X^!yCI`Y-Gs2WL+g%6cR`BJadq&+4&)f@o3AZJ^(-dslSCagg|8s&;g^c{V6d$cM4-4mq*!!Bk+=s*yiQjBg?T+Nx`E=v#EdCu*XCb<>ii`Pp|m4sB@qYDXMguN!Gl0P zGg!g#n<)kmg_x#h_bahm=V*JgJ@l7vmwy;k4X~FS;Lhzhd%m#U=?jnj=}5<3ov9`d zIBm{2Lwu6yGk(Jla9T-7oMF2JpJ z4TxJ+*DhkDH(2O7?}@Ggy;FG zauMM{*GH#rJ`fY!AFDUO6v4O3Aw$pse;8e}&@ps8w0kr3zGPl+!(ayUTEvfO=c3B& z{vTrV*@~(M1qb5}+bS2njWu1EL-tgl5+EeLEh5sBj<1aj6feuUocnF;&hz`kwJd>C#iC44v0lGt2j!9amEkFuN@Mj5``Yxbe9jg z2l?t24L(Bjg$KTX*>|{a`py!kX*148epTR}_hT}PbGS-*V7l-%$i%b z62jtF!>9}7jD(oo0WMHZ5?kTtP05U~U^&T)x6U82&ds%%{_rfdOlhvhU>Opw?;VO! z^m930&0U=l$g7Zl#U7C6`Xv8COT_e>?V>^V@yv74b7-+H!{;}*M7ska(Z1A=HvEF8 z?hL-dA7wow`06AA|7s@t6>gmAH&e^|^TQhKr-CzDS!eo9?m&A~xdp$949A+isK&U! zV}Lp_uXy@EI}K@5-_{gdH-pBO)1LZrDLj=KWIb(@E`udWKT=*S>z&!ka?Q`L(euun z-%p1+%>_1hS1l8E#UNpvBhG`y)6F`*_G^C2&CO0kb3y9;!WVu-23tmw6u}aAM*>>r z5vCv5fp;NT9)Qptfrshb0JDN*lpih@xHF~!)fF>w1DUiPT{2(wqP_hi?}f~^p6_yF zxH|-dncy;x*TlGTSrMtv$Zu`eJ`|#Tz|tpk(kshm6P0uZv3^4bjG^EGN3Iz46MW=0s&_J}$Yi zpVrL47vpOO>V`trR*6BGF7IfK9NpZi3VyrBuHhuvho_@d*}V*pr2Xsk&t;7^^Pj~& zcxF8LE|(tIZ?-o;u`v|Hw!BtzXkn)3(CTV&p>+b_gw;?wcopklA(FsIJhs5PST8@V2RkaIa%qnkj+0O_R-HlL}nw5gMVMQD2(C7k8k^(6mAF;!vzrNFXxuKR=`nEd;!N%(@gKV_K@E_VoH&l z`ppg%+QvK%A34@s)n-5}QJa*`3=F2=M3@4!waV?ggGs6fKF}RCS4BJC@|irEmkxU; zXmvRjLSB2()=xpNRFMex**b4%g*BAHJ=1P(#Q{|Rt(szsE8J2-vI_Bqf$?w=;~-NV z_9qEB_HA{T(L4nNE@F$K9rma`WY&~!7p|R;(dXE>;03q##|Tr!iyhgzH1Xw>s(w z+0}neFbn%gd~`#pXct%q6uD@J_=tkZCZko(UlpsttA~V^wp|qo@#fdwJ=iLcTVK*jIAm zmwW(U{1~*GJ=nqdIuoc+Y|-1=?ymW0KbA3U63z9RdC#|(pcicaS?@0W8PXY$=i+T) z?3g}}les*hsK~TB zP!lS#uRU_ht>=jf$H1C>qDp+mlkE|GD6Z-ib_aPJvFbf1z3G0)!IdEvY3O;bJ~azg zN|Z<5LTn=Cm>4Rd8z-`OoF_#qG?+5(eZ*y{!y*5C4kOEO^&MKVJN*{*+L+@ZS{)hc z#r0wm^KkKN%IEZ7OKYmCu-xAY_7QPBQ}@>0h%1>iWhTVrg)EB6L13D3(|ug^aw zIV_JY81r&%srzw`EN8rw2d(`uI!F(=SNwa)gn^+q@r9 zVGPZmX2)sX>nR~@6>Q-L+bPb5hO})+)t-EkY;KXByF5A(?;NE>MFE@BKCq+< zhn`AcH93uiAI$xpfAkQ?B}uMWz4)G>NLdi;68M&l3YDp!65FBT_JoPX%+GWOoA&2l zpEqXT$y~kb9V?VF!#9~j6VOwc7Ofgp3~l)a)-`NZhne1O71{L0n$cQ_yUDE-Et_T z;r7KF!y#HuvQHlVzSD|p`5V&9uHTkHY|C1YX@-ANpK%#c6P@vJEgt_RoSOdpA^Vi0 z4oz=vcL9}%p)Mqr43&1Y(hfYQyZTjqNA!Q#wSamnPz@=b`GwlWRDtoKk=Lk^tH#t4l)zUT5|gUajj-{bDIK4SJYzDO?(Lm4VLD zjEyj!@1&mqlvAVij-R#(y^1rEr{Rp3QJO6ZvccyrHUy^F?>*5cVv*&vpm9g`=0Za> z2N?xh13xPL8kkn3k8Hz}xq{`gYG+g*{W!Uf9-+4Og;r*q^OB|T3@iyd(@xxwrps&A$M&Cu(d6{L2& zDmHDAwrtj0(@j)OD~j8_H-6cz&Bb?hKhHKUo*le7Zz=!e%RLc_hk>SEar_y7F0g;jH;F>hGBP8J|B~E5r{6`%5MrG z;&Qe<^ENZ9V4KjoXz-|h8@D@SajPa?IwrdNt} zzt24yJlD0me!e-`;F78x>1rj(Yx}mNEV$`&AlQh&OLgpreY+d;LLcI+01ek#} zaGh^?$@oM40$Z!m*3tfR^+6oc z4g7(bM$)gqj-*Kwqvu$eKDw48h9(FZUdw6sM{ zpZJF_mki7ueQf>mX`P5t@LYt)^_YhoOEGG!7i?bmwA>O=RKFh!`vsLUPUUXQNKlaD zO(vXk1SBX~l{BTc>F8&8N3@;t&IsI9efIm!dAlpvKzcZ=LOx)lADTGr3*}0Gu#FIT zb2ZxHj8nY(!)sQ8fqRdXE2pZLb#j83bsY8oT3-?2f-+@!A=InFQk+|`4Y47 z;yH9nQr*h5d2l$^)QY^2TE$j^HSB0rCD}7puUz@E_@wldRH@e67tw$-xdrMI(s3Bm zXDI~kd%0BW^e)?4s#W=w=8>5nEDccOnT(kjj?x;!bh!`{#ULBQ+}E}nX4m`QkPJsv z{y|iw{&rhm8T4h?p~?^G%dPXiCqdqqOZGQDz!Lzw_JjP||#cz|6-$39!wEwTI@>_$6K=xHdn#Pk< z8OnV62NhIr1KsqM?jv1$DqI*Bdz7%7Jxbt;iolzbs|0$#dwgvi@T;!cfA7qGSX#+m}uJvkdmUzYIWtsYN^;yWDJ6*^6kX6w} z-jShY&fSo|61Mcd*Zm}#hG6_38N}mcjdzeAP7<3#&VhPDxv*^0>px668g{=P4@&5< zV02zn(K&fC7inrzb3#5bgBH^51ou`%DP&mXYU>vq)BfJ>&tb#KoBpNUoJC@mV?eh> zBFcao)-?i`_K0cN#631Nr*1jSC166n)8!bB`F_~}Ut%U>Ox{pgPmPS;lX>$#^UWX3 zvZQNlV)z>{*vjyDvi?KUQ3I5I~#0#x;^{mL$}zJnj^$ijl-T@ zL75Mb#Z2zGJCt>vMfQQQjrrjao#1iD*M*DjF9!e0OfLd`B$HTd)f?^AkT81SlkYQI zjYp3%D}LZp%s5zCX7W8P+V@Gf(*k=UPH3gyg?!81^;>7w{ya1DT;8(&dRus|os|VJ z20KaEF{l#zr1E%S%aF-*lwGstoZY<2d)3E(W=smZ)wmuA5v5~Bh}=vLiYuNMF6|Ya zAx=G#+*6G+zN>ks|6`f?A14JNZ(wnK*`EQ*?)ayjkQ&D}x(F5@6h_|K$@KR?1S${# z9;iZ}vppimP`T1875}ElckNC_Ov5MWZ8+^VJGEs57uUO1)?Yse>!zQ2d{k=D^sI)5uBmr<7~$ z!Y6-s_L&~5-nsyfQ6PwaDenR<9rAV+Iw0ieko;LTHsaUSRV3+ESpvU$8;@7L{lZ4TKJ?r`LWuV}2FIz%9jlZ>_(5{RWj zY*`v<@(-yKr#bBO<7qf$d_vJs?ZWR#lxC@X3l~HQIyMxZU;e-tUO4xjK~8HdY)qZuO$^=CTH}D= zg;7PT2TIJY<94#X-;)$s-um)VleCSd-O8pD$Io!f^ADBn`*Lyu)+wH~eq+ zf)W{83w7`H84hjweBx9>NS|0pENVhf31&6rYQdry}x?Y94>1vVj#pO;<7JYIt!#iSGgvWr2Y@RBH}k&)D_N$_8Ou$LUQf{ zDosznyvKfeLX73}@4PlRtwVB722xf^OBWe4qs{tthlb}5+&_V`>EgBW*B$)bGM1J9 zqwoQCpY!hy-o&bo0U9D1Kyw)oT#KzlEfq5;h((Foj%#AIP^%L3IgArQVSWBU>~b_? zr^`YXZO?{lj))zC9-Bg@mpin5UG{v{ee7K))WIssy;&d*@m-_@WCGb@cpQw zUMRtV@vzO!3*eM>Rp#zeUZT;vUN# zcL;xoal1=>OSD@9P;ZmdEUIdg3N(grv<-LDSjNeC=V}=-VqV<+R}OA?*$6@MQOq%O_jCNhZAJh`l!B+=-0A-y#IaTVlvkfavvMupFxwjf|sb( zSzOVJz+d5yYYnE(8==-OtvdUYj4VN-A%&)s;O;~YO$VMTc#BN$3LQKA5vdTZ$w}s}n23=nr38PeS zTbzO{zWUKh*_ow#@XLO~V5N5d2F4hthfWw~TivM&LNxe)U9aqzd+R<^_gcd9RGoDn z|DYjE#QZ0SZ44~U!kMr}C`RoE%K_AIDMk2{>AkAfk&3JRuV{f>Ci8bmO$pK;$yg~! zI=sn=o7`EnZPs?cx<%CI&5U3e*J+9*q;&1qMS!1V(MnooLo`*pkqxzjkNhJeUfzCT z@IuZqy{2GXH?wcIAb-bhR<#E%v7jD1TZqx+{Cn$+-Ot%1^H;b#tT$vvJ3iP;kZPv- zc^Iu-8v0E_t_In^Ts`X7j~)Nu+Kl?u(*`B69$c>C3WBUxUc-qgpRf)n{X`xFOb0)AMw{=%oRPiaTs$;?m?1`_d4}LmpqY$8vxx zwq*AekUh=&g3ZVTN=wkUdbv~_mR0?^CU|qmeR7KEop5h%--8!R4rW}I2tD8p+RNGm zXWO&`m~vVvR_*2DpHD(sRZvyQ+k?r&YWrk_vah~f-)HN<-~p_Aqd=~O#RAz| z?bfX@V{Y6ydVeY|?BXHc6DYzRnadB`(#Z4_X7DiCxv2onM@^_ViJQ-UF?w9UD&^OR zytR}nU((AP`1alV98IrjuZcbm7xU6sv2yZ>Q;%=OXN$~y{~~n#nFPf39)P6@8#O@K zRR9h%yo(&kUlz)slpzL)SK2H8!oCPQRt8Vr_yDXSfs?6*ms>>BlV@ivKOHWiq})dvj{ zW7a2qGvjrR=%0o5a%Ccoe!N7he8Cx^@EdoAKzs1JACm*lqw9`+>$PY4FMC8y8?Em< zWj9c$n#;BD6OoGC13Csmy3VnD2$9k^4y}Y=Klj6PAoog#z*Q42Rvy=5hAFKog_&>r z;3Pl65B+iC_*czAd)_BP z8=uPdA)5x8%bB;S5atK+=jyMr{`7LG))wc)6y&4&JN10KO`%iPU)cgod7oWKjQljA`+TpNquy~@c^h>#O3IfnJT(hs5ks~XG|5@*zP{C_b@$ny(71m6 zj~ZpWXG;8d7RAP1fbOttykMe4@XWEyt1Z58B^QB??$Toj^!2wJ7d$rsBTr}VWyB0~ zIES^?$2ecw0(Pc6;hN6Sc<1)pGWdmh?%i6naV5ke&_L>!V943V=AxEW+i0=(=b4Wb zr%xEm!m@Mk=y>n=m?eHGk^y#2Bl!lL0fbaMw znTE6ckP}m!)F6lv3dX8VZ1z zpv?hLH}_uXBgbhDSf|$eY+q&;{qEo=eU)QmHA+rhwBR993k)h)Ic_4HpX*ot(JReZ3?O4vUAMcL9Nsiu+*YOzBy8 zx`hnt^e>`?YMbcPlD}XVxTn6!Ge`1*WEdIc%B5HNP^!x>gJPMEv)D}FBh9%X zHDNw*CGeNx&OIU|nSJs-VkQ{2S!?LhFdW6wt2N=voSBr&-19IZql}LSbTbI;c2vJ_ zW`twvZczUEM}0WupHZ!z0mj&f|E4LrOG~znEb`b85{$o z4UneEm2;%w^k|?HSO@{gD|qs`t}W-l>W0>1N_WG>K_~Vkw!40>-SZ_Ss|Qjfmkfc8g7p3-FQlTmZw^k&p&M$3hgZgpJT)_7$2SlzTvM`g^IEr+)Tx zS>^ikzzd4tX$qf=?L;!l*8&=wu!B3d_0AuS6T0Q6;3xi3f$(7L zi91UZ{7VRyV|R(cJ5V7c+kQ?Cr}}iEEqD3kq0rylw-1IbmcV5SXi3(bvBVn&0 zl$|K)NDV4;=+^!ep|WHBF#m^;s2#7L-1=sj$keI7uzc!Q{Ib2z?m=NIMgzx1d={~r z0-BsX_5Duey0Aj&?q2EP>_f*4{j#^-FfVbWpii@Co4>F((e<1x4A`F9;Tuj`mFyd= za%yX?=IpnDQz_5st*z&<&U@H3GjmX$kzbI1#H&}1x^ggqct-em8$VMZ1AT{S#bvNP zh`T(INkf|DwJaVx%L~a{m;k~@q%wiwO3`yM@3PU_*l0|B9W!*)!dE@!rxnzB%h~P; zTc}|dd)xwH2@(z~ZO0Oajy};F8~78pWt6eJ)4IqS8;zaW&j9yvz_aYh^!JGMmoW$(aDFB`?hwva_+Oiv+AFso=SOEun+qk zcz6V=CAeC4AAxNH5EG>t5JMG=?16>3l#Tcn)-Ge$g4_Oe`lZ2_O~3yMxD|&6cQ&3K zJT`a@!3soTeAaH_Q~?Lo<&5AQYbEbz?R0TWcFS<`15^^G7@IEKTy>ImZaD92ouNr6 zaX6Nb5WVcwc}9MfBW$M{YW5h1RR z%p~t`atvf&sF~cWcV}?!Suia(9U>Ae2zHViiN25_W4!JH|idc9rYnl zMsrz=$|Ynpz2~U2)%m+^hxGCRhPARh#;pI?)|(zWm+tt@@BQ$)Y?!C+R-Q3x^9{{7 z3LHjbAT44O^4hKsp@C)NI#Tt%h$EK%wX&5dVHnix+mms*bFSWV+xTYgsc2+q5c*gt zpQ=;L$^cIK4Gy!P!$oDIXzk*+VyWI~jE0L0snR>ME_Z$QuRmT8i`y>A3fG--K3T_QEyt6@ zU1D_A_Kp)Xj7HF58-Bal?lUU`LQi)sJk;vvJu}erI+)%>k4lssrjij0r8F9u8B$B9 z;$AO@+U(IRAI=Zv?#Vsju;joFFYBPvqv7FGfGZXzV1BXVR|RFiP z+;I6MWxu77;yR2Y!Sx%#Bm0ZJDDR8pFIJ)N%#ZI0L*_4NGerN+l+$JPatrsyLqMDAV{`%cauP}QW!vSQ#j&`gWB zv-X_BXcwusi}3>Y>!5U|4b93s1qryH?FxxxASsxKdRY!DLv|@hupf*!ctX|wuK>D zYAjJE*h0E_h@;D@qUwJS)i`c3^1{GR=%T=RsN&WMlrc$Oht$k6rEFAEklep4B!+NG z!JDzg4{z-jUHq!DdcEO6`S|M&qYIec+FOXGUN*)8CqkZehKtK&kt=oP# zuo9^{^A6WanRT%n!Mhb};TVX6_yzz`;ksau!Ql{p(6Qypd6g(j$$QsG{RyrkoIQ+D zZ(&AQ0o6$E#9~{N!Bj6(!m<)Crpp?8$mz!aQA5>mdId{&rR8Dq3z+D>b{FbrWcPh{ zr?BF*hpuXTU9)kWp{(LwC`-R!_i3WYr;Vo} z7s6HD+S723GEk1ANV@Dalp48-g~@$yNlmAF(8%(r(PN(~$AiMlQNFJ82Po*SbMz0I zQ}@y<;}L+XgRusvn$77ib-XSRYImPPyeT4gjk51Um7YD)L8*dAVe4Rr`M|D6O5&t!b3T}38-=1bcZjsWNQ9q%(pZ>x;ex69Y z>cX58PcH+{$(4pAIH}K$5q^>Qm$tFvvB8g_ms>j- zfi7kYE<_jCr)f(GE+Sg|+ggt6#$%>`ZyNhz(Z)uJ2W_zeydB9A?YX1d_P|%1<6!h%Y4`W9i;5|6JdF&21&@cX?ZD5xTFQY&7Y1zf43o zb3Hu0#CX`uhnjP28I|1igY(NVdG-f%Fyx_zZ29@#K;^snEZ2LSVW=Ut(}fEy@fGb) zA?@8Q*`A&{81mqyr#&o=c7f1$ao%lX#XFrLPKhNTcmv~+xomwZvE^%c)KR{#LmiJc zw;q(^`v))jCz)~Oj)S?dv5NUY`>3GM((uGqujI%*--qwLvVHIKJm~9O&`*Fkl4Iax zxow)@!xjaCF6O20=r2}a z&l3Aqe{OD+in9fgn0Bl3F8Yfg!?vdWxYy;#_Zoj;#$f}S?`cYTT<~QY)hF}7za^@G zDRiNI#ih;tI#ZPP`gtiG%0rWJ2m2=BEk0fK=ulun?htfmgpzVN`nIzfh#sk5!i_Pa zggG1~mYuqgWTzjp;kjF6p}<8e>RV_SAlTj!c9(hC)hK3y-EKYaR*}V z^7n$?mNtC%EK8PN-y6D4OZ=t1KIs=HUt1>$63!>mVp@<{Mx)>V)+j`eklr%Q0nA{((? zka+M36puB_?m5))#NGMbnOLd9DcXn?Dzl$ZC6PS6J{eq{jlV&fl%5sAfWC?J-kp#s z=?7l)Df^3j59j1w2&8zg+QQ-x7>;=2@Z_nQYe%YgtZbsMg+z+BHXN|#(KtqkN_hLA zfHz>~13VWYvXZEd)*5B=R0r$0J3(_dTOTdilUb8L$JQS%+Azl^sxioqmhtIZ`BmOICo!2g&_KA4PZLDI* zQqHv(!lq7QiCDh+Hi-*i>4f6zNwV5d7Vllos#F-%Dp5Px6{=O6=DAY4%J zE)=uJ6&9aAoj;oY$*kB%lof8xjLC|mlEpwK#1*e26Z8eo8u(AUvv!Jqj;%{@t?mLi z6g+HLlWa78g4Qkk>gOLvT)1$s^T99|##|D4Ma}gi&yV`>_N; zE~rqbvanvr(;d+-2cufLTtFd(2mH$Er_e z;_=-IHgdK+gwadQEvgL?zL{O5`(5LfMmy%b(7=6dk(kY;5cQ29Ryfnnn&}E6qu8#1 zjf;lw|19gF9(QSjPs3sB3X~-Gf+x~UZmb>*$5;>0;u|0(ISRqvdpaU=qD4Xjly)!n z$13geaFw35U-2wm#xr@&eQnp#>RLH)>f4M8qe^2XvQxugl%@?1-XM4R~hU+Y!A1e503}3icl;jKs6DmDM2l-l~)M z=T6O+!m{GRWrt zwn(e-{C1n@>sr9UWa&?6=E3mEgA|KuCfQ}AF(XmYq1^q7gL0A7FTaJ_f&0&7usS-8 z{JlQ7c-L|;y9BE^eF%xQkUfnZN*lFIzRK5o!~{b^;vibCzI?&upD^>ZXrFyJUbHPk zAS=JyTLmx}c+trUk(??YjTsMJk%J4Zkq3&_x8wiqLv&n)yQlqrZy_B^nEv`f4h*bm3N8KyJcIBUI zE6#biX)MhhqXf*W!uZY&?lm^z?#RuA{Q-hnXjb9bbCf$0QzD#GhjyHOesfv#8PY7$ zQ0z3+#y!U~o)@oHS4~+UBxt3Ag|Cs!>u%DwF%y!kTdZl)kzxY3{ zC>eNTz-skbJ}pShN`EXpoL)=QQZ79I>Ts^xL&bd*5c$2Ah3)j=TV8`p{#h_a(N*T`9;}2Z)N<~C_H>-$z1_-KgIDi zddSBe@$o;f>5xXr=L z-(|umfF!*;sm7{?6GiSdd{Pf3I^;0_>hyws!#||xA z8m4*HZbPDi?2~s_wU}2$F>>3Rn0ozM8^Sl&nZv`w{hW}C2?O@?GjIAVHSWIAP3dK= z*NmPO$VAx?W)#2RZ^~a=ZV$Dn_D_H+9AaO(hxrzoK0SH1MJ4vSSb4H^%aM+{2DB+) ztb|aG^Z-JSVKax(zJxhFo{P&b3ZYicpww?^*RH?TqwpL~6+)CcNjn7Qv5L{H^0WPRnea;GY&5`MA|5+ex!1 zC}o6Gr8AUSv==B^cZv9&gARqzYoOXwMCzaaeim-YKUzC!PB)4j81U!bibsleu9rWPx4;PIrr#*wFn zhn*Vf%JBgRO8lkmqdY`Vey4=;M1knw*%v1D8TuG}-X`KF@IZLouxoeQs0va4{apAwA@IT&%yXN4c#&u=EvyK)U3E=w#tqL%f>$U2ZdzhSV8EE5WHVxmIqsy>;TT^ zRnom;sFxl~m3)d#30s@e68o;K_FCqi)&9tPWM zKe#~E+Jh<~ZL~05vg;a6GuCSGaG-qNvCEjdFNl6&PUHxEKeMO$_^R@RsW~Uy30lCcK*UEYh#Hl~I z`{RTk)~S}Wp77t{xe(8b46Mh*Ec!99w2Ow5H}^@APc!A~P6nDNM!b$N<$80z1q+4B zb7jZ3bbc1NYzsm;wnG`#5dXtcc9d%qm_z}|>!eHt3s&3<$62H>RGqs1^30bj8rcDV za8}K}7M|>GW+|1e{@KidiU2p3eHP;Qk7RDNW`4KM(zaRoc_v@S&of^CdinRCn15qH z$ulCxAJ_yQ#DO((+O2WU;G`i2?Lw*e79Gm{={Q}lzH#~UlDec9HHq?8;E6)~2-4pL zgHvQcsVnRN+COF$KKoJ6*JhT~PlV>_4@jI6cs8IbI+bYZ`f|up!8L$FmQba~M$^)V tR1Y|^+&nb5e7--5Jv343$>aU Date: Mon, 27 Jul 2026 15:24:51 +0800 Subject: [PATCH 107/240] [Model] Optimize MiniCPM-o multimodal encoder memory (#5188) Signed-off-by: xsmccc <247721718+xsmccc@users.noreply.github.com> Co-authored-by: xsmccc <247721718+xsmccc@users.noreply.github.com> --- .../minicpmo_4_5/test_vision_batching.py | 175 ++++++++++++++++++ .../minicpmo_4_5/minicpmo_4_5_omni_llm.py | 47 +++-- 2 files changed, 211 insertions(+), 11 deletions(-) create mode 100644 tests/model_executor/models/minicpmo_4_5/test_vision_batching.py diff --git a/tests/model_executor/models/minicpmo_4_5/test_vision_batching.py b/tests/model_executor/models/minicpmo_4_5/test_vision_batching.py new file mode 100644 index 000000000..250d63b12 --- /dev/null +++ b/tests/model_executor/models/minicpmo_4_5/test_vision_batching.py @@ -0,0 +1,175 @@ +from dataclasses import dataclass + +import pytest +import torch + +from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni_llm import ( + MiniCPMO45OmniLLMForConditionalGeneration, +) + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +@dataclass +class _VisionEncoderOutput: + last_hidden_state: torch.Tensor + + +@dataclass +class _VisionConfig: + vision_batch_size: int + + +@dataclass +class _AudioConfig: + audio_chunk_length: float = 0 + audio_pool_step: int = 1 + + +@dataclass +class _FakeConv: + weight: torch.Tensor + + +@dataclass +class _AudioEncoderOutput: + last_hidden_state: torch.Tensor + hidden_states: tuple[torch.Tensor, ...] | None + + +class _FakeVisionEncoder: + def __init__(self) -> None: + self.batch_sizes: list[int] = [] + + def __call__( + self, + pixel_values: torch.Tensor, + *, + patch_attention_mask: torch.Tensor, + tgt_sizes: torch.Tensor, + ) -> _VisionEncoderOutput: + batch_size = pixel_values.shape[0] + self.batch_sizes.append(batch_size) + assert patch_attention_mask.shape[0] == batch_size + assert tgt_sizes.shape[0] == batch_size + + per_item = pixel_values.sum(dim=(1, 2, 3)) + hidden_states = per_item[:, None, None].expand(-1, 3, 4).clone() + return _VisionEncoderOutput(last_hidden_state=hidden_states) + + +class _FakeResampler: + def __init__(self) -> None: + self.batch_sizes: list[int] = [] + + def __call__(self, hidden_states: torch.Tensor, tgt_sizes: torch.Tensor) -> torch.Tensor: + self.batch_sizes.append(hidden_states.shape[0]) + return hidden_states + tgt_sizes.sum(dim=1)[:, None, None] + + +class _FakeAudioEncoder: + def __init__(self) -> None: + self.conv1 = _FakeConv(weight=torch.empty(1, dtype=torch.float32)) + self.output_hidden_states: list[bool] = [] + + def __call__( + self, + input_features: torch.Tensor, + *, + attention_mask: torch.Tensor, + output_hidden_states: bool, + ) -> _AudioEncoderOutput: + self.output_hidden_states.append(output_hidden_states) + assert attention_mask.shape == (1, 1, 2, 2) + final = torch.full((1, 2, 4), 2.0) + earlier = torch.full((1, 2, 4), 1.0) + return _AudioEncoderOutput( + last_hidden_state=final, + hidden_states=(earlier, final) if output_hidden_states else None, + ) + + +class _FakeVisionModel: + def __init__(self, vision_batch_size: int) -> None: + self.config = _VisionConfig(vision_batch_size=vision_batch_size) + self.vpm = _FakeVisionEncoder() + self.resampler = _FakeResampler() + + +class _FakeAudioModel: + def __init__(self, audio_encoder: _FakeAudioEncoder, audio_encoder_layer: int) -> None: + self.config = _AudioConfig() + self.apm = audio_encoder + self.audio_projection_layer = torch.nn.Identity() + self.audio_avg_pooler = torch.nn.Identity() + self.audio_encoder_layer = audio_encoder_layer + + def _get_feat_extract_output_lengths( + self, + input_lengths: torch.LongTensor, + ) -> tuple[torch.LongTensor, torch.LongTensor]: + input_lengths_after_cnn = (input_lengths - 1) // 2 + 1 + input_lengths_after_pooling = ( + input_lengths_after_cnn - self.config.audio_pool_step + ) // self.config.audio_pool_step + 1 + return input_lengths_after_cnn, input_lengths_after_pooling.to(dtype=torch.int32) + + +def _run_vision_encoder(vision_batch_size: int) -> tuple[torch.Tensor, list[int], list[int]]: + model = _FakeVisionModel(vision_batch_size) + pixel_values = [ + torch.full((3, 2, length), fill_value=index + 1, dtype=torch.float32) + for index, length in enumerate((4, 3, 2, 4, 1)) + ] + tgt_sizes = torch.tensor([[1, 2], [1, 2], [1, 1], [2, 2], [1, 1]]) + + result = MiniCPMO45OmniLLMForConditionalGeneration.get_vision_hidden_states( + model, + {"pixel_values": pixel_values, "tgt_sizes": tgt_sizes}, + ) + return result, model.vpm.batch_sizes, model.resampler.batch_sizes + + +def test_vision_encoder_batches_vpm_before_resampling() -> None: + chunked, vision_batches, resampler_batches = _run_vision_encoder(2) + unchunked, unchunked_vision_batches, unchunked_resampler_batches = _run_vision_encoder(16) + + assert vision_batches == [2, 2, 1] + assert resampler_batches == [5] + assert unchunked_vision_batches == [5] + assert unchunked_resampler_batches == [5] + torch.testing.assert_close(chunked, unchunked, rtol=0, atol=0) + + +@pytest.mark.parametrize("vision_batch_size", [0, -2]) +def test_vision_encoder_clamps_nonpositive_batch_size(vision_batch_size: int) -> None: + chunked, vision_batches, resampler_batches = _run_vision_encoder(vision_batch_size) + unchunked, _, _ = _run_vision_encoder(16) + + assert vision_batches == [1, 1, 1, 1, 1] + assert resampler_batches == [5] + torch.testing.assert_close(chunked, unchunked, rtol=0, atol=0) + + +@pytest.mark.parametrize( + ("audio_encoder_layer", "expected_value", "expected_hidden_states"), + [(-1, 2.0, False), (0, 1.0, True)], +) +def test_audio_encoder_retains_layers_only_for_nonfinal_selection( + audio_encoder_layer: int, + expected_value: float, + expected_hidden_states: bool, +) -> None: + audio_encoder = _FakeAudioEncoder() + model = _FakeAudioModel(audio_encoder, audio_encoder_layer) + + result = MiniCPMO45OmniLLMForConditionalGeneration.get_audio_hidden_states( + model, + { + "audio_features": torch.zeros((1, 80, 4)), + "audio_feature_lens": [torch.tensor([4])], + }, + ) + + assert audio_encoder.output_hidden_states == [expected_hidden_states] + torch.testing.assert_close(result[0], torch.full((2, 4), expected_value)) diff --git a/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py b/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py index e59742ad7..6f6841db3 100644 --- a/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py +++ b/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py @@ -4170,6 +4170,8 @@ class MiniCPMO45OmniLLMForConditionalGeneration(nn.Module, SupportsMultiModal, S device = pixel_values[0].device dtype = pixel_values[0].dtype + # VPM batching bounds encoder activations below, while this padding + # buffer intentionally remains sized for the full input batch. all_pixel_values = torch.zeros((B, 3, P, L), dtype=dtype, device=device) for i, pixel_values_item in enumerate(pixel_values): L_item = pixel_values_item.shape[-1] @@ -4183,16 +4185,29 @@ class MiniCPMO45OmniLLMForConditionalGeneration(nn.Module, SupportsMultiModal, S for i, num_patches_item in enumerate(num_patches): patch_attn_mask[i, :num_patches_item] = True - vision_embedding = self.vpm( - all_pixel_values, - patch_attention_mask=patch_attn_mask.unsqueeze(1), - tgt_sizes=tgt_sizes, - ) + patch_attn_mask = patch_attn_mask.unsqueeze(1) - if not isinstance(vision_embedding, torch.Tensor): - vision_embedding = vision_embedding.last_hidden_state + def encode_vision(start: int, end: int) -> torch.Tensor: + chunk_tgt_sizes = tgt_sizes[start:end] + vision_embedding = self.vpm( + all_pixel_values[start:end], + patch_attention_mask=patch_attn_mask[start:end], + tgt_sizes=chunk_tgt_sizes, + ) - return self.resampler(vision_embedding, tgt_sizes) + if not isinstance(vision_embedding, torch.Tensor): + vision_embedding = vision_embedding.last_hidden_state + + return vision_embedding + + vision_batch_size = max(1, int(self.config.vision_batch_size)) + if B <= vision_batch_size: + return self.resampler(encode_vision(0, B), tgt_sizes) + + vision_embeddings = [ + encode_vision(start, min(start + vision_batch_size, B)) for start in range(0, B, vision_batch_size) + ] + return self.resampler(torch.cat(vision_embeddings, dim=0), tgt_sizes) def _process_vision_input( self, @@ -4309,9 +4324,19 @@ class MiniCPMO45OmniLLMForConditionalGeneration(nn.Module, SupportsMultiModal, S audio_attention_mask_ = torch.logical_or(audio_attention_mask_, torch.logical_not(chunk_mask)) audio_attention_mask[audio_attention_mask_] = float("-inf") - audio_states = self.apm(wavforms, attention_mask=audio_attention_mask, output_hidden_states=True).hidden_states[ - self.audio_encoder_layer - ] + selects_final_layer = self.audio_encoder_layer == -1 + audio_outputs = self.apm( + wavforms, + attention_mask=audio_attention_mask, + output_hidden_states=not selects_final_layer, + ) + if selects_final_layer: + audio_states = audio_outputs.last_hidden_state + else: + # Whisper follows the Hugging Face hidden-state ordering: the + # embedding output, followed by each encoder layer output. + assert audio_outputs.hidden_states is not None + audio_states = audio_outputs.hidden_states[self.audio_encoder_layer] audio_embeds = self.audio_projection_layer(audio_states) audio_embeds = audio_embeds.transpose(1, 2) From 776424f0de8a69fc2f5da0c4f3cf767f96855f90 Mon Sep 17 00:00:00 2001 From: Zhou Taichang Date: Mon, 27 Jul 2026 16:24:47 +0800 Subject: [PATCH 108/240] [Refactor][WIP] Add missing test for modelrunner (G1/N) (#5333) Signed-off-by: Taichang Zhou Co-authored-by: Claude Opus 4.8 (1M context) --- tests/worker/test_capture_replay_contract.py | 79 +++++ .../test_fa3_scheduler_metadata_size.py | 85 ++++++ tests/worker/test_gpu_ar_model_runner.py | 92 ++++++ tests/worker/test_memory_utils.py | 71 +++++ tests/worker/test_payload_span.py | 166 +++++++++++ tests/worker/test_sampler_history_backfill.py | 142 +++++++++ tests/worker/test_worker_base.py | 237 +++++++++++++++ tools/baselines/omni_hash_baseline.py | 276 ++++++++++++++++++ tools/baselines/qwen3_omni_hash.baseline | 48 +++ 9 files changed, 1196 insertions(+) create mode 100644 tests/worker/test_capture_replay_contract.py create mode 100644 tests/worker/test_fa3_scheduler_metadata_size.py create mode 100644 tests/worker/test_memory_utils.py create mode 100644 tests/worker/test_payload_span.py create mode 100644 tests/worker/test_sampler_history_backfill.py create mode 100644 tests/worker/test_worker_base.py create mode 100644 tools/baselines/omni_hash_baseline.py create mode 100644 tools/baselines/qwen3_omni_hash.baseline diff --git a/tests/worker/test_capture_replay_contract.py b/tests/worker/test_capture_replay_contract.py new file mode 100644 index 000000000..504aaa7b0 --- /dev/null +++ b/tests/worker/test_capture_replay_contract.py @@ -0,0 +1,79 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Static coupling check for the ``has_preprocess`` capture/replay buffers. + +For models with ``has_preprocess``, CUDA-graph *capture* (``_dummy_run``) and +*replay* (``_preprocess``) MUST read from the same pre-allocated buffers +(``self.input_ids.gpu`` + ``self.inputs_embeds.gpu``); otherwise replay reads a +buffer capture never wrote and produces garbage. This is a source-level check +(no GPU) that catches a one-sided edit to either branch. + +The check is AST-scoped: it inspects ONLY the body of the ``has_preprocess`` +guarded branch (not a line window), so a buffer read removed from that branch +cannot be masked by an adjacent ``else``/``elif`` that happens to read it. It +matches by attribute chain (``self.input_ids.gpu``), so local variable renames +do not break it. + +Scope: the BASE runner (``OmniGPUModelRunner``) only. The generation runner is +KNOWN to lack the ``has_preprocess`` branch; extend this +check to the generation file when it adds the branch. +""" + +import ast +import inspect +import textwrap + +import pytest + +from vllm_omni.worker.gpu_model_runner import OmniGPUModelRunner + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + +_REQUIRED_BUFFERS = {"self.input_ids.gpu", "self.inputs_embeds.gpu"} + + +def _attr_chain(node: ast.AST) -> str | None: + """Reconstruct a dotted ``self.x.y`` chain from an ast.Attribute node.""" + parts: list[str] = [] + cur = node + while isinstance(cur, ast.Attribute): + parts.append(cur.attr) + cur = cur.value + if isinstance(cur, ast.Name): + parts.append(cur.id) + return ".".join(reversed(parts)) + return None + + +def _attr_chains_in(nodes: list[ast.stmt]) -> set[str]: + chains: set[str] = set() + for stmt in nodes: + for inner in ast.walk(stmt): + if isinstance(inner, ast.Attribute): + chain = _attr_chain(inner) + if chain is not None: + chains.add(chain) + return chains + + +def _has_preprocess_branch_reads_both_buffers(method) -> bool: + """True iff SOME ``if has_preprocess:`` branch body reads BOTH buffers. + + Only the guarded branch's own ``body`` is inspected (elif/else excluded), + which is what makes a one-sided buffer removal fail. + """ + src = textwrap.dedent(inspect.getsource(method)) + tree = ast.parse(src) + guarded = [node for node in ast.walk(tree) if isinstance(node, ast.If) and "has_preprocess" in ast.dump(node.test)] + assert guarded, f"{method.__qualname__}: has_preprocess branch removed entirely" + return any(_REQUIRED_BUFFERS <= _attr_chains_in(node.body) for node in guarded) + + +@pytest.mark.parametrize("method_name", ["_dummy_run", "_preprocess"]) +def test_has_preprocess_branch_reads_shared_buffers(method_name): + method = getattr(OmniGPUModelRunner, method_name) + assert _has_preprocess_branch_reads_both_buffers(method), ( + f"{OmniGPUModelRunner.__name__}.{method_name}: the has_preprocess branch no " + "longer reads BOTH self.input_ids.gpu and self.inputs_embeds.gpu — capture " + "and replay would diverge." + ) diff --git a/tests/worker/test_fa3_scheduler_metadata_size.py b/tests/worker/test_fa3_scheduler_metadata_size.py new file mode 100644 index 000000000..5e308bed7 --- /dev/null +++ b/tests/worker/test_fa3_scheduler_metadata_size.py @@ -0,0 +1,85 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Characterization of ``OmniGPUModelRunner.initialize_metadata_builders``. + +Upstream ``FlashAttentionMetadataBuilder`` pre-allocates ``scheduler_metadata`` +for only ``max_num_seqs + 1`` entries, but FA3 with split scheduling +(``max_num_splits > 1``) may need ``max_num_seqs * max_num_splits + 1`` during +CUDA-graph capture. ``OmniGPUModelRunner`` re-sizes the buffer up to that +requirement after upstream init — this test pins that Omni-specific workaround +exactly. + +It drives the Omni runner's OWN method (with the upstream ``super()`` call +stubbed and a fake builder standing in), so a change to the resize is caught and +a failure is unambiguously an Omni-runner regression. Pure CPU — no model, no +CUDA, no flashinfer. +""" + +from types import SimpleNamespace + +import pytest +import torch + +from vllm_omni.worker.gpu_model_runner import OmniGPUModelRunner + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + +# The class ``super().initialize_metadata_builders`` dispatches to. +_UPSTREAM_RUNNER = OmniGPUModelRunner.__mro__[1] + + +def _builder(*, prealloc: int | None, max_num_splits: int) -> SimpleNamespace: + sm = None if prealloc is None else torch.zeros(prealloc, dtype=torch.int32) + return SimpleNamespace(scheduler_metadata=sm, max_num_splits=max_num_splits) + + +def _make_runner(monkeypatch, *, max_num_seqs: int, builders: list) -> OmniGPUModelRunner: + # Stub the upstream initialize_metadata_builders; our fake runner already + # carries the attn_groups the Omni override iterates over. + monkeypatch.setattr(_UPSTREAM_RUNNER, "initialize_metadata_builders", lambda self, *a, **k: None) + runner = object.__new__(OmniGPUModelRunner) + runner.scheduler_config = SimpleNamespace(max_num_seqs=max_num_seqs) + runner.cache_config = SimpleNamespace(enable_prefix_caching=False) # skip omni_prefix_cache branch + runner.attn_groups = [[SimpleNamespace(metadata_builders=builders)]] + return runner + + +def test_resizes_up_when_split_scheduling_needs_more(monkeypatch): + max_num_seqs, splits, prealloc = 32, 4, 33 # 33 == upstream default (max_num_seqs + 1) + builder = _builder(prealloc=prealloc, max_num_splits=splits) + runner = _make_runner(monkeypatch, max_num_seqs=max_num_seqs, builders=[builder]) + + OmniGPUModelRunner.initialize_metadata_builders(runner, None, None) + + assert builder.scheduler_metadata.shape[0] == max_num_seqs * splits + 1 + assert builder.scheduler_metadata.dtype == torch.int32 + + +def test_no_resize_when_not_split_scheduling(monkeypatch): + builder = _builder(prealloc=33, max_num_splits=1) # max_num_splits <= 1 -> workaround skipped + runner = _make_runner(monkeypatch, max_num_seqs=32, builders=[builder]) + original = builder.scheduler_metadata + + OmniGPUModelRunner.initialize_metadata_builders(runner, None, None) + + assert builder.scheduler_metadata is original # untouched + + +def test_no_resize_when_prealloc_already_sufficient(monkeypatch): + max_num_seqs, splits = 32, 4 + builder = _builder(prealloc=max_num_seqs * splits + 1, max_num_splits=splits) # already big enough + runner = _make_runner(monkeypatch, max_num_seqs=max_num_seqs, builders=[builder]) + original = builder.scheduler_metadata + + OmniGPUModelRunner.initialize_metadata_builders(runner, None, None) + + assert builder.scheduler_metadata is original + + +def test_no_crash_when_builder_has_no_scheduler_metadata(monkeypatch): + builder = _builder(prealloc=None, max_num_splits=4) # scheduler_metadata is None + runner = _make_runner(monkeypatch, max_num_seqs=32, builders=[builder]) + + OmniGPUModelRunner.initialize_metadata_builders(runner, None, None) + + assert builder.scheduler_metadata is None diff --git a/tests/worker/test_gpu_ar_model_runner.py b/tests/worker/test_gpu_ar_model_runner.py index daab10ba0..a1f885c15 100644 --- a/tests/worker/test_gpu_ar_model_runner.py +++ b/tests/worker/test_gpu_ar_model_runner.py @@ -802,3 +802,95 @@ def test_build_omni_output_falls_back_to_mm_cpu_without_prefix_merge(monkeypatch assert torch.equal(output.inter_stage_outputs[0]["codes.audio"], codes[0:1]) assert torch.equal(output.inter_stage_outputs[1]["codes.audio"], codes[1:2]) assert output.multimodal_outputs is None + + +# --- builder gap-fill: contract corners not covered by the tests above --- +def test_build_omni_output_never_leaks_internal_pooler_output_on_wire(monkeypatch): + """The internal pooler_output feeds full-payload accumulation only; the wire + object's ``pooler_output`` is always None (builder NOTE at gpu_ar:1738).""" + runner = _make_async_output_runner() + + monkeypatch.setattr( + GPUARModelRunner, + "_resolve_pooler_payload_req_ids", + lambda self, req_ids: ("audio", req_ids), + ) + monkeypatch.setattr(GPUARModelRunner, "_should_accumulate_full_payload_output", lambda self: True) + monkeypatch.setattr( + GPUARModelRunner, + "accumulate_full_payload_output", + lambda self, rid, payload, request: None, + ) + monkeypatch.setattr(GPUARModelRunner, "get_omni_connector_output", lambda self: None) + + output = GPUARModelRunner._build_omni_model_runner_output_from_snapshot( + runner, + scheduler_output=SimpleNamespace( + total_num_scheduled_tokens=3, + num_scheduled_tokens={"r1": 1, "r2": 2}, + ), + hidden_states=torch.tensor([[1.0], [2.0], [3.0]]), + staged_hidden_states_cpu=None, + multimodal_outputs={"foo": torch.tensor([10.0, 20.0, 30.0])}, + req_ids_output_copy=["r1", "r2"], + req_id_to_index_output_copy={"r1": 0, "r2": 1}, + valid_sampled_token_ids=[[101], [102]], + logprobs_lists=None, + prompt_logprobs_dict={}, + num_nans_in_logits=None, + kv_connector_output=None, + ec_connector_output=None, + cudagraph_stats=None, + kv_extracted_req_ids=None, + num_scheduled_tokens_np=np.array([1, 2], dtype=np.int32), + query_start_loc_cpu=torch.tensor([0, 1], dtype=torch.long), + ) + + assert output.pooler_output is None + + +def test_build_omni_output_splits_client_mm_from_inter_stage_keys(monkeypatch): + """The inter-stage vs client-mm partition: a client-root key ("audio") lands in + ``multimodal_outputs`` while inter-stage keys ("hidden") stay in + ``inter_stage_outputs`` (partition_flat_payload / _CLIENT_MM_ROOT_KEYS).""" + runner = _make_async_output_runner() + + monkeypatch.setattr( + GPUARModelRunner, + "_resolve_pooler_payload_req_ids", + lambda self, req_ids: ("audio", req_ids), + ) + monkeypatch.setattr(GPUARModelRunner, "_should_accumulate_full_payload_output", lambda self: False) + monkeypatch.setattr(GPUARModelRunner, "get_omni_connector_output", lambda self: None) + + output = GPUARModelRunner._build_omni_model_runner_output_from_snapshot( + runner, + scheduler_output=SimpleNamespace( + total_num_scheduled_tokens=3, + num_scheduled_tokens={"r1": 1, "r2": 2}, + ), + hidden_states=torch.tensor([[1.0], [2.0], [3.0]]), + staged_hidden_states_cpu=None, + # "audio" root is a client mm key; "codes.audio" root ("codes") is inter-stage. + multimodal_outputs={"audio": torch.tensor([10.0, 20.0, 30.0])}, + req_ids_output_copy=["r1", "r2"], + req_id_to_index_output_copy={"r1": 0, "r2": 1}, + valid_sampled_token_ids=[[101], [102]], + logprobs_lists=None, + prompt_logprobs_dict={}, + num_nans_in_logits=None, + kv_connector_output=None, + ec_connector_output=None, + cudagraph_stats=None, + kv_extracted_req_ids=None, + num_scheduled_tokens_np=np.array([1, 2], dtype=np.int32), + query_start_loc_cpu=torch.tensor([0, 1], dtype=torch.long), + ) + + # Client mm key routed to the wire multimodal_outputs, per request. + assert output.multimodal_outputs is not None + assert "audio" in output.multimodal_outputs[0] + # Inter-stage hidden stayed on inter_stage_outputs and did NOT leak to client. + assert output.inter_stage_outputs is not None + assert "hidden" in output.inter_stage_outputs[0] + assert "audio" not in output.inter_stage_outputs[0] diff --git a/tests/worker/test_memory_utils.py b/tests/worker/test_memory_utils.py new file mode 100644 index 000000000..672fa4648 --- /dev/null +++ b/tests/worker/test_memory_utils.py @@ -0,0 +1,71 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Characterization tests for ``vllm_omni.worker.memory_utils``. + +``request_memory_tolerant`` is still live in both workers +(``gpu_ar_worker.py`` / ``gpu_generation_worker.py``); a later change makes its cap loud +but must not change the numeric contract. These tests pin that contract with a +mocked ``MemorySnapshot`` — pure CPU, no GPU / NVML. +""" + +import math +from types import SimpleNamespace + +import pytest + +from vllm_omni.worker.memory_utils import request_memory_tolerant + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + +GIB = 1024**3 + + +def _snapshot(total: int, free: int) -> SimpleNamespace: + # Only the fields request_memory_tolerant reads (incl. device_ for the log). + return SimpleNamespace(total_memory=total, free_memory=free, device_="cuda:0") + + +def _cache_config(util: float) -> SimpleNamespace: + return SimpleNamespace(gpu_memory_utilization=util) + + +def test_passes_through_requested_when_free_suffices(): + snap = _snapshot(total=40 * GIB, free=40 * GIB) + cfg = _cache_config(0.5) + + out = request_memory_tolerant(snap, cfg) + + assert out == math.ceil(40 * GIB * 0.5) + + +def test_caps_to_free_when_insufficient(): + cfg = _cache_config(0.9) + requested = math.ceil(40 * GIB * 0.9) + snap = _snapshot(total=40 * GIB, free=10 * GIB) # free < requested + + out = request_memory_tolerant(snap, cfg) + + assert out == 10 * GIB + assert out < requested + + +def test_boundary_free_equals_requested_is_not_capped(): + util = 0.5 + total = 40 * GIB + requested = math.ceil(total * util) + snap = _snapshot(total=total, free=requested) # strict `<`, so no cap at equality + + out = request_memory_tolerant(snap, _cache_config(util)) + + assert out == requested + + +def test_requested_uses_ceil_of_total_times_util(): + # 7 GiB * 0.3333... rounds UP (math.ceil), not down. + total = 7 * GIB + util = 1 / 3 + snap = _snapshot(total=total, free=total) + + out = request_memory_tolerant(snap, _cache_config(util)) + + assert out == math.ceil(total * util) diff --git a/tests/worker/test_payload_span.py b/tests/worker/test_payload_span.py new file mode 100644 index 000000000..5eb4d8319 --- /dev/null +++ b/tests/worker/test_payload_span.py @@ -0,0 +1,166 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Characterization tests for ``vllm_omni.worker.payload_span``. + +These pin the CURRENT behaviour of the thinker-decode span helpers so the later +worker refactors that touch span +plumbing have a regression net. Pure CPU — no model, no CUDA. +""" + +import pytest +import torch + +from vllm_omni.worker.payload_span import ( + get_tensor_span, + get_tensor_span_row, + merge_tensor_spans, +) + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +def _rows(n: int, *, start_val: int = 0, hidden: int = 1) -> torch.Tensor: + """A ``[n, hidden]`` tensor whose first column counts up from ``start_val``. + + ``[n, hidden]`` mirrors the connector async-chunk shape (tokens x hidden). + """ + base = torch.arange(start_val, start_val + n, dtype=torch.float32).unsqueeze(1) + return base.repeat(1, hidden) + + +# --------------------------------------------------------------------------- # +# get_tensor_span # +# --------------------------------------------------------------------------- # +def test_get_tensor_span_valid_returns_triplet(): + tensor = _rows(3, hidden=4) + payload = {"emb": tensor, "start": 5, "end": 8} + + span = get_tensor_span(payload, tensor_key="emb", start_key="start", end_key="end") + + assert span is not None + got_tensor, start, end = span + assert got_tensor is tensor + assert (start, end) == (5, 8) + + +def test_get_tensor_span_non_tensor_returns_none(): + payload = {"emb": None, "start": 0, "end": 0} + assert get_tensor_span(payload, tensor_key="emb", start_key="start", end_key="end") is None + + +def test_get_tensor_span_non_int_bounds_returns_none(): + tensor = _rows(2) + # start is a float, not an int -> rejected. + payload = {"emb": tensor, "start": 0.0, "end": 2} + assert get_tensor_span(payload, tensor_key="emb", start_key="start", end_key="end") is None + + +def test_get_tensor_span_negative_start_returns_none(): + tensor = _rows(1) + payload = {"emb": tensor, "start": -1, "end": 0} + assert get_tensor_span(payload, tensor_key="emb", start_key="start", end_key="end") is None + + +def test_get_tensor_span_end_before_start_returns_none(): + tensor = _rows(2) + payload = {"emb": tensor, "start": 5, "end": 3} + assert get_tensor_span(payload, tensor_key="emb", start_key="start", end_key="end") is None + + +def test_get_tensor_span_length_mismatch_returns_none(): + tensor = _rows(3) # tensor.shape[0] == 3 + payload = {"emb": tensor, "start": 0, "end": 2} # end - start == 2 + assert get_tensor_span(payload, tensor_key="emb", start_key="start", end_key="end") is None + + +# --------------------------------------------------------------------------- # +# merge_tensor_spans # +# --------------------------------------------------------------------------- # +def test_merge_none_operand_returns_none(): + span = (_rows(2), 0, 2) + assert merge_tensor_spans(None, span) is None + assert merge_tensor_spans(span, None) is None + + +def test_merge_adjacent_spans_concatenates(): + existing = (_rows(3, start_val=0), 0, 3) + incoming = (_rows(2, start_val=3), 3, 5) + + merged = merge_tensor_spans(existing, incoming) + + assert merged is not None + tensor, start, end = merged + assert (start, end) == (0, 5) + assert tensor.shape[0] == 5 + assert tensor[:, 0].tolist() == [0.0, 1.0, 2.0, 3.0, 4.0] + + +def test_merge_overlapping_spans_trims_incoming(): + # existing covers [0, 3); incoming covers [2, 5) -> 1 row of overlap. + existing = (_rows(3, start_val=0), 0, 3) + incoming = (_rows(3, start_val=100), 2, 5) + + merged = merge_tensor_spans(existing, incoming) + + assert merged is not None + tensor, start, end = merged + assert (start, end) == (0, 5) + # First 3 rows from existing, then incoming's tail (overlap=1 row trimmed). + assert tensor[:, 0].tolist() == [0.0, 1.0, 2.0, 101.0, 102.0] + + +def test_merge_fully_overlapping_incoming_keeps_existing(): + # incoming [1, 3) is fully covered by existing [0, 3): overlap >= incoming len. + existing = (_rows(3, start_val=0), 0, 3) + incoming = (_rows(2, start_val=100), 1, 3) + + merged = merge_tensor_spans(existing, incoming) + + assert merged is not None + tensor, start, end = merged + assert (start, end) == (0, 3) + assert tensor is existing[0] # returned unchanged + + +def test_merge_non_contiguous_spans_returns_none(): + # gap between existing end (3) and incoming start (5). + existing = (_rows(3, start_val=0), 0, 3) + incoming = (_rows(2, start_val=100), 5, 7) + + assert merge_tensor_spans(existing, incoming) is None + + +def test_merge_casts_incoming_to_existing_dtype_and_device(): + existing = (_rows(2, start_val=0).to(torch.float32), 0, 2) + incoming = (torch.arange(2, 4, dtype=torch.int64).unsqueeze(1), 2, 4) + + merged = merge_tensor_spans(existing, incoming) + + assert merged is not None + tensor, _, _ = merged + assert tensor.dtype == torch.float32 + assert tensor.device == existing[0].device + + +# --------------------------------------------------------------------------- # +# get_tensor_span_row # +# --------------------------------------------------------------------------- # +def test_get_tensor_span_row_in_range(): + span = (_rows(3, start_val=10), 5, 8) # rows map to indices 5, 6, 7 + row = get_tensor_span_row(span, 6) + assert row is not None + assert row[0].item() == 11.0 # second row (start_val 10 + 1) + + +def test_get_tensor_span_row_none_span_returns_none(): + assert get_tensor_span_row(None, 0) is None + + +def test_get_tensor_span_row_below_start_returns_none(): + span = (_rows(3), 5, 8) + assert get_tensor_span_row(span, 4) is None + + +def test_get_tensor_span_row_at_or_above_end_returns_none(): + span = (_rows(3), 5, 8) + assert get_tensor_span_row(span, 8) is None diff --git a/tests/worker/test_sampler_history_backfill.py b/tests/worker/test_sampler_history_backfill.py new file mode 100644 index 000000000..5f1b520b6 --- /dev/null +++ b/tests/worker/test_sampler_history_backfill.py @@ -0,0 +1,142 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Golden characterization of ``_build_model_sampler_output_token_ids``. + +The base runner (``OmniGPUModelRunner``) and the AR runner +(``GPUARModelRunner``) implement the SAME async-placeholder backfill, but the +AR copy adds a trailing crop step: after backfill it truncates every history at +the first remaining ``-1``, while the base copy KEEPS residual ``-1``s. + +A later change collapses these two copies into one; these goldens pin today's divergence +exactly (crop vs keep) so that merge is provably behaviour-preserving. Pure CPU. +""" + +from types import SimpleNamespace + +import pytest +import torch + +from vllm_omni.worker.gpu_ar_model_runner import GPUARModelRunner +from vllm_omni.worker.gpu_model_runner import OmniGPUModelRunner + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +def _runner( + cls, + *, + req_ids, + req_output_token_ids, + sampled_token_ids_cpu, + prev_req_id_to_index, +): + """Build a fake runner exposing only the ``input_batch`` fields the method reads.""" + runner = object.__new__(cls) + event = SimpleNamespace(synchronize=lambda: None) + runner.input_batch = SimpleNamespace( + req_output_token_ids=req_output_token_ids, + req_ids=req_ids, + sampled_token_ids_cpu=(None if sampled_token_ids_cpu is None else torch.tensor(sampled_token_ids_cpu)), + async_copy_ready_event=event, + prev_req_id_to_index=prev_req_id_to_index, + ) + return runner + + +def _build(cls, **kwargs): + return cls._build_model_sampler_output_token_ids(_runner(cls, **kwargs)) + + +# --------------------------------------------------------------------------- # +# Shared behaviour — base and AR agree # +# --------------------------------------------------------------------------- # +@pytest.mark.parametrize("cls", [OmniGPUModelRunner, GPUARModelRunner]) +def test_full_backfill_no_residual_placeholder(cls): + # One trailing placeholder, one sampled token -> fully resolved for both. + out = _build( + cls, + req_ids=["r0"], + req_output_token_ids=[[10, 11, -1]], + sampled_token_ids_cpu=[[42]], + prev_req_id_to_index={"r0": 0}, + ) + assert out == [[10, 11, 42]] + + +@pytest.mark.parametrize("cls", [OmniGPUModelRunner, GPUARModelRunner]) +def test_spec_decode_trailing_pad_in_sampled_is_trimmed(cls): + # sampled row has a trailing -1 pad; only the real prefix [42, 43] is used. + out = _build( + cls, + req_ids=["r0"], + req_output_token_ids=[[10, -1, -1]], + sampled_token_ids_cpu=[[42, 43, -1]], + prev_req_id_to_index={"r0": 0}, + ) + assert out == [[10, 42, 43]] + + +@pytest.mark.parametrize("cls", [OmniGPUModelRunner, GPUARModelRunner]) +def test_no_trailing_placeholder_is_untouched(cls): + out = _build( + cls, + req_ids=["r0"], + req_output_token_ids=[[10, 11, 12]], + sampled_token_ids_cpu=[[42]], + prev_req_id_to_index={"r0": 0}, + ) + assert out == [[10, 11, 12]] + + +@pytest.mark.parametrize("cls", [OmniGPUModelRunner, GPUARModelRunner]) +def test_early_return_when_sampled_absent_keeps_placeholders(cls): + # sampled_token_ids_cpu is None -> both copies return BEFORE the crop step, + # so even the AR copy keeps residual -1 here. + out = _build( + cls, + req_ids=["r0"], + req_output_token_ids=[[10, -1]], + sampled_token_ids_cpu=None, + prev_req_id_to_index={"r0": 0}, + ) + assert out == [[10, -1]] + + +# --------------------------------------------------------------------------- # +# Divergence — base KEEPS residual -1, AR CROPS at the first -1 # +# --------------------------------------------------------------------------- # +_RESIDUAL_CASE = dict( + req_ids=["r0"], + req_output_token_ids=[[10, -1, -1]], # two placeholders + sampled_token_ids_cpu=[[42]], # only one sampled token -> one -1 remains + prev_req_id_to_index={"r0": 0}, +) + + +def test_base_keeps_residual_placeholder(): + out = _build(OmniGPUModelRunner, **_RESIDUAL_CASE) + assert out == [[10, 42, -1]] + + +def test_ar_crops_residual_placeholder(): + out = _build(GPUARModelRunner, **_RESIDUAL_CASE) + assert out == [[10, 42]] + + +_ABSENT_FROM_PREV_CASE = dict( + req_ids=["r0"], + req_output_token_ids=[[10, -1]], + sampled_token_ids_cpu=[[42]], + prev_req_id_to_index={}, # r0 not present last step -> no backfill happens +) + + +def test_base_keeps_placeholder_when_request_absent_from_prev_step(): + out = _build(OmniGPUModelRunner, **_ABSENT_FROM_PREV_CASE) + assert out == [[10, -1]] + + +def test_ar_crops_placeholder_when_request_absent_from_prev_step(): + # No backfill occurred, but the AR crop pass still truncates at the -1. + out = _build(GPUARModelRunner, **_ABSENT_FROM_PREV_CASE) + assert out == [[10]] diff --git a/tests/worker/test_worker_base.py b/tests/worker/test_worker_base.py new file mode 100644 index 000000000..d470d86a8 --- /dev/null +++ b/tests/worker/test_worker_base.py @@ -0,0 +1,237 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Characterization tests for ``vllm_omni.worker.base.OmniGPUWorkerBase``. + +Pins the CURRENT behaviour of ``determine_available_memory`` (BOTH the NVML / +process-scoped arm AND the profiling-fallback arm — this base still carries the +NVML branch), plus the +memory-pool / sleep / wake-up plumbing that a later change may touch. + +Workers are constructed via ``object.__new__`` with only the attributes each +method reads; module-level collaborators are monkeypatched. Pure CPU. +""" + +from contextlib import contextmanager, nullcontext +from types import SimpleNamespace + +import pytest +import torch + +import vllm_omni.worker.base as base +from vllm_omni.worker.base import OmniGPUWorkerBase + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + +GIB = 1024**3 + + +def _fake_memory_profiling(*, non_torch: int, torch_peak: int): + """A stand-in for ``vllm.utils.mem_utils.memory_profiling`` context manager.""" + + @contextmanager + def _mp(snapshot, weights_memory): # noqa: ARG001 - signature parity only + yield SimpleNamespace( + non_torch_increase=non_torch, + torch_peak_increase=torch_peak, + ) + + return _mp + + +def _make_worker(*, requested_memory: int, kv_cache_memory_bytes: int = 0, model_memory_usage: int = 0): + worker = object.__new__(OmniGPUWorkerBase) + worker.cache_config = SimpleNamespace(kv_cache_memory_bytes=kv_cache_memory_bytes) + worker.model_runner = SimpleNamespace( + profile_run=lambda: None, + model_memory_usage=model_memory_usage, + ) + worker.init_snapshot = object() # unused once memory_profiling is faked + worker.requested_memory = requested_memory + worker.local_rank = 0 + return worker + + +# --------------------------------------------------------------------------- # +# determine_available_memory # +# --------------------------------------------------------------------------- # +def test_determine_available_memory_nvml_arm(monkeypatch): + """NVML available -> available = requested - per-process memory.""" + worker = _make_worker(requested_memory=30 * GIB, model_memory_usage=10 * GIB) + monkeypatch.setattr(base, "memory_profiling", _fake_memory_profiling(non_torch=1 * GIB, torch_peak=2 * GIB)) + monkeypatch.setattr(base, "is_process_scoped_memory_available", lambda: True) + monkeypatch.setattr(base, "detect_pid_host", lambda: True) + monkeypatch.setattr(base, "get_process_gpu_memory", lambda local_rank: 8 * GIB) + + out = OmniGPUWorkerBase.determine_available_memory(worker) + + # NVML arm ignores profiled usage; uses process_memory directly. + assert out == 30 * GIB - 8 * GIB + + +def test_determine_available_memory_profiling_fallback_arm(monkeypatch): + """NVML unavailable -> available = requested - (weights + peak + non_torch).""" + worker = _make_worker(requested_memory=30 * GIB, model_memory_usage=10 * GIB) + monkeypatch.setattr(base, "memory_profiling", _fake_memory_profiling(non_torch=1 * GIB, torch_peak=2 * GIB)) + monkeypatch.setattr(base, "is_process_scoped_memory_available", lambda: False) + # detect_pid_host is short-circuited by the `and`, but pin it anyway. + monkeypatch.setattr(base, "detect_pid_host", lambda: True) + + out = OmniGPUWorkerBase.determine_available_memory(worker) + + profiled = 10 * GIB + 2 * GIB + 1 * GIB + assert out == 30 * GIB - profiled + + +def test_determine_available_memory_falls_back_when_not_pid_host(monkeypatch): + """detect_pid_host False also routes to the profiling fallback.""" + worker = _make_worker(requested_memory=20 * GIB, model_memory_usage=5 * GIB) + monkeypatch.setattr(base, "memory_profiling", _fake_memory_profiling(non_torch=0, torch_peak=0)) + monkeypatch.setattr(base, "is_process_scoped_memory_available", lambda: True) + monkeypatch.setattr(base, "detect_pid_host", lambda: False) + + out = OmniGPUWorkerBase.determine_available_memory(worker) + + assert out == 20 * GIB - 5 * GIB + + +def test_determine_available_memory_clamps_to_zero(monkeypatch): + """Over-subscription clamps the KV budget to 0, never negative.""" + worker = _make_worker(requested_memory=5 * GIB, model_memory_usage=0) + monkeypatch.setattr(base, "memory_profiling", _fake_memory_profiling(non_torch=0, torch_peak=0)) + monkeypatch.setattr(base, "is_process_scoped_memory_available", lambda: True) + monkeypatch.setattr(base, "detect_pid_host", lambda: True) + monkeypatch.setattr(base, "get_process_gpu_memory", lambda local_rank: 8 * GIB) + + out = OmniGPUWorkerBase.determine_available_memory(worker) + + assert out == 0 + + +def test_determine_available_memory_kv_cache_short_circuit(monkeypatch): + """A pre-set kv_cache_memory_bytes short-circuits profiling and is returned.""" + worker = _make_worker(requested_memory=30 * GIB, kv_cache_memory_bytes=7 * GIB) + profile_calls = [] + worker.model_runner.profile_run = lambda: profile_calls.append(True) + # is_rocm gates only an extra synchronize on the short-circuit path. + monkeypatch.setattr(base, "current_omni_platform", SimpleNamespace(is_rocm=lambda: False)) + + out = OmniGPUWorkerBase.determine_available_memory(worker) + + assert out == 7 * GIB + assert profile_calls == [True] # profile_run still runs before returning + + +def test_determine_available_memory_kv_short_circuit_syncs_on_rocm(monkeypatch): + """On ROCm the short-circuit adds an accelerator synchronize (a later change may remove it).""" + worker = _make_worker(requested_memory=30 * GIB, kv_cache_memory_bytes=7 * GIB) + monkeypatch.setattr(base, "current_omni_platform", SimpleNamespace(is_rocm=lambda: True)) + synced = [] + monkeypatch.setattr(torch.accelerator, "synchronize", lambda: synced.append(True)) + + out = OmniGPUWorkerBase.determine_available_memory(worker) + + assert out == 7 * GIB + assert synced == [True] # ROCm path synchronizes; non-ROCm path (above) does not + + +# --------------------------------------------------------------------------- # +# _maybe_get_memory_pool_context # +# --------------------------------------------------------------------------- # +def test_memory_pool_context_disabled_returns_nullcontext(): + worker = object.__new__(OmniGPUWorkerBase) + worker.rank = 0 + worker.cache_config = SimpleNamespace(enable_sleep_mode=False) + # No vllm_config attribute -> v1_config_enabled stays False. + + ctx = OmniGPUWorkerBase._maybe_get_memory_pool_context(worker, "weights") + + assert isinstance(ctx, type(nullcontext())) + + +def test_memory_pool_context_enabled_uses_cumem_pool_with_tag(monkeypatch): + import vllm.device_allocator.cumem as cumem_mod + + worker = object.__new__(OmniGPUWorkerBase) + worker.rank = 0 + worker.cache_config = SimpleNamespace(enable_sleep_mode=True) + monkeypatch.setattr(base, "current_omni_platform", SimpleNamespace(synchronize=lambda: None)) + + recorded = {} + + class FakeAllocator: + @classmethod + def get_instance(cls): + return cls() + + def use_memory_pool(self, tag): + recorded["tag"] = tag + return "POOL_CTX" + + monkeypatch.setattr(cumem_mod, "CuMemAllocator", FakeAllocator) + + ctx = OmniGPUWorkerBase._maybe_get_memory_pool_context(worker, "weights") + + assert ctx == "POOL_CTX" + assert recorded["tag"] == "weights" + + +# --------------------------------------------------------------------------- # +# sleep / wake_up # +# --------------------------------------------------------------------------- # +def _fake_platform(): + return SimpleNamespace( + get_current_memory_usage=lambda device: 0, + empty_cache=lambda: None, + synchronize=lambda: None, + ) + + +@pytest.mark.parametrize( + ("level", "expected_tags"), + [(1, ("weights",)), (2, tuple())], +) +def test_sleep_offload_tags_by_level(monkeypatch, level, expected_tags): + import vllm.device_allocator.cumem as cumem_mod + + worker = object.__new__(OmniGPUWorkerBase) + worker.rank = 0 + worker.device = "cuda:0" + monkeypatch.setattr(base, "current_omni_platform", _fake_platform()) + + calls = {} + + class FakeAllocator: + @classmethod + def get_instance(cls): + return cls() + + def sleep(self, offload_tags): + calls["offload_tags"] = offload_tags + + monkeypatch.setattr(cumem_mod, "CuMemAllocator", FakeAllocator) + + assert OmniGPUWorkerBase.sleep(worker, level=level) is True + assert calls["offload_tags"] == expected_tags + + +def test_wake_up_forwards_tags_to_allocator(monkeypatch): + import vllm.device_allocator.cumem as cumem_mod + + worker = object.__new__(OmniGPUWorkerBase) + worker.rank = 0 + monkeypatch.setattr(base, "current_omni_platform", SimpleNamespace(synchronize=lambda: None)) + + calls = {} + + class FakeAllocator: + @classmethod + def get_instance(cls): + return cls() + + def wake_up(self, tags): + calls["tags"] = tags + + monkeypatch.setattr(cumem_mod, "CuMemAllocator", FakeAllocator) + + assert OmniGPUWorkerBase.wake_up(worker, tags=["weights"]) is True + assert calls["tags"] == ["weights"] diff --git a/tools/baselines/omni_hash_baseline.py b/tools/baselines/omni_hash_baseline.py new file mode 100644 index 000000000..1bb93099b --- /dev/null +++ b/tools/baselines/omni_hash_baseline.py @@ -0,0 +1,276 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Record / verify a deterministic text+audio output hash baseline for an Omni model. + +This is the cheap "output is bit-identical" gate for the behaviour-preserving +worker/model-runner changes: record a baseline on the bench box +BEFORE any code moves, then re-run with ``--check`` after each refactor. + +Determinism: **batch size 1** (one prompt per ``generate`` call) + **fixed seed**. +Thinker and code2wav run greedy (temperature 0); the talker's speech sampler needs +temperature to emit valid codec tokens, so it runs at its natural sampling with a +fixed seed (reproducible via seed + batch-1, not pure-greedy). 3 fixed prompts, run +with async output OFF and ON (6 entries total). Hashes are only meaningful WITHIN a single recorded +environment (driver/cuda/torch/gpu), so the baseline JSON records the env and +``--check`` refuses to compare across a changed env unless ``--force-env``. + +Baseline JSON schema (results keyed by BOTH prompt_id and async_mode so the +matrix is self-identifying):: + + { + "model": "...", + "commit": "", + "env": {"driver": "", "cuda": "", "torch": "...", "gpu": "..."}, + "results": [ + {"prompt_id": 0, "async_mode": false, "text_sha": "...", "wav_sha": "..."}, + ... # exactly len(FIXED_PROMPTS) * 2 entries + ] + } + +The file content is JSON, but committed baselines use a NON-``.json`` extension +(``.baseline``) because the repo ``.gitignore`` excludes ``*.json``; ``--check`` +loads by content, so the extension does not matter to the script. + +Usage:: + + # record (run twice, confirm the two runs are identical before committing) + python tools/baselines/omni_hash_baseline.py --yaml \ + --out tools/baselines/_hash.baseline + # verify after a refactor (same bench box) + python tools/baselines/omni_hash_baseline.py --yaml \ + --check tools/baselines/_hash.baseline +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import subprocess +import sys + +import torch +from vllm.sampling_params import SamplingParams + +from vllm_omni.entrypoints.omni import Omni + +SEED = 42 +DEFAULT_SYSTEM = ( + "You are Qwen, a virtual human developed by the Qwen Team, Alibaba Group, " + "capable of perceiving auditory and visual inputs, as well as generating " + "text and speech." +) + +# 3 fixed prompts — do NOT edit once a baseline is recorded (prompt_id is positional). +FIXED_PROMPTS = [ + "Explain the system architecture for a scalable audio generation pipeline. Answer in 15 words.", + "Describe how ocean tides work. Answer in 15 words.", + "List three benefits of unit tests. Answer in 15 words.", +] +ASYNC_MODES = (False, True) + + +def _text_prompt(question: str) -> dict: + return { + "prompt": ( + f"<|im_start|>system\n{DEFAULT_SYSTEM}<|im_end|>\n" + f"<|im_start|>user\n{question}<|im_end|>\n" + f"<|im_start|>assistant\n" + ) + } + + +def _sha(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def _git_commit() -> str: + try: + return subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip() + except Exception: + return "unknown" + + +def _driver_version() -> str: + """The actual NVIDIA driver version (NOT torch.version.cuda, which is the build).""" + try: + out = subprocess.check_output(["nvidia-smi", "--query-gpu=driver_version", "--format=csv,noheader"], text=True) + return out.strip().splitlines()[0].strip() + except Exception: + return "unknown" + + +def _env_fingerprint() -> dict: + gpu = torch.cuda.get_device_name(0) if torch.cuda.is_available() else "cpu" + return { + "driver": _driver_version(), + "cuda": str(torch.version.cuda), + "torch": torch.__version__, + "gpu": gpu, + } + + +def _sampling_params() -> list[SamplingParams]: + # thinker + code2wav: greedy. talker: seeded (temperature>0) — its speech sampler + # needs temperature to emit valid codec tokens; the fixed seed keeps it reproducible. + thinker = SamplingParams( + temperature=0.0, top_p=1.0, top_k=-1, max_tokens=128, seed=SEED, detokenize=True, repetition_penalty=1.1 + ) + talker = SamplingParams( + temperature=0.9, top_p=0.8, top_k=40, max_tokens=256, seed=SEED, detokenize=True, repetition_penalty=1.05 + ) + code2wav = SamplingParams( + temperature=0.0, top_p=1.0, top_k=-1, max_tokens=256, seed=SEED, detokenize=True, repetition_penalty=1.1 + ) + return [thinker, talker, code2wav] + + +def _audio_bytes(audio) -> bytes: + """Deterministic bytes for one request's audio, handling chunked (list) output. + + async-chunk mode can emit a list of audio chunks instead of a single tensor; + concatenate them along the last axis in emit order. + """ + if isinstance(audio, (list, tuple)): + chunks = [a for a in audio if a is not None] + if not chunks: + return b"" + audio = torch.cat([torch.as_tensor(a).detach().to(torch.float32).cpu() for a in chunks], dim=-1) + if audio is None: + return b"" + return torch.as_tensor(audio).detach().to(torch.float32).cpu().numpy().tobytes() + + +def _run_once(model: str, yaml_path: str, *, async_mode: bool) -> list[dict]: + """One deterministic pass, batch size 1 per prompt; returns per-prompt hashes.""" + # ``async_chunk`` is the repo's documented async-output toggle for streaming + # (see tests/e2e/accuracy/qwen3_omni). Confirm the exact knob at bench time. + omni = Omni( + model, + stage_configs_path=yaml_path, + stage_init_timeout=1800, + init_timeout=3600, + async_chunk=async_mode, + ) + results: list[dict] = [] + try: + for prompt_id, question in enumerate(FIXED_PROMPTS): + text: str | None = None + wav: bytes = b"" + # Batch size 1: submit a single prompt so scheduling is deterministic. + for step in omni.generate([_text_prompt(question)], _sampling_params(), py_generator=False): + out = step.request_output + if step.final_output_type == "text": + text = out.outputs[0].text # exact text, no strip -> truly bit-identical + elif step.final_output_type == "audio": + wav = _audio_bytes(out.outputs[0].multimodal_output["audio"]) + if text is None: + raise RuntimeError(f"prompt_id={prompt_id} async={async_mode}: produced no text output") + if not wav: + raise RuntimeError(f"prompt_id={prompt_id} async={async_mode}: produced no/empty audio output") + results.append( + { + "prompt_id": prompt_id, + "async_mode": async_mode, + "text_sha": _sha(text.encode("utf-8")), + "wav_sha": _sha(wav), + } + ) + finally: + omni.close() + return results + + +def _record(model: str, yaml_path: str) -> dict: + results: list[dict] = [] + for async_mode in ASYNC_MODES: + results.extend(_run_once(model, yaml_path, async_mode=async_mode)) + return {"model": model, "commit": _git_commit(), "env": _env_fingerprint(), "results": results} + + +def _key(record: dict): + return (record["prompt_id"], record["async_mode"]) + + +def _expected_keys() -> set: + return {(pid, mode) for pid in range(len(FIXED_PROMPTS)) for mode in ASYNC_MODES} + + +def _check(model: str, yaml_path: str, baseline_path: str, *, force_env: bool) -> int: + with open(baseline_path) as fh: + baseline = json.load(fh) + + now = _record(model, yaml_path) + if not force_env and baseline.get("env") != now["env"]: + print( + f"ENV MISMATCH: baseline={baseline.get('env')} now={now['env']} — hashes are " + "env-specific. Re-record on this box or pass --force-env.", + file=sys.stderr, + ) + return 2 + + problems: list[str] = [] + + def _index_unique(records: list[dict], label: str) -> dict: + by_key: dict = {} + for record in records: + key = _key(record) + if key in by_key: + problems.append(f"{label}: duplicate entry for (prompt_id={key[0]}, async={key[1]})") + by_key[key] = record + return by_key + + base_by_key = _index_unique(baseline["results"], "baseline") + now_by_key = _index_unique(now["results"], "current-run") + expected = _expected_keys() + + # Guard against duplicate-collapse: the record count must match the key count. + if len(baseline["results"]) != len(expected): + problems.append(f"baseline has {len(baseline['results'])} records, expected exactly {len(expected)}") + if set(now_by_key) != expected: + problems.append(f"current-run key set {sorted(set(now_by_key))} != expected {sorted(expected)}") + if set(base_by_key) != set(now_by_key): + missing = sorted(set(base_by_key) - set(now_by_key)) + extra = sorted(set(now_by_key) - set(base_by_key)) + problems.append(f"baseline/run key-set mismatch: missing-from-run={missing} extra-in-run={extra}") + for key in sorted(set(base_by_key) & set(now_by_key)): + b, r = base_by_key[key], now_by_key[key] + if (b["text_sha"], b["wav_sha"]) != (r["text_sha"], r["wav_sha"]): + problems.append( + f"HASH MISMATCH (prompt_id={key[0]}, async={key[1]}): " + f"text {b['text_sha'][:8]}!={r['text_sha'][:8]} / wav {b['wav_sha'][:8]}!={r['wav_sha'][:8]}" + ) + + if problems: + for problem in problems: + print(problem, file=sys.stderr) + return 1 + print(f"OK: all {len(expected)} (prompt_id, async_mode) entries match the baseline.") + return 0 + + +def main() -> int: + p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("model") + p.add_argument("--yaml", required=True, help="Stage deploy config path") + p.add_argument("--out", help="Write a new baseline JSON to this path") + p.add_argument("--check", metavar="BASELINE_JSON", help="Verify current output against this baseline") + p.add_argument("--force-env", action="store_true", help="Compare hashes even if the recorded env differs") + args = p.parse_args() + + if args.check: + return _check(args.model, args.yaml, args.check, force_env=args.force_env) + + baseline = _record(args.model, args.yaml) + payload = json.dumps(baseline, indent=2) + if args.out: + with open(args.out, "w") as fh: + fh.write(payload) + print(f"Wrote baseline: {args.out}") + else: + print(payload) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/baselines/qwen3_omni_hash.baseline b/tools/baselines/qwen3_omni_hash.baseline new file mode 100644 index 000000000..7fed33016 --- /dev/null +++ b/tools/baselines/qwen3_omni_hash.baseline @@ -0,0 +1,48 @@ +{ + "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct", + "commit": "24eb4ab89cc2ad3f8cdd83f57b66ce7786a0d65c", + "env": { + "driver": "570.133.20", + "cuda": "13.0", + "torch": "2.11.0+cu130", + "gpu": "NVIDIA L20X" + }, + "results": [ + { + "prompt_id": 0, + "async_mode": false, + "text_sha": "e4bf6ed906c0dada879874a9cc09f9ba3a4399149262a831a01fd53e6a8fd043", + "wav_sha": "20a203a149005f948afd485bbea780fa06c109f3689ebb6b3110f51b3237bd35" + }, + { + "prompt_id": 1, + "async_mode": false, + "text_sha": "2e8a6b141a9747bd68aadc2931f278b89712ad7ad7ae12b25b39e193b241bf68", + "wav_sha": "1839a759bb5d8671b6352b2e3110dab5edfa718e38ff073d379176201f1eac8c" + }, + { + "prompt_id": 2, + "async_mode": false, + "text_sha": "e0aa5a513a5162024f35afee3830aeb67f2e378381726adb9d3a66b984aa652d", + "wav_sha": "74338df3e8357bc9d0c05bf141b63c45251b12807de34495097edb07fc7b19f8" + }, + { + "prompt_id": 0, + "async_mode": true, + "text_sha": "e4bf6ed906c0dada879874a9cc09f9ba3a4399149262a831a01fd53e6a8fd043", + "wav_sha": "e09f862f54b553cc7c64e1631b821a768167768812b3f2cd3d0c69d0da991ee7" + }, + { + "prompt_id": 1, + "async_mode": true, + "text_sha": "2e8a6b141a9747bd68aadc2931f278b89712ad7ad7ae12b25b39e193b241bf68", + "wav_sha": "9115d481f62a4b771c4c3d8b89d5c5a01149f002096289ab1da3f034fa388a2b" + }, + { + "prompt_id": 2, + "async_mode": true, + "text_sha": "e0aa5a513a5162024f35afee3830aeb67f2e378381726adb9d3a66b984aa652d", + "wav_sha": "6a5985c9d4ca1596518c291228cd29239e76fcaf310c1cdb5921276b7efce801" + } + ] +} From 5b230541c90b4ba883d3739dd1ed708f9275b656 Mon Sep 17 00:00:00 2001 From: rein yang <73573651+R2-Y@users.noreply.github.com> Date: Mon, 27 Jul 2026 16:37:25 +0800 Subject: [PATCH 109/240] [Deploy] Reduce MiniCPM-o single-GPU startup memory (#5447) Signed-off-by: R2-Y --- examples/online_serving/minicpmo/README.md | 11 ++++++----- recipes/OpenBMB/MiniCPM-o-4_5.md | 13 ++++++++----- .../models/minicpmo_4_5/test_pipeline.py | 7 ++++--- vllm_omni/deploy/minicpmo_4_5.yaml | 10 ++++++---- 4 files changed, 24 insertions(+), 17 deletions(-) diff --git a/examples/online_serving/minicpmo/README.md b/examples/online_serving/minicpmo/README.md index 148fbb5e2..134ddd62c 100644 --- a/examples/online_serving/minicpmo/README.md +++ b/examples/online_serving/minicpmo/README.md @@ -23,10 +23,11 @@ including `librosa`. The deploy config auto-loads via `--omni`. The default `vllm_omni/deploy/minicpmo_4_5.yaml` keeps all three stages on -logical device 0 with memory budgets of 65%, 15%, and 15%. For throughput, -`minicpmo_4_5_batching.yaml` gives the Thinker GPU 0 (90%) and colocates the -Talker (55%) and Code2Wav (35%) on GPU 1. Each stage admits at most four -concurrent sequences. +logical device 0 with memory budgets of 55%, 22%, and 22%. The profile admits +at most four sequences per stage and bounds startup video profiling to 32 +frames per video. For throughput,`minicpmo_4_5_batching.yaml` gives the Thinker +GPU 0 (90%) and colocates the Talker (55%) and Code2Wav (35%) on GPU 1. That +profile admits at most four concurrent sequences per stage. | deploy config | GPUs | Notes | |---|---|---| @@ -60,7 +61,7 @@ checkpoint path. To start the experimental native duplex backend, use ```bash vllm serve openbmb/MiniCPM-o-4_5 --omni --trust-remote-code --port 8099 \ - --stage-overrides '{"0": {"gpu_memory_utilization": 0.65}}' + --stage-overrides '{"0": {"gpu_memory_utilization": 0.55}}' ``` ## Send multimodal requests diff --git a/recipes/OpenBMB/MiniCPM-o-4_5.md b/recipes/OpenBMB/MiniCPM-o-4_5.md index dddc9fac4..97b3c6d6d 100644 --- a/recipes/OpenBMB/MiniCPM-o-4_5.md +++ b/recipes/OpenBMB/MiniCPM-o-4_5.md @@ -79,9 +79,9 @@ a fallback because it would duplicate state machines and correctness paths. The default [`vllm_omni/deploy/minicpmo_4_5.yaml`](../../vllm_omni/deploy/minicpmo_4_5.yaml) co-locates Thinker, codec-only Talker, and Code2Wav on GPU 0. Their -`gpu_memory_utilization` budgets are 0.65, 0.15, and 0.15. This layout -minimizes the GPU count; use a large-memory accelerator and leave the -remaining 5% for runtime overhead. +`gpu_memory_utilization` budgets are 0.55, 0.22, and 0.22. All stages admit +up to four sequences. Startup video profiling is bounded to 32 frames per +video. Use the two-GPU profile for production throughput. #### Environment @@ -202,8 +202,8 @@ speech output (TTS)"** checkbox on / off. #### Notes -- Memory budget: Thinker, Talker, and Code2Wav reserve 0.65, 0.15, and - 0.15 of GPU 0. The larger Thinker share protects its multimodal KV cache; +- Memory budget: Thinker, Talker, and Code2Wav reserve 0.55, 0.22, and + 0.22 of GPU 0. The larger Thinker share protects its multimodal KV cache; all three model processes still share one CUDA device. - `--trust-remote-code` is required — the HF repo ships a custom `MiniCPMO` config / model class. @@ -214,6 +214,9 @@ speech output (TTS)"** checkbox on / off. contention. Talker AR state and Code2Wav caches are request-owned; Code2Wav batches only exact-shape-compatible chunks and does not fall back to serial decode. +- `limit_mm_per_prompt.video.num_frames: 32` bounds startup dummy profiling, + not runtime media decoding. Use `media_io_kwargs.video.num_frames` when a + matching URL/file video sampling limit is required. - `StageRequestStats.batch_size` is a request-scoped placeholder, not the scheduler's execution batch. - Single-GPU co-location trades throughput for hardware density: Stage 0/1 diff --git a/tests/model_executor/models/minicpmo_4_5/test_pipeline.py b/tests/model_executor/models/minicpmo_4_5/test_pipeline.py index 1b8bd669f..968df04a9 100644 --- a/tests/model_executor/models/minicpmo_4_5/test_pipeline.py +++ b/tests/model_executor/models/minicpmo_4_5/test_pipeline.py @@ -159,10 +159,11 @@ class TestDeployTopology: if filename == "minicpmo_4_5.yaml": assert [stage.yaml_engine_args["max_num_seqs"] for stage in stages] == [4, 4, 4] assert [stage.yaml_engine_args["gpu_memory_utilization"] for stage in stages] == [ - 0.65, - 0.15, - 0.15, + 0.55, + 0.22, + 0.22, ] + assert stages[0].yaml_engine_args["limit_mm_per_prompt"] == {"video": {"count": 1, "num_frames": 32}} elif filename in {"minicpmo_4_5_batching.yaml", "minicpmo_4_5_2gpu.yaml"}: assert [stage.yaml_engine_args["gpu_memory_utilization"] for stage in stages] == [ 0.9, diff --git a/vllm_omni/deploy/minicpmo_4_5.yaml b/vllm_omni/deploy/minicpmo_4_5.yaml index da84667af..ea1fdaf2f 100644 --- a/vllm_omni/deploy/minicpmo_4_5.yaml +++ b/vllm_omni/deploy/minicpmo_4_5.yaml @@ -18,13 +18,15 @@ connectors: stages: - stage_id: 0 max_num_seqs: 4 - gpu_memory_utilization: 0.65 + gpu_memory_utilization: 0.55 enforce_eager: false tensor_parallel_size: 1 max_num_batched_tokens: 16384 devices: "0" limit_mm_per_prompt: - video: 1 + video: + count: 1 + num_frames: 32 default_sampling_params: temperature: 0.0 top_p: 1.0 @@ -36,7 +38,7 @@ stages: - stage_id: 1 max_num_seqs: 4 - gpu_memory_utilization: 0.15 + gpu_memory_utilization: 0.22 enforce_eager: false max_num_batched_tokens: 8192 devices: "0" @@ -56,7 +58,7 @@ stages: - stage_id: 2 max_num_seqs: 4 - gpu_memory_utilization: 0.15 + gpu_memory_utilization: 0.22 enforce_eager: true enable_chunked_prefill: false max_num_batched_tokens: 65536 From 88b7ca2ffa6ad9192a9175b36b15948e1f78febd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B1=AA=E5=BF=97=E9=B9=8F?= Date: Mon, 27 Jul 2026 17:01:54 +0800 Subject: [PATCH 110/240] [Model] Support MammothModa2-Dev (#5411) Signed-off-by: princepride --- docs/models/supported_models.md | 1 + .../text_to_image/text_to_image.py | 12 + ...ammothModa2-Preview.md => MammothModa2.md} | 98 +++++--- recipes/README.md | 2 +- tests/model_extras/test_model_extras.py | 33 ++- .../test_mammoth_moda2_config.py | 31 +++ .../mammoth_moda2/mammothmoda2_dit_model.py | 34 ++- .../pipeline_mammothmoda2_dit.py | 41 +++- .../models/mammoth_moda2/mammoth_moda2.py | 226 +++++++++++++++++- vllm_omni/model_executor/models/registry.py | 10 + .../model_extras/mammothmodal2_preview.py | 13 +- vllm_omni/model_extras/registry.py | 5 + .../configs/mammoth_moda2.py | 83 +++++++ 13 files changed, 525 insertions(+), 64 deletions(-) rename recipes/MammothModa2/{MammothModa2-Preview.md => MammothModa2.md} (69%) diff --git a/docs/models/supported_models.md b/docs/models/supported_models.md index ca3c38a91..9f99592fd 100644 --- a/docs/models/supported_models.md +++ b/docs/models/supported_models.md @@ -61,6 +61,7 @@ th { | `OmniVoiceModel` | OmniVoice | `k2-fsa/OmniVoice` | ✅︎ | | | | | `VoxCPM2TalkerForConditionalGeneration` | VoxCPM2 | `openbmb/VoxCPM2` | ✅︎ | | | | | `MammothModa2ForConditionalGeneration` | MammothModa2-Preview | `bytedance-research/MammothModa2-Preview` | ✅︎ | ✅︎ | | | +| `MammothModa2ForConditionalGeneration` | MammothModa2-Dev (AR-only image understanding) | `bytedance-research/MammothModa2-Dev` | ✅︎ | | | | | `Flux2KleinPipeline` | FLUX.2-klein | `black-forest-labs/FLUX.2-klein-4B`, `black-forest-labs/FLUX.2-klein-9B` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | | `FluxKontextPipeline` | FLUX.1-Kontext-dev | `black-forest-labs/FLUX.1-Kontext-dev` | ✅︎ | ✅︎ | | | | `FluxPipeline` | FLUX.1-dev | `black-forest-labs/FLUX.1-dev` | ✅︎ | ✅︎ | | ✅︎ | diff --git a/examples/offline_inference/text_to_image/text_to_image.py b/examples/offline_inference/text_to_image/text_to_image.py index d01b2973f..81acebe00 100644 --- a/examples/offline_inference/text_to_image/text_to_image.py +++ b/examples/offline_inference/text_to_image/text_to_image.py @@ -556,9 +556,21 @@ def main(): elif init_non_diffusion and hasattr(params, "extra_args"): if params.extra_args is None: params.extra_args = {} + params.extra_args.update(diffusion_params.extra_args or {}) if args.seed is not None and hasattr(params, "seed"): params.seed = args.seed + # MammothModa2's AR stage emits one visual token per grid cell, + # one EOL token per row, and one final look-ahead token whose hidden + # state is unavailable. Size the first stage from the prompt metadata + # instead of SamplingParams' default of 16 tokens. + prompt_info = prompt_dict.get("additional_information", {}) + if idx == 0 and prompt_info.get("omni_task") == ["t2i"]: + ar_width = int(prompt_info.get("ar_width", [0])[0]) + ar_height = int(prompt_info.get("ar_height", [0])[0]) + if ar_width > 0 and ar_height > 0: + params.max_tokens = ar_height * (ar_width + 1) + 1 + if not diffusion_replaced and len(sampling_params_list) == 1: sampling_params_list = [diffusion_params] diff --git a/recipes/MammothModa2/MammothModa2-Preview.md b/recipes/MammothModa2/MammothModa2.md similarity index 69% rename from recipes/MammothModa2/MammothModa2-Preview.md rename to recipes/MammothModa2/MammothModa2.md index c94560254..60c391dec 100644 --- a/recipes/MammothModa2/MammothModa2-Preview.md +++ b/recipes/MammothModa2/MammothModa2.md @@ -1,13 +1,12 @@ -# MammothModa2-Preview +# MammothModa2 -> MammothModa2-Preview generation and understanding through shared offline examples +> MammothModa2-Preview and MammothModa2-Dev unified understanding and generation ## Summary - Vendor: ByteDance Research -- Model: `bytedance-research/MammothModa2-Preview` -- Task: Text-to-image generation (AR → DiT two-stage pipeline), plus - image-to-text understanding (AR stage) +- Models: `bytedance-research/MammothModa2-Preview`, `bytedance-research/MammothModa2-Dev` +- Tasks: Preview and Dev text-to-image (AR → DiT); Dev text/image understanding - Mode: Offline inference - Maintainer: Community @@ -29,7 +28,9 @@ flags. Image size uses the standard `--height` / `--width` flags. - Upstream model: [`bytedance-research/MammothModa2-Preview`](https://huggingface.co/bytedance-research/MammothModa2-Preview) -- Related text-to-image example: +- Dev model: + [`bytedance-research/MammothModa2-Dev`](https://huggingface.co/bytedance-research/MammothModa2-Dev) +- Related offline example: [`examples/offline_inference/text_to_image/text_to_image.py`](../../examples/offline_inference/text_to_image/text_to_image.py) - Related T2T/I2T example: [`examples/offline_inference/x_to_text/x_to_text.py`](../../examples/offline_inference/x_to_text/x_to_text.py) @@ -131,10 +132,24 @@ ls -lh mammoth_t2i.png python -c "from PIL import Image; print(Image.open('mammoth_t2i.png').size)" ``` -### Image-to-Text (I2T, simplest) +## MammothModa2-Dev unified inference -For the simplest offline image-understanding run, use a short inline Python -snippet from the repository root: +MammothModa2-Dev uses a Qwen3-VL AR backbone, while MammothModa2-Preview uses +Qwen2.5-VL. vLLM-Omni selects the matching implementation from the nested +`llm_config.model_type`; no checkpoint edits or `trust_remote_code` flag are +required. + +Text-to-text and image-to-text use the AR-only deploy. Text-to-image loads the +Qwen3 generation experts (`gen_mlp`), extra visual vocabulary and image head, +then sends the generated visual tokens and hidden states to the DiT stage. + +Download the checkpoint: + +```bash +hf download bytedance-research/MammothModa2-Dev --local-dir ./MammothModa2-Dev +``` + +Run image-to-text inference from the repository root: ```python from PIL import Image @@ -142,35 +157,44 @@ from vllm import SamplingParams from vllm.multimodal.image import convert_image_mode from vllm_omni import Omni -def main(): - prompt = ( - "<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n" - "<|im_start|>user\n<|vision_start|><|image_pad|><|vision_end|>" - "Summarize this image.<|im_end|>\n" - "<|im_start|>assistant\n" - ) +prompt = ( + "<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n" + "<|im_start|>user\n<|vision_start|><|image_pad|><|vision_end|>" + "Summarize this image.<|im_end|>\n" + "<|im_start|>assistant\n" +) - omni = Omni( - model="./MammothModa2-Preview", - deploy_config="vllm_omni/deploy/mammoth_moda2_ar.yaml", - ) - try: - outputs = list( - omni.generate( - [{ - "prompt": prompt, - "multi_modal_data": {"image": convert_image_mode(Image.open("./image.png"), "RGB")}, - "additional_information": {"omni_task": ["chat"]}, - }], - [SamplingParams(temperature=0.2, top_p=0.9, top_k=-1, max_tokens=512, seed=42)], - ) +omni = Omni( + model="./MammothModa2-Dev", + deploy_config="vllm_omni/deploy/mammoth_moda2_ar.yaml", +) +try: + outputs = list( + omni.generate( + [{ + "prompt": prompt, + "multi_modal_data": { + "image": convert_image_mode(Image.open("./image.png"), "RGB"), + }, + "additional_information": {"omni_task": ["chat"]}, + }], + [SamplingParams( + temperature=0.2, + top_p=0.9, + top_k=-1, + max_tokens=512, + seed=42, + )], ) - finally: - omni.close() + ) +finally: + omni.close() - ro = getattr(outputs[-1], "request_output", outputs[-1]) - print(ro.outputs[0].text.strip()) - -if __name__ == "__main__": - main() +request_output = getattr(outputs[-1], "request_output", outputs[-1]) +print(request_output.outputs[0].text.strip()) ``` + +The Dev checkpoint is approximately 47.55 GiB on disk. In the verified AR-only +run, loaded model weights used approximately 16.97 GiB of GPU memory before KV +and encoder caches. Allow additional GPU memory for those caches and the input +image. diff --git a/recipes/README.md b/recipes/README.md index 13c2320be..94273caa3 100644 --- a/recipes/README.md +++ b/recipes/README.md @@ -48,7 +48,7 @@ recipes/ | [`krea/Krea-2.md`](./krea/Krea-2.md) | Text-to-image (Turbo + Raw), offline + online, with LoRA | 1x H100 80GB | | [`LTX/LTX-2.md`](./LTX/LTX-2.md) | Text-to-video and image-to-video serving | 1x H200 141GB | | [`LTX/LTX-2.3.md`](./LTX/LTX-2.3.md) | Text-to-video with audio generation (22B) | 1x GPU (96GB VRAM) | -| [`MammothModa2/MammothModa2-Preview.md`](./MammothModa2/MammothModa2-Preview.md) | Text-to-image with the shared offline image example (AR → DiT) | 1x L40S 48GB / 1x ≥40GB GPU | +| [`MammothModa2/MammothModa2.md`](./MammothModa2/MammothModa2.md) | Preview and Dev text-to-image (AR → DiT); Dev text/image understanding | Preview: 1x L40S 48GB / 1x ≥40GB GPU; Dev: 1x NVIDIA GPU with sufficient cache headroom | | [`mistralai/Voxtral-TTS.md`](./mistralai/Voxtral-TTS.md) | Online serving for TTS | 1x RTX 4090 24GB | | [`Robbyant/LingBot-Video.md`](./Robbyant/LingBot-Video.md) | Native dense and MoE text-to-video serving | Dense: 1x L20X; MoE: 1x L20X (~67.7 GiB peak) | | [`cosmos3/Cosmos3-Nano.md`](./cosmos3/Cosmos3-Nano.md) | Text-to-image, text-to-video, image-to-video, video-to-video generation, text to video with sound, action policy | 1x H200 141GB / B300 | diff --git a/tests/model_extras/test_model_extras.py b/tests/model_extras/test_model_extras.py index 7c03616bc..47c108235 100644 --- a/tests/model_extras/test_model_extras.py +++ b/tests/model_extras/test_model_extras.py @@ -533,15 +533,29 @@ def test_declared_extra_args_apply_to_existing_sampling_params() -> None: @pytest.mark.core_model @pytest.mark.cpu def test_mammothmoda2_extra_registry_declares_request_and_response_params() -> None: - assert get_extra_body_params("MammothModa2DiTPipeline") == frozenset( - { - "text_guidance_scale", - "cfg_range", - "num_inference_steps", - } + for model_class_name in ( + "MammothModa2DiTPipeline", + "MammothModa2ForConditionalGeneration", + "Mammothmoda2Model", + ): + assert get_extra_body_params(model_class_name) == frozenset( + { + "text_guidance_scale", + "cfg_range", + "num_inference_steps", + } + ) + assert get_extra_output_params(model_class_name) == frozenset() + assert should_init_extra_args_for_non_diffusion_stages(model_class_name) is True + + wrapper_prompt = build_text_to_image_prompt( + "MammothModa2ForConditionalGeneration", + prompt="a cat", + negative_prompt=None, + height=256, + width=256, ) - assert get_extra_output_params("MammothModa2DiTPipeline") == frozenset() - assert should_init_extra_args_for_non_diffusion_stages("MammothModa2DiTPipeline") is True + assert wrapper_prompt["additional_information"]["omni_task"] == ["t2i"] @pytest.mark.core_model @@ -568,5 +582,8 @@ def test_mammothmoda2_text_to_image_prompt_builder() -> None: "ar_height": [32], "image_height": [512], "image_width": [768], + "eol_token_id": [152064], + "visual_token_start_id": [152072], + "visual_token_end_id": [168456], }, } diff --git a/tests/unit/mammoth_moda2/test_mammoth_moda2_config.py b/tests/unit/mammoth_moda2/test_mammoth_moda2_config.py index 77b68ca3f..6e35a94f6 100644 --- a/tests/unit/mammoth_moda2/test_mammoth_moda2_config.py +++ b/tests/unit/mammoth_moda2/test_mammoth_moda2_config.py @@ -16,8 +16,12 @@ from vllm_omni.transformers_utils.configs.mammoth_moda2 import ( Mammothmoda2Config, Mammothmoda2Qwen2_5_VLConfig, Mammothmoda2Qwen2_5_VLTextConfig, + Mammothmoda2Qwen3VLConfig, + Mammothmoda2Qwen3VLTextConfig, ) +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + @pytest.mark.cpu class TestMammothmoda2Config: @@ -89,3 +93,30 @@ class TestMammothmoda2Qwen2_5_VLConfig: assert hasattr(text_config, "vocab_size") assert hasattr(text_config, "image_token_id") assert hasattr(text_config, "video_token_id") + + +@pytest.mark.cpu +class TestMammothmoda2Qwen3VLConfig: + def test_dev_config_uses_qwen3_vl_subconfigs(self): + config = Mammothmoda2Config( + llm_config={ + "model_type": "mammothmoda2_qwen3_vl", + "text_config": { + "model_type": "mammothmoda2_qwen3_vl_text", + "vocab_size": 151936, + "gen_vocab_size": 32800, + "gen_vocab_start_index": 152064, + }, + "vision_config": { + "model_type": "mammothmoda2_qwen3_vl_vision", + "deepstack_visual_indexes": [8, 16, 24], + }, + } + ) + + assert isinstance(config.llm_config, Mammothmoda2Qwen3VLConfig) + assert isinstance(config.get_text_config(), Mammothmoda2Qwen3VLTextConfig) + # vLLM validates sampling against the complete base + visual vocabulary. + assert config.get_text_config().vocab_size == 152064 + 32800 + assert config.get_text_config().gen_vocab_start_index == 152064 + assert config.llm_config.vision_config.deepstack_visual_indexes == [8, 16, 24] diff --git a/vllm_omni/diffusion/models/mammoth_moda2/mammothmoda2_dit_model.py b/vllm_omni/diffusion/models/mammoth_moda2/mammothmoda2_dit_model.py index ed42e24ca..000abe051 100644 --- a/vllm_omni/diffusion/models/mammoth_moda2/mammothmoda2_dit_model.py +++ b/vllm_omni/diffusion/models/mammoth_moda2/mammothmoda2_dit_model.py @@ -189,6 +189,7 @@ class SimpleQFormerImageRefiner(nn.Module): def __init__( self, hidden_size: int, + output_hidden_size: int | None = None, num_queries: int = 128, num_layers: int = 2, num_heads: int | None = None, @@ -196,6 +197,8 @@ class SimpleQFormerImageRefiner(nn.Module): norm_eps: float = 1e-5, ) -> None: super().__init__() + input_hidden_size = hidden_size + hidden_size = output_hidden_size or hidden_size self.hidden_size = hidden_size self.num_queries = num_queries # ensure num_heads divides hidden_size @@ -203,8 +206,8 @@ class SimpleQFormerImageRefiner(nn.Module): num_heads = max(1, hidden_size // 128) self.num_heads = self._choose_valid_num_heads(hidden_size, num_heads) self.input_proj = nn.Sequential( - Qwen2RMSNorm(hidden_size, eps=norm_eps), - nn.Linear(hidden_size, hidden_size, bias=True), + Qwen2RMSNorm(input_hidden_size, eps=norm_eps), + nn.Linear(input_hidden_size, hidden_size, bias=True), ) # Learnable query embeddings @@ -662,11 +665,32 @@ class Transformer2DModel(ModelMixin, ConfigMixin): batch_size: int, height: int, width: int, + ar_image_hidden_states: torch.Tensor | None = None, + ar_image_attention_mask: torch.Tensor | None = None, ): device = hidden_states.device p = self.config.patch_size temb, text_hidden_states = self.time_caption_embed(timestep, text_hidden_states, hidden_states.dtype) + image_embedder = getattr(self.time_caption_embed, "image_embedder", None) + if image_embedder is not None and ar_image_hidden_states is not None: + if ar_image_attention_mask is None: + ar_image_attention_mask = torch.ones( + ar_image_hidden_states.shape[:2], + dtype=torch.bool, + device=ar_image_hidden_states.device, + ) + image_hidden_states = image_embedder( + ar_image_hidden_states, + ~ar_image_attention_mask.bool(), + ) + image_attention_mask = torch.ones( + image_hidden_states.shape[:2], + dtype=torch.bool, + device=image_hidden_states.device, + ) + text_hidden_states = torch.cat([text_hidden_states, image_hidden_states], dim=1) + text_attention_mask = torch.cat([text_attention_mask, image_attention_mask], dim=1) img_tokens = rearrange(hidden_states, "b c (h p1) (w p2) -> b (h w) (p1 p2 c)", p1=p, p2=p) img_tokens = self.x_embedder(img_tokens) @@ -699,6 +723,7 @@ class Transformer2DModel(ModelMixin, ConfigMixin): return ( temb, text_hidden_states, + text_attention_mask, img_tokens, img_mask, img_len, @@ -740,6 +765,8 @@ class Transformer2DModel(ModelMixin, ConfigMixin): freqs_cis: torch.Tensor, text_attention_mask: torch.Tensor, ref_image_hidden_states: list[list[torch.Tensor]] | None = None, + ar_image_hidden_states: torch.Tensor | None = None, + ar_image_attention_mask: torch.Tensor | None = None, return_dict: bool = False, ) -> torch.Tensor: batch_size, height, width = self._validate_inputs( @@ -749,6 +776,7 @@ class Transformer2DModel(ModelMixin, ConfigMixin): ( temb, text_hidden_states, + text_attention_mask, img_tokens, img_mask, img_len, @@ -766,6 +794,8 @@ class Transformer2DModel(ModelMixin, ConfigMixin): batch_size, height, width, + ar_image_hidden_states, + ar_image_attention_mask, ) text_hidden_states, img_tokens = self._apply_refiners( diff --git a/vllm_omni/diffusion/models/mammoth_moda2/pipeline_mammothmoda2_dit.py b/vllm_omni/diffusion/models/mammoth_moda2/pipeline_mammothmoda2_dit.py index 614c8268c..5298d5514 100644 --- a/vllm_omni/diffusion/models/mammoth_moda2/pipeline_mammothmoda2_dit.py +++ b/vllm_omni/diffusion/models/mammoth_moda2/pipeline_mammothmoda2_dit.py @@ -42,6 +42,7 @@ class MammothModa2DiTPipeline(nn.Module, SupportsComponentDiscovery): hf_to_vllm_mapper = WeightsMapper( orig_to_new_prefix={ "llm_model.": None, + "gen_tokenizer.": None, } ) @@ -77,11 +78,23 @@ class MammothModa2DiTPipeline(nn.Module, SupportsComponentDiscovery): ) self._reinit_caption_embedder(llm_hidden_size) - # Optional: image condition refiner (Q-Former) - if self.config.gen_image_condition_refiner_config is not None: + # Optional image condition Q-Former. Preview stores it as a standalone + # module; Dev stores it under the DiT timestep/caption embedder. + llm_model_type = getattr(self.config.llm_config, "model_type", "") + refiner_config = self.config.gen_image_condition_refiner_config + if refiner_config is not None and llm_model_type == "mammothmoda2_qwen3_vl": + dit_hidden_size = int(self.gen_transformer.hidden_size) + self.gen_transformer.time_caption_embed.image_embedder = SimpleQFormerImageRefiner( + hidden_size=llm_hidden_size, + output_hidden_size=dit_hidden_size, + num_heads=max(1, dit_hidden_size // 128), + **refiner_config, + ) + self.gen_image_condition_refiner = None + elif refiner_config is not None: self.gen_image_condition_refiner = SimpleQFormerImageRefiner( hidden_size=llm_hidden_size, - **self.config.gen_image_condition_refiner_config, + **refiner_config, ) else: self.gen_image_condition_refiner = None @@ -237,6 +250,13 @@ class MammothModa2DiTPipeline(nn.Module, SupportsComponentDiscovery): text_cond = _ensure_2d(text_cond, "text_prompt_embeds") image_cond = _ensure_2d(image_cond, "image_prompt_embeds") + if image_cond.shape[0] == 0: + answer_token_ids = info.get("full_token_ids", [])[int(info.get("answer_start_index", [0])[0]) :] + raise ValueError( + "MammothModa2 AR stage produced no visual-token hidden states; " + "the DiT stage requires at least one generated visual token. " + f"Generated token ids: {answer_token_ids[:32]}" + ) text_cond = text_cond.to(device=model_device, dtype=target_dtype, non_blocking=True).contiguous() image_cond = image_cond.to(device=model_device, dtype=target_dtype, non_blocking=True).contiguous() @@ -263,8 +283,17 @@ class MammothModa2DiTPipeline(nn.Module, SupportsComponentDiscovery): device=image_embeds.device, ) - prompt_embeds = torch.cat([text_embeds, image_embeds], dim=1) - prompt_attention_mask = torch.cat([text_attention_mask, image_attention_mask], dim=1) + nested_image_embedder = getattr(self.gen_transformer.time_caption_embed, "image_embedder", None) + if nested_image_embedder is None: + prompt_embeds = torch.cat([text_embeds, image_embeds], dim=1) + prompt_attention_mask = torch.cat([text_attention_mask, image_attention_mask], dim=1) + ar_image_embeds = None + ar_image_attention_mask = None + else: + prompt_embeds = text_embeds + prompt_attention_mask = text_attention_mask + ar_image_embeds = image_embeds + ar_image_attention_mask = image_attention_mask # Prepare negative prompt (for CFG). If none provided, fall back to unconditional. negative_prompt_embeds = None @@ -337,6 +366,8 @@ class MammothModa2DiTPipeline(nn.Module, SupportsComponentDiscovery): text_hidden_states=prompt_embeds, text_attention_mask=prompt_attention_mask, ref_image_hidden_states=None, + ar_image_hidden_states=ar_image_embeds, + ar_image_attention_mask=ar_image_attention_mask, freqs_cis=self.gen_freqs_cis, ) guidance_scale = text_guidance_scale if cfg_range[0] <= i / total_steps <= cfg_range[1] else 1.0 diff --git a/vllm_omni/model_executor/models/mammoth_moda2/mammoth_moda2.py b/vllm_omni/model_executor/models/mammoth_moda2/mammoth_moda2.py index 2fe616cfd..47c2b5e37 100644 --- a/vllm_omni/model_executor/models/mammoth_moda2/mammoth_moda2.py +++ b/vllm_omni/model_executor/models/mammoth_moda2/mammoth_moda2.py @@ -6,7 +6,7 @@ from typing import Any import torch from torch import nn -from transformers import Qwen2Config +from transformers import Qwen2Config, Qwen3VLProcessor from transformers.models.qwen2_5_vl.processing_qwen2_5_vl import Qwen2_5_VLProcessor from vllm.config import CacheConfig, VllmConfig from vllm.distributed import get_pp_group @@ -30,7 +30,15 @@ from vllm.model_executor.models.qwen2_5_vl import ( Qwen2_5_VLProcessingInfo, ) from vllm.model_executor.models.qwen2_vl import Qwen2VLMultiModalDataParser +from vllm.model_executor.models.qwen3 import Qwen3DecoderLayer, Qwen3MLP +from vllm.model_executor.models.qwen3_vl import ( + Qwen3VLDummyInputsBuilder, + Qwen3VLForConditionalGeneration, + Qwen3VLMultiModalProcessor, + Qwen3VLProcessingInfo, +) from vllm.model_executor.models.utils import ( + AutoWeightsLoader, PPMissingLayer, WeightsMapper, init_vllm_registered_model, @@ -51,6 +59,18 @@ from vllm_omni.model_executor.models.utils import add_prefix_to_loaded_weights from vllm_omni.transformers_utils.configs.mammoth_moda2 import Mammothmoda2Config +def _runtime_meta(runtime_info: dict[str, Any]) -> dict[str, Any]: + """Accept both legacy flat metadata and the canonical nested payload.""" + meta = runtime_info.get("meta") + return meta if isinstance(meta, dict) else runtime_info + + +def _first_int(value: Any) -> int: + if isinstance(value, (list, tuple)): + value = value[0] + return int(value) + + def moe_enable(moe_type, layer_type, layer_idx) -> bool: """Determine if MoE should be enabled for a specific layer type and index. @@ -291,6 +311,7 @@ class MammothModa2Qwen2ForCausalLM(nn.Module, SupportsPP): self.gen_vocab_size = int(getattr(config, "gen_vocab_size", 0) or 0) self.base_vocab_size = int(self.gen_vocab_start_index) if self.extra_gen_vocab else int(config.vocab_size) + self.org_base_vocab_size = int(getattr(config, "base_vocab_size", self.base_vocab_size)) # The configuration level (hf_text_config.vocab_size) has been extended to base+gen # by the upstream config class. Use config.vocab_size as the total vocab size. self.total_vocab_size = int(getattr(config, "vocab_size", self.base_vocab_size)) @@ -299,6 +320,7 @@ class MammothModa2Qwen2ForCausalLM(nn.Module, SupportsPP): self.embed_tokens = VocabParallelEmbedding( self.base_vocab_size, config.hidden_size, + org_num_embeddings=self.org_base_vocab_size, quant_config=quant_config, prefix=f"{prefix}.embed_tokens", ) @@ -322,6 +344,7 @@ class MammothModa2Qwen2ForCausalLM(nn.Module, SupportsPP): self.lm_head = ParallelLMHead( self.base_vocab_size, config.hidden_size, + org_num_embeddings=self.org_base_vocab_size, quant_config=quant_config, prefix=f"{prefix}.lm_head", ) @@ -425,6 +448,7 @@ class MammothModa2Qwen2ForCausalLM(nn.Module, SupportsPP): positions: torch.Tensor, intermediate_tensors: IntermediateTensors | None = None, inputs_embeds: torch.Tensor | None = None, + deepstack_input_embeds: IntermediateTensors | None = None, ) -> torch.Tensor | IntermediateTensors: if get_pp_group().is_first_rank: if inputs_embeds is not None: @@ -446,8 +470,17 @@ class MammothModa2Qwen2ForCausalLM(nn.Module, SupportsPP): residual = intermediate_tensors["residual"] gen_token_mask = intermediate_tensors.tensors.get("gen_token_mask") - for idx, layer in enumerate(islice(self.layers, self.start_layer, self.end_layer)): + for idx, layer in enumerate( + islice(self.layers, self.start_layer, self.end_layer), + start=self.start_layer, + ): hidden_states, residual = layer(positions, hidden_states, residual, gen_token_mask) + # The vision encoder's deepstack_visual_indexes select which vision + # layers produce these features. The resulting tensors are keyed + # 0..N-1 and, as in Qwen3-Omni, are injected into the first N + # language-model layers. + if deepstack_input_embeds is not None and idx < len(deepstack_input_embeds): + hidden_states = hidden_states + deepstack_input_embeds[f"deepstack_input_embeds_{idx}"] if not get_pp_group().is_last_rank: tensors = {"hidden_states": hidden_states, "residual": residual} @@ -532,6 +565,65 @@ class MammothModa2Qwen2ForCausalLM(nn.Module, SupportsPP): return loaded_params +class Mammoth3DecoderLayer(Qwen3DecoderLayer): + """Qwen3 decoder layer with MammothModa2's generation MLP expert.""" + + def __init__( + self, + config, + layer_idx: int, + cache_config: CacheConfig | None = None, + quant_config: QuantizationConfig | None = None, + prefix: str = "", + ) -> None: + super().__init__(config, cache_config, quant_config, prefix) + self.moe_enable = moe_enable(config.moe_type, "ffn", layer_idx) + self.gen_mlp = ( + Qwen3MLP( + hidden_size=self.hidden_size, + intermediate_size=config.intermediate_size, + hidden_act=config.hidden_act, + quant_config=quant_config, + prefix=f"{prefix}.gen_mlp", + ) + if self.moe_enable + else None + ) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + residual: torch.Tensor | None, + gen_token_mask: torch.Tensor | None = None, + ) -> tuple[torch.Tensor, torch.Tensor]: + if residual is None: + residual = hidden_states + hidden_states = self.input_layernorm(hidden_states) + else: + hidden_states, residual = self.input_layernorm(hidden_states, residual) + hidden_states = self.self_attn(positions=positions, hidden_states=hidden_states) + hidden_states, residual = self.post_attention_layernorm(hidden_states, residual) + hidden_states = moe_forward( + hidden_states, + self.mlp, + self.gen_mlp, + gen_token_mask if self.moe_enable else None, + ) + return hidden_states, residual + + +class MammothModa2Qwen3ForCausalLM(MammothModa2Qwen2ForCausalLM): + """MammothModa2 generation-aware language model using Qwen3 attention.""" + + def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None: + super().__init__( + vllm_config=vllm_config, + prefix=prefix, + decoder_layer_type=Mammoth3DecoderLayer, + ) + + @MULTIMODAL_REGISTRY.register_processor( MammothModa2ARMultiModalProcessor, info=MammothModa2ARProcessingInfo, @@ -604,17 +696,17 @@ class MammothModa2ARForConditionalGeneration(Qwen2_5_VLForConditionalGeneration) num_reqs = int(logits.shape[0]) for i in range(num_reqs): runtime_info = runtime_infos[i] if isinstance(runtime_infos[i], dict) else {} - meta = runtime_info.get("meta", {}) + meta = _runtime_meta(runtime_info) omni_task = meta.get("omni_task") if not isinstance(omni_task, list) or not omni_task or omni_task[0] != "t2i": # Text/understanding/chat: forbid sampling from the extra gen vocab. logits[i, self.language_model.base_vocab_size :] = neg_inf continue - ar_width = meta["ar_width"][0] - eol_token_id = meta["eol_token_id"][0] - visual_start = meta["visual_token_start_id"][0] - visual_end = meta["visual_token_end_id"][0] + ar_width = _first_int(meta["ar_width"]) + eol_token_id = _first_int(meta["eol_token_id"]) + visual_start = _first_int(meta["visual_token_start_id"]) + visual_end = _first_int(meta["visual_token_end_id"]) generated_len = runtime_info["generated_len"] row = logits[i] @@ -682,6 +774,100 @@ class MammothModa2ARForConditionalGeneration(Qwen2_5_VLForConditionalGeneration) return logits +class MammothModa2Qwen3ProcessingInfo(Qwen3VLProcessingInfo): + """Qwen3-VL processing using MammothModa2's nested AR configuration.""" + + def get_hf_config(self): + mammoth_cfg: Mammothmoda2Config = self.ctx.get_hf_config(Mammothmoda2Config) + return mammoth_cfg.llm_config + + def get_hf_processor(self, **kwargs: object) -> Qwen3VLProcessor: + # The MammothModa2 repository does not ship tokenizer_config.json, so + # AutoProcessor otherwise resolves only the image processor and leaves + # Qwen3-VL's image placeholder undefined. + return self.ctx.get_hf_processor( + Qwen3VLProcessor, + use_fast=kwargs.pop("use_fast", True), + **kwargs, + ) + + def get_supported_mm_limits(self) -> Mapping[str, int | None]: + return {"image": None} + + +@MULTIMODAL_REGISTRY.register_processor( + Qwen3VLMultiModalProcessor, + info=MammothModa2Qwen3ProcessingInfo, + dummy_inputs=Qwen3VLDummyInputsBuilder, +) +class MammothModa2Qwen3ARForConditionalGeneration(Qwen3VLForConditionalGeneration): + """MammothModa2-Dev Qwen3-VL stage for understanding and generation.""" + + hf_to_vllm_mapper = WeightsMapper( + orig_to_new_prefix={ + "llm_model.model.visual.": "visual.", + "llm_model.lm_head.": "language_model.lm_head.", + "llm_model.model.language_model.gen_embed_tokens.": "language_model.gen_embed_tokens.", + "llm_model.model.language_model.": "language_model.", + "llm_model.gen_head.": "language_model.gen_head.", + "gen_image_condition_refiner.": None, + "gen_transformer.": None, + "gen_vae.": None, + "gen_tokenizer.": None, + } + ) + + def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None: + mammoth_cfg = vllm_config.model_config.hf_config + ar_hf_config = getattr(mammoth_cfg, "llm_config", mammoth_cfg) + ar_vllm_config = vllm_config.with_hf_config( + ar_hf_config, + architectures=vllm_config.model_config.architectures, + ) + super().__init__(vllm_config=ar_vllm_config, prefix=prefix) + text_config = ar_hf_config.text_config + self.language_model = MammothModa2Qwen3ForCausalLM( + vllm_config=ar_vllm_config.with_hf_config(text_config), + prefix=maybe_prefix(prefix, "language_model"), + ) + self.make_empty_intermediate_tensors = self.language_model.make_empty_intermediate_tensors + self._last_runtime_additional_information: list[dict[str, Any]] | None = None + + def forward(self, *args, **kwargs): + runtime_infos = kwargs.get("runtime_additional_information") + self._last_runtime_additional_information = runtime_infos if isinstance(runtime_infos, list) else None + return super().forward(*args, **kwargs) + + def compute_logits(self, hidden_states: torch.Tensor): + logits = super().compute_logits(hidden_states) + if not isinstance(logits, torch.Tensor) or self._last_runtime_additional_information is None: + return logits + neg_inf = -float("inf") + for i, runtime_info in enumerate(self._last_runtime_additional_information): + meta = _runtime_meta(runtime_info) if isinstance(runtime_info, dict) else {} + if (meta.get("omni_task") or [None])[0] != "t2i": + logits[i, self.language_model.base_vocab_size :] = neg_inf + continue + ar_width = _first_int(meta["ar_width"]) + eol_token_id = _first_int(meta["eol_token_id"]) + visual_start = _first_int(meta["visual_token_start_id"]) + visual_end = _first_int(meta["visual_token_end_id"]) + row = logits[i] + if int(runtime_info["generated_len"]) % (ar_width + 1) == ar_width: + eol_logit = row[eol_token_id].clone() + row.fill_(neg_inf) + row[eol_token_id] = eol_logit + else: + row[:visual_start] = neg_inf + row[visual_end + 1 :] = neg_inf + row[eol_token_id] = neg_inf + return logits + + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: + loader = AutoWeightsLoader(self) + return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) + + @MULTIMODAL_REGISTRY.register_processor( MammothModa2ARMultiModalProcessor, info=MammothModa2ARProcessingInfo, @@ -716,7 +902,11 @@ class MammothModa2ForConditionalGeneration(nn.Module, SupportsMultiModal, Suppor vllm_config=vllm_config, prefix=maybe_prefix(prefix, "ar"), hf_config=cfg.llm_config if hasattr(cfg, "llm_config") else cfg.text_config, - architectures=["MammothModa2ARForConditionalGeneration"], + architectures=[ + "MammothModa2Qwen3ARForConditionalGeneration" + if getattr(cfg.llm_config, "model_type", "") == "mammothmoda2_qwen3_vl" + else "MammothModa2ARForConditionalGeneration" + ], ) self.model = self.ar elif self.model_stage == "dit": @@ -750,6 +940,26 @@ class MammothModa2ForConditionalGeneration(nn.Module, SupportsMultiModal, Suppor return self.model.get_language_model() return self.model + def embed_input_ids( + self, + input_ids: torch.Tensor, + multimodal_embeddings=None, + *, + is_multimodal: torch.Tensor | None = None, + ) -> torch.Tensor: + if isinstance(self.model, MammothModa2Qwen3ARForConditionalGeneration): + return self.model.embed_input_ids( + input_ids, + multimodal_embeddings=multimodal_embeddings, + is_multimodal=is_multimodal, + ) + return SupportsMultiModal.embed_input_ids( + self, + input_ids, + multimodal_embeddings=multimodal_embeddings, + is_multimodal=is_multimodal, + ) + def get_multimodal_embeddings(self, **kwargs: object): # Backward compatibility: route through embed_multimodal. return self.embed_multimodal(**kwargs) diff --git a/vllm_omni/model_executor/models/registry.py b/vllm_omni/model_executor/models/registry.py index 6f0302499..2c3a4c1d7 100644 --- a/vllm_omni/model_executor/models/registry.py +++ b/vllm_omni/model_executor/models/registry.py @@ -99,6 +99,16 @@ _OMNI_MODELS = { "mammoth_moda2", "MammothModa2ARForConditionalGeneration", ), + "MammothModa2Qwen3ARForConditionalGeneration": ( + "mammoth_moda2", + "mammoth_moda2", + "MammothModa2Qwen3ARForConditionalGeneration", + ), + "MammothModa2Qwen3ForCausalLM": ( + "mammoth_moda2", + "mammoth_moda2", + "MammothModa2Qwen3ForCausalLM", + ), "MammothModa2DiTPipeline": ( "mammoth_moda2", "pipeline_mammothmoda2_dit", diff --git a/vllm_omni/model_extras/mammothmodal2_preview.py b/vllm_omni/model_extras/mammothmodal2_preview.py index ddce493f6..96f3fb939 100644 --- a/vllm_omni/model_extras/mammothmodal2_preview.py +++ b/vllm_omni/model_extras/mammothmodal2_preview.py @@ -9,6 +9,11 @@ from typing import Any # bespoke MammothModa2 example used to convert image dimensions -> AR grid dims. _PATCH_SIZE = 16 +# Shared by Preview and Dev (see ``t2i_generation_config.json``). +_EOL_TOKEN_ID = 152064 +_VISUAL_TOKEN_START_ID = 152072 +_VISUAL_TOKEN_END_ID = 168456 + _AR_SYSTEM_PROMPT = "You are a helpful image generator." @@ -26,9 +31,8 @@ def build_text_to_image_prompt( Model-specific sampling knobs (``text_guidance_scale``, ``cfg_range``, ``num_inference_steps``) flow separately via ``extra_body`` -> ``extra_args``. - Config-derived token ids (``eol_token_id``, ``visual_token_start_id``, - ``visual_token_end_id``, ``visual_ids``) are sourced inside the AR stage - rather than passed by the caller -- see the ar2dit stage input processor. + The structural token ids are included in ``additional_information`` so the + AR sampler can apply the same constraints for both Preview and Dev. MammothModa2 t2i uses classifier-free guidance via ``text_guidance_scale`` and has no explicit negative-prompt path, so ``negative_prompt`` is accepted @@ -52,6 +56,9 @@ def build_text_to_image_prompt( "ar_height": [ar_height], "image_height": [h], "image_width": [w], + "eol_token_id": [_EOL_TOKEN_ID], + "visual_token_start_id": [_VISUAL_TOKEN_START_ID], + "visual_token_end_id": [_VISUAL_TOKEN_END_ID], }, } diff --git a/vllm_omni/model_extras/registry.py b/vllm_omni/model_extras/registry.py index 44051c081..edf28101c 100644 --- a/vllm_omni/model_extras/registry.py +++ b/vllm_omni/model_extras/registry.py @@ -243,6 +243,11 @@ _EXTRA_SPECS: dict[str, dict[str, Any]] = { }, } +# Multi-stage discovery reports the top-level wrapper rather than its DiT +# submodule, so both names must resolve to the same request adapters. +_EXTRA_SPECS["MammothModa2ForConditionalGeneration"] = _EXTRA_SPECS["MammothModa2DiTPipeline"] +_EXTRA_SPECS["Mammothmoda2Model"] = _EXTRA_SPECS["MammothModa2DiTPipeline"] + def _get_spec(model_class_name: str | None) -> dict[str, Any] | None: if not model_class_name: diff --git a/vllm_omni/transformers_utils/configs/mammoth_moda2.py b/vllm_omni/transformers_utils/configs/mammoth_moda2.py index 1c6352264..d0848d1ca 100644 --- a/vllm_omni/transformers_utils/configs/mammoth_moda2.py +++ b/vllm_omni/transformers_utils/configs/mammoth_moda2.py @@ -8,6 +8,11 @@ from transformers.models.qwen2_5_vl.configuration_qwen2_5_vl import ( Qwen2_5_VLTextConfig, Qwen2_5_VLVisionConfig, ) +from transformers.models.qwen3_vl.configuration_qwen3_vl import ( + Qwen3VLConfig, + Qwen3VLTextConfig, + Qwen3VLVisionConfig, +) from vllm_omni.tokenizers.mammoth_moda2_tokenizer import MammothUTokenizer @@ -16,6 +21,9 @@ __all__ = [ "Mammothmoda2Qwen2_5_VLConfig", "Mammothmoda2Qwen2_5_VLTextConfig", "Mammothmoda2Qwen2_5_VLVisionConfig", + "Mammothmoda2Qwen3VLConfig", + "Mammothmoda2Qwen3VLTextConfig", + "Mammothmoda2Qwen3VLVisionConfig", ] @@ -209,6 +217,77 @@ class Mammothmoda2Qwen2_5_VLConfig(Qwen2_5_VLConfig): self.tokenizer_class = "MammothUTokenizer" +class Mammothmoda2Qwen3VLVisionConfig(Qwen3VLVisionConfig): + model_type = "mammothmoda2_qwen3_vl_vision" + base_config_key = "vision_config" + + +class Mammothmoda2Qwen3VLTextConfig(Qwen3VLTextConfig): + model_type = "mammothmoda2_qwen3_vl_text" + base_config_key = "text_config" + + def __init__( + self, + extra_gen_vocab: bool = True, + gen_vocab_size: int = 32800, + gen_vocab_start_index: int | None = None, + moe_type: str = "ffn", + **kwargs, + ) -> None: + super().__init__(**kwargs) + self.base_vocab_size = int(self.vocab_size) + self.extra_gen_vocab = extra_gen_vocab + self.gen_vocab_size = gen_vocab_size + self.gen_vocab_start_index = self.vocab_size if gen_vocab_start_index is None else gen_vocab_start_index + self.moe_type = moe_type + if self.extra_gen_vocab: + self.vocab_size = int(self.gen_vocab_start_index) + int(self.gen_vocab_size) + + +class Mammothmoda2Qwen3VLConfig(Qwen3VLConfig): + """MammothModa2-Dev AR config backed by Qwen3-VL.""" + + model_type = "mammothmoda2_qwen3_vl" + sub_configs = { + "vision_config": Mammothmoda2Qwen3VLVisionConfig, + "text_config": Mammothmoda2Qwen3VLTextConfig, + } + keys_to_ignore_at_inference = ["past_key_values"] + + def __init__( + self, + text_config: dict | PretrainedConfig | None = None, + vision_config: dict | PretrainedConfig | None = None, + extra_gen_vocab: bool = True, + gen_vocab_size: int = 32800, + gen_vocab_start_index: int | None = None, + moe_type: str = "ffn", + **kwargs, + ) -> None: + if isinstance(vision_config, dict): + vision_config = self.sub_configs["vision_config"](**vision_config) + elif vision_config is None: + vision_config = self.sub_configs["vision_config"]() + extras = { + "extra_gen_vocab": extra_gen_vocab, + "gen_vocab_size": gen_vocab_size, + "gen_vocab_start_index": gen_vocab_start_index, + "moe_type": moe_type, + } + if isinstance(text_config, dict): + for key, value in extras.items(): + text_config.setdefault(key, value) + text_config = self.sub_configs["text_config"](**text_config) + elif text_config is None: + text_config = self.sub_configs["text_config"](**extras) + super().__init__(text_config=text_config, vision_config=vision_config, **kwargs) + self.extra_gen_vocab = getattr(self.text_config, "extra_gen_vocab", extra_gen_vocab) + self.gen_vocab_size = getattr(self.text_config, "gen_vocab_size", gen_vocab_size) + self.gen_vocab_start_index = getattr(self.text_config, "gen_vocab_start_index", gen_vocab_start_index) + self.moe_type = getattr(self.text_config, "moe_type", moe_type) + self.tokenizer_class = "MammothUTokenizer" + + class Mammothmoda2Config(PretrainedConfig): """Top-level MammothModa2 composition configuration""" @@ -289,6 +368,10 @@ AutoConfig.register(Mammothmoda2Config.model_type, Mammothmoda2Config) AutoConfig.register(Mammothmoda2Qwen2_5_VLConfig.model_type, Mammothmoda2Qwen2_5_VLConfig) AutoConfig.register(Mammothmoda2Qwen2_5_VLTextConfig.model_type, Mammothmoda2Qwen2_5_VLTextConfig) AutoConfig.register(Mammothmoda2Qwen2_5_VLVisionConfig.model_type, Mammothmoda2Qwen2_5_VLVisionConfig) +AutoConfig.register(Mammothmoda2Qwen3VLConfig.model_type, Mammothmoda2Qwen3VLConfig) +AutoConfig.register(Mammothmoda2Qwen3VLTextConfig.model_type, Mammothmoda2Qwen3VLTextConfig) +AutoConfig.register(Mammothmoda2Qwen3VLVisionConfig.model_type, Mammothmoda2Qwen3VLVisionConfig) # Register tokenizer_type -> the configs & AutoTokenizer AutoTokenizer.register(config_class=Mammothmoda2Config, slow_tokenizer_class=MammothUTokenizer) AutoTokenizer.register(config_class=Mammothmoda2Qwen2_5_VLConfig, slow_tokenizer_class=MammothUTokenizer) +AutoTokenizer.register(config_class=Mammothmoda2Qwen3VLConfig, slow_tokenizer_class=MammothUTokenizer) From 8001bb155dae5798a1ae891ae2529a314c6ee99a Mon Sep 17 00:00:00 2001 From: zzh <67720593+BLANKETusers@users.noreply.github.com> Date: Mon, 27 Jul 2026 17:15:08 +0800 Subject: [PATCH 111/240] Fix missing sequence_parallel_size in deploy config output (#5449) Signed-off-by: zzh <943967662@qq.com> --- tests/e2e/accuracy/test_hunyuan_image3_pixel_accuracy.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/e2e/accuracy/test_hunyuan_image3_pixel_accuracy.py b/tests/e2e/accuracy/test_hunyuan_image3_pixel_accuracy.py index 74fb14086..197044ef3 100644 --- a/tests/e2e/accuracy/test_hunyuan_image3_pixel_accuracy.py +++ b/tests/e2e/accuracy/test_hunyuan_image3_pixel_accuracy.py @@ -112,6 +112,7 @@ def _write_deploy_config(path: Path) -> None: f"the number of devices ({num_devices}): {parallel_config}" ) + parallel_config["sequence_parallel_size"] = sequence_parallel_size config_text = yaml.dump(config, default_flow_style=False, sort_keys=False) path.write_text(config_text) logger.info( From e9bc7fc06a39eb7abc57e2b984497ab493df0b4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B1=AA=E5=BF=97=E9=B9=8F?= Date: Mon, 27 Jul 2026 22:11:49 +0800 Subject: [PATCH 112/240] [Refactor]: Mammothmoda2-dev use x_to_text.py in examples (#5454) Signed-off-by: princepride --- .../offline_inference/x_to_text/README.md | 12 +++++ recipes/MammothModa2/MammothModa2.md | 54 +++++-------------- 2 files changed, 26 insertions(+), 40 deletions(-) diff --git a/examples/offline_inference/x_to_text/README.md b/examples/offline_inference/x_to_text/README.md index fb3f7ae14..de668b6ee 100644 --- a/examples/offline_inference/x_to_text/README.md +++ b/examples/offline_inference/x_to_text/README.md @@ -11,6 +11,7 @@ Generate text from text or image inputs with vLLM-Omni's shared offline entrypoi | `ByteDance-Seed/BAGEL-7B-MoT` | Yes | Yes | `vllm_omni/deploy/bagel.yaml` | | `tencent/HunyuanImage-3.0-Instruct` | Yes | Yes | `vllm_omni/deploy/hunyuan_image3_ar.yaml` | | `bytedance-research/MammothModa2-Preview` | Yes | Yes | `vllm_omni/deploy/mammoth_moda2_ar.yaml` | +| `bytedance-research/MammothModa2-Dev` | Yes | Yes | `vllm_omni/deploy/mammoth_moda2_ar.yaml` | The script recognizes these model families from `config.json`, applies the model-specific prompt format, and selects an AR-only deploy for HunyuanImage-3 @@ -42,6 +43,14 @@ python examples/offline_inference/x_to_text/x_to_text.py \ --prompt "Explain multimodal inference in three concise sentences." ``` +The same command supports the Dev checkpoint: + +```bash +python examples/offline_inference/x_to_text/x_to_text.py \ + --model bytedance-research/MammothModa2-Dev \ + --prompt "Explain multimodal inference in three concise sentences." +``` + ## Image-To-Text Pass one image with `--image`. The default question can be replaced with any @@ -70,6 +79,9 @@ python examples/offline_inference/x_to_text/x_to_text.py \ --prompt "Describe this image in detail." ``` +Use `bytedance-research/MammothModa2-Dev` in the same command to run I2T with +the Dev checkpoint. + ## Key Arguments | Argument | Default | Description | diff --git a/recipes/MammothModa2/MammothModa2.md b/recipes/MammothModa2/MammothModa2.md index 60c391dec..7056a6f8f 100644 --- a/recipes/MammothModa2/MammothModa2.md +++ b/recipes/MammothModa2/MammothModa2.md @@ -149,49 +149,23 @@ Download the checkpoint: hf download bytedance-research/MammothModa2-Dev --local-dir ./MammothModa2-Dev ``` -Run image-to-text inference from the repository root: +Run text-to-text through the shared understanding example. It recognizes the +Dev checkpoint as MammothModa2 and automatically selects +`mammoth_moda2_ar.yaml`: -```python -from PIL import Image -from vllm import SamplingParams -from vllm.multimodal.image import convert_image_mode -from vllm_omni import Omni +```bash +python examples/offline_inference/x_to_text/x_to_text.py \ + --model ./MammothModa2-Dev \ + --prompt "Explain multimodal generation in three sentences." +``` -prompt = ( - "<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n" - "<|im_start|>user\n<|vision_start|><|image_pad|><|vision_end|>" - "Summarize this image.<|im_end|>\n" - "<|im_start|>assistant\n" -) +Add an image for image-to-text or image summarization: -omni = Omni( - model="./MammothModa2-Dev", - deploy_config="vllm_omni/deploy/mammoth_moda2_ar.yaml", -) -try: - outputs = list( - omni.generate( - [{ - "prompt": prompt, - "multi_modal_data": { - "image": convert_image_mode(Image.open("./image.png"), "RGB"), - }, - "additional_information": {"omni_task": ["chat"]}, - }], - [SamplingParams( - temperature=0.2, - top_p=0.9, - top_k=-1, - max_tokens=512, - seed=42, - )], - ) - ) -finally: - omni.close() - -request_output = getattr(outputs[-1], "request_output", outputs[-1]) -print(request_output.outputs[0].text.strip()) +```bash +python examples/offline_inference/x_to_text/x_to_text.py \ + --model ./MammothModa2-Dev \ + --image ./image.png \ + --prompt "Please summarize the content of this image." ``` The Dev checkpoint is approximately 47.55 GiB on disk. In the verified AR-only From 82172b112d727c32a8ad8504ba9d3e11b6b50c38 Mon Sep 17 00:00:00 2001 From: "ruirui(rein) yang" <73573651+R2-Y@users.noreply.github.com> Date: Mon, 27 Jul 2026 22:19:46 +0800 Subject: [PATCH 113/240] [BugFix][MiniCPM-o] Restore audio output without async chunk (#5455) Signed-off-by: R2-Y Signed-off-by: amy-why-3459 Co-authored-by: amy-why-3459 --- .../sched/test_omni_scheduling_coordinator.py | 1 + .../omni_connectors/test_adapter_and_flow.py | 21 +++- .../test_async_omni_engine_stage_init.py | 23 +++++ .../models/minicpmo_4_5/test_pipeline.py | 25 +++-- .../test_minicpmo_4_5_async_chunk.py | 63 ++++++++++++ .../core/sched/omni_scheduling_coordinator.py | 2 + vllm_omni/engine/stage_init_utils.py | 14 ++- .../models/minicpmo_4_5/pipeline.py | 2 + .../minicpmo_4_5_omni.py | 99 ++++++++++++++++++- 9 files changed, 233 insertions(+), 17 deletions(-) diff --git a/tests/core/sched/test_omni_scheduling_coordinator.py b/tests/core/sched/test_omni_scheduling_coordinator.py index 3685fa5a2..37794edae 100644 --- a/tests/core/sched/test_omni_scheduling_coordinator.py +++ b/tests/core/sched/test_omni_scheduling_coordinator.py @@ -119,6 +119,7 @@ class TestFullPayloadCoordinatorSelection(unittest.TestCase): ("CovoAudioForConditionalGeneration", "code2wav"), ("MiMoAudioModel", "code2wav"), ("Qwen3TTSCode2Wav", "code2wav"), + ("MiniCPMO45Code2Wav", "code2wav"), ("CosyVoice3Model", "cosyvoice3_code2wav"), ("IndexTTS2S2MelDecoder", "indextts2_s2mel_decoder"), ("DyninOmniForConditionalGeneration", "token2image"), diff --git a/tests/distributed/omni_connectors/test_adapter_and_flow.py b/tests/distributed/omni_connectors/test_adapter_and_flow.py index 471d3108a..1cabaf3b3 100644 --- a/tests/distributed/omni_connectors/test_adapter_and_flow.py +++ b/tests/distributed/omni_connectors/test_adapter_and_flow.py @@ -218,7 +218,8 @@ def test_get_connectors_for_stage(): assert stage_2_config["from_stage_1"]["spec"]["name"] == "C2" -def test_outgoing_only_async_stage_uses_sender_connector_spec(): +@pytest.mark.parametrize("async_chunk", [False, True]) +def test_outgoing_stage_uses_sender_connector_spec(async_chunk: bool): config = OmniTransferConfig( connectors={ ("1", "2"): ConnectorSpec( @@ -228,12 +229,28 @@ def test_outgoing_only_async_stage_uses_sender_connector_spec(): } ) - spec = get_stage_connector_spec(config, stage_id=1, async_chunk=True) + spec = get_stage_connector_spec(config, stage_id=1, async_chunk=async_chunk) assert spec["name"] == "SharedMemoryConnector" assert spec["extra"] == {"codec_chunk_frames": 25, "role": "sender"} +def test_full_payload_consumer_uses_receiver_connector_spec(): + config = OmniTransferConfig( + connectors={ + ("1", "2"): ConnectorSpec( + name="SharedMemoryConnector", + extra={"codec_chunk_frames": 25}, + ) + } + ) + + spec = get_stage_connector_spec(config, stage_id=2, async_chunk=False) + + assert spec["name"] == "SharedMemoryConnector" + assert spec["extra"] == {"codec_chunk_frames": 25, "role": "receiver"} + + def test_recv_with_missing_metadata(mocker: MockerFixture): """Test recv when queue payload is malformed (missing metadata).""" # Connector expects metadata but task doesn't have it diff --git a/tests/engine/test_async_omni_engine_stage_init.py b/tests/engine/test_async_omni_engine_stage_init.py index e98c58d74..db7d53b79 100644 --- a/tests/engine/test_async_omni_engine_stage_init.py +++ b/tests/engine/test_async_omni_engine_stage_init.py @@ -609,6 +609,29 @@ def test_build_engine_args_stage_model_overrides_parent_model(): assert engine_args["model"] == "/stage/model" +def test_build_engine_args_keeps_full_payload_connector_spec(): + from vllm_omni.engine.stage_init_utils import build_engine_args_dict + + stage_cfg = types.SimpleNamespace( + stage_id=1, + stage_type="llm", + engine_args={"async_chunk": False}, + default_sampling_params={}, + ) + connector_spec = { + "name": "SharedMemoryConnector", + "extra": {"role": "sender"}, + } + + engine_args = build_engine_args_dict( + stage_cfg, + "/parent/model", + stage_connector_spec=connector_spec, + ) + + assert engine_args["stage_connector_spec"] == connector_spec + + def test_build_engine_args_keeps_stage_owned_tokenizer_subdir(tmp_path): from vllm_omni.engine.stage_init_utils import build_engine_args_dict diff --git a/tests/model_executor/models/minicpmo_4_5/test_pipeline.py b/tests/model_executor/models/minicpmo_4_5/test_pipeline.py index 968df04a9..2442cad44 100644 --- a/tests/model_executor/models/minicpmo_4_5/test_pipeline.py +++ b/tests/model_executor/models/minicpmo_4_5/test_pipeline.py @@ -6,7 +6,7 @@ Covers: - pipeline declared in the central registry - lazy loader returns the expected ``PipelineConfig`` - split 3-stage topology with no fused compatibility registration - - stage 1 routes through ``llm2tts`` and the async Code2Wav producer + - stage 1 routes through ``llm2tts`` and mode-specific Code2Wav producers - ``hf_architectures`` covers both the shared ``MiniCPMO`` alias and the explicit 4.5 arch - ``hf_config_predicate`` selects MiniCPM-o 4.5 only and rejects 2.6 @@ -105,7 +105,9 @@ class TestPipelineTopology: assert talker.engine_output_type == "latent" # scope KV cache / mrope sizing to talker sub-config assert talker.hf_config_name == "tts_config" - assert talker.custom_process_next_stage_input_func is None + assert talker.custom_process_next_stage_input_func == ( + "vllm_omni.model_executor.stage_input_processors.minicpmo_4_5_omni.tts2code2wav_full_payload" + ) assert talker.async_chunk_process_next_stage_input_func == ( "vllm_omni.model_executor.stage_input_processors.minicpmo_4_5_omni.tts2code2wav_async_chunk" ) @@ -130,7 +132,9 @@ class TestPipelineTopology: assert code2wav.final_output_type == "audio" assert code2wav.engine_output_type == "audio" assert code2wav.model_arch == "MiniCPMO45Code2Wav" - assert code2wav.sync_process_input_func is None + assert code2wav.sync_process_input_func == ( + "vllm_omni.model_executor.stage_input_processors.minicpmo_4_5_omni.tts2code2wav_token_only" + ) class TestDeployTopology: @@ -156,6 +160,8 @@ class TestDeployTopology: assert connector["extra"]["codec_left_context_frames"] == 3 assert connector["extra"]["connector_get_max_wait_first_chunk"] == 3000 assert connector["extra"]["connector_get_max_wait"] == 300 + expected_processor = "tts2code2wav_async_chunk" if deploy.async_chunk else "tts2code2wav_full_payload" + assert stages[1].yaml_engine_args["custom_process_next_stage_input_func"].endswith(expected_processor) if filename == "minicpmo_4_5.yaml": assert [stage.yaml_engine_args["max_num_seqs"] for stage in stages] == [4, 4, 4] assert [stage.yaml_engine_args["gpu_memory_utilization"] for stage in stages] == [ @@ -171,14 +177,21 @@ class TestDeployTopology: 0.35, ] - def test_pipeline_exposes_no_full_payload_or_token_placeholder_hooks(self) -> None: + def test_pipeline_exposes_full_and_async_payload_hooks(self) -> None: pipeline = OMNI_PIPELINES[_PIPELINE_KEY] talker = pipeline.get_stage(1) code2wav = pipeline.get_stage(2) assert talker is not None assert code2wav is not None - assert talker.custom_process_next_stage_input_func is None - assert code2wav.sync_process_input_func is None + assert talker.custom_process_next_stage_input_func == ( + "vllm_omni.model_executor.stage_input_processors.minicpmo_4_5_omni.tts2code2wav_full_payload" + ) + assert talker.async_chunk_process_next_stage_input_func == ( + "vllm_omni.model_executor.stage_input_processors.minicpmo_4_5_omni.tts2code2wav_async_chunk" + ) + assert code2wav.sync_process_input_func == ( + "vllm_omni.model_executor.stage_input_processors.minicpmo_4_5_omni.tts2code2wav_token_only" + ) def test_code2wav_model_is_lazily_registered() -> None: diff --git a/tests/model_executor/stage_input_processors/test_minicpmo_4_5_async_chunk.py b/tests/model_executor/stage_input_processors/test_minicpmo_4_5_async_chunk.py index 4f17b2c64..25124ed6f 100644 --- a/tests/model_executor/stage_input_processors/test_minicpmo_4_5_async_chunk.py +++ b/tests/model_executor/stage_input_processors/test_minicpmo_4_5_async_chunk.py @@ -10,6 +10,8 @@ from vllm.v1.request import RequestStatus from vllm_omni.model_executor.stage_input_processors.minicpmo_4_5_omni import ( tts2code2wav_async_chunk, + tts2code2wav_full_payload, + tts2code2wav_token_only, ) pytestmark = [pytest.mark.core_model, pytest.mark.cpu] @@ -132,6 +134,67 @@ def test_first_chunk_forwards_reference_voice_and_duplex_identity() -> None: assert payload.meta.turn_end is True +def test_full_payload_forwards_all_codes_and_request_metadata() -> None: + manager = _manager() + request = _request("req") + request.additional_information = { + "codes": {"ref": [0.1, -0.1]}, + "meta": { + "ref_audio_sr": 16000, + "native_duplex_segment_text": "hello", + "segment_end": True, + "turn_end": True, + }, + "duplex": {"epoch": 3, "model_turn_id": 7}, + } + + payload = tts2code2wav_full_payload( + transfer_manager=manager, + pooling_output={ + "codes.audio": torch.arange(7, dtype=torch.long).reshape(-1, 1), + "meta.finished": torch.tensor(True), + }, + request=request, + ) + + assert _codes(payload) == [4218, 4218, 4218, *range(7)] + assert payload.codes.ref.tolist() == pytest.approx([0.1, -0.1]) + assert payload.meta.request_id == "req" + assert payload.meta.chunk_seq == 0 + assert payload.meta.code_flat_numel == 10 + assert payload.meta.codec_chunk_frames == 7 + assert payload.meta.codec_left_context_frames == 3 + assert payload.meta.left_context_size == 3 + assert payload.meta.last_chunk is True + assert payload.meta.finished.item() is True + assert payload.meta.ref_audio_sr == 16000 + assert payload.meta.native_duplex_segment_text == "hello" + assert payload.meta.duplex_epoch == 3 + assert payload.meta.duplex_turn_id == 7 + assert payload.meta.segment_end is True + assert payload.meta.turn_end is True + + +def test_sync_token_only_reserves_codec_and_silence_slots() -> None: + output = SimpleNamespace( + finished=True, + outputs=[ + SimpleNamespace( + multimodal_output={ + "codes.audio": torch.arange(7, dtype=torch.long).reshape(-1, 1), + "meta.finished": torch.tensor(True), + } + ) + ], + ) + + prompts = tts2code2wav_token_only([output]) + + assert len(prompts) == 1 + assert prompts[0]["prompt_token_ids"] == [0] * 10 + assert prompts[0]["additional_information"] is None + + def test_empty_final_releases_wait_gate_once() -> None: manager = _manager() request = _request("req") diff --git a/vllm_omni/core/sched/omni_scheduling_coordinator.py b/vllm_omni/core/sched/omni_scheduling_coordinator.py index fc21be991..59df59a26 100644 --- a/vllm_omni/core/sched/omni_scheduling_coordinator.py +++ b/vllm_omni/core/sched/omni_scheduling_coordinator.py @@ -47,6 +47,8 @@ _FULL_PAYLOAD_INPUT_STAGES: frozenset[tuple[str, str]] = frozenset( # qwen3_tts: Qwen3TTSTalkerForConditionalGeneration (Stage 0) # -> Qwen3TTSCode2Wav (Stage 1). Stage 1 is the consumer. ("Qwen3TTSCode2Wav", "code2wav"), + # minicpmo_4_5: Talker (Stage 1) -> Code2Wav (Stage 2). + ("MiniCPMO45Code2Wav", "code2wav"), # cosyvoice3: cosyvoice3_talker (Stage 0) -> cosyvoice3_code2wav (Stage 1). ("CosyVoice3Model", "cosyvoice3_code2wav"), # indextts2: indextts2_talker (Stage 0) -> indextts2_s2mel_decoder diff --git a/vllm_omni/engine/stage_init_utils.py b/vllm_omni/engine/stage_init_utils.py index 9861d7b38..3b1bb93af 100644 --- a/vllm_omni/engine/stage_init_utils.py +++ b/vllm_omni/engine/stage_init_utils.py @@ -713,7 +713,7 @@ def build_engine_args_dict( # Stage id must come from stage config instead of inherited CLI kwargs # (e.g. `--stage-id` defaulting to None). engine_args_dict["stage_id"] = stage_id - if engine_args_dict.get("async_chunk", False): + if stage_connector_spec: engine_args_dict["stage_connector_spec"] = dict(stage_connector_spec or {}) if stage_type == "diffusion": @@ -1054,19 +1054,17 @@ def get_stage_connector_spec( stage_id: int, async_chunk: bool, ) -> dict[str, Any]: - """Return the first connector spec for the stage when async chunking is enabled.""" + """Return the first connector spec for a stage data-plane edge.""" from vllm_omni.distributed.omni_connectors import get_stage_connector_config - if not async_chunk: - return {} - stage_connectors_cfg = get_stage_connector_config(omni_transfer_config, stage_id) for cfg in stage_connectors_cfg.values(): return dict(cfg.get("spec", {})) - # A stage can be an async producer without consuming chunks itself. Keep - # its connector for save_async(), but mark it sender-only so the scheduler - # does not park normal orchestrator-provided inputs waiting for a chunk. + # A producer does not consume connector data itself. Keep its connector + # for both async-chunk and terminal full-payload sends, but mark it + # sender-only so the scheduler does not park orchestrator-provided inputs + # waiting for an upstream payload. target_stage = str(stage_id) for (from_stage, _to_stage), spec in getattr(omni_transfer_config, "connectors", {}).items(): if from_stage == target_stage: diff --git a/vllm_omni/model_executor/models/minicpmo_4_5/pipeline.py b/vllm_omni/model_executor/models/minicpmo_4_5/pipeline.py index 31d02c7cc..b8fed05f1 100644 --- a/vllm_omni/model_executor/models/minicpmo_4_5/pipeline.py +++ b/vllm_omni/model_executor/models/minicpmo_4_5/pipeline.py @@ -60,6 +60,7 @@ MINICPMO_4_5_PIPELINE = PipelineConfig( hf_config_name="tts_config", engine_output_type="latent", custom_process_input_func=f"{_PROC}.llm2tts", + custom_process_next_stage_input_func=f"{_PROC}.tts2code2wav_full_payload", async_chunk_process_next_stage_input_func=f"{_PROC}.tts2code2wav_async_chunk", sampling_constraints={"detokenize": False}, ), @@ -72,6 +73,7 @@ MINICPMO_4_5_PIPELINE = PipelineConfig( final_output_type="audio", engine_output_type="audio", model_arch="MiniCPMO45Code2Wav", + sync_process_input_func=f"{_PROC}.tts2code2wav_token_only", sampling_constraints={"detokenize": True}, ), ), diff --git a/vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py b/vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py index 4f052b68d..950d5434d 100644 --- a/vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py +++ b/vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py @@ -167,7 +167,7 @@ def _extract_codec_delta(pooling_output: Any, request_id: str) -> list[int]: if routed_id is not None and str(routed_id) != request_id: return [] codes = pooling_output.get("codes") - audio = codes.get("audio") if isinstance(codes, Mapping) else None + audio = codes.get("audio") if isinstance(codes, Mapping) else pooling_output.get("codes.audio") return _codec_scalars(audio) if isinstance(pooling_output, Sequence) and not isinstance( pooling_output, @@ -341,6 +341,103 @@ def tts2code2wav_async_chunk( ) +def tts2code2wav_full_payload( + transfer_manager: Any, + pooling_output: Any, + request: Any, +) -> OmniPayloadStruct: + """Build one terminal Code2Wav payload when async chunks are disabled.""" + external_id = getattr(request, "external_req_id", None) + internal_id = getattr(request, "request_id", None) + request_id = str(external_id if external_id is not None else internal_id) + codes = _extract_codec_delta(pooling_output, request_id) + _, left_context_frames = _codec_config(transfer_manager) + context = [_MINICPMO45_SILENCE_CODE] * left_context_frames if codes else [] + output_codes = [*context, *codes] + + request_info = getattr(request, "additional_information", None) + if not isinstance(request_info, Mapping): + request_info = {} + codes_info = request_info.get("codes") + if not isinstance(codes_info, Mapping): + codes_info = {} + meta_info = request_info.get("meta") + if not isinstance(meta_info, Mapping): + meta_info = {} + duplex_info = request_info.get("duplex") + if not isinstance(duplex_info, Mapping): + duplex_info = {} + + ref_audio = codes_info.get("ref") + finished = torch.tensor(True, dtype=torch.bool) + return OmniPayloadStruct( + codes=CodesStruct( + audio=torch.tensor(output_codes, dtype=torch.long), + ref=torch.as_tensor(ref_audio, dtype=torch.float32).reshape(-1) if ref_audio is not None else None, + ), + meta=_MiniCPMO45MetaStruct( + request_id=request_id, + chunk_seq=0, + cache_epoch=0, + code_flat_numel=len(output_codes), + codec_chunk_frames=len(codes), + codec_left_context_frames=len(context), + left_context_size=len(context), + last_chunk=True, + stream_finished=finished, + finished=finished, + req_id=[request_id], + ref_audio_sr=_coerce_int(meta_info.get("ref_audio_sr")), + native_duplex_segment_text=( + str(meta_info["native_duplex_segment_text"]) + if isinstance(meta_info.get("native_duplex_segment_text"), str) + else None + ), + duplex_turn_id=_coerce_int(duplex_info.get("model_turn_id", duplex_info.get("turn_id"))), + duplex_epoch=_coerce_int(duplex_info.get("epoch")), + segment_end=bool(meta_info.get("segment_end", False)), + turn_end=bool(meta_info.get("turn_end", False)), + tts_is_last_chunk=True, + ), + request_id=request_id, + ) + + +def tts2code2wav_token_only( + source_outputs: list[Any], + _prompt: Any = None, + _requires_multimodal_data: bool = False, +) -> list[OmniTokensPrompt]: + """Build the sync Code2Wav placeholder; codec data arrives by connector.""" + code2wav_inputs: list[OmniTokensPrompt] = [] + for talker_output in source_outputs: + if not talker_output.finished: + continue + output = talker_output.outputs[0] + multimodal_output = getattr(output, "multimodal_output", None) + codes = multimodal_output.get("codes") if isinstance(multimodal_output, Mapping) else None + audio = ( + codes.get("audio") + if isinstance(codes, Mapping) + else multimodal_output.get("codes.audio") + if isinstance(multimodal_output, Mapping) + else None + ) + codec_codes = _codec_scalars(audio) + if not codec_codes: + continue + prompt_len = len(codec_codes) + 3 + code2wav_inputs.append( + OmniTokensPrompt( + prompt_token_ids=[0] * prompt_len, + additional_information=None, + multi_modal_data=None, + mm_processor_kwargs=None, + ) + ) + return code2wav_inputs + + def _special_token_ids_from_mm_output(mm_output): if not isinstance(mm_output, Mapping): return {} From e108cbb51049046275deefa5eb7a9668d22133be Mon Sep 17 00:00:00 2001 From: jiangmengyu18 <56633611+jiangmengyu18@users.noreply.github.com> Date: Mon, 27 Jul 2026 22:41:34 +0800 Subject: [PATCH 114/240] [NPU][HunyuanImage3] Adapt diffusion FusedMoE for vLLM >= 0.24.0 (#5167) Signed-off-by: betta18 Signed-off-by: jiangmengyu18 <56633611+jiangmengyu18@users.noreply.github.com> Co-authored-by: betta18 Co-authored-by: Hongsheng Liu --- .../test_expert_parallel_layout.py | 6 +- tests/diffusion/layers/test_fused_moe.py | 215 ++++++++++++++++++ .../hunyuan_image3/test_hunyuan_fused_moe.py | 197 ---------------- .../diffusion/distributed/parallel_state.py | 7 +- .../fused_moe.py} | 65 +----- .../models/hunyuan_image3/__init__.py | 3 +- .../hunyuan_image3_transformer.py | 9 +- .../worker/diffusion_model_runner.py | 6 + vllm_omni/platforms/interface.py | 43 +++- vllm_omni/platforms/npu/layers/__init__.py | 2 + .../fused_moe.py} | 107 +++++---- vllm_omni/platforms/npu/platform.py | 37 ++- 12 files changed, 379 insertions(+), 318 deletions(-) create mode 100644 tests/diffusion/layers/test_fused_moe.py delete mode 100644 tests/diffusion/models/hunyuan_image3/test_hunyuan_fused_moe.py rename vllm_omni/diffusion/{models/hunyuan_image3/hunyuan_fused_moe.py => layers/fused_moe.py} (66%) create mode 100644 vllm_omni/platforms/npu/layers/__init__.py rename vllm_omni/platforms/npu/{models/hunyuan_fused_moe.py => layers/fused_moe.py} (52%) diff --git a/tests/diffusion/distributed/test_expert_parallel_layout.py b/tests/diffusion/distributed/test_expert_parallel_layout.py index c4d3c6ae7..bea7e9ccf 100644 --- a/tests/diffusion/distributed/test_expert_parallel_layout.py +++ b/tests/diffusion/distributed/test_expert_parallel_layout.py @@ -193,7 +193,7 @@ def test_moe_ep_maps_diffusion_sp_cfg_dp_to_vllm_groups(monkeypatch): @pytest.mark.cpu @pytest.mark.core_model def test_cfg_parallel_keeps_diffusion_dp_without_ep(monkeypatch): - """vLLM DP should keep diffusion DP when expert parallelism is not enabled.""" + """Without EP, keep diffusion DP and retain rank metadata for MC2.""" local_rank = 0 world_size = 8 @@ -249,7 +249,9 @@ def test_cfg_parallel_keeps_diffusion_dp_without_ep(monkeypatch): assert parallel_state.vllm_parallel_state._DP.local_group == [0, 4] assert parallel_state.vllm_parallel_state._PCP is None assert parallel_state.vllm_parallel_state._EP is None - assert parallel_state._EXPERT_PARALLEL_GROUP_RANKS is None + assert parallel_state._EXPERT_PARALLEL_GROUP_RANKS == [ + [0, 1, 2, 3, 4, 5, 6, 7], + ] @pytest.mark.cpu diff --git a/tests/diffusion/layers/test_fused_moe.py b/tests/diffusion/layers/test_fused_moe.py new file mode 100644 index 000000000..983e146ef --- /dev/null +++ b/tests/diffusion/layers/test_fused_moe.py @@ -0,0 +1,215 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Unit tests for the diffusion FusedMoE adapter.""" + +import sys +from types import SimpleNamespace + +import pytest +import torch + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +class TestSetForwardContextNumTokens: + """Test _set_forward_context_num_tokens defensive fix for vLLM 0.18.0.""" + + def test_sets_num_tokens_when_context_available(self, mocker): + """num_tokens should be set on ForwardContext when available.""" + import vllm_omni.diffusion.layers.fused_moe as fused_moe + + mock_ctx = mocker.MagicMock() + del mock_ctx.in_profile_run # simulate missing attr + mocker.patch.object(fused_moe._vllm_fc, "is_forward_context_available", return_value=True) + mocker.patch.object(fused_moe._vllm_fc, "get_forward_context", return_value=mock_ctx) + + fused_moe._set_forward_context_num_tokens(1024) + + assert mock_ctx.num_tokens == 1024 + assert mock_ctx.in_profile_run is False + + def test_sets_in_profile_run_only_if_missing(self, mocker): + """in_profile_run should not be overwritten if already set.""" + import vllm_omni.diffusion.layers.fused_moe as fused_moe + + mock_ctx = mocker.MagicMock() + mock_ctx.in_profile_run = True # already set + mocker.patch.object(fused_moe._vllm_fc, "is_forward_context_available", return_value=True) + mocker.patch.object(fused_moe._vllm_fc, "get_forward_context", return_value=mock_ctx) + + fused_moe._set_forward_context_num_tokens(512) + + assert mock_ctx.num_tokens == 512 + assert mock_ctx.in_profile_run is True # not overwritten + + def test_noop_when_context_unavailable(self, mocker): + """Should do nothing when ForwardContext is not available.""" + import vllm_omni.diffusion.layers.fused_moe as fused_moe + + mocker.patch.object(fused_moe._vllm_fc, "is_forward_context_available", return_value=False) + mock_get = mocker.patch.object(fused_moe._vllm_fc, "get_forward_context") + + fused_moe._set_forward_context_num_tokens(256) + + mock_get.assert_not_called() + + +class TestSetForwardContextDPMetadata: + """Test diffusion MoE DP metadata setup for vLLM 0.24 dispatch.""" + + def test_sets_dp_metadata_from_vllm_dp_group(self, mocker): + import vllm_omni.diffusion.layers.fused_moe as fused_moe + + mock_ctx = mocker.MagicMock() + mock_ctx.dp_metadata = None + fake_group = mocker.MagicMock() + fake_group.world_size = 2 + fake_group.cpu_group = object() + + def _all_gather_object(output, obj, group): + assert group is fake_group.cpu_group + output[:] = [obj, obj + 1] + + mocker.patch.object(fused_moe._vllm_fc, "is_forward_context_available", return_value=True) + mocker.patch.object(fused_moe._vllm_fc, "get_forward_context", return_value=mock_ctx) + mocker.patch.object(fused_moe._vllm_ps, "_DP", fake_group, create=True) + mocker.patch.object(fused_moe.torch.distributed, "all_gather_object", side_effect=_all_gather_object) + + fused_moe._set_forward_context_dp_metadata(1024) + + assert mock_ctx.dp_metadata is not None + assert mock_ctx.dp_metadata.num_tokens_across_dp_cpu.tolist() == [1024, 1025] + + def test_keeps_existing_dp_metadata(self, mocker): + import vllm_omni.diffusion.layers.fused_moe as fused_moe + + existing_metadata = object() + mock_ctx = mocker.MagicMock() + mock_ctx.dp_metadata = existing_metadata + mocker.patch.object(fused_moe._vllm_fc, "is_forward_context_available", return_value=True) + mocker.patch.object(fused_moe._vllm_fc, "get_forward_context", return_value=mock_ctx) + mock_all_gather = mocker.patch.object(fused_moe.torch.distributed, "all_gather_object") + + fused_moe._set_forward_context_dp_metadata(1024) + + assert mock_ctx.dp_metadata is existing_metadata + mock_all_gather.assert_not_called() + + +class TestFusedMoEFactory: + """Test FusedMoE factory __new__ and mapping delegation.""" + + def test_new_returns_vllm_runner_and_registers_hooks(self, mocker, monkeypatch): + import vllm_omni.diffusion.layers.fused_moe as fused_moe + + mock_platform = mocker.MagicMock() + mocker.patch.object(fused_moe, "current_omni_platform", mock_platform) + + mock_runner = mocker.MagicMock() + mock_vllm_fused_moe = mocker.MagicMock(return_value=mock_runner) + monkeypatch.setitem( + sys.modules, + "vllm.model_executor.layers.fused_moe", + SimpleNamespace(FusedMoE=mock_vllm_fused_moe), + ) + + result = fused_moe.FusedMoE(prefix="test", a=1, reduce_results=True) + + assert result is mock_runner + mock_platform.prepare_diffusion_op_runtime.assert_called_once_with("fused_moe") + mock_vllm_fused_moe.assert_called_once_with(prefix="test", a=1) + mock_runner.register_forward_pre_hook.assert_called_once_with( + mocker.ANY, + with_kwargs=True, + ) + mock_platform.register_additional_diffusion_fused_moe_hooks.assert_called_once_with(mock_runner) + + def test_num_tokens_pre_hook_uses_kwargs_hidden_states(self, mocker, monkeypatch): + import vllm_omni.diffusion.layers.fused_moe as fused_moe + + captured_hook = None + + class MockRunner: + def register_forward_pre_hook(self, hook, *, with_kwargs): + nonlocal captured_hook + assert with_kwargs is True + captured_hook = hook + + mock_runner = MockRunner() + monkeypatch.setitem( + sys.modules, + "vllm.model_executor.layers.fused_moe", + SimpleNamespace(FusedMoE=mocker.MagicMock(return_value=mock_runner)), + ) + mocker.patch.object(fused_moe, "current_omni_platform", mocker.MagicMock()) + mock_set_num_tokens = mocker.patch.object(fused_moe, "_set_forward_context_num_tokens") + mock_set_dp_metadata = mocker.patch.object(fused_moe, "_set_forward_context_dp_metadata") + + fused_moe.FusedMoE(prefix="") + assert captured_hook is not None + + hidden_states = torch.empty(17, 32) + captured_hook(mock_runner, (), {"hidden_states": hidden_states}) + + mock_set_num_tokens.assert_called_once_with(17) + mock_set_dp_metadata.assert_called_once_with(17) + + def test_num_tokens_pre_hook_uses_positional_hidden_states(self, mocker, monkeypatch): + import vllm_omni.diffusion.layers.fused_moe as fused_moe + + captured_hook = None + + class MockRunner: + def register_forward_pre_hook(self, hook, *, with_kwargs): + nonlocal captured_hook + assert with_kwargs is True + captured_hook = hook + + mock_runner = MockRunner() + monkeypatch.setitem( + sys.modules, + "vllm.model_executor.layers.fused_moe", + SimpleNamespace(FusedMoE=mocker.MagicMock(return_value=mock_runner)), + ) + mocker.patch.object(fused_moe, "current_omni_platform", mocker.MagicMock()) + mock_set_num_tokens = mocker.patch.object(fused_moe, "_set_forward_context_num_tokens") + mock_set_dp_metadata = mocker.patch.object(fused_moe, "_set_forward_context_dp_metadata") + + fused_moe.FusedMoE(prefix="") + assert captured_hook is not None + + hidden_states = torch.empty(23, 32) + captured_hook(mock_runner, (hidden_states,), {}) + + mock_set_num_tokens.assert_called_once_with(23) + mock_set_dp_metadata.assert_called_once_with(23) + + def test_make_expert_params_mapping_delegates_to_vllm_function(self, mocker, monkeypatch): + import vllm_omni.diffusion.layers.fused_moe as fused_moe + + expected_mapping = [("a", "b", 0, "c")] + mock_mapping_fn = mocker.MagicMock(return_value=expected_mapping) + monkeypatch.setitem( + sys.modules, + "vllm.model_executor.layers.fused_moe", + SimpleNamespace(fused_moe_make_expert_params_mapping=mock_mapping_fn), + ) + + result = fused_moe.FusedMoE.make_expert_params_mapping( + model=None, + ckpt_gate_proj_name="gate", + ckpt_down_proj_name="down", + ckpt_up_proj_name="up", + num_experts=4, + num_redundant_experts=0, + ) + + assert result == expected_mapping + mock_mapping_fn.assert_called_once_with( + None, + ckpt_gate_proj_name="gate", + ckpt_down_proj_name="down", + ckpt_up_proj_name="up", + num_experts=4, + num_redundant_experts=0, + ) diff --git a/tests/diffusion/models/hunyuan_image3/test_hunyuan_fused_moe.py b/tests/diffusion/models/hunyuan_image3/test_hunyuan_fused_moe.py deleted file mode 100644 index e8ade2e51..000000000 --- a/tests/diffusion/models/hunyuan_image3/test_hunyuan_fused_moe.py +++ /dev/null @@ -1,197 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Unit tests for HunyuanFusedMoE (Support HunyuanImage3 Diffusion Model, 5a779b4).""" - -import pytest - -pytestmark = [pytest.mark.core_model, pytest.mark.cpu] - - -class TestSetForwardContextNumTokens: - """Test _set_forward_context_num_tokens defensive fix for vLLM 0.18.0.""" - - def test_sets_num_tokens_when_context_available(self, mocker): - """num_tokens should be set on ForwardContext when available.""" - import vllm_omni.diffusion.models.hunyuan_image3.hunyuan_fused_moe as hunyuan_moe - - mock_ctx = mocker.MagicMock() - del mock_ctx.in_profile_run # simulate missing attr - mocker.patch.object(hunyuan_moe._vllm_fc, "is_forward_context_available", return_value=True) - mocker.patch.object(hunyuan_moe._vllm_fc, "get_forward_context", return_value=mock_ctx) - - hunyuan_moe._set_forward_context_num_tokens(1024) - - assert mock_ctx.num_tokens == 1024 - assert mock_ctx.in_profile_run is False - - def test_sets_in_profile_run_only_if_missing(self, mocker): - """in_profile_run should not be overwritten if already set.""" - import vllm_omni.diffusion.models.hunyuan_image3.hunyuan_fused_moe as hunyuan_moe - - mock_ctx = mocker.MagicMock() - mock_ctx.in_profile_run = True # already set - mocker.patch.object(hunyuan_moe._vllm_fc, "is_forward_context_available", return_value=True) - mocker.patch.object(hunyuan_moe._vllm_fc, "get_forward_context", return_value=mock_ctx) - - hunyuan_moe._set_forward_context_num_tokens(512) - - assert mock_ctx.num_tokens == 512 - assert mock_ctx.in_profile_run is True # not overwritten - - def test_noop_when_context_unavailable(self, mocker): - """Should do nothing when ForwardContext is not available.""" - import vllm_omni.diffusion.models.hunyuan_image3.hunyuan_fused_moe as hunyuan_moe - - mocker.patch.object(hunyuan_moe._vllm_fc, "is_forward_context_available", return_value=False) - mock_get = mocker.patch.object(hunyuan_moe._vllm_fc, "get_forward_context") - - hunyuan_moe._set_forward_context_num_tokens(256) - - mock_get.assert_not_called() - - -class TestSetForwardContextDPMetadata: - """Test diffusion MoE DP metadata setup for vLLM 0.24 all2all dispatch.""" - - def test_sets_dp_metadata_from_vllm_dp_group(self, mocker): - import vllm_omni.diffusion.models.hunyuan_image3.hunyuan_fused_moe as hunyuan_moe - - mock_ctx = mocker.MagicMock() - mock_ctx.dp_metadata = None - fake_group = mocker.MagicMock() - fake_group.world_size = 2 - fake_group.cpu_group = object() - - def _all_gather_object(output, obj, group): - assert group is fake_group.cpu_group - output[:] = [obj, obj + 1] - - mocker.patch.object(hunyuan_moe._vllm_fc, "is_forward_context_available", return_value=True) - mocker.patch.object(hunyuan_moe._vllm_fc, "get_forward_context", return_value=mock_ctx) - mocker.patch.object(hunyuan_moe._vllm_ps, "_DP", fake_group, create=True) - mocker.patch.object(hunyuan_moe.torch.distributed, "all_gather_object", side_effect=_all_gather_object) - - hunyuan_moe._set_forward_context_dp_metadata(1024) - - assert mock_ctx.dp_metadata is not None - assert mock_ctx.dp_metadata.num_tokens_across_dp_cpu.tolist() == [1024, 1025] - - def test_keeps_existing_dp_metadata(self, mocker): - import vllm_omni.diffusion.models.hunyuan_image3.hunyuan_fused_moe as hunyuan_moe - - existing_metadata = object() - mock_ctx = mocker.MagicMock() - mock_ctx.dp_metadata = existing_metadata - mocker.patch.object(hunyuan_moe._vllm_fc, "is_forward_context_available", return_value=True) - mocker.patch.object(hunyuan_moe._vllm_fc, "get_forward_context", return_value=mock_ctx) - mock_all_gather = mocker.patch.object(hunyuan_moe.torch.distributed, "all_gather_object") - - hunyuan_moe._set_forward_context_dp_metadata(1024) - - assert mock_ctx.dp_metadata is existing_metadata - mock_all_gather.assert_not_called() - - -class TestHunyuanFusedMoEPlatformDispatch: - """Test platform dispatch via platform qualname hooks.""" - - def test_default_platform_uses_default_impl_qualname(self, mocker): - """HunyuanFusedMoE should resolve the impl class from the platform hook.""" - import vllm_omni.diffusion.models.hunyuan_image3.hunyuan_fused_moe as hunyuan_moe - - mock_platform = mocker.MagicMock() - mock_platform.get_diffusion_model_impl_qualname.return_value = ( - "vllm_omni.diffusion.models.hunyuan_image3.hunyuan_fused_moe.HunyuanFusedMoEDefault" - ) - - mocker.patch.object( - hunyuan_moe, - "current_omni_platform", - mock_platform, - ) - mock_resolve = mocker.patch.object(hunyuan_moe, "resolve_obj_by_qualname") - mock_impl = mocker.MagicMock() - mock_resolve.return_value = mock_impl - - from vllm_omni.diffusion.models.hunyuan_image3.hunyuan_fused_moe import ( - HunyuanFusedMoE, - ) - - HunyuanFusedMoE(prefix="") - - mock_platform.prepare_diffusion_op_runtime.assert_called_once_with("hunyuan_fused_moe") - mock_platform.get_diffusion_model_impl_qualname.assert_called_once_with("hunyuan_fused_moe") - mock_resolve.assert_called_once_with( - "vllm_omni.diffusion.models.hunyuan_image3.hunyuan_fused_moe.HunyuanFusedMoEDefault" - ) - mock_impl.assert_called_once_with(prefix="") - - -class TestHunyuanFusedMoEFactory: - """Test HunyuanFusedMoE factory __new__ and make_expert_params_mapping delegation.""" - - def test_new_delegates_to_impl_class(self, mocker): - """HunyuanFusedMoE(prefix=..., **kwargs) should instantiate and return impl instance.""" - import vllm_omni.diffusion.models.hunyuan_image3.hunyuan_fused_moe as hunyuan_moe - - class MockImpl: - def __init__(self, *, prefix: str = "", **kwargs): - self.prefix = prefix - self.kwargs = kwargs - - mock_platform = mocker.MagicMock() - mock_platform.get_diffusion_model_impl_qualname.return_value = "mock.impl.Qualname" - mocker.patch.object(hunyuan_moe, "current_omni_platform", mock_platform) - - mock_impl_class = mocker.MagicMock(return_value=MockImpl(prefix="test", a=1)) - mocker.patch.object(hunyuan_moe, "resolve_obj_by_qualname", return_value=mock_impl_class) - - from vllm_omni.diffusion.models.hunyuan_image3.hunyuan_fused_moe import ( - HunyuanFusedMoE, - ) - - result = HunyuanFusedMoE(prefix="test", a=1) - - assert isinstance(result, MockImpl) - assert result.prefix == "test" - assert result.kwargs == {"a": 1} - mock_platform.prepare_diffusion_op_runtime.assert_called_once_with("hunyuan_fused_moe") - mock_platform.get_diffusion_model_impl_qualname.assert_called_once_with("hunyuan_fused_moe") - mock_impl_class.assert_called_once_with(prefix="test", a=1) - - def test_make_expert_params_mapping_delegates_to_impl(self, mocker): - """make_expert_params_mapping should delegate to impl class method.""" - import vllm_omni.diffusion.models.hunyuan_image3.hunyuan_fused_moe as hunyuan_moe - - expected_mapping = [("a", "b", 0, "c")] - mock_platform = mocker.MagicMock() - mock_platform.get_diffusion_model_impl_qualname.return_value = "mock.impl.Qualname" - mocker.patch.object(hunyuan_moe, "current_omni_platform", mock_platform) - - mock_impl_class = mocker.MagicMock() - mock_impl_class.make_expert_params_mapping = mocker.MagicMock(return_value=expected_mapping) - mocker.patch.object(hunyuan_moe, "resolve_obj_by_qualname", return_value=mock_impl_class) - - from vllm_omni.diffusion.models.hunyuan_image3.hunyuan_fused_moe import ( - HunyuanFusedMoE, - ) - - result = HunyuanFusedMoE.make_expert_params_mapping( - model=None, - ckpt_gate_proj_name="gate", - ckpt_down_proj_name="down", - ckpt_up_proj_name="up", - num_experts=4, - num_redundant_experts=0, - ) - - assert result == expected_mapping - mock_platform.get_diffusion_model_impl_qualname.assert_called_once_with("hunyuan_fused_moe") - mock_impl_class.make_expert_params_mapping.assert_called_once_with( - None, - ckpt_gate_proj_name="gate", - ckpt_down_proj_name="down", - ckpt_up_proj_name="up", - num_experts=4, - num_redundant_experts=0, - ) diff --git a/vllm_omni/diffusion/distributed/parallel_state.py b/vllm_omni/diffusion/distributed/parallel_state.py index 4b531fec9..20fcc9dea 100644 --- a/vllm_omni/diffusion/distributed/parallel_state.py +++ b/vllm_omni/diffusion/distributed/parallel_state.py @@ -312,7 +312,6 @@ def get_allgather_parallel_rank(): def get_expert_parallel_group_ranks() -> list[list[int]]: - assert vllm_parallel_state._EP is not None, "expert parallel group is not initialized" assert _EXPERT_PARALLEL_GROUP_RANKS is not None, "expert parallel group ranks are not initialized" return _EXPERT_PARALLEL_GROUP_RANKS @@ -948,16 +947,14 @@ def initialize_model_parallel( parallel_mode="fully_shard", ) - _EXPERT_PARALLEL_GROUP_RANKS = None + _EXPERT_PARALLEL_GROUP_RANKS = rank_generator.get_ranks("tp-sp-cfg-dp") if use_moe_parallel_mapping: - ep_group_ranks = rank_generator.get_ranks("tp-sp-cfg-dp") vllm_parallel_state._EP = init_vllm_model_parallel_group( - group_ranks=ep_group_ranks, + group_ranks=_EXPERT_PARALLEL_GROUP_RANKS, local_rank=get_world_group().local_rank, backend=backend, group_name="ep", ) - _EXPERT_PARALLEL_GROUP_RANKS = ep_group_ranks init_dit_group(dit_parallel_size, backend) diff --git a/vllm_omni/diffusion/models/hunyuan_image3/hunyuan_fused_moe.py b/vllm_omni/diffusion/layers/fused_moe.py similarity index 66% rename from vllm_omni/diffusion/models/hunyuan_image3/hunyuan_fused_moe.py rename to vllm_omni/diffusion/layers/fused_moe.py index fc1317a48..c5ee02487 100644 --- a/vllm_omni/diffusion/models/hunyuan_image3/hunyuan_fused_moe.py +++ b/vllm_omni/diffusion/layers/fused_moe.py @@ -6,7 +6,6 @@ from typing import Any import torch import vllm.distributed.parallel_state as _vllm_ps import vllm.forward_context as _vllm_fc -from vllm.utils.import_utils import resolve_obj_by_qualname from vllm_omni.platforms import current_omni_platform @@ -50,18 +49,18 @@ def _set_forward_context_dp_metadata(num_tokens: int) -> None: forward_context.dp_metadata = _vllm_fc.DPMetadata(torch.tensor(gathered_num_tokens, dtype=torch.int64)) -class HunyuanFusedMoEDefault: - """Adapter that configures the upstream FusedMoE ``MoERunner`` for HunyuanImage3. +class FusedMoE: + """Adapter that configures the upstream FusedMoE ``MoERunner`` for diffusion models. Upstream commit dc68bd8c41 refactored FusedMoE from a class (``nn.Module``) into a factory function that returns a ``MoERunner`` instance, whose expert weights live in a ``routed_experts`` submodule (``...experts.routed_experts.w13_weight`` / ``...w2_weight``). - This adapter builds that runner, installs the omni-specific forward-context - setup and one-shot kernel-initialisation hook, and returns the runner - *directly* from ``__new__`` so the parent MoE block registers it as a real - ``nn.Module`` submodule. + This adapter builds that runner, installs the diffusion forward-context + setup and additional platform hooks, and returns the runner *directly* from + ``__new__`` so the parent MoE block registers it as a real ``nn.Module`` + submodule. Returning the runner (rather than wrapping it in a plain object that holds it in an attribute) is required for correctness: a non-Module wrapper hides the @@ -71,9 +70,11 @@ class HunyuanFusedMoEDefault: """ def __new__(cls, *, prefix: str = "", **kwargs: Any) -> Any: + current_omni_platform.prepare_diffusion_op_runtime("fused_moe") + # Current vLLM FusedMoE handles output reduction internally. kwargs.pop("reduce_results", None) - # FusedMoE is now a factory function — call it to get a MoERunner. + # FusedMoE is now a factory function; call it to get a MoERunner. from vllm.model_executor.layers.fused_moe import FusedMoE as _FusedMoE moe_runner = _FusedMoE(prefix=prefix, **kwargs) @@ -91,21 +92,7 @@ class HunyuanFusedMoEDefault: _set_forward_context_dp_metadata(hidden_states.shape[0]) moe_runner.register_forward_pre_hook(_num_tokens_pre_hook, with_kwargs=True) - - # One-shot lazy kernel initialisation on the first forward (no-op unless - # the runner exposes an uninitialised quant_method). Mirrors the prior - # wrapper behaviour exactly, just bound to the runner module. - init_handle: Any = None - - def _kernel_init_pre_hook(module: Any, args: Any, kwargs: Any) -> None: - nonlocal init_handle - quant_method = getattr(module, "quant_method", None) - if quant_method is not None and getattr(quant_method, "moe_kernel", None) is None: - quant_method.process_weights_after_loading(module) - if init_handle is not None: - init_handle.remove() - - init_handle = moe_runner.register_forward_pre_hook(_kernel_init_pre_hook, with_kwargs=True) + current_omni_platform.register_additional_diffusion_fused_moe_hooks(moe_runner) return moe_runner @@ -138,35 +125,3 @@ class HunyuanFusedMoEDefault: num_experts=num_experts, num_redundant_experts=num_redundant_experts, ) - - -class HunyuanFusedMoE: - def __new__(cls, *, prefix: str = "", **kwargs: Any) -> Any: - op_name = "hunyuan_fused_moe" - current_omni_platform.prepare_diffusion_op_runtime(op_name) - impl = resolve_obj_by_qualname( - current_omni_platform.get_diffusion_model_impl_qualname(op_name), - ) - return impl(prefix=prefix, **kwargs) - - @classmethod - def make_expert_params_mapping( - cls, - model: Any, - ckpt_gate_proj_name: str, - ckpt_down_proj_name: str, - ckpt_up_proj_name: str, - num_experts: int, - num_redundant_experts: int = 0, - ) -> list[tuple[str, str, int, str]]: - impl = resolve_obj_by_qualname( - current_omni_platform.get_diffusion_model_impl_qualname("hunyuan_fused_moe"), - ) - return impl.make_expert_params_mapping( - model, - ckpt_gate_proj_name=ckpt_gate_proj_name, - ckpt_down_proj_name=ckpt_down_proj_name, - ckpt_up_proj_name=ckpt_up_proj_name, - num_experts=num_experts, - num_redundant_experts=num_redundant_experts, - ) diff --git a/vllm_omni/diffusion/models/hunyuan_image3/__init__.py b/vllm_omni/diffusion/models/hunyuan_image3/__init__.py index 6612bd855..625895d99 100644 --- a/vllm_omni/diffusion/models/hunyuan_image3/__init__.py +++ b/vllm_omni/diffusion/models/hunyuan_image3/__init__.py @@ -2,7 +2,6 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """Hunyuan Image 3 diffusion model components.""" -from vllm_omni.diffusion.models.hunyuan_image3.hunyuan_fused_moe import HunyuanFusedMoE from vllm_omni.diffusion.models.hunyuan_image3.hunyuan_image3_transformer import ( HunyuanImage3Model, HunyuanImage3Text2ImagePipeline, @@ -11,4 +10,4 @@ from vllm_omni.diffusion.models.hunyuan_image3.pipeline_hunyuan_image3 import ( HunyuanImage3Pipeline, ) -__all__ = ["HunyuanImage3Pipeline", "HunyuanImage3Model", "HunyuanImage3Text2ImagePipeline", "HunyuanFusedMoE"] +__all__ = ["HunyuanImage3Pipeline", "HunyuanImage3Model", "HunyuanImage3Text2ImagePipeline"] diff --git a/vllm_omni/diffusion/models/hunyuan_image3/hunyuan_image3_transformer.py b/vllm_omni/diffusion/models/hunyuan_image3/hunyuan_image3_transformer.py index 5f72a607d..f749d73d3 100644 --- a/vllm_omni/diffusion/models/hunyuan_image3/hunyuan_image3_transformer.py +++ b/vllm_omni/diffusion/models/hunyuan_image3/hunyuan_image3_transformer.py @@ -76,11 +76,12 @@ from vllm_omni.diffusion.distributed.sp_plan import ( ) from vllm_omni.diffusion.distributed.utils import get_local_device from vllm_omni.diffusion.forward_context import set_forward_context_denoise_step_idx +from vllm_omni.diffusion.layers.fused_moe import FusedMoE from vllm_omni.diffusion.layers.norm import RMSNorm from vllm_omni.diffusion.layers.rope import RotaryEmbedding -from vllm_omni.diffusion.models.hunyuan_image3.hunyuan_fused_moe import HunyuanFusedMoE from vllm_omni.diffusion.utils.kv_utils import repeat_kv from vllm_omni.model_executor.layers.timestep_embedding import timestep_embedding +from vllm_omni.platforms import current_omni_platform logger = logging.getLogger(__name__) @@ -1620,7 +1621,7 @@ class HunYuanSparseMoeBlock(nn.Module): self.shared_mlp = None enable_expert_parallel = get_current_vllm_config().parallel_config.enable_expert_parallel - self.experts = HunyuanFusedMoE( + self.experts = FusedMoE( shared_experts=self.shared_mlp, num_experts=self.n_routed_experts, top_k=top_k, @@ -2100,7 +2101,7 @@ class HunyuanImage3Model(nn.Module): if _is_moe(self.config): # Params for weights, fp8 weight scales, fp8 activation scales # (param_name, weight_name, expert_id, shard_id) - fused_moe_expert_mapping = HunyuanFusedMoE.make_expert_params_mapping( + fused_moe_expert_mapping = FusedMoE.make_expert_params_mapping( self, ckpt_gate_proj_name="gate_proj", ckpt_down_proj_name="down_proj", @@ -2381,6 +2382,8 @@ class HunyuanImage3Model(nn.Module): ar_kv_reuse_len: int = 0, full_attn_spans: list[list[tuple[int, int]]] | None = None, ) -> tuple | BaseModelOutputWithPast: + current_omni_platform.reset_diffusion_fused_moe_forward_context() + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states diff --git a/vllm_omni/diffusion/worker/diffusion_model_runner.py b/vllm_omni/diffusion/worker/diffusion_model_runner.py index c54b721f8..be68f071f 100644 --- a/vllm_omni/diffusion/worker/diffusion_model_runner.py +++ b/vllm_omni/diffusion/worker/diffusion_model_runner.py @@ -205,6 +205,12 @@ class DiffusionModelRunner(OmniConnectorModelRunnerMixin): if load_format == "dummy": return + current_omni_platform.init_diffusion_model_runner_runtime( + vllm_config=self.vllm_config, + od_config=self.od_config, + device=self.device, + ) + load_device = ( "cpu" if self.od_config.enable_cpu_offload or self.od_config.enable_layerwise_offload else str(self.device) ) diff --git a/vllm_omni/platforms/interface.py b/vllm_omni/platforms/interface.py index 54725b553..c4e4c6e37 100644 --- a/vllm_omni/platforms/interface.py +++ b/vllm_omni/platforms/interface.py @@ -70,13 +70,31 @@ class OmniPlatform(Platform): raise NotImplementedError @classmethod - def get_diffusion_model_impl_qualname(cls, op_name: str) -> str: - if op_name == "hunyuan_fused_moe": - return "vllm_omni.diffusion.models.hunyuan_image3.hunyuan_fused_moe.HunyuanFusedMoEDefault" - raise NotImplementedError(f"Unsupported diffusion model op: {op_name}") + def prepare_diffusion_op_runtime(cls, op_name: str, **kwargs: Any) -> None: + return None @classmethod - def prepare_diffusion_op_runtime(cls, op_name: str, **kwargs: Any) -> None: + def register_additional_diffusion_fused_moe_hooks(cls, moe_runner: Any) -> None: + # One-shot lazy kernel initialisation on the first forward (no-op unless + # the runner exposes an uninitialised quant_method). Mirrors the prior + # wrapper behaviour exactly, just bound to the runner module. + init_handle: Any = None + + def _kernel_init_pre_hook(module: Any, args: Any, kwargs: Any) -> None: + nonlocal init_handle + quant_method = getattr(module, "quant_method", None) + if quant_method is not None and getattr(quant_method, "moe_kernel", None) is None: + quant_method.process_weights_after_loading(module) + if init_handle is not None: + init_handle.remove() + + init_handle = moe_runner.register_forward_pre_hook( + _kernel_init_pre_hook, + with_kwargs=True, + ) + + @classmethod + def reset_diffusion_fused_moe_forward_context(cls) -> None: return None @classmethod @@ -137,10 +155,23 @@ class OmniPlatform(Platform): return "vllm_omni.diffusion.worker.diffusion_model_runner.DiffusionModelRunner" @classmethod - def init_diffusion_worker_vllm_config(cls, vllm_config: Any) -> None: + def init_diffusion_worker_vllm_config( + cls, + vllm_config: Any, + ) -> None: """Initialize platform-specific state for diffusion worker VllmConfig.""" return None + @classmethod + def init_diffusion_model_runner_runtime( + cls, + vllm_config: Any, + od_config: Any, + device: torch.device, + ) -> None: + """Initialize platform-specific runtime state for diffusion model runners.""" + return None + @classmethod def get_torch_device(cls, local_rank: int | None = None) -> torch.device: raise NotImplementedError diff --git a/vllm_omni/platforms/npu/layers/__init__.py b/vllm_omni/platforms/npu/layers/__init__.py new file mode 100644 index 000000000..208f01a7c --- /dev/null +++ b/vllm_omni/platforms/npu/layers/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project diff --git a/vllm_omni/platforms/npu/models/hunyuan_fused_moe.py b/vllm_omni/platforms/npu/layers/fused_moe.py similarity index 52% rename from vllm_omni/platforms/npu/models/hunyuan_fused_moe.py rename to vllm_omni/platforms/npu/layers/fused_moe.py index 251eac2cf..e17610701 100644 --- a/vllm_omni/platforms/npu/models/hunyuan_fused_moe.py +++ b/vllm_omni/platforms/npu/layers/fused_moe.py @@ -4,14 +4,14 @@ from typing import Any import torch +import vllm.distributed.parallel_state as vllm_parallel_state import vllm.forward_context as _vllm_fc from vllm.config import VllmConfig -from vllm.distributed import get_ep_group +from vllm.distributed import get_dp_group, get_ep_group from vllm.distributed.parallel_state import ( init_model_parallel_group as vllm_init_model_parallel_group, ) from vllm_ascend.ascend_forward_context import MoECommType -from vllm_ascend.ops.fused_moe.fused_moe import AscendFusedMoE from vllm_ascend.ops.fused_moe.moe_comm_method import _MoECommMethods from vllm_ascend.utils import AscendDeviceType, get_ascend_device_type @@ -29,17 +29,6 @@ def _ensure_forward_context_attr(name: str, annotation: Any, default: Any) -> No setattr(_vllm_fc.ForwardContext, name, default) -def _set_hunyuan_fused_moe_forward_context(num_tokens: int) -> None: - if not _vllm_fc.is_forward_context_available(): - return - - forward_context = _vllm_fc.get_forward_context() - forward_context.num_tokens = num_tokens - forward_context.moe_comm_type = _select_moe_comm_method(vllm_config=omni_get_ctx().vllm_config) - forward_context.moe_comm_method = _MoECommMethods.get(forward_context.moe_comm_type) - forward_context.flash_comm_v1_enabled = False - - def _init_mc2_group_for_diffusion( backend: str, local_rank: int, @@ -74,16 +63,16 @@ def _select_moe_comm_method(vllm_config: VllmConfig) -> MoECommType | None: return moe_comm_type -def prepare_hunyuan_fused_moe_runtime() -> None: +def prepare_fused_moe_runtime() -> None: vllm_config = omni_get_ctx().vllm_config - if vllm_config.parallel_config.enable_expert_parallel: - backend = torch.distributed.get_backend(get_world_group().device_group) - local_rank = get_world_group().local_rank - _init_mc2_group_for_diffusion( - backend=backend, - local_rank=local_rank, - group_ranks=get_expert_parallel_group_ranks(), - ) + + backend = torch.distributed.get_backend(get_world_group().device_group) + local_rank = get_world_group().local_rank + _init_mc2_group_for_diffusion( + backend=backend, + local_rank=local_rank, + group_ranks=get_expert_parallel_group_ranks(), + ) moe_comm_type = _select_moe_comm_method(vllm_config=vllm_config) _ensure_forward_context_attr("num_tokens", int | None, None) @@ -91,24 +80,66 @@ def prepare_hunyuan_fused_moe_runtime() -> None: _ensure_forward_context_attr("moe_comm_type", MoECommType | None, moe_comm_type) _ensure_forward_context_attr("moe_comm_method", Any, _MoECommMethods.get(moe_comm_type)) _ensure_forward_context_attr("flash_comm_v1_enabled", bool, False) + _ensure_forward_context_attr("max_tokens_across_dp", int | None, None) + _ensure_forward_context_attr("max_tokens_across_pcp", int | None, None) -# NOTE: vLLM v0.20.0 folded SharedFusedMoE into FusedMoE, and vllm-ascend in turn -# removed AscendSharedFusedMoE — the shared-experts / gate / multistream-overlap -# paths now live directly on AscendFusedMoE and are activated by passing -# shared_experts= as a kwarg. -class AscendHunyuanFusedMoE(AscendFusedMoE): - def __init__(self, *, prefix: str = "", **kwargs: Any) -> None: - super().__init__(prefix=prefix, **kwargs) - self._prefix = prefix +def reset_fused_moe_forward_context() -> None: + try: + forward_context = _vllm_fc.get_forward_context() + except AssertionError: + return - def forward(self, hidden_states: Any, router_logits: Any) -> Any: - _set_hunyuan_fused_moe_forward_context(hidden_states.shape[0]) - return super().forward(hidden_states, router_logits) + # MoE input token counts can change between transformer forwards, for + # example when text tokens are present only in some forwards. Clear cached + # DP/PCP padding bounds so the first MoE layer recomputes them. + forward_context.max_tokens_across_dp = None + forward_context.max_tokens_across_pcp = None - def __del__(self): - import vllm_ascend.distributed.parallel_state as vllm_ascend_parallel_state - if vllm_ascend_parallel_state._MC2: - vllm_ascend_parallel_state._MC2.destroy() - vllm_ascend_parallel_state._MC2 = None +def _set_max_tokens(forward_context: Any, hidden_states: torch.Tensor) -> None: + if ( + getattr(forward_context, "max_tokens_across_dp", None) is not None + or getattr(forward_context, "max_tokens_across_pcp", None) is not None + ): + return + + num_tokens_before_pcp = hidden_states.shape[0] + dp_metadata = getattr(forward_context, "dp_metadata", None) + if dp_metadata is not None: + max_tokens_across_dp = int(dp_metadata.num_tokens_across_dp_cpu.max().item()) + forward_context.max_tokens_across_dp = max_tokens_across_dp + # DP prepare pads each rank to max_tokens_across_dp, then gathers + # all DP ranks before the PCP gather. + num_tokens_before_pcp = max_tokens_across_dp * get_dp_group().world_size + + pcp_group = getattr(vllm_parallel_state, "_PCP", None) + if pcp_group is None or getattr(pcp_group, "world_size", 1) <= 1: + return + + gathered_num_tokens: list[int | None] = [None] * pcp_group.world_size + torch.distributed.all_gather_object( + gathered_num_tokens, + int(num_tokens_before_pcp), + group=pcp_group.cpu_group, + ) + if any(num_tokens is None for num_tokens in gathered_num_tokens): + raise RuntimeError(f"Failed to gather MoE PCP token counts: {gathered_num_tokens}") + forward_context.max_tokens_across_pcp = max(gathered_num_tokens) + + +def fused_moe_forward_context_pre_hook( + module: Any, + args: Any, + kwargs: Any, +) -> None: + forward_context = _vllm_fc.get_forward_context() + + hidden_states = kwargs.get("hidden_states") + if hidden_states is None and args: + hidden_states = args[0] + if hidden_states is not None: + _set_max_tokens(forward_context, hidden_states) + + forward_context.moe_comm_type = _select_moe_comm_method(vllm_config=omni_get_ctx().vllm_config) + forward_context.moe_comm_method = _MoECommMethods.get(forward_context.moe_comm_type) diff --git a/vllm_omni/platforms/npu/platform.py b/vllm_omni/platforms/npu/platform.py index 5133fa98e..42ff0b418 100644 --- a/vllm_omni/platforms/npu/platform.py +++ b/vllm_omni/platforms/npu/platform.py @@ -33,6 +33,8 @@ class NPUOmniPlatform(OmniPlatform, NPUPlatform): # conv2d convolution operator in the code2wav module of Qwen3-TTS not being able to run on Aclnn def __init__(self) -> None: + from vllm_ascend.utils import adapt_patch + from vllm_omni.platforms.npu._310p import apply_patches as apply_310p_patches from vllm_omni.platforms.npu.models.qwen3_tts_code2wav import ( apply_qwen3_tts_code2wav_patch, @@ -41,6 +43,7 @@ class NPUOmniPlatform(OmniPlatform, NPUPlatform): apply_qwen3_tts_tokenizer_v2_patch, ) + adapt_patch(is_global_patch=True) apply_qwen3_tts_code2wav_patch() apply_qwen3_tts_tokenizer_v2_patch() apply_310p_patches() @@ -73,26 +76,40 @@ class NPUOmniPlatform(OmniPlatform, NPUPlatform): init_ascend_config(vllm_config) + @classmethod + def init_diffusion_model_runner_runtime(cls, vllm_config: Any, od_config: Any, device: torch.device) -> None: + from vllm_ascend.ascend_forward_context import set_mc2_mask, set_mc2_tokens_capacity + + set_mc2_tokens_capacity(vllm_config, od_config.max_num_seqs, 1) + set_mc2_mask(vllm_config, device) + @classmethod def get_default_stage_config_path(cls) -> str: return "vllm_omni/deploy" - @classmethod - def get_diffusion_model_impl_qualname(cls, op_name: str) -> str: - if op_name == "hunyuan_fused_moe": - return "vllm_omni.platforms.npu.models.hunyuan_fused_moe.AscendHunyuanFusedMoE" - return super().get_diffusion_model_impl_qualname(op_name) - @classmethod def prepare_diffusion_op_runtime(cls, op_name: str, **kwargs: Any) -> None: - if op_name != "hunyuan_fused_moe": + if op_name != "fused_moe": return - from vllm_omni.platforms.npu.models.hunyuan_fused_moe import ( - prepare_hunyuan_fused_moe_runtime, + from vllm_omni.platforms.npu.layers.fused_moe import prepare_fused_moe_runtime + + prepare_fused_moe_runtime() + + @classmethod + def register_additional_diffusion_fused_moe_hooks(cls, moe_runner: Any) -> None: + from vllm_omni.platforms.npu.layers.fused_moe import fused_moe_forward_context_pre_hook + + moe_runner.register_forward_pre_hook( + fused_moe_forward_context_pre_hook, + with_kwargs=True, ) - prepare_hunyuan_fused_moe_runtime() + @classmethod + def reset_diffusion_fused_moe_forward_context(cls) -> None: + from vllm_omni.platforms.npu.layers.fused_moe import reset_fused_moe_forward_context + + reset_fused_moe_forward_context() @classmethod def get_diffusion_packed_modules_mapping( From 4dd1b47f478a09f3b1ab3841e30a38bcc81fd480 Mon Sep 17 00:00:00 2001 From: Wang fuyin Date: Mon, 27 Jul 2026 22:44:06 +0800 Subject: [PATCH 115/240] [Config] Read stage metadata from VllmOmniConfig (#5343) Signed-off-by: Acerak01-fy Co-authored-by: Hongsheng Liu --- .../test_async_omni_engine_stage_init.py | 8 +- tests/engine/test_single_stage_mode.py | 10 +- tests/engine/test_stage_metadata.py | 160 ++++++++++++++++++ tests/entrypoints/test_serve.py | 4 +- vllm_omni/engine/stage_engine_startup.py | 5 +- vllm_omni/engine/stage_init_utils.py | 72 +++++++- vllm_omni/engine/stage_runtime.py | 14 +- vllm_omni/platforms/rocm/platform.py | 2 +- 8 files changed, 256 insertions(+), 19 deletions(-) create mode 100644 tests/engine/test_stage_metadata.py diff --git a/tests/engine/test_async_omni_engine_stage_init.py b/tests/engine/test_async_omni_engine_stage_init.py index db7d53b79..87edc594b 100644 --- a/tests/engine/test_async_omni_engine_stage_init.py +++ b/tests/engine/test_async_omni_engine_stage_init.py @@ -342,7 +342,7 @@ def test_build_logical_stage_init_plans_applies_replica_device_splits(monkeypatc monkeypatch.setattr( runtime_mod, - "extract_stage_metadata", + "extract_legacy_stage_metadata", lambda cfg: types.SimpleNamespace(**metadata_by_stage[cfg.stage_id].__dict__), ) monkeypatch.setattr(runtime_mod, "get_stage_connector_spec", lambda **_: {}) @@ -836,9 +836,9 @@ def test_resolve_stage_configs_does_not_inject_diffusion_attention_into_llm_stag assert not hasattr(stage_configs[0].engine_args, "diffusion_attention_config") -def test_extract_stage_metadata_rocm_does_not_inject_diffusion_attention(monkeypatch): +def test_extract_legacy_stage_metadata_rocm_does_not_inject_diffusion_attention(monkeypatch): """ROCm default attention logic only applies to LLM stages, not diffusion.""" - from vllm_omni.engine.stage_init_utils import extract_stage_metadata + from vllm_omni.engine.stage_init_utils import extract_legacy_stage_metadata monkeypatch.setattr("vllm_omni.engine.stage_init_utils.current_omni_platform.is_rocm", lambda: True) @@ -852,7 +852,7 @@ def test_extract_stage_metadata_rocm_does_not_inject_diffusion_attention(monkeyp final_output_type=None, ) - metadata = extract_stage_metadata(stage_cfg) + metadata = extract_legacy_stage_metadata(stage_cfg) assert metadata.stage_type == "diffusion" assert "diffusion_attention_config" not in stage_cfg.engine_args diff --git a/tests/engine/test_single_stage_mode.py b/tests/engine/test_single_stage_mode.py index dc06c0c8d..7cf4c9d75 100644 --- a/tests/engine/test_single_stage_mode.py +++ b/tests/engine/test_single_stage_mode.py @@ -450,7 +450,7 @@ class TestSingleStageInitialization: monkeypatch = pytest.MonkeyPatch() monkeypatch.setattr( runtime_mod, - "extract_stage_metadata", + "extract_legacy_stage_metadata", lambda cfg: SimpleNamespace( stage_id=cfg.stage_id, stage_type=getattr(cfg, "stage_type", "llm"), @@ -477,7 +477,7 @@ class TestSingleStageInitialization: monkeypatch = pytest.MonkeyPatch() monkeypatch.setattr( runtime_mod, - "extract_stage_metadata", + "extract_legacy_stage_metadata", lambda cfg: SimpleNamespace( stage_id=cfg.stage_id, stage_type=getattr(cfg, "stage_type", "llm"), @@ -550,7 +550,7 @@ class TestSingleStageInitialization: monkeypatch = pytest.MonkeyPatch() monkeypatch.setattr( runtime_mod, - "extract_stage_metadata", + "extract_legacy_stage_metadata", lambda cfg: SimpleNamespace( stage_id=cfg.stage_id, stage_type="llm", @@ -609,7 +609,7 @@ class TestSingleStageInitialization: monkeypatch = pytest.MonkeyPatch() monkeypatch.setattr( runtime_mod, - "extract_stage_metadata", + "extract_legacy_stage_metadata", lambda cfg: SimpleNamespace( stage_id=cfg.stage_id, stage_type="diffusion", @@ -943,7 +943,7 @@ class TestSingleStageReplicaInitialization: plan = _make_diffusion_plan(1, stage_id=1, launch_mode="remote").replicas[0] sentinel_client = SimpleNamespace() - mocker.patch.object(runtime_mod, "extract_stage_metadata", return_value=remote_metadata) + mocker.patch.object(runtime_mod, "extract_legacy_stage_metadata", return_value=remote_metadata) mock_connect = mocker.patch.object(runtime_mod, "connect_remote_diffusion_proc", side_effect=_fake_connect) mock_from_addresses = mocker.patch( "vllm_omni.diffusion.stage_diffusion_client.StageDiffusionClient.from_addresses", diff --git a/tests/engine/test_stage_metadata.py b/tests/engine/test_stage_metadata.py new file mode 100644 index 000000000..f56bbc2a9 --- /dev/null +++ b/tests/engine/test_stage_metadata.py @@ -0,0 +1,160 @@ +import copy +import operator + +import pytest +from vllm.sampling_params import SamplingParams + +from vllm_omni.config.omni_config import VllmOmniConfig +from vllm_omni.config.stage_config import ( + DeployConfig, + PipelineConfig, + StageDeployConfig, + StageExecutionType, + StagePipelineConfig, + merge_pipeline_deploy, +) +from vllm_omni.engine.stage_init_utils import ( + extract_legacy_stage_metadata, + extract_stage_metadata, + extract_stage_metadata_from_omni_stage_config, +) +from vllm_omni.inputs.data import OmniDiffusionSamplingParams + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +def _metadata_inputs() -> tuple[PipelineConfig, DeployConfig]: + pipeline = PipelineConfig( + model_type="metadata-test", + stages=( + StagePipelineConfig( + stage_id=0, + model_stage="thinker", + execution_type=StageExecutionType.LLM_AR, + owns_tokenizer=True, + requires_multimodal_data=True, + engine_output_type="token_ids", + custom_process_input_func="operator.add", + prompt_expand_func="operator.mul", + ), + StagePipelineConfig( + stage_id=1, + model_stage="talker", + execution_type=StageExecutionType.LLM_GENERATION, + input_sources=(0,), + engine_output_type="audio_tokens", + custom_process_input_func="operator.sub", + sync_process_input_func="operator.floordiv", + ), + StagePipelineConfig( + stage_id=2, + model_stage="dit", + execution_type=StageExecutionType.DIFFUSION, + input_sources=(1,), + final_output=True, + final_output_type="image", + custom_process_input_func="operator.neg", + cfg_kv_collect_func="operator.concat", + ), + ), + ) + deploy = DeployConfig( + async_chunk=False, + stages=[ + StageDeployConfig( + stage_id=0, + devices="0", + env={"STAGE": "thinker"}, + default_sampling_params={"temperature": 0.25}, + ), + StageDeployConfig( + stage_id=1, + devices="1,2", + num_replicas=2, + env={"STAGE": "talker"}, + default_sampling_params={"temperature": 0.75}, + ), + StageDeployConfig( + stage_id=2, + devices="3", + env={"STAGE": "dit"}, + default_sampling_params={"seed": 7}, + ), + ], + ) + return pipeline, deploy + + +def test_extract_stage_metadata_matches_legacy_projection(): + pipeline, deploy = _metadata_inputs() + omni_config = VllmOmniConfig.from_pipeline_config( + pipeline, + user_deploy_config=copy.deepcopy(deploy), + ) + legacy_configs = [ + stage.to_omegaconf() + for stage in merge_pipeline_deploy( + pipeline, + copy.deepcopy(deploy), + ) + ] + + parity_fields = ( + "stage_id", + "stage_type", + "engine_output_type", + "is_comprehension", + "requires_multimodal_data", + "engine_input_source", + "final_output", + "final_output_type", + "custom_process_input_func", + "model_stage", + "prompt_expand_func", + "cfg_kv_collect_func", + ) + for legacy_config in legacy_configs: + stage_id = legacy_config.stage_id + omni_stage_config = omni_config.stage_by_id(stage_id) + structured = extract_stage_metadata_from_omni_stage_config(omni_stage_config) + legacy = extract_legacy_stage_metadata(legacy_config) + + assert {field: getattr(structured, field) for field in parity_fields} == { + field: getattr(legacy, field) for field in parity_fields + } + assert type(structured.default_sampling_params) is type(legacy.default_sampling_params) + assert structured.runtime_cfg is omni_stage_config.runtime_config + for runtime_field in ("devices", "num_replicas", "env"): + assert getattr(structured.runtime_cfg, runtime_field) == getattr(legacy.runtime_cfg, runtime_field) + + thinker = extract_stage_metadata_from_omni_stage_config(omni_config.stage_by_id(0)) + talker = extract_stage_metadata_from_omni_stage_config(omni_config.stage_by_id(1)) + diffusion = extract_stage_metadata_from_omni_stage_config(omni_config.stage_by_id(2)) + + assert isinstance(thinker.default_sampling_params, SamplingParams) + assert thinker.default_sampling_params.temperature == 0.25 + assert thinker.custom_process_input_func is operator.add + assert thinker.prompt_expand_func is operator.mul + + assert isinstance(talker.default_sampling_params, SamplingParams) + assert talker.default_sampling_params.temperature == 0.75 + assert talker.custom_process_input_func is operator.floordiv + + assert isinstance(diffusion.default_sampling_params, OmniDiffusionSamplingParams) + assert diffusion.default_sampling_params.seed == 7 + assert diffusion.custom_process_input_func is operator.neg + assert diffusion.cfg_kv_collect_func is operator.concat + + +def test_extract_stage_metadata_preserves_legacy_one_argument_api(): + pipeline, deploy = _metadata_inputs() + legacy_config = merge_pipeline_deploy( + pipeline, + deploy, + )[0].to_omegaconf() + + metadata = extract_stage_metadata(legacy_config) + + assert metadata.stage_id == 0 + assert metadata.model_stage == "thinker" + assert metadata.custom_process_input_func is operator.add diff --git a/tests/entrypoints/test_serve.py b/tests/entrypoints/test_serve.py index 61ffed3d5..bbed8fc08 100644 --- a/tests/entrypoints/test_serve.py +++ b/tests/entrypoints/test_serve.py @@ -377,7 +377,7 @@ def test_run_headless_diffusion_registers_and_spawns_proc(mocker: MockerFixture) return_value=(None, None, None), ) mocker.patch( - "vllm_omni.engine.stage_init_utils.extract_stage_metadata", + "vllm_omni.engine.stage_init_utils.extract_legacy_stage_metadata", return_value=SimpleNamespace(stage_id=1, stage_type="diffusion"), ) mock_inject = mocker.patch("vllm_omni.engine.stage_init_utils.inject_kv_stage_info") @@ -457,7 +457,7 @@ def test_run_headless_diffusion_raises_on_nonzero_proc_exit(mocker: MockerFixtur return_value=(None, None, None), ) mocker.patch( - "vllm_omni.engine.stage_init_utils.extract_stage_metadata", + "vllm_omni.engine.stage_init_utils.extract_legacy_stage_metadata", return_value=SimpleNamespace(stage_id=1, stage_type="diffusion"), ) mocker.patch("vllm_omni.engine.stage_init_utils.inject_kv_stage_info") diff --git a/vllm_omni/engine/stage_engine_startup.py b/vllm_omni/engine/stage_engine_startup.py index 7a55bb93f..d23d9f3e0 100644 --- a/vllm_omni/engine/stage_engine_startup.py +++ b/vllm_omni/engine/stage_engine_startup.py @@ -1347,7 +1347,10 @@ def launch_headless_diffusion_replicas( stage_id, ) - metadata = stage_init_utils.extract_stage_metadata(stage_cfg) + # Headless diffusion startup and its downstream helpers still consume the + # legacy StageConfig shape; switch this with the coordinated RFC #4021 + # stage-init cutover. + metadata = stage_init_utils.extract_legacy_stage_metadata(stage_cfg) if omni_conn_cfg: inject_omni_kv_config(stage_cfg, omni_conn_cfg, omni_from, omni_to) # Headless single-stage launch must still infer cross-stage TP topology diff --git a/vllm_omni/engine/stage_init_utils.py b/vllm_omni/engine/stage_init_utils.py index 3b1bb93af..2310daebe 100644 --- a/vllm_omni/engine/stage_init_utils.py +++ b/vllm_omni/engine/stage_init_utils.py @@ -25,6 +25,8 @@ from vllm.usage.usage_lib import UsageContext from vllm.v1.engine.input_processor import InputProcessor from vllm.v1.executor import Executor +from vllm_omni.config.omni_config import BaseVllmOmniStageConfig +from vllm_omni.config.stage_config import StageType from vllm_omni.diffusion.data import OmniDiffusionConfig from vllm_omni.engine.arg_utils import OmniEngineArgs from vllm_omni.entrypoints.stage_utils import _to_dict, set_stage_devices @@ -346,8 +348,12 @@ class StageMetadata: replica_id: int = 0 -def extract_stage_metadata(stage_config: Any) -> StageMetadata: - """Pure data extraction from a stage_config object.""" +def extract_legacy_stage_metadata(stage_config: Any) -> StageMetadata: + """Extract metadata through the active production legacy path. + + Keep production callers on this path until RFC #4021 migrates the + engine-argument and stage-init consumers together. + """ stage_id: int = stage_config.stage_id stage_type: Literal["llm", "diffusion"] = _get_attr_or_item(stage_config, "stage_type", "llm") engine_args = stage_config.engine_args @@ -432,6 +438,68 @@ def extract_stage_metadata(stage_config: Any) -> StageMetadata: ) +def extract_stage_metadata(stage_config: Any) -> StageMetadata: + """Preserve the legacy one-argument API for external callers.""" + return extract_legacy_stage_metadata(stage_config) + + +def _resolve_omni_metadata_hook(path: str | None) -> Callable | None: + if not path: + return None + module_path, function_name = path.rsplit(".", 1) + return getattr(importlib.import_module(module_path), function_name) + + +def extract_stage_metadata_from_omni_stage_config( + stage_config: BaseVllmOmniStageConfig, +) -> StageMetadata: + """Project one typed stage config into metadata for a future cutover. + + This projection is not used by production startup yet. Current replica + layout, engine-argument, remote-diffusion, and platform setup paths still + require the legacy StageConfig/OmegaConf shape. + """ + stage_type: Literal["llm", "diffusion"] = "diffusion" if stage_config.stage_type == StageType.DIFFUSION else "llm" + sampling_params_cls = SamplingParams if stage_type == "llm" else OmniDiffusionSamplingParams + sampling_params: OmniSamplingParams = sampling_params_cls( + **(stage_config.model_config.default_sampling_params or {}) + ) + custom_process_input_func = _resolve_omni_metadata_hook(stage_config.custom_process_input_func) + + if stage_type == "diffusion": + return StageMetadata( + stage_id=stage_config.stage_id, + stage_type="diffusion", + engine_output_type=None, + is_comprehension=False, + requires_multimodal_data=False, + engine_input_source=stage_config.input_sources, + final_output=stage_config.final_output, + final_output_type=stage_config.final_output_type, + default_sampling_params=sampling_params, + custom_process_input_func=custom_process_input_func, + model_stage=stage_config.model_stage, + runtime_cfg=stage_config.runtime_config, + cfg_kv_collect_func=_resolve_omni_metadata_hook(stage_config.cfg_kv_collect_func), + ) + + return StageMetadata( + stage_id=stage_config.stage_id, + stage_type="llm", + engine_output_type=stage_config.engine_output_type, + is_comprehension=stage_config.is_comprehension, + requires_multimodal_data=stage_config.requires_multimodal_data, + engine_input_source=stage_config.input_sources, + final_output=stage_config.final_output, + final_output_type=stage_config.final_output_type, + default_sampling_params=sampling_params, + custom_process_input_func=custom_process_input_func, + model_stage=stage_config.model_stage, + runtime_cfg=stage_config.runtime_config, + prompt_expand_func=_resolve_omni_metadata_hook(stage_config.prompt_expand_func), + ) + + def prepare_engine_environment() -> None: """One-time global setup: load plugins, set multiprocessing spawn method.""" from vllm_omni.plugins import load_omni_general_plugins diff --git a/vllm_omni/engine/stage_runtime.py b/vllm_omni/engine/stage_runtime.py index 4ee4ccc61..1f2024fd2 100644 --- a/vllm_omni/engine/stage_runtime.py +++ b/vllm_omni/engine/stage_runtime.py @@ -51,7 +51,7 @@ from vllm_omni.engine.stage_init_utils import ( build_llm_stage_output_processor, build_vllm_config, compute_replica_layout, - extract_stage_metadata, + extract_legacy_stage_metadata, get_stage_connector_spec, inject_kv_stage_info, inject_omni_kv_connector_config, @@ -339,8 +339,12 @@ class StageRuntime: """Build startup plans for every logical stage and replica.""" stage_plans: list[LogicalStageInitPlan] = [] + # RFC #4021 transition boundary: stage planning still relies on legacy + # StageConfig.runtime and StageConfig.engine_args for replica and engine + # setup. Keep metadata extraction on the legacy path until the + # coordinated stage-init cutover. for stage_idx, stage_cfg in enumerate(self._stage_configs): - base_metadata = extract_stage_metadata(stage_cfg) + base_metadata = extract_legacy_stage_metadata(stage_cfg) stage_id = int(base_metadata.stage_id) if stage_id != stage_idx: raise ValueError( @@ -392,7 +396,7 @@ class StageRuntime: if stage_idx in replica_devices_map: replica_cfg.runtime.devices = replica_devices_map[stage_idx][replica_id] - replica_metadata = extract_stage_metadata(replica_cfg) + replica_metadata = extract_legacy_stage_metadata(replica_cfg) replica_metadata.replica_id = replica_id if launch_mode == "remote" and replica_metadata.stage_type != "diffusion": replica_metadata.runtime_cfg = None @@ -865,8 +869,10 @@ class DistStageRuntime(StageRuntime): if registered_stage_cfg is None: raise ValueError(f"Remote stage {plan.metadata.stage_id} registered without stage config") + # Remote diffusion registration still transports the legacy mapping + # shape. Reconstruct and project that shape until its RFC #4021 cutover. metadata = ( - extract_stage_metadata(OmegaConf.create(registered_stage_cfg)) + extract_legacy_stage_metadata(OmegaConf.create(registered_stage_cfg)) if plan.metadata.stage_type == "diffusion" else copy.deepcopy(plan.metadata) ) diff --git a/vllm_omni/platforms/rocm/platform.py b/vllm_omni/platforms/rocm/platform.py index 38a14c325..f44bb86fa 100644 --- a/vllm_omni/platforms/rocm/platform.py +++ b/vllm_omni/platforms/rocm/platform.py @@ -30,7 +30,7 @@ class RocmOmniPlatform(OmniPlatform, RocmPlatform): when the selected_backend is not specified. So the behaviour of the attention backend overriding logic currently lives in - extract_stage_metadata in `vllm_omni/engine/stage_init_utils.py` + extract_legacy_stage_metadata in `vllm_omni/engine/stage_init_utils.py` ``` if current_omni_platform.is_rocm(): From 7762d664d8550bd21cd71f092c56b53e679d5a74 Mon Sep 17 00:00:00 2001 From: SYLAR <125541396+lishunyang12@users.noreply.github.com> Date: Mon, 27 Jul 2026 22:48:11 +0800 Subject: [PATCH 116/240] [Attention] Add trtllm diffusion attention backend with Skip-Softmax (#5283) Signed-off-by: lishunyang12 Co-authored-by: lishunyang12 --- docs/.nav.yml | 1 + docs/design/feature/skip_softmax.md | 88 +++++ docs/design/module/dit_module.md | 4 +- .../diffusion/attention_backends.md | 62 ++- recipes/README.md | 1 + recipes/Wan-AI/Wan2.2-T2V.md | 186 +++++++++ .../attention/test_attention_config.py | 45 ++- tests/diffusion/attention/test_trtllm_attn.py | 358 ++++++++++++++++++ .../attention/test_trtllm_calibration.py | 224 +++++++++++ .../diffusion/attention/backends/registry.py | 1 + .../attention/backends/trtllm_attn.py | 226 +++++++++++ .../attention/backends/trtllm_calibration.py | 152 ++++++++ vllm_omni/diffusion/attention/layer.py | 16 +- vllm_omni/diffusion/attention/selector.py | 11 +- vllm_omni/diffusion/data.py | 91 ++++- vllm_omni/diffusion/forward_context.py | 16 + .../model_loader/diffusers_loader.py | 9 + vllm_omni/diffusion/model_metadata.py | 5 + .../models/wan2_2/pipeline_wan2_2.py | 5 +- .../models/wan2_2/pipeline_wan2_2_i2v.py | 5 +- .../models/wan2_2/pipeline_wan2_2_s2v.py | 12 +- .../models/wan2_2/pipeline_wan2_2_vace.py | 3 +- vllm_omni/entrypoints/cli/serve.py | 3 +- vllm_omni/platforms/cuda/platform.py | 40 ++ 24 files changed, 1512 insertions(+), 52 deletions(-) create mode 100644 docs/design/feature/skip_softmax.md create mode 100644 recipes/Wan-AI/Wan2.2-T2V.md create mode 100644 tests/diffusion/attention/test_trtllm_attn.py create mode 100644 tests/diffusion/attention/test_trtllm_calibration.py create mode 100644 vllm_omni/diffusion/attention/backends/trtllm_attn.py create mode 100644 vllm_omni/diffusion/attention/backends/trtllm_calibration.py diff --git a/docs/.nav.yml b/docs/.nav.yml index 4cb9d20ed..1da3deb94 100644 --- a/docs/.nav.yml +++ b/docs/.nav.yml @@ -116,6 +116,7 @@ nav: - design/feature/hsdp.md - design/feature/cache_dit.md - design/feature/teacache.md + - design/feature/skip_softmax.md - design/feature/async_chunk.md - design/feature/vae_parallel.md - design/feature/diffusion_step_execution.md diff --git a/docs/design/feature/skip_softmax.md b/docs/design/feature/skip_softmax.md new file mode 100644 index 000000000..8e6375ee8 --- /dev/null +++ b/docs/design/feature/skip_softmax.md @@ -0,0 +1,88 @@ +# Skip-Softmax + +Skip-Softmax is the sparse-attention mode of the `TRTLLM_ATTN` backend. Usage — the config keys +and how to pick an operating point — is in +[Diffusion Attention Backends](../../user_guide/diffusion/attention_backends.md). This page explains +the algorithm. + +## Motivation + +In a long attention row (a video DiT can have tens of thousands of keys), the softmax weight +concentrates on a small fraction of the keys; the rest receive near-zero weight and barely move the +output. Computing softmax and the value-weighted sum over those keys is wasted work. Skip-Softmax +detects, per block of keys, when a block cannot matter and skips its softmax and its value +multiply. + +It is approximate: a skipped block still carries a small non-zero contribution, so the mode is +opt-in and off by default. + +## The online-softmax pass + +Attention is computed in a single streaming pass over the keys, in tiles of 128. Per query row the +kernel maintains three running values: + +- `m` — the largest score seen so far, +- `l` — the running denominator `Σ exp(sⱼ − m)`, +- `O` — the running numerator `Σ exp(sⱼ − m)·vⱼ`, + +and returns `O / l` at the end. For each key tile it computes the tile's scores `Q · K_jᵀ · scale`, +updates `m`, and accumulates that tile's contribution into `l` and `O`. Rescaling `l` and `O` when +`m` grows keeps the single pass numerically exact. + +## The skip test + +Once a tile's scores are known, its largest score `tile_max` is compared against the running +maximum: + +```text +if exp(tile_max − running_max) < threshold: + skip this tile # do not compute its softmax weights or its P·V accumulation +``` + +`exp(tile_max − running_max)` is an upper bound on the softmax weight any key in the tile can +receive: if even the tile's best key is far below the current maximum, every key in the tile is +negligible, and both the softmax (the exponentials) and the `P·V` accumulation for that tile can be +dropped. The tile's contribution to `l` and `O` is simply skipped. + +## What this bounds + +Two properties of the test shape the achievable speedup: + +- **`Q · K_jᵀ` always runs.** The test needs `tile_max`, which comes from the tile's scores, so the + score matmul is never skipped — only the softmax and the `P·V` accumulation are. The score matmul + and the value matmul are comparable in cost, so skipping every eligible tile removes at most + roughly half the attention work; the kernel-level speedup is bounded well under 2×, not unbounded. + +- **The decision is per tile, not per key.** A tile is skipped only when *all* of its keys are + collectively negligible; a single important key keeps the whole tile. How many tiles actually + qualify depends on the data and rounds down to tile granularity, so `target_sparsity` selects an + operating point on a calibrated curve — it is not a promise that a fixed fraction of tiles is + skipped. + +## The threshold + +The per-tile threshold is normalized by sequence length: + +```text +threshold = factor / seqlen +``` + +`factor` comes from one of two sources: + +- **Calibrated** — `factor = a · exp(b · target_sparsity)`, where `a` and `b` are fit per model (and + per expert, for a multi-expert model) so that a given `target_sparsity` lands near that fraction of + skipped tiles on the calibration data. +- **Direct** — `factor = skip_softmax.threshold · seqlen`, the calibration-free path; the user sets + the threshold themselves. + +## Timestep gating + +`disabled_until_timestep = D` keeps the mode off during the early, high-noise denoise steps and +turns it on once the normalized timestep `t` drops to `t ≤ D` (`t` runs `1.0` → `0.0` over the +schedule). The early steps set the global structure of the output and their errors propagate through +every later step, so keeping them dense costs a few skipped-tile opportunities but protects fidelity. + +`t` is the scheduler's own timestep divided by `num_train_timesteps`, published by the pipeline via +`DenoiseProgressMixin.record_denoise_step`. It is deliberately not derived from the step index, +because schedulers space their steps non-uniformly. A pipeline that does not publish a timestep +stays dense when `disabled_until_timestep` is set. diff --git a/docs/design/module/dit_module.md b/docs/design/module/dit_module.md index ace355427..aabdb78b0 100644 --- a/docs/design/module/dit_module.md +++ b/docs/design/module/dit_module.md @@ -498,7 +498,7 @@ class Attention(nn.Module): role_category=role_category, ) self.attn_impl_cls = attn_backend_cls.get_impl_cls() - self.attention = self.attn_impl_cls(..., backend_kwargs=spec.extra if spec else None) + self.attention = self.attn_impl_cls(..., backend_kwargs=spec.backend_kwargs() if spec else None) ``` **Available Backends**: @@ -540,7 +540,7 @@ def get_attn_backend_for_role(role, head_size, attention_config=None, role_categ 4. **Platform default** — `current_omni_platform.get_diffusion_attn_backend_cls(...)` picks the best-available kernel for the hardware. -`AttentionSpec.extra` is forwarded to the backend constructor as `backend_kwargs`, so backend-specific parameters (e.g. SparseBlock `block_size`) can be set without changing model code. +A backend that needs configuration exposes it as a typed field on `AttentionSpec`; `AttentionSpec.backend_kwargs()` serializes it into the `backend_kwargs` the impl consumes. Today the only such field is `skip_softmax` (TRTLLM_ATTN), so e.g. `target_sparsity` can be set without changing model code. For the user-facing CLI surface, see [Diffusion Attention Backends](../../user_guide/diffusion/attention_backends.md). For the role declaration contract on the model side, see [Adding a Diffusion Model](../../contributing/model/adding_diffusion_model.md). diff --git a/docs/user_guide/diffusion/attention_backends.md b/docs/user_guide/diffusion/attention_backends.md index 50e02971f..e54955c04 100644 --- a/docs/user_guide/diffusion/attention_backends.md +++ b/docs/user_guide/diffusion/attention_backends.md @@ -14,6 +14,7 @@ The full set of backends and their platform defaults is in the **Backend Options | Value | Notes | |---|---| +| `TRTLLM_ATTN` | FlashInfer's trtllm-gen FMHA (TensorRT-LLM's generated kernels, vendored by FlashInfer). BF16, GQA native, `head_dim=128`. Datacenter Blackwell only (sm_100 / sm_103). Supports **Skip-Softmax** sparse attention — see below. Requires `flashinfer`. | | `FLASH_ATTN` | Wraps FlashAttention 2. Default on Hopper / Ada / Ampere when `flash-attn` is installed. | | `CUDNN_ATTN` | Pins `sdpa_kernel([CUDNN_ATTENTION])`. Default on Blackwell (sm_10x / sm_12x) with cuDNN ≥ 9.5. Wins on mask-heavy DiTs (HunyuanVideo-1.5: 2× e2e vs SDPA). | | `FLASHINFER_ATTN` | Calls FlashInfer's dense `single_prefill_with_kv_cache` directly with `custom_mask` for non-causal masked attention. Used as Blackwell fallback when cuDNN is unavailable. Requires `flashinfer`. | @@ -68,11 +69,12 @@ vllm-omni serve \ --diffusion-attention-config '{"default":{"backend":"FLASH_ATTN"},"per_role":{"cross":{"backend":"TORCH_SDPA"}}}' ``` -Backends may also accept backend-specific parameters via `extra`: +A backend that needs configuration exposes it as a typed field on the spec. Today the only +one is `TRTLLM_ATTN`'s Skip-Softmax (see [below](#trtllm_attn-backend-and-skip-softmax)): ```bash ---diffusion-attention-config.per_role.self.backend SPARSE_BLOCK \ ---diffusion-attention-config.per_role.self.extra.block_size 128 +--diffusion-attention-config.default.backend TRTLLM_ATTN \ +--diffusion-attention-config.default.skip_softmax.target_sparsity 0.5 ``` ### Programmatic API @@ -101,10 +103,15 @@ A plain dict is also accepted and normalized to `AttentionConfig`. Auto-route preference, in order: -1. `CUDNN_ATTN` — when cuDNN ≥ 9.5 is available (ships in PyTorch 2.5+ wheels) -2. `FLASHINFER_ATTN` — when `flashinfer` is installed but cuDNN < 9.5 -3. `FLASH_ATTN` — when `flash-attn` is installed with the Blackwell CUTE kernel -4. `TORCH_SDPA` — last resort +1. `TRTLLM_ATTN` — on **datacenter** Blackwell (sm_100 / sm_103) when `flashinfer` is installed, the model's `head_dim` is 128, **and the model is mask-free** (see below) +2. `CUDNN_ATTN` — when cuDNN ≥ 9.5 is available (ships in PyTorch 2.5+ wheels) +3. `FLASHINFER_ATTN` — when `flashinfer` is installed but cuDNN < 9.5 +4. `FLASH_ATTN` — when `flash-attn` is installed with the Blackwell CUTE kernel +5. `TORCH_SDPA` — last resort + +`TRTLLM_ATTN` is skipped on workstation Blackwell (sm_120 / sm_121) and for any `head_dim != 128`, so those GPUs keep the `CUDNN_ATTN` route described below. + +`TRTLLM_ATTN` outranks `CUDNN_ATTN` on datacenter Blackwell because it is the only backend that can enable Skip-Softmax, not because its dense kernel is faster — dense, the two are comparable. It cannot honor an attention mask, so the auto-default only applies to pipelines verified mask-free (the Wan family); mask-using pipelines keep `CUDNN_ATTN`. `TRTLLM_ATTN` stays available everywhere via explicit `--diffusion-attention-backend TRTLLM_ATTN` (which raises clearly if a mask is received). The startup log line `Defaulting to diffusion attention backend CUDNN_ATTN (Blackwell sm_120, cuDNN 91002)` confirms the route. @@ -119,6 +126,47 @@ Auto-route preference: `CUDNN_ATTN` and `FLASHINFER_ATTN` are still selectable via env var on these GPUs but are not in the auto-route — FlashAttention 2 is the well-tuned path on pre-Blackwell hardware. +## TRTLLM_ATTN Backend and Skip-Softmax + +`TRTLLM_ATTN` runs FlashInfer's trtllm-gen FMHA. Dense it is a BF16 backend like the others; on top of +that it can enable **Skip-Softmax**, a sparse-attention mode that trades a little fidelity for speed +(algorithm: [design doc](../../design/feature/skip_softmax.md)). + +Enable it through the typed `skip_softmax` block on the attention spec: + +| Key | Valid values | Meaning | +|---|---|---| +| `target_sparsity` | finite, `[0, 1]` | Operating point on a calibrated curve. Needs a calibration for the model. | +| `threshold` | finite, `≥ 0` | Direct threshold, no calibration needed. Mutually exclusive with `target_sparsity`. | +| `disabled_until_timestep` | finite, `[0, 1]` | Holds early, high-noise steps dense; skip turns on once the normalized timestep `t ≤ D`. | + +```bash +vllm-omni serve Wan-AI/Wan2.2-T2V-A14B-Diffusers \ + --diffusion-attention-config '{"default": {"backend": "TRTLLM_ATTN", + "skip_softmax": {"target_sparsity": 0.65, "disabled_until_timestep": 0.86}}}' +``` + +Programmatically the same block is a typed `SkipSoftmaxSpec` (values validated at construction): + +```python +from vllm_omni.diffusion.data import AttentionConfig, AttentionSpec, SkipSoftmaxSpec + +AttentionConfig( + default=AttentionSpec( + backend="TRTLLM_ATTN", + skip_softmax=SkipSoftmaxSpec(target_sparsity=0.65, disabled_until_timestep=0.86), + ), +) +``` + +**Start at `target_sparsity=0.65, disabled_until_timestep=0.86`** and raise `target_sparsity` only +as far as your quality bar allows — the output diverges from dense as it climbs. The gain grows with +sequence length, since Skip-Softmax only accelerates attention. Dense BF16 is `TRTLLM_ATTN` with no +`skip_softmax` block. + +Requires datacenter Blackwell with `head_dim == 128`; elsewhere, or without FlashInfer, selecting +`TRTLLM_ATTN` raises rather than silently degrading. + ## End-to-End Benchmark (BF16, sm_120 RTX Pro 6000 Blackwell) Same prompt and seed across runs. `Total generation time` from `text_to_video.py` / `text_to_image.py`. diff --git a/recipes/README.md b/recipes/README.md index 94273caa3..e5e8473ee 100644 --- a/recipes/README.md +++ b/recipes/README.md @@ -66,6 +66,7 @@ recipes/ | [`StabilityAI/Stable-Audio-Open.md`](./StabilityAI/Stable-Audio-Open.md) | Offline + online text-to-audio generation (Stable Audio Open) | 1x RTX 4090 24GB | | [`Tencent/Covo-Audio-Chat.md`](./Tencent/Covo-Audio-Chat.md) | Online serving for audio chat | 1x A100 80GB | | [`Tencent/HunyuanImage-3.0-Instruct.md`](./Tencent/HunyuanImage-3.0-Instruct.md) | DiT-only text-to-image serving and benchmark, including ModelOpt mixed FP8/NVFP4 | 4x H100/H800 80GB / 2x B200 | +| [`Wan-AI/Wan2.2-T2V.md`](./Wan-AI/Wan2.2-T2V.md) | Text-to-video serving (Wan2.2 14B) with Skip-Softmax sparse attention | 1x datacenter Blackwell (B200/B300) | | [`Wan-AI/Wan2.2-I2V.md`](./Wan-AI/Wan2.2-I2V.md) | Image-to-video serving (Wan2.2 14B) | 8x Ascend NPU (A2/A3) | | [`Wan-AI/Wan2.2-S2V.md`](./Wan-AI/Wan2.2-S2V.md) | Speech-to-video serving (Wan2.2 14B) | 2x A100/H100 80GB | | [`Wan-AI/Wan2.1-VACE.md`](./Wan-AI/Wan2.1-VACE.md) | Unified T2V, I2V, V2LF, FLF2V, inpaint, and R2V | 1x RTX 5090 (1.3B) / 1x L40S 48GB with layerwise offload (14B) | diff --git a/recipes/Wan-AI/Wan2.2-T2V.md b/recipes/Wan-AI/Wan2.2-T2V.md new file mode 100644 index 000000000..640129a47 --- /dev/null +++ b/recipes/Wan-AI/Wan2.2-T2V.md @@ -0,0 +1,186 @@ +# Wan2.2 Text-to-Video + +> Text-to-video serving (Wan2.2 14B), with optional Skip-Softmax sparse attention on Blackwell + +## Summary + +- Vendor: Wan-AI +- Model: `Wan-AI/Wan2.2-T2V-A14B-Diffusers` +- Task: Text-to-video generation +- Mode: Online serving with the OpenAI-compatible API (offline `Omni` also supported) +- Maintainer: Community + +## When to use this recipe + +Use this recipe to deploy the Wan2.2 14B text-to-video model with vLLM-Omni. The +standard plan is multi-card serving with sequence/CFG parallelism (below). On +**datacenter Blackwell** (sm_100 / sm_103) you can additionally enable +**Skip-Softmax** through the `TRTLLM_ATTN` backend for extra speed — see the +Blackwell section. + +## References + +- Upstream model card: +- Attention backends guide: [`docs/user_guide/diffusion/attention_backends.md`](../../docs/user_guide/diffusion/attention_backends.md) +- Skip-Softmax design: [`docs/design/feature/skip_softmax.md`](../../docs/design/feature/skip_softmax.md) + +## Hardware Support + +## GPU + +### 8x NVIDIA H20 / H100 / A100 (standard multi-card serving) + +The recommended parallel strategy depends on whether the checkpoint needs +classifier-free guidance: + +1. **Distilled model (no CFG)** — higher throughput; use when the checkpoint does + not require negative-prompt computation. +2. **Official open-source model (with CFG)** — uses CFG parallelism to run the + negative and positive samples for the original released weights. + +#### Environment + +- OS: Linux +- Python: 3.10+ +- Driver / runtime: NVIDIA driver with a CUDA runtime supported by your PyTorch build +- vLLM version: Match the repository requirements for your checkout +- vLLM-Omni version or commit: Use the commit you are deploying from + +#### Command + +**Distilled model (no CFG, recommended for distilled checkpoints):** + +```bash +CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 \ +vllm serve Wan-AI/Wan2.2-T2V-A14B-Diffusers \ + --omni \ + --use-hsdp \ + --usp 8 \ + --vae-patch-parallel-size 8 \ + --vae-use-tiling +``` + +**Official open-source model (with CFG):** + +```bash +CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 \ +vllm serve Wan-AI/Wan2.2-T2V-A14B-Diffusers \ + --omni \ + --use-hsdp \ + --cfg-parallel-size 2 \ + --usp 4 \ + --vae-patch-parallel-size 8 \ + --vae-use-tiling +``` + +The official model splits the positive and negative CFG branches with +`--cfg-parallel-size 2`, keeping Ulysses sequence parallelism at `--usp 4` so the +two branches cover all 8 GPUs (`usp * cfg = 8`). + +#### Verification + +For online serving client examples and request formats, see +[`examples/online_serving/text_to_video`](../../examples/online_serving/text_to_video). + +#### Notes + +- **Key flags:** + - `--omni` — enables vLLM-Omni diffusion serving. + - `--use-hsdp` — Hybrid Sharded Data Parallelism for the 14B DiT weights. + - `--usp ` — Unified (Ulysses) Sequence Parallelism degree. + - `--cfg-parallel-size ` — CFG parallelism; set to 2 for the official model, + omit for distilled checkpoints. + - `--vae-patch-parallel-size 8` / `--vae-use-tiling` — parallel + tiled VAE + decoding; disabling patch parallelism can significantly increase VAE latency. + +## Accelerating with TRTLLM_ATTN + Skip-Softmax (datacenter Blackwell) + +On **datacenter Blackwell** (sm_100 / sm_103), the mask-free Wan pipeline +auto-routes to the `TRTLLM_ATTN` backend, which can enable **Skip-Softmax** — a +sparse-attention mode that trades a little fidelity for speed. This composes with +the multi-card plan above (Ulysses SP is compatible; ring SP is not). + +### Environment + +- Same as above, plus: datacenter Blackwell (sm_100 / sm_103), `head_dim == 128`, + and `flashinfer` (already a hard dependency of vLLM — no extra install). + +### Command + +Dense BF16 (Skip-Softmax off — `TRTLLM_ATTN` is already the Blackwell auto-route +for mask-free Wan, so no flags are needed): + +```bash +vllm serve Wan-AI/Wan2.2-T2V-A14B-Diffusers --omni +``` + +With Skip-Softmax (calibrated curve — the `(a, b)` curve is read from the +checkpoint's `sparse_attention_config`; you only pick the operating point): + +```bash +vllm serve Wan-AI/Wan2.2-T2V-A14B-Diffusers \ + --omni \ + --diffusion-attention-config '{"default": {"backend": "TRTLLM_ATTN", + "skip_softmax": {"target_sparsity": 0.65, "disabled_until_timestep": 0.86}}}' +``` + +Offline, the same config is a typed object (or the equivalent dict) — values are +validated at construction: + +```python +from vllm_omni import Omni +from vllm_omni.diffusion.data import AttentionConfig, AttentionSpec, SkipSoftmaxSpec + +llm = Omni( + model="Wan-AI/Wan2.2-T2V-A14B-Diffusers", + diffusion_attention_config=AttentionConfig( + default=AttentionSpec( + backend="TRTLLM_ATTN", + skip_softmax=SkipSoftmaxSpec(target_sparsity=0.65, disabled_until_timestep=0.86), + ), + ), +) +``` + +Calibration-free path (no checkpoint curve needed — set an absolute threshold +instead of `target_sparsity`): + +```bash +--diffusion-attention-config '{"default": {"backend": "TRTLLM_ATTN", + "skip_softmax": {"threshold": 0.02, "disabled_until_timestep": 0.86}}}' +``` + +### Skip-Softmax controls + +| Key | Valid values | Meaning | +|---|---|---| +| `target_sparsity` | finite, `[0, 1]` | Operating point on the checkpoint's calibrated curve (`a·exp(b·s)`). | +| `threshold` | finite, `≥ 0` | Direct skip threshold, no calibration needed. Mutually exclusive with `target_sparsity`. | +| `disabled_until_timestep` | finite, `[0, 1]` | Keeps early, high-noise steps dense; skip turns on once the normalized timestep `t ≤ D`. | + +**Start at `target_sparsity=0.65, disabled_until_timestep=0.86`** (near-lossless) +and raise `target_sparsity` only as far as your quality bar allows — the output +diverges from dense as it climbs. Gate the noisy early steps first (`D`), then +raise sparsity; the timestep gate buys more speed per unit of divergence than +sparsity does. The gain grows with sequence length, since Skip-Softmax only +accelerates attention. + +Measured on B300 / SM103, 1280×720 / 81f / 50 steps, torch.compile, official +ModelOpt calibration (speedup and divergence relative to dense): + +| config | speedup | LPIPS (divergence vs dense) | +|---|---|---| +| dense | 1.000x | - | +| `s=0.65, D=0.86` | 1.062x | 0.041 | +| `s=0.75, D=1.00` | 1.181x | 0.377 | +| `s=1.00` | 1.202x | 0.597 | + +### Notes + +- **Requirements:** datacenter Blackwell (sm_100 / sm_103), `head_dim == 128`, and + `flashinfer`. Elsewhere, or without FlashInfer, selecting `TRTLLM_ATTN` raises + rather than silently degrading. +- **Known limitations:** + - Skip-Softmax is not supported under ring sequence parallelism; use Ulysses SP + (`--usp`), or run Skip-Softmax without ring. + - `TRTLLM_ATTN` is BF16-only (no attention-level quantization). diff --git a/tests/diffusion/attention/test_attention_config.py b/tests/diffusion/attention/test_attention_config.py index a4c67851a..26a93e901 100644 --- a/tests/diffusion/attention/test_attention_config.py +++ b/tests/diffusion/attention/test_attention_config.py @@ -31,21 +31,27 @@ from vllm_omni.diffusion.data import ( parse_attention_config, ) +pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu] + class TestAttentionSpec: - def test_construct_no_extra(self): + def test_construct_no_skip_softmax(self): spec = AttentionSpec(backend="FLASH_ATTN") - assert spec.extra == {} + assert spec.skip_softmax is None - def test_mapping_extra_normalized(self): - spec = AttentionSpec(backend="SAGE_ATTN", extra={"quant": "int8"}) - assert spec.backend == "SAGE_ATTN" - assert spec.extra == {"quant": "int8"} + def test_skip_softmax_mapping_coerced(self): + spec = AttentionSpec(backend="TRTLLM_ATTN", skip_softmax={"target_sparsity": 0.5}) + assert spec.backend == "TRTLLM_ATTN" + assert spec.skip_softmax.target_sparsity == 0.5 def test_invalid_backend_type(self): with pytest.raises(TypeError): AttentionSpec(backend=123) # type: ignore[arg-type] + def test_skip_softmax_rejected_on_non_trtllm(self): + with pytest.raises(ValueError, match="only supported by the TRTLLM_ATTN"): + AttentionSpec(backend="TORCH_SDPA", skip_softmax={"target_sparsity": 0.5}) + class TestAttentionConfig: def test_empty_config(self): @@ -57,13 +63,13 @@ class TestAttentionConfig: config = AttentionConfig( default={"backend": "FLASH_ATTN"}, per_role={ - "self": {"backend": "SPARSE_BLOCK", "extra": {"block_size": 128}}, + "self": {"backend": "TRTLLM_ATTN", "skip_softmax": {"target_sparsity": 0.5}}, "cross": "SAGE_ATTN", }, ) assert config.default.backend == "FLASH_ATTN" - assert config.per_role["self"].backend == "SPARSE_BLOCK" - assert config.per_role["self"].extra == {"block_size": 128} + assert config.per_role["self"].backend == "TRTLLM_ATTN" + assert config.per_role["self"].skip_softmax.target_sparsity == 0.5 assert config.per_role["cross"].backend == "SAGE_ATTN" def test_constructor_flattens_nested_per_role_tree(self): @@ -178,10 +184,10 @@ class TestAttentionConfig: config = AttentionConfig( default=AttentionSpec(backend="FLASH_ATTN"), per_role={ - "self": AttentionSpec(backend="SPARSE_BLOCK", extra={"block_size": 128}), + "self": AttentionSpec(backend="SPARSE_BLOCK"), "cross": AttentionSpec(backend="SAGE_ATTN"), "ltx2.audio_self": AttentionSpec(backend="FLASH_ATTN"), - "ltx2.audio_to_video": AttentionSpec(backend="FLASH_ATTN", extra={"causal_window": 64}), + "ltx2.audio_to_video": AttentionSpec(backend="FLASH_ATTN"), }, ) @@ -200,7 +206,6 @@ class TestAttentionConfig: # audio-to-video → exact match spec, _ = config.resolve_with_source("ltx2.audio_to_video", "cross") assert spec.backend == "FLASH_ATTN" - assert spec.extra == {"causal_window": 64} # video-to-audio → category fallback to "cross" assert config.resolve_with_source("ltx2.video_to_audio", "cross")[0].backend == "SAGE_ATTN" @@ -364,12 +369,13 @@ class TestAttentionInitUsesCurrentDiffusionConfig: head_size, attention_config=None, role_category=None, + allow_trtllm_default=False, ): captured["role"] = role captured["head_size"] = head_size captured["role_category"] = role_category captured["attention_config"] = attention_config - return _FakeBackend, AttentionSpec(backend="TORCH_SDPA", extra={"block_size": 128}) + return _FakeBackend, AttentionSpec(backend="TRTLLM_ATTN", skip_softmax={"target_sparsity": 0.5}) class _FakeRingParallelAttention: def __init__(self, sp_group, attn_backend_pref=None): @@ -391,7 +397,7 @@ class TestAttentionInitUsesCurrentDiffusionConfig: od_config = SimpleNamespace( diffusion_attention_config=AttentionConfig( default=AttentionSpec(backend="FLASH_ATTN"), - per_role={"cross": AttentionSpec(backend="TORCH_SDPA", extra={"block_size": 128})}, + per_role={"cross": AttentionSpec(backend="TRTLLM_ATTN", skip_softmax={"target_sparsity": 0.5})}, ), parallel_config=SimpleNamespace(ring_degree=2), diffusion_kv_cache_dtype=None, @@ -414,12 +420,12 @@ class TestAttentionInitUsesCurrentDiffusionConfig: assert captured["role_category"] == "cross" assert captured["head_size"] == 64 assert captured["attention_config"] is od_config.diffusion_attention_config - assert attn.backend_pref == "TORCH_SDPA" - assert attn.attention.kwargs["backend_kwargs"] == {"block_size": 128} + assert attn.backend_pref == "TRTLLM_ATTN" + assert attn.attention.kwargs["backend_kwargs"] == {"target_sparsity": 0.5} assert attn.attention.kwargs["qkv_layout"] == "BSND" assert attn.use_ring is True assert attn.ring_runner is not None - assert attn.ring_runner.attn_backend_pref == "TORCH_SDPA" + assert attn.ring_runner.attn_backend_pref == "TRTLLM_ATTN" class TestDiffusionKvCacheQuantization: @@ -453,7 +459,10 @@ class TestDiffusionKvCacheQuantization: monkeypatch.setattr( layer_mod, "get_attn_backend_for_role", - lambda role, head_size, attention_config=None, role_category=None: (_FakeBackend, None), + lambda role, head_size, attention_config=None, role_category=None, allow_trtllm_default=False: ( + _FakeBackend, + None, + ), ) monkeypatch.setattr(layer_mod.SDPABackend, "get_impl_cls", staticmethod(lambda: _FakeAttentionImpl)) monkeypatch.setattr(layer_mod, "build_parallel_attention_strategy", lambda **kwargs: object()) diff --git a/tests/diffusion/attention/test_trtllm_attn.py b/tests/diffusion/attention/test_trtllm_attn.py new file mode 100644 index 000000000..75d04c2b8 --- /dev/null +++ b/tests/diffusion/attention/test_trtllm_attn.py @@ -0,0 +1,358 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import math + +import pytest +import torch +import torch.nn.functional as F + +from vllm_omni.diffusion.attention.backends import trtllm_attn as tg +from vllm_omni.diffusion.attention.backends.trtllm_attn import TrtllmAttentionImpl + +pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cuda] + + +def _has_trtllm_attn() -> bool: + if not torch.cuda.is_available(): + return False + if torch.cuda.get_device_capability()[0] < 10: + return False + try: + import flashinfer # noqa: F401 + except Exception: + return False + return tg.HAS_FLASHINFER + + +requires_trtllm_attn = pytest.mark.skipif( + not _has_trtllm_attn(), reason="requires Blackwell SM100+ GPU with flashinfer" +) + + +def _impl(**bk): + return TrtllmAttentionImpl(8, 128, 1.0 / math.sqrt(128), causal=False, num_kv_heads=8, backend_kwargs=bk) + + +def test_skip_config_pure_resolution(): + from vllm_omni.diffusion.attention.backends.trtllm_attn import SkipSoftmaxConfig + + assert not SkipSoftmaxConfig().enabled + assert SkipSoftmaxConfig().resolve_factor(4096, timestep=0.3) is None + + cfg = SkipSoftmaxConfig.from_backend_kwargs({"skip_softmax_threshold": 0.01, "disabled_until_timestep": 0.6}) + assert cfg.enabled and cfg.gated + assert cfg.resolve_factor(4096, timestep=0.9) is None + assert cfg.resolve_factor(4096, timestep=0.3) == pytest.approx(0.01 * 4096) + assert cfg.resolve_factor(4096, timestep=None) == pytest.approx(0.01 * 4096) + + +def test_skip_factor_none_without_curve(): + assert _impl(target_sparsity=0.5)._resolve_skip_factor(4096) is None + + +def _calibrated(a, b, **bk): + impl = _impl(**bk) + impl.set_layer_calibration(a, b) + return impl + + +def test_skip_factor_from_curve(): + assert _calibrated(100.0, 0.0, target_sparsity=0.5)._resolve_skip_factor(4096) == pytest.approx(100.0) + f2 = _calibrated(2.0, 1.0, target_sparsity=0.5)._resolve_skip_factor(4096) + assert f2 == pytest.approx(2.0 * math.exp(0.5)) + + +def test_skip_factor_none_until_stamped(): + assert _impl(target_sparsity=0.5)._resolve_skip_factor(4096) is None + + +def test_skip_factor_from_threshold_no_calibration(): + impl = _impl(skip_softmax_threshold=0.01) + assert impl._resolve_skip_factor(4096) == pytest.approx(0.01 * 4096) + assert impl._resolve_skip_factor(8192) == pytest.approx(0.01 * 8192) + + impl2 = _calibrated(100.0, 0.0, skip_softmax_threshold=0.02, target_sparsity=0.5) + assert impl2._resolve_skip_factor(4096) == pytest.approx(0.02 * 4096) + + +def test_skip_timestep_gate(monkeypatch): + from vllm_omni.diffusion import forward_context as fc + + impl = _calibrated(100.0, 0.0, target_sparsity=0.5, disabled_until_timestep=0.6) + ctx = fc.ForwardContext(denoise_timestep=0.9) + monkeypatch.setattr(fc, "_forward_context", ctx) + assert impl._resolve_skip_factor(4096) is None + ctx.denoise_timestep = 0.3 + assert impl._resolve_skip_factor(4096) == pytest.approx(100.0) + + +def test_denoise_progress_mixin(monkeypatch): + import types + + from vllm_omni.diffusion import forward_context as fc + + class _Pipe(fc.DenoiseProgressMixin): + scheduler = types.SimpleNamespace(config=types.SimpleNamespace(num_train_timesteps=1000)) + + ctx = fc.ForwardContext() + monkeypatch.setattr(fc, "_forward_context", ctx) + _Pipe().record_denoise_step(3, 250) + assert ctx.denoise_step_idx == 3 + assert ctx.denoise_timestep == pytest.approx(0.25) + + _Pipe().record_denoise_step(4) + assert ctx.denoise_step_idx == 4 and ctx.denoise_timestep == pytest.approx(0.25) + + +@pytest.mark.parametrize( + "bk, field", + [ + ({"target_sparsity": 1.5}, "target_sparsity"), + ({"target_sparsity": -0.1}, "target_sparsity"), + ({"disabled_until_timestep": 1.2}, "disabled_until_timestep"), + ({"skip_softmax_threshold": -1.0}, "skip_softmax_threshold"), + ({"target_sparsity": float("nan")}, "target_sparsity"), + ({"skip_softmax_threshold": float("inf")}, "skip_softmax_threshold"), + ], +) +def test_skip_control_validation_rejects_out_of_range(bk, field): + from vllm_omni.diffusion.attention.backends.trtllm_attn import SkipSoftmaxConfig + + with pytest.raises(ValueError, match=field): + SkipSoftmaxConfig.from_backend_kwargs(bk) + + +def test_skip_control_validation_accepts_valid(): + from vllm_omni.diffusion.attention.backends.trtllm_attn import SkipSoftmaxConfig + + cfg = SkipSoftmaxConfig.from_backend_kwargs( + {"target_sparsity": 0.65, "disabled_until_timestep": 0.86, "skip_softmax_threshold": 0.01} + ) + assert cfg.target_sparsity == pytest.approx(0.65) + assert cfg.disabled_until_timestep == pytest.approx(0.86) + assert cfg.threshold == pytest.approx(0.01) + + +def test_ring_sp_with_skip_softmax_rejected(): + import types + + from vllm_omni.diffusion.attention.layer import Attention + + fake = types.SimpleNamespace(attention=_impl(target_sparsity=0.5), ring_runner=None) + with pytest.raises(NotImplementedError, match="ring sequence parallelism"): + Attention._run_ring_attention(fake, None, None, None, None) + + +def test_mask_free_allowlist_gates_trtllm_default(): + from vllm_omni.diffusion.model_metadata import get_diffusion_model_metadata as meta + + assert meta("WanPipeline").attention_mask_free is True + assert meta("WanVACEPipeline").attention_mask_free is True + assert meta("FluxPipeline").attention_mask_free is False + assert meta(None).attention_mask_free is False + + +def test_masked_layer_raises(): + b, s, h, d = 1, 8, 8, 128 + q, k, v = (torch.zeros(b, s, h, d) for _ in range(3)) + mask = torch.ones(b, s, dtype=torch.bool) + mask[:, s // 2 :] = False + + class _Meta: + attn_mask = mask + + with pytest.raises(ValueError, match="does not support attn_mask"): + _impl().forward_cuda(q, k, v, _Meta()) + + +def test_propagate_calibration_045_schema(monkeypatch): + import types + + from vllm_omni.diffusion.data import ( + AttentionConfig, + AttentionSpec, + OmniDiffusionConfig, + TransformerConfig, + ) + + cfg = AttentionConfig(default=AttentionSpec(backend="TRTLLM_ATTN")) + tf = TransformerConfig.from_dict( + { + "sparse_attention_config": { + "config_groups": { + "group_0": { + "algorithm": "skip_softmax", + "targets": ["WanAttention"], + "ignore": ["blocks.0.attn2", "blocks.1.attn2"], + "threshold_scale_factor": { + "formula": "a * exp(b * target_sparsity)", + "coefficients": {"a": 2142.7, "b": 4.28}, + }, + "target_sparsity": 0.5, + } + } + } + } + ) + monkeypatch.setattr( + "vllm_omni.diffusion.attention.backends.trtllm_calibration.parse_secondary_expert_calibration", + lambda *a, **k: None, + ) + OmniDiffusionConfig._propagate_skip_softmax_calibration( + types.SimpleNamespace( + diffusion_attention_config=cfg, + model="/x", + ), + tf, + ) + spec, _ = cfg.resolve_with_source("self") + cal = spec.skip_calibration["by_expert"]["transformer"] + assert cal["a"] == pytest.approx(2142.7) + assert cal["b"] == pytest.approx(4.28) + assert cal["ignore"] == ["blocks.0.attn2", "blocks.1.attn2"] + assert "transformer_2" not in spec.skip_calibration["by_expert"] + + +def test_target_sparsity_without_calibration_raises(): + import types + + from vllm_omni.diffusion.data import ( + AttentionConfig, + AttentionSpec, + OmniDiffusionConfig, + TransformerConfig, + ) + + def _cfg(skip_softmax): + return types.SimpleNamespace( + diffusion_attention_config=AttentionConfig( + default=AttentionSpec(backend="TRTLLM_ATTN", skip_softmax=skip_softmax) + ), + model="/models/no-such-calibration", + ) + + with pytest.raises(ValueError, match="no skip-softmax calibration"): + OmniDiffusionConfig._propagate_skip_softmax_calibration( + _cfg({"target_sparsity": 0.5}), TransformerConfig.from_dict({}) + ) + + OmniDiffusionConfig._propagate_skip_softmax_calibration(_cfg({"threshold": 0.02}), TransformerConfig.from_dict({})) + + +def _sdpa_ref(q, k, v, scale): + qp, kp, vp = (x.permute(0, 2, 1, 3) for x in (q, k, v)) + return F.scaled_dot_product_attention(qp, kp, vp, is_causal=False, scale=scale).permute(0, 2, 1, 3).float() + + +@requires_trtllm_attn +def test_bf16_dense_matches_sdpa(): + torch.manual_seed(0) + B, S, H, D = 2, 256, 8, 128 + scale = 1.0 / math.sqrt(D) + q, k, v = (torch.randn(B, S, H, D, device="cuda", dtype=torch.bfloat16) for _ in range(3)) + out = _impl().forward_cuda(q, k, v, None).float() + ref = _sdpa_ref(q, k, v, scale) + rel = (out - ref).abs().mean() / ref.abs().mean() + assert rel < 0.01, f"BF16 dense rel err {rel:.4f} too high" + + +@requires_trtllm_attn +def test_skip_tiny_threshold_matches_dense(): + torch.manual_seed(0) + B, S, H, D = 2, 512, 8, 128 + scale = 1.0 / math.sqrt(D) + q, k, v = (torch.randn(B, S, H, D, device="cuda", dtype=torch.bfloat16) for _ in range(3)) + + out = _impl(skip_softmax_threshold=1e-9).forward_cuda(q, k, v, None).float() + ref = _sdpa_ref(q, k, v, scale) + rel = (out - ref).abs().mean() / ref.abs().mean() + assert rel < 0.02, f"skip(tiny) rel err {rel:.4f} should be ~dense" + + +@requires_trtllm_attn +def test_skip_threshold_engages(): + torch.manual_seed(0) + B, S, H, D = 2, 512, 8, 128 + q, k, v = (torch.randn(B, S, H, D, device="cuda", dtype=torch.bfloat16) for _ in range(3)) + dense = _impl().forward_cuda(q, k, v, None).float() + skipped = _impl(skip_softmax_threshold=2.0).forward_cuda(q, k, v, None).float() + assert torch.isfinite(skipped).all() + assert (skipped - dense).abs().mean() / dense.abs().mean() > 0.02 + + +@requires_trtllm_attn +def test_gqa_matches_sdpa(): + torch.manual_seed(0) + B, S, Hq, Hkv, D = 1, 256, 64, 8, 128 + scale = 1.0 / math.sqrt(D) + q = torch.randn(B, S, Hq, D, device="cuda", dtype=torch.bfloat16) + k = torch.randn(B, S, Hkv, D, device="cuda", dtype=torch.bfloat16) + v = torch.randn(B, S, Hkv, D, device="cuda", dtype=torch.bfloat16) + out = ( + TrtllmAttentionImpl(Hq, D, scale, causal=False, num_kv_heads=Hkv, backend_kwargs={}) + .forward_cuda(q, k, v, None) + .float() + ) + qp, kp, vp = (x.permute(0, 2, 1, 3) for x in (q, k, v)) + ref = ( + F.scaled_dot_product_attention(qp, kp, vp, is_causal=False, scale=scale, enable_gqa=True) + .permute(0, 2, 1, 3) + .float() + ) + rel = (out - ref).abs().mean() / ref.abs().mean() + assert rel < 0.01, f"GQA rel err {rel:.4f} vs SDPA(GQA) too high" + + +@requires_trtllm_attn +def test_unsupported_head_dim_raises(): + q = k = v = torch.randn(1, 32, 8, 64, device="cuda", dtype=torch.bfloat16) + with pytest.raises(Exception, match="[Uu]nsupported head dim"): + _impl().forward_cuda(q, k, v, None) + + +def test_skip_end_to_end_config_path(monkeypatch): + import types + + from vllm_omni.diffusion import forward_context as fc + from vllm_omni.diffusion.data import ( + AttentionConfig, + AttentionSpec, + OmniDiffusionConfig, + TransformerConfig, + ) + + cfg = AttentionConfig( + default=AttentionSpec( + backend="TRTLLM_ATTN", + skip_softmax={"target_sparsity": 0.65, "disabled_until_timestep": 0.86}, + ) + ) + + tf = TransformerConfig.from_dict( + {"sparse_attention_config": {"threshold_scale_factor": {"prefill": {"a": 7.93, "b": 8.61}}}} + ) + monkeypatch.setattr( + "vllm_omni.diffusion.attention.backends.trtllm_calibration.parse_secondary_expert_calibration", + lambda *a, **k: None, + ) + OmniDiffusionConfig._propagate_skip_softmax_calibration( + types.SimpleNamespace( + diffusion_attention_config=cfg, + model="/x", + ), + tf, + ) + spec, _ = cfg.resolve_with_source("self") + cal = spec.skip_calibration["by_expert"]["transformer"] + assert cal["a"] == 7.93 and cal["b"] == 8.61 + + impl = TrtllmAttentionImpl( + 8, 128, 1.0 / math.sqrt(128), causal=False, num_kv_heads=8, backend_kwargs=spec.backend_kwargs() + ) + impl.set_layer_calibration(cal["a"], cal["b"]) + expected = 7.93 * math.exp(8.61 * 0.65) + + ctx = fc.ForwardContext(denoise_timestep=0.3) + monkeypatch.setattr(fc, "_forward_context", ctx) + assert impl._resolve_skip_factor(4096) == pytest.approx(expected) diff --git a/tests/diffusion/attention/test_trtllm_calibration.py b/tests/diffusion/attention/test_trtllm_calibration.py new file mode 100644 index 000000000..4db7eaa16 --- /dev/null +++ b/tests/diffusion/attention/test_trtllm_calibration.py @@ -0,0 +1,224 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import pytest + +from vllm_omni.diffusion.attention.backends.trtllm_calibration import ( + is_ignored, + layer_match_names, + parse_sparse_attention_config, + resolve_layer_calibration, + select_expert, +) + +pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu] + + +def test_layer_match_names_strips_expert_prefix(): + names = layer_match_names("transformer_2.blocks.5.attn1") + assert "transformer_2.blocks.5.attn1" in names + assert "blocks.5.attn1" in names + + +def test_layer_match_names_strips_orig_mod(): + names = layer_match_names("transformer.blocks.5._orig_mod.attn1") + assert "transformer.blocks.5.attn1" in names or "blocks.5.attn1" in names + + +def test_is_ignored_matches_relative_pattern(): + assert is_ignored("transformer.blocks.5.attn2", ["blocks.*.attn2"]) + assert not is_ignored("transformer.blocks.5.attn1", ["blocks.*.attn2"]) + + +def test_is_ignored_empty_patterns(): + assert not is_ignored("transformer.blocks.5.attn1", []) + assert not is_ignored("transformer.blocks.5.attn1", None) + + +def test_is_ignored_ancestor_match_with_attn_suffix(): + assert is_ignored("transformer.blocks.0.attn2.attn", ["blocks.0.attn2"]) + + assert not is_ignored("transformer.blocks.11.attn2.attn", ["blocks.1.attn2"]) + + assert not is_ignored("transformer.blocks.5.attn1.attn", ["blocks.0.attn2", "blocks.1.attn2"]) + + +def test_select_expert_longest_prefix_wins(): + keys = ("transformer", "transformer_2") + assert select_expert("transformer_2.blocks.5.attn1", keys) == "transformer_2" + assert select_expert("transformer.blocks.5.attn1", keys) == "transformer" + + +def test_select_expert_no_match(): + assert select_expert("blocks.5.attn1", ("transformer", "transformer_2")) is None + + +def _two_expert_calib(): + return { + "by_expert": { + "transformer": { + "a": 2142.7, + "b": 4.28, + "target_sparsity": 0.45, + "formula": "a*exp(b*target_sparsity)", + "ignore": ["blocks.*.attn2"], + }, + "transformer_2": { + "a": 314.68, + "b": 6.17, + "target_sparsity": 0.45, + "formula": "a*exp(b*target_sparsity)", + "ignore": ["blocks.*.attn2"], + }, + } + } + + +def test_resolve_routes_to_correct_expert(): + calib = _two_expert_calib() + hi = resolve_layer_calibration("transformer.blocks.5.attn1", calib) + lo = resolve_layer_calibration("transformer_2.blocks.5.attn1", calib) + assert hi["a"] == pytest.approx(2142.7) + assert lo["a"] == pytest.approx(314.68) + + +def test_resolve_ignored_layer_is_dense(): + assert resolve_layer_calibration("transformer.blocks.5.attn2", _two_expert_calib()) is None + + +def test_resolve_single_transformer(): + calib = { + "a": 104.76, + "b": 7.81, + "target_sparsity": 0.45, + "formula": "a*exp(b*target_sparsity)", + "ignore": ["refiner_blocks.*"], + } + assert resolve_layer_calibration("transformer_blocks.5.attn", calib)["a"] == pytest.approx(104.76) + assert resolve_layer_calibration("refiner_blocks.0.attn", calib) is None + + +def test_resolve_none_calibration(): + assert resolve_layer_calibration("transformer.blocks.5.attn1", None) is None + + +def test_resolve_unmatched_component_is_uncalibrated(): + calib = {"by_expert": {"transformer": {"a": 1.0, "b": 2.0, "ignore": []}}} + assert resolve_layer_calibration("transformer.blocks.5.attn1", calib)["a"] == pytest.approx(1.0) + assert resolve_layer_calibration("transformer_2.blocks.5.attn1", calib) is None + assert resolve_layer_calibration("blocks.5.attn1", calib) is None + + +def test_parse_045_config_groups_schema(): + sac = { + "config_groups": { + "group_0": { + "algorithm": "skip_softmax", + "ignore": ["blocks.0.attn1"], + "target_sparsity": 0.45, + "threshold_scale_factor": { + "formula": "a * exp(b * target_sparsity)", + "coefficients": {"a": 2142.7, "b": 4.28}, + }, + } + } + } + parsed = parse_sparse_attention_config(sac) + assert parsed["a"] == pytest.approx(2142.7) + assert parsed["b"] == pytest.approx(4.28) + assert parsed["ignore"] == ["blocks.0.attn1"] + + +def test_parse_unsupported_formula_raises(): + sac = { + "config_groups": { + "group_0": { + "algorithm": "skip_softmax", + "threshold_scale_factor": {"formula": "a * b ** target_sparsity", "coefficients": {"a": 1.0, "b": 2.0}}, + } + } + } + with pytest.raises(ValueError, match="unsupported skip-softmax formula"): + parse_sparse_attention_config(sac) + + +def test_parse_empty_returns_none(): + assert parse_sparse_attention_config(None) is None + assert parse_sparse_attention_config({}) is None + + +class _FakeImpl: + def __init__(self): + self.stamped = None + + def set_layer_calibration(self, a, b): + self.stamped = (a, b) + + +def _fake_pipeline(): + import torch.nn as nn + + def _attn_layer(): + layer = nn.Module() + layer.attention = _FakeImpl() + return layer + + def _wan_attn(): + wa = nn.Module() + wa.attn = _attn_layer() + return wa + + def _transformer(): + tf = nn.Module() + blocks = nn.ModuleList() + for _ in range(3): + blk = nn.Module() + blk.attn1 = _wan_attn() + blk.attn2 = _wan_attn() + blocks.append(blk) + tf.blocks = blocks + return tf + + pipe = nn.Module() + pipe.transformer = _transformer() + pipe.transformer_2 = _transformer() + return pipe + + +def test_apply_to_pipeline_routes_and_ignores(): + from vllm_omni.diffusion.attention.backends.trtllm_calibration import apply_to_pipeline + + pipe = _fake_pipeline() + + calib = { + "by_expert": { + "transformer": { + "a": 2142.7, + "b": 4.28, + "target_sparsity": 0.5, + "formula": "a*exp(b*target_sparsity)", + "ignore": ["blocks.*.attn2", "blocks.0.attn1"], + }, + "transformer_2": { + "a": 314.68, + "b": 6.17, + "target_sparsity": 0.5, + "formula": "a*exp(b*target_sparsity)", + "ignore": ["blocks.*.attn2", "blocks.0.attn1"], + }, + } + } + stamped = apply_to_pipeline(pipe, calib) + + assert stamped == 4 + + def impl(expert, blk, attn): + return getattr(getattr(pipe, expert).blocks[blk], attn).attn.attention + + assert impl("transformer", 1, "attn1").stamped == pytest.approx((2142.7, 4.28)) + + assert impl("transformer_2", 1, "attn1").stamped == pytest.approx((314.68, 6.17)) + + assert impl("transformer", 1, "attn2").stamped is None + + assert impl("transformer", 0, "attn1").stamped is None diff --git a/vllm_omni/diffusion/attention/backends/registry.py b/vllm_omni/diffusion/attention/backends/registry.py index a822d78fb..92d3b6926 100644 --- a/vllm_omni/diffusion/attention/backends/registry.py +++ b/vllm_omni/diffusion/attention/backends/registry.py @@ -64,6 +64,7 @@ class DiffusionAttentionBackendEnum(Enum, metaclass=_DiffusionBackendEnumMeta): FLASHINFER_ATTN = "vllm_omni.diffusion.attention.backends.flashinfer_attn.FlashInferAttentionBackend" FLASH_ATTN_HUB = "vllm_omni.diffusion.attention.backends.flash_attn_hub.FlashAttentionHubBackend" FLASH_ATTN_3_HUB = "vllm_omni.diffusion.attention.backends.flash_attn_hub.FlashAttention3HubBackend" + TRTLLM_ATTN = "vllm_omni.diffusion.attention.backends.trtllm_attn.TrtllmAttentionBackend" def get_path(self, include_classname: bool = True) -> str: """Get the class path for this backend (respects overrides). diff --git a/vllm_omni/diffusion/attention/backends/trtllm_attn.py b/vllm_omni/diffusion/attention/backends/trtllm_attn.py new file mode 100644 index 000000000..9e3b305f1 --- /dev/null +++ b/vllm_omni/diffusion/attention/backends/trtllm_attn.py @@ -0,0 +1,226 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import math +from dataclasses import dataclass, replace + +import torch +from vllm.logger import init_logger + +from vllm_omni.diffusion.attention.backends.abstract import ( + AttentionBackend, + AttentionImpl, + AttentionMetadata, +) + +logger = init_logger(__name__) + + +def _validate_control(value, name: str, lo: float, hi: float | None) -> float | None: + if value is None: + return None + v = float(value) + if not math.isfinite(v) or v < lo or (hi is not None and v > hi): + rng = f"in [{lo}, {hi}]" if hi is not None else f">= {lo}" + raise ValueError(f"{name} must be finite and {rng}; got {value!r}.") + return v + + +@dataclass(frozen=True) +class SkipSoftmaxConfig: + threshold: float | None = None + target_sparsity: float | None = None + disabled_until_timestep: float = 0.0 + a: float | None = None + b: float | None = None + + @classmethod + def from_backend_kwargs(cls, backend_kwargs: dict | None) -> "SkipSoftmaxConfig": + bk = backend_kwargs or {} + return cls( + threshold=_validate_control(bk.get("skip_softmax_threshold"), "skip_softmax_threshold", 0.0, None), + target_sparsity=_validate_control(bk.get("target_sparsity"), "target_sparsity", 0.0, 1.0), + disabled_until_timestep=_validate_control( + bk.get("disabled_until_timestep", 0.0), "disabled_until_timestep", 0.0, 1.0 + ), + ) + + @property + def enabled(self) -> bool: + return self.threshold is not None or ( + self.target_sparsity is not None and self.a is not None and self.b is not None + ) + + @property + def configured(self) -> bool: + return self.threshold is not None or self.target_sparsity is not None + + @property + def gated(self) -> bool: + return self.disabled_until_timestep > 0.0 + + def resolve_factor(self, seqlen: int, timestep: float | None) -> float | None: + if self.threshold is not None: + factor = self.threshold * seqlen + elif self.target_sparsity is not None and self.a is not None and self.b is not None: + factor = self.a * math.exp(self.b * self.target_sparsity) + else: + return None + if self.gated and timestep is not None and timestep > self.disabled_until_timestep: + return None + return factor + + +try: + from flashinfer.prefill import trtllm_ragged_attention_deepseek + + HAS_FLASHINFER = True +except Exception as e: # pragma: no cover - import guard + HAS_FLASHINFER = False + logger.warning( + "FlashInfer is unavailable; TRTLLM_ATTN backend will not work. Reason: %s", + e, + ) + + +def _workspace_bytes() -> int: + import vllm.envs as envs + + return getattr(envs, "VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE", 394 * 1024 * 1024) + + +class TrtllmAttentionBackend(AttentionBackend): + accept_output_buffer: bool = True + + @staticmethod + def get_supported_head_sizes() -> list[int]: + return [128] + + @staticmethod + def get_name() -> str: + return "TRTLLM_ATTN" + + @staticmethod + def get_impl_cls() -> type["TrtllmAttentionImpl"]: + return TrtllmAttentionImpl + + +class TrtllmAttentionImpl(AttentionImpl): + _workspace: torch.Tensor | None = None + + def __init__( + self, + num_heads: int, + head_size: int, + softmax_scale: float, + causal: bool = False, + num_kv_heads: int | None = None, + prefix: str = "", + qkv_layout: str | None = None, + backend_kwargs: dict | None = None, + **extra_impl_args, + ) -> None: + self.num_heads = num_heads + self.head_size = head_size + self.softmax_scale = softmax_scale + self.causal = causal + self.num_kv_heads = num_kv_heads if num_kv_heads is not None else num_heads + + self.skip = SkipSoftmaxConfig.from_backend_kwargs(backend_kwargs) + self._warned_missing_timestep = False + + def set_layer_calibration(self, a: float, b: float) -> None: + self.skip = replace(self.skip, a=a, b=b) + + def _resolve_skip_factor(self, seqlen: int) -> float | None: + if not self.skip.enabled: + return None + + timestep = None + if self.skip.gated: + from vllm_omni.diffusion.forward_context import get_forward_context + + timestep = getattr(get_forward_context(), "denoise_timestep", None) + if timestep is None: + if not self._warned_missing_timestep: + logger.warning( + "TRTLLM skip: disabled_until_timestep=%s set but this pipeline does not " + "publish denoise_timestep; staying dense. Have the pipeline call " + "DenoiseProgressMixin.record_denoise_step to enable timestep gating.", + self.skip.disabled_until_timestep, + ) + self._warned_missing_timestep = True + return None + return self.skip.resolve_factor(seqlen, timestep) + + @classmethod + def _get_workspace(cls, device: torch.device) -> torch.Tensor: + nbytes = _workspace_bytes() + ws = cls._workspace + if ws is None or ws.device != device or ws.numel() < nbytes: + ws = torch.zeros(nbytes, dtype=torch.uint8, device=device) + cls._workspace = ws + return ws + + def forward_cuda( + self, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + attn_metadata: AttentionMetadata | None = None, + ) -> torch.Tensor: + attn_mask = getattr(attn_metadata, "attn_mask", None) if attn_metadata is not None else None + if attn_mask is not None: + raise ValueError( + "TRTLLM_ATTN does not support attn_mask (Wan sets one only under SP with " + "mask_sp_padding=True and a non-divisible seq len). Either set " + "parallel_config.mask_sp_padding=False, or select a mask-capable backend " + "(e.g. CUDNN_ATTN / TORCH_SDPA)." + ) + + if not HAS_FLASHINFER: + raise ImportError( + "TRTLLM_ATTN backend requires flashinfer. Install it or select " + "another backend via --diffusion-attention-backend." + ) + + batch, q_len, num_q_heads, head_dim = query.shape + kv_len, num_kv_heads = key.shape[1], key.shape[2] + + device = query.device + + q = query.reshape(batch * q_len, num_q_heads, head_dim).contiguous() + k = key.reshape(batch * kv_len, num_kv_heads, head_dim).contiguous() + v = value.reshape(batch * kv_len, num_kv_heads, head_dim).contiguous() + + seq_lens = torch.full((batch,), kv_len, dtype=torch.int32, device=device) + cu_seq_lens_q = torch.arange(0, (batch + 1) * q_len, step=q_len, dtype=torch.int32, device=device) + cu_seq_lens_kv = torch.arange(0, (batch + 1) * kv_len, step=kv_len, dtype=torch.int32, device=device) + workspace = self._get_workspace(device) + + bmm1_scale = self.softmax_scale + bmm2_scale = 1.0 + + _skip_factor = self._resolve_skip_factor(kv_len) + + out = trtllm_ragged_attention_deepseek( + query=q, + key=k, + value=v, + workspace_buffer=workspace, + seq_lens=seq_lens, + max_q_len=q_len, + max_kv_len=kv_len, + bmm1_scale=bmm1_scale, + bmm2_scale=bmm2_scale, + o_sf_scale=-1.0, + batch_size=batch, + window_left=-1, + cum_seq_lens_q=cu_seq_lens_q, + cum_seq_lens_kv=cu_seq_lens_kv, + enable_pdl=False, + is_causal=self.causal, + return_lse=False, + skip_softmax_threshold_scale_factor=_skip_factor, + ) + return out.reshape(batch, q_len, num_q_heads, head_dim) diff --git a/vllm_omni/diffusion/attention/backends/trtllm_calibration.py b/vllm_omni/diffusion/attention/backends/trtllm_calibration.py new file mode 100644 index 000000000..71b26a53f --- /dev/null +++ b/vllm_omni/diffusion/attention/backends/trtllm_calibration.py @@ -0,0 +1,152 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from __future__ import annotations + +import fnmatch +import logging +from typing import Any + +logger = logging.getLogger(__name__) + +_SUPPORTED_FORMULA = "a*exp(b*target_sparsity)" + + +def parse_sparse_attention_config(sac: dict | None) -> dict | None: + if not sac: + return None + group: dict = {} + for g in (sac.get("config_groups") or {}).values(): + if isinstance(g, dict) and (g.get("algorithm") == "skip_softmax" or g.get("threshold_scale_factor")): + group = g + break + tsf = group.get("threshold_scale_factor") or sac.get("threshold_scale_factor") or {} + ab = tsf.get("coefficients") or tsf.get("prefill") or {} + if "a" not in ab or "b" not in ab: + return None + + formula = (tsf.get("formula") or _SUPPORTED_FORMULA).replace(" ", "") + if formula != _SUPPORTED_FORMULA: + raise ValueError( + f"unsupported skip-softmax formula {tsf.get('formula')!r}; this backend implements '{_SUPPORTED_FORMULA}'." + ) + + return {"a": float(ab["a"]), "b": float(ab["b"]), "ignore": list(group.get("ignore") or [])} + + +def parse_secondary_expert_calibration(model: str | None) -> dict | None: + from vllm.transformers_utils.config import get_hf_file_to_dict + + if not model: + return None + try: + cfg = get_hf_file_to_dict("transformer_2/config.json", model) + return parse_sparse_attention_config(cfg.get("sparse_attention_config")) if cfg else None + except Exception as exc: # noqa: BLE001 + logger.warning( + "Could not read transformer_2 skip-softmax calibration (%s); transformer_2 will stay uncalibrated (dense).", + exc, + ) + return None + + +def propagate_skip_softmax_calibration(specs: list, model: str | None, tf_config: Any) -> None: + calibration = parse_sparse_attention_config( + tf_config.get("sparse_attention_config") if tf_config is not None else None + ) + if calibration is not None: + by_expert = {"transformer": calibration} + ckpt2 = parse_secondary_expert_calibration(model) + if ckpt2 is not None: + by_expert["transformer_2"] = ckpt2 + calibration = {"by_expert": by_expert} + + if calibration is None: + for spec in specs: + ss = spec.skip_softmax + if ss is not None and ss.target_sparsity is not None and ss.threshold is None: + raise ValueError( + f"target_sparsity was requested but the checkpoint for model '{model}' carries no " + f"skip-softmax calibration. Either set skip_softmax.threshold for the " + f"calibration-free path or load a calibrated checkpoint." + ) + return + + for spec in specs: + if spec.skip_calibration is None: + spec.skip_calibration = calibration + + experts = list(calibration["by_expert"]) + logger.info("Loaded skip-softmax calibration from checkpoint (%d curve(s): %s).", len(experts), ", ".join(experts)) + + +def apply_skip_softmax_calibration(attention_config: Any, model: Any) -> None: + cfg = attention_config + specs = getattr(cfg, "per_role", None) + calibration = None + for spec in (getattr(cfg, "default", None), *(specs.values() if specs else ())): + calibration = calibration or getattr(spec, "skip_calibration", None) + if not calibration: + return + + stamped = apply_to_pipeline(model, calibration) + logger.info("Skip-softmax: stamped calibration onto %d attention layer(s).", stamped) + + +_EXPERT_PREFIXES = ("transformer.", "transformer_2.") + + +def layer_match_names(module_name: str) -> tuple[str, ...]: + names = {module_name, module_name.replace("._orig_mod.", ".")} + for name in tuple(names): + for pfx in _EXPERT_PREFIXES: + if name.startswith(pfx): + names.add(name[len(pfx) :]) + return tuple(names) + + +def is_ignored(module_name: str, ignore_patterns) -> bool: + names = layer_match_names(module_name) + for p in ignore_patterns or (): + for n in names: + if fnmatch.fnmatch(n, p) or fnmatch.fnmatch(n, p + ".*"): + return True + return False + + +def select_expert(module_name: str, expert_keys) -> str | None: + for key in sorted(expert_keys or (), key=len, reverse=True): + if module_name == key or module_name.startswith(key + "."): + return key + return None + + +def resolve_layer_calibration(module_name: str, calibration: dict) -> dict | None: + if not calibration: + return None + by_expert = calibration.get("by_expert") + if by_expert: + key = select_expert(module_name, by_expert.keys()) + if key is None: + return None + entry = by_expert[key] + else: + entry = calibration + if is_ignored(module_name, entry.get("ignore")): + return None + return {"a": entry.get("a"), "b": entry.get("b")} + + +def apply_to_pipeline(pipeline, calibration: dict) -> int: + if not calibration: + return 0 + stamped = 0 + for name, module in pipeline.named_modules(): + impl = getattr(module, "attention", None) + set_layer = getattr(impl, "set_layer_calibration", None) + if set_layer is None: + continue + per = resolve_layer_calibration(name, calibration) + if per and per.get("a") is not None and per.get("b") is not None: + set_layer(per["a"], per["b"]) + stamped += 1 + return stamped diff --git a/vllm_omni/diffusion/attention/layer.py b/vllm_omni/diffusion/attention/layer.py index a11aecbb8..77d65f79e 100644 --- a/vllm_omni/diffusion/attention/layer.py +++ b/vllm_omni/diffusion/attention/layer.py @@ -77,14 +77,20 @@ class Attention(nn.Module): config = get_current_diffusion_config_or_none() attention_config = config.diffusion_attention_config if config is not None else None + from vllm_omni.diffusion.model_metadata import get_diffusion_model_metadata + + model_class_name = getattr(config, "model_class_name", None) if config is not None else None + allow_trtllm_default = get_diffusion_model_metadata(model_class_name).attention_mask_free + attn_backend_cls, spec = get_attn_backend_for_role( role=role, head_size=head_size, attention_config=attention_config, role_category=role_category, + allow_trtllm_default=allow_trtllm_default, ) if spec is not None: - backend_kwargs = spec.extra or None + backend_kwargs = spec.backend_kwargs() self.backend_pref = spec.backend logger.debug("Attention(role=%s) → backend=%s", role, spec.backend) else: @@ -99,6 +105,7 @@ class Attention(nn.Module): causal=causal, num_kv_heads=num_kv_heads, qkv_layout=qkv_layout, + prefix=prefix, backend_kwargs=backend_kwargs, ) # Instantiate fallback backend for float32 support @@ -310,6 +317,13 @@ class Attention(nn.Module): ) def _run_ring_attention(self, query, key, value, attn_metadata): + skip = getattr(self.attention, "skip", None) + if skip is not None and getattr(skip, "configured", False): + raise NotImplementedError( + "Skip-Softmax (TRTLLM_ATTN) is not supported with ring sequence parallelism: " + "the ring path bypasses the backend, so the skip config would be silently ignored. " + "Use Ulysses SP instead, or remove the skip_softmax config." + ) # Delegate to RingParallelAttention strategy if available if self.ring_runner is not None: return self.ring_runner.run_attention( diff --git a/vllm_omni/diffusion/attention/selector.py b/vllm_omni/diffusion/attention/selector.py index 6eede02c2..9d357b4f6 100644 --- a/vllm_omni/diffusion/attention/selector.py +++ b/vllm_omni/diffusion/attention/selector.py @@ -51,18 +51,20 @@ def _load_backend_cls(cls_path: str) -> type[AttentionBackend]: def _cached_get_backend_cls( backend_name: str | None, head_size: int, + allow_trtllm_default: bool = True, ) -> type[AttentionBackend]: - """Cache backend class resolution by (backend_name, head_size). + """Cache backend class resolution by (backend_name, head_size, allow_trtllm_default). This ensures platform validation (compute capability checks, package - availability, etc.) runs only once per unique (backend_name, head_size) - combination, avoiding repeated log messages. + availability, etc.) runs only once per unique combination, avoiding + repeated log messages. """ from vllm_omni.platforms import current_omni_platform backend_cls_path = current_omni_platform.get_diffusion_attn_backend_cls( selected_backend=backend_name, head_size=head_size, + allow_trtllm_default=allow_trtllm_default, ) return _load_backend_cls(backend_cls_path) @@ -97,6 +99,7 @@ def get_attn_backend_for_role( head_size: int, attention_config: AttentionConfig | None = None, role_category: str | None = None, + allow_trtllm_default: bool = True, ) -> tuple[type[AttentionBackend], AttentionSpec | None]: """ Get attention backend for a specific attention role. @@ -140,7 +143,7 @@ def get_attn_backend_for_role( return backend_cls, spec # 2. Platform default - backend_cls = _cached_get_backend_cls(None, head_size) + backend_cls = _cached_get_backend_cls(None, head_size, allow_trtllm_default) _log_backend_resolution( role=role, role_category=role_category, diff --git a/vllm_omni/diffusion/data.py b/vllm_omni/diffusion/data.py index 9e049b211..d3880f900 100644 --- a/vllm_omni/diffusion/data.py +++ b/vllm_omni/diffusion/data.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import copy +import math import os import random from collections.abc import Callable, Mapping @@ -1070,6 +1071,19 @@ class OmniDiffusionConfig: """ self.tf_model_config = tf_config self._propagate_quantization_from_tf_config(tf_config) + self._propagate_skip_softmax_calibration(tf_config) + + def _propagate_skip_softmax_calibration(self, tf_config: "TransformerConfig") -> None: + cfg = getattr(self, "diffusion_attention_config", None) + if not isinstance(cfg, AttentionConfig): + return + specs = [s for s in (cfg.default, *cfg.per_role.values()) if s is not None] + + from vllm_omni.diffusion.attention.backends.trtllm_calibration import ( + propagate_skip_softmax_calibration, + ) + + propagate_skip_softmax_calibration(specs, self.model, tf_config) def update_multimodal_support(self) -> None: # Resolve serving-visible multimodal behavior from shared metadata @@ -1323,23 +1337,78 @@ class DiffusionRequestAbortedError(RuntimeError): """Raised when a diffusion request ends via user-visible abort.""" +def _in_range(value: Any, name: str, lo: float, hi: float | None) -> float | None: + """Validate an optional numeric control at config-parse time.""" + if value is None: + return None + x = float(value) + if not math.isfinite(x) or x < lo or (hi is not None and x > hi): + rng = f"in [{lo}, {hi}]" if hi is not None else f">= {lo}" + raise ValueError(f"{name} must be finite and {rng}; got {value!r}.") + return x + + +@dataclass +class SkipSoftmaxSpec: + """User-facing skip-softmax controls for the TRTLLM_ATTN backend. + + ``target_sparsity`` uses the checkpoint's calibrated curve (a·exp(b·s)); ``threshold`` + is the calibration-free absolute skip threshold. They are mutually exclusive. + """ + + target_sparsity: float | None = None + threshold: float | None = None + disabled_until_timestep: float = 0.0 + + def __post_init__(self) -> None: + self.target_sparsity = _in_range(self.target_sparsity, "skip_softmax.target_sparsity", 0.0, 1.0) + self.threshold = _in_range(self.threshold, "skip_softmax.threshold", 0.0, None) + self.disabled_until_timestep = ( + _in_range(self.disabled_until_timestep, "skip_softmax.disabled_until_timestep", 0.0, 1.0) or 0.0 + ) + if self.target_sparsity is not None and self.threshold is not None: + raise ValueError("skip_softmax: set either target_sparsity or threshold, not both.") + + @dataclass class AttentionSpec: - """Specifies a backend and its backend-specific parameters for one attention role.""" + """Specifies a backend and its typed backend-specific config for one attention role.""" - backend: str # registry name, e.g. "FLASH_ATTN" - extra: dict[str, Any] = field(default_factory=dict) # backend-specific kwargs + backend: str + skip_softmax: SkipSoftmaxSpec | None = None + skip_calibration: dict | None = field(default=None, repr=False) def __post_init__(self) -> None: if not isinstance(self.backend, str): raise TypeError(f"Expected str for AttentionSpec.backend, got {type(self.backend)!r}") + self.skip_softmax = self._coerce_skip_softmax(self.skip_softmax) + if self.skip_softmax is not None and self.backend.upper() != "TRTLLM_ATTN": + raise ValueError( + f"skip_softmax is only supported by the TRTLLM_ATTN backend, but backend={self.backend!r}. " + f"Remove skip_softmax or set backend to TRTLLM_ATTN." + ) - if self.extra is None: - self.extra = {} - elif isinstance(self.extra, Mapping): - self.extra = dict(self.extra) - else: - raise TypeError(f"Expected dict for AttentionSpec.extra, got {type(self.extra)!r}") + @staticmethod + def _coerce_skip_softmax(value: Any) -> SkipSoftmaxSpec | None: + if value is None or isinstance(value, SkipSoftmaxSpec): + return value + if isinstance(value, Mapping): + return SkipSoftmaxSpec(**dict(value)) + raise TypeError(f"Expected dict or SkipSoftmaxSpec for skip_softmax, got {type(value)!r}") + + def backend_kwargs(self) -> dict[str, Any] | None: + """Serialize typed backend config into the kwargs dict the backend impl consumes.""" + if self.skip_softmax is None: + return None + ss = self.skip_softmax + kw: dict[str, Any] = {} + if ss.threshold is not None: + kw["skip_softmax_threshold"] = ss.threshold + if ss.target_sparsity is not None: + kw["target_sparsity"] = ss.target_sparsity + if ss.disabled_until_timestep: + kw["disabled_until_timestep"] = ss.disabled_until_timestep + return kw or None @dataclass @@ -1408,13 +1477,13 @@ class AttentionConfig: normalized[role] = node return - spec_keys = {"backend", "extra"} + spec_keys = {"backend", "skip_softmax"} node_dict = dict(node) node_keys = set(node_dict) if node_keys & spec_keys: if not node_keys <= spec_keys: raise ValueError( - f"Invalid per_role entry for role {role!r}: cannot mix backend/extra with nested role keys." + f"Invalid per_role entry for role {role!r}: cannot mix backend/skip_softmax with nested role keys." ) normalized[role] = node_dict return diff --git a/vllm_omni/diffusion/forward_context.py b/vllm_omni/diffusion/forward_context.py index ea230ed9b..a8e23756f 100644 --- a/vllm_omni/diffusion/forward_context.py +++ b/vllm_omni/diffusion/forward_context.py @@ -28,6 +28,7 @@ class ForwardContext: attn_metadata: dict[str, AttentionMetadata] | list[dict[str, AttentionMetadata]] | None = None split_text_embed_in_sp: bool = False denoise_step_idx: int | None = None + denoise_timestep: float | None = None # Per-request reference latent for img2img DiT models (e.g. Ming) ref_latent: torch.Tensor | None = None # whether to split the text embed in sequence parallel, if True, the text embed will be split in sequence parallel @@ -214,6 +215,21 @@ def set_forward_context_denoise_step_idx(step_idx: int | None) -> None: _forward_context.denoise_step_idx = step_idx +class DenoiseProgressMixin: + def record_denoise_step( + self, + step_idx: int | None, + timestep=None, + scheduler=None, + ) -> None: + set_forward_context_denoise_step_idx(step_idx) + if timestep is None or _forward_context is None: + return + scheduler = scheduler if scheduler is not None else getattr(self, "scheduler", None) + ntt = getattr(getattr(scheduler, "config", None), "num_train_timesteps", None) + _forward_context.denoise_timestep = float(timestep) / ntt if ntt else None + + def set_forward_context_ref_latent(ref_latent: torch.Tensor | None) -> None: """Set the per-request reference latent on the active ForwardContext. diff --git a/vllm_omni/diffusion/model_loader/diffusers_loader.py b/vllm_omni/diffusion/model_loader/diffusers_loader.py index bb048451a..9864d8dbf 100644 --- a/vllm_omni/diffusion/model_loader/diffusers_loader.py +++ b/vllm_omni/diffusion/model_loader/diffusers_loader.py @@ -378,6 +378,7 @@ class DiffusersPipelineLoader: model.to("cpu") logger.info("Quantization complete, offloaded model back to CPU") + self._apply_skip_softmax_calibration(model) return model.eval() @staticmethod @@ -391,6 +392,14 @@ class DiffusersPipelineLoader: return True return False + def _apply_skip_softmax_calibration(self, model: nn.Module) -> None: + from vllm_omni.diffusion.attention.backends.trtllm_calibration import ( + apply_skip_softmax_calibration, + ) + + cfg = getattr(self.od_config, "diffusion_attention_config", None) + apply_skip_softmax_calibration(cfg, model) + def _process_weights_after_loading(self, model: nn.Module, target_device: torch.device) -> None: """Process weights after loading for quantization methods. diff --git a/vllm_omni/diffusion/model_metadata.py b/vllm_omni/diffusion/model_metadata.py index 7e02cf5d8..4952539b7 100644 --- a/vllm_omni/diffusion/model_metadata.py +++ b/vllm_omni/diffusion/model_metadata.py @@ -10,6 +10,7 @@ class DiffusionModelMetadata: # config/model plumbing can read it without importing concrete pipelines. supports_multimodal_inputs: bool = False max_multimodal_image_inputs: int | None = None + attention_mask_free: bool = False QWEN_IMAGE_EDIT_PLUS_MAX_INPUT_IMAGES = 4 @@ -35,6 +36,10 @@ _DIFFUSION_MODEL_METADATA: dict[str, DiffusionModelMetadata] = { supports_multimodal_inputs=True, max_multimodal_image_inputs=BOOGU_IMAGE_MAX_INPUT_IMAGES, ), + "WanPipeline": DiffusionModelMetadata(attention_mask_free=True), + "WanImageToVideoPipeline": DiffusionModelMetadata(attention_mask_free=True), + "WanVACEPipeline": DiffusionModelMetadata(attention_mask_free=True), + "WanS2VPipeline": DiffusionModelMetadata(attention_mask_free=True), } diff --git a/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2.py b/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2.py index d3726e087..e3617f84d 100644 --- a/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2.py +++ b/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2.py @@ -24,7 +24,7 @@ from vllm_omni.diffusion.distributed.autoencoders.autoencoder_kl_wan import Dist from vllm_omni.diffusion.distributed.cfg_parallel import CFGParallelMixin from vllm_omni.diffusion.distributed.pipeline_parallel import AsyncLatents, PipelineParallelMixin from vllm_omni.diffusion.distributed.utils import get_local_device -from vllm_omni.diffusion.forward_context import set_forward_context_denoise_step_idx +from vllm_omni.diffusion.forward_context import DenoiseProgressMixin from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader from vllm_omni.diffusion.model_loader.hub_prefetch import from_pretrained_with_prefetch, prefetch_subfolders from vllm_omni.diffusion.models.dmd2 import DMD2PipelineMixin @@ -264,6 +264,7 @@ class Wan22Pipeline( PipelineParallelMixin, CFGParallelMixin, ProgressBarMixin, + DenoiseProgressMixin, DiffusionPipelineProfilerMixin, SupportsComponentDiscovery, ): @@ -460,7 +461,7 @@ class Wan22Pipeline( with self.progress_bar(total=len(timesteps)) as pbar: for step_idx, t in enumerate(timesteps): self._current_timestep = t - set_forward_context_denoise_step_idx(step_idx) + self.record_denoise_step(step_idx, t) # Select model based on timestep and boundary_ratio # High noise stage (t >= boundary_timestep): use transformer diff --git a/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_i2v.py b/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_i2v.py index 8b042b595..d8c1d4a8c 100644 --- a/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_i2v.py +++ b/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_i2v.py @@ -24,7 +24,7 @@ from vllm_omni.diffusion.distributed.autoencoders.autoencoder_kl_wan import Dist from vllm_omni.diffusion.distributed.cfg_parallel import CFGParallelMixin from vllm_omni.diffusion.distributed.pipeline_parallel import AsyncLatents, PipelineParallelMixin from vllm_omni.diffusion.distributed.utils import get_local_device -from vllm_omni.diffusion.forward_context import set_forward_context_denoise_step_idx +from vllm_omni.diffusion.forward_context import DenoiseProgressMixin from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader from vllm_omni.diffusion.model_loader.hub_prefetch import from_pretrained_with_prefetch, prefetch_subfolders from vllm_omni.diffusion.models.dmd2 import DMD2PipelineMixin @@ -144,6 +144,7 @@ class Wan22I2VPipeline( CFGParallelMixin, ProgressBarMixin, DiffusionPipelineProfilerMixin, + DenoiseProgressMixin, SupportsComponentDiscovery, ): """ @@ -354,7 +355,7 @@ class Wan22I2VPipeline( current_model = self.transformer_2 current_guidance_scale = guidance_high - set_forward_context_denoise_step_idx(step_idx) + self.record_denoise_step(step_idx, t) # Prepare latent input if self.expand_timesteps: diff --git a/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_s2v.py b/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_s2v.py index 47b8e02d1..52a42cb79 100644 --- a/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_s2v.py +++ b/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_s2v.py @@ -29,6 +29,7 @@ from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig from vllm_omni.diffusion.distributed.autoencoders.autoencoder_kl_wan import DistributedAutoencoderKLWan from vllm_omni.diffusion.distributed.cfg_parallel import CFGParallelMixin from vllm_omni.diffusion.distributed.utils import get_local_device +from vllm_omni.diffusion.forward_context import DenoiseProgressMixin from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader from vllm_omni.diffusion.models.interface import SupportAudioInput, SupportImageInput from vllm_omni.diffusion.models.progress_bar import ProgressBarMixin @@ -462,7 +463,13 @@ def _load_wan_t5_as_umt5( class Wan22S2VPipeline( - nn.Module, SupportImageInput, SupportAudioInput, CFGParallelMixin, ProgressBarMixin, DiffusionPipelineProfilerMixin + nn.Module, + SupportImageInput, + SupportAudioInput, + CFGParallelMixin, + DenoiseProgressMixin, + ProgressBarMixin, + DiffusionPipelineProfilerMixin, ): """ Wan2.2 Speech-to-Video Pipeline. @@ -993,8 +1000,9 @@ class Wan22S2VPipeline( do_true_cfg = self.do_classifier_free_guidance and negative_prompt_embeds is not None with self.progress_bar(total=len(timesteps)) as pbar: - for t in timesteps: + for step_idx, t in enumerate(timesteps): self._current_timestep = t + self.record_denoise_step(step_idx, t) latent_model_input = [latents.to(device)] timestep = t.unsqueeze(0).to(device) if t.dim() == 0 else t.to(device) diff --git a/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_vace.py b/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_vace.py index 4173b8601..d0cdb1250 100644 --- a/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_vace.py +++ b/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_vace.py @@ -25,7 +25,6 @@ from vllm.model_executor.layers.quantization.base_config import QuantizationConf from vllm.model_executor.models.utils import AutoWeightsLoader from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig -from vllm_omni.diffusion.forward_context import set_forward_context_denoise_step_idx from vllm_omni.diffusion.models.interface import SupportImageInput from vllm_omni.diffusion.models.wan2_2.pipeline_wan2_2 import ( Wan22Pipeline, @@ -204,7 +203,7 @@ class Wan22VACEPipeline(Wan22Pipeline, SupportImageInput): with self.progress_bar(total=len(timesteps)) as pbar: for step_idx, t in enumerate(timesteps): self._current_timestep = t - set_forward_context_denoise_step_idx(step_idx) + self.record_denoise_step(step_idx, t) if boundary_timestep is not None and t < boundary_timestep and self.transformer_2 is not None: current_model = self.transformer_2 diff --git a/vllm_omni/entrypoints/cli/serve.py b/vllm_omni/entrypoints/cli/serve.py index 98c086d86..ce0549597 100644 --- a/vllm_omni/entrypoints/cli/serve.py +++ b/vllm_omni/entrypoints/cli/serve.py @@ -561,7 +561,8 @@ class OmniServeCommand(CLISubcommand): help="Diffusion attention config. Accepts JSON or vLLM-style dotted flags. " "Examples: " "--diffusion-attention-config.default.backend FLASH_ATTN, " - "--diffusion-attention-config.per_role.self.backend SPARSE_BLOCK, " + "--diffusion-attention-config.default.backend TRTLLM_ATTN " + "--diffusion-attention-config.default.skip_softmax.target_sparsity 0.5, " "--diffusion-attention-config.per_role.cross.backend SAGE_ATTN, " '--diffusion-attention-config \'{"default": {"backend": "FLASH_ATTN"}, ' '"per_role": {"cross": {"backend": "SAGE_ATTN"}}}\'.', diff --git a/vllm_omni/platforms/cuda/platform.py b/vllm_omni/platforms/cuda/platform.py index 7be135352..4f203cfc2 100644 --- a/vllm_omni/platforms/cuda/platform.py +++ b/vllm_omni/platforms/cuda/platform.py @@ -57,6 +57,7 @@ class CudaOmniPlatform(OmniPlatform, CudaPlatformBase): cls, selected_backend: str | None, head_size: int, + allow_trtllm_default: bool = True, ) -> str: from vllm_omni.diffusion.envs import PACKAGES_CHECKER @@ -119,6 +120,18 @@ class CudaOmniPlatform(OmniPlatform, CudaPlatformBase): # not abort startup — just treat FlashInfer as unavailable. logger.debug("FlashInfer probe failed (%s); treating as unavailable", e) + # TRTLLM_ATTN needs the trtllm-gen kernel specifically, not just any FlashInfer + # wheel. Probe the actual symbol so a released wheel lacking it does not get + # auto-routed to TRTLLM_ATTN and then crash on the first forward. + trtllm_gen_available = False + if flashinfer_available: + try: + from flashinfer.prefill import trtllm_ragged_attention_deepseek # noqa: F401 + + trtllm_gen_available = True + except Exception as e: + logger.debug("trtllm-gen kernel probe failed (%s); treating as unavailable", e) + if selected_backend is not None: backend_upper = selected_backend.upper() if backend_upper in ("FLASH_ATTN_HUB", "FLASH_ATTN_3_HUB"): @@ -172,10 +185,37 @@ class CudaOmniPlatform(OmniPlatform, CudaPlatformBase): "SageAttention/sageattention3_blackwell. Falling back to TORCH_SDPA backend." ) return DiffusionAttentionBackendEnum.TORCH_SDPA.get_path() + if backend_upper == "TRTLLM_ATTN": + trtllm_attn_supported = compute_capability is not None and compute_capability.major == 10 + if not trtllm_attn_supported: + raise ValueError( + "TRTLLM_ATTN diffusion attention backend requires a datacenter " + "Blackwell GPU (SM100 / SM103, compute capability 10.x). Select a " + "different --diffusion-attention-backend." + ) + if not flashinfer_available: + raise ValueError( + "TRTLLM_ATTN diffusion attention backend requires flashinfer, which is not installed." + ) backend = DiffusionAttentionBackendEnum[backend_upper] logger.debug("Using diffusion attention backend '%s'", backend_upper) return backend.get_path() + trtllm_attn_default_ok = ( + allow_trtllm_default + and compute_capability is not None + and compute_capability.major == 10 + and head_size == 128 + and trtllm_gen_available + ) + if trtllm_attn_default_ok: + logger.info( + "Defaulting to diffusion attention backend TRTLLM_ATTN (datacenter Blackwell %s, head_dim %d)", + sm_str, + head_size, + ) + return DiffusionAttentionBackendEnum.TRTLLM_ATTN.get_path() + if is_blackwell and cudnn_blackwell_ready: logger.info( "Defaulting to diffusion attention backend CUDNN_ATTN (Blackwell %s, cuDNN %d)", From fb89ab43ead813afa8a5580f8aa986f2b111771c Mon Sep 17 00:00:00 2001 From: LHXuuu Date: Mon, 27 Jul 2026 22:55:47 +0800 Subject: [PATCH 117/240] [Refactor][Ming-TTS] Move speaker extraction and caching to Stage-0 (#5351) Signed-off-by: LHXuuu Co-authored-by: Yuanheng Zhao <54058983+yuanheng-zhao@users.noreply.github.com> --- .../openai_api/test_serving_speech.py | 147 ++++++++++++------ .../ming_tts/test_ming_shared_modules.py | 123 +++++++++++++++ .../entrypoints/openai/serving_speech.py | 78 ++++++---- .../openai/tts_adapters/ming_tts.py | 47 ++---- .../models/ming_tts/config_ming_tts.py | 6 + .../models/ming_tts/constants.py | 3 + .../models/ming_tts/ming_tts.py | 15 +- .../models/ming_tts/prompt_assembly.py | 14 +- .../models/ming_tts/speaker_extractor.py | 130 ++++++++++++++++ 9 files changed, 440 insertions(+), 123 deletions(-) diff --git a/tests/entrypoints/openai_api/test_serving_speech.py b/tests/entrypoints/openai_api/test_serving_speech.py index aab1acfbc..b6c7c7c87 100644 --- a/tests/entrypoints/openai_api/test_serving_speech.py +++ b/tests/entrypoints/openai_api/test_serving_speech.py @@ -45,7 +45,13 @@ from vllm_omni.model_executor.models.fish_speech.prompt_utils import ( FISH_TEXT_ONLY_SYSTEM_PROMPT, build_fish_voice_clone_prompt_ids, ) -from vllm_omni.model_executor.models.ming_tts.constants import SPEAKER_EMBEDDING_DIM +from vllm_omni.model_executor.models.ming_tts.constants import ( + KEY_SPEAKER_EMBEDDING, + KEY_SPEAKER_SAMPLE_RATES, + KEY_SPEAKER_WAVEFORM, + KEY_SPEAKER_WAVEFORM_LENGTHS, + SPEAKER_EMBEDDING_DIM, +) from vllm_omni.outputs import OmniRequestOutput pytestmark = [pytest.mark.core_model, pytest.mark.cpu] @@ -1014,7 +1020,6 @@ class TestTTSMethods: right = np.full(32000, -0.25, dtype=np.float32) samples = np.stack((left, right), axis=-1) mocker.patch("soundfile.read", return_value=(samples, 16000)) - mock_extract = mocker.patch.object(speech_server, "_extract_ming_speaker_embeddings_from_ref_audio") saved: dict[str, object] = {} def fake_save_file(tensors, path, metadata=None): @@ -1038,7 +1043,6 @@ class TestTTSMethods: ) ) - mock_extract.assert_not_called() tensors = saved["tensors"] metadata = saved["metadata"] assert isinstance(tensors, dict) @@ -1669,16 +1673,20 @@ class TestTTSMethods: mock_get_emb.assert_called_once_with("emb_voice") mock_get_audio.assert_not_called() - mock_prompt.assert_called_once_with(req, ref_audio_data=None) + mock_prompt.assert_called_once_with( + req, + ref_audio_data=None, + voice_name=None, + voice_created_at=0, + ) prompt_request = mock_prompt.call_args.args[0] assert prompt_request.speaker_embedding == fake_embedding assert len(prompt_request.speaker_embedding) == SPEAKER_EMBEDDING_DIM assert prepared.prompt["additional_information"]["speaker_count"] == 1 assert prepared.model_type == "ming_tts" - def test_ming_adapter_uploaded_audio_uses_cached_embedding(self, speech_server, mocker: MockerFixture): - """Cached Ming audio voices skip repeated speaker extraction.""" - speech_server._speaker_cache.clear() + def test_ming_adapter_uploaded_audio_forwards_worker_cache_metadata(self, speech_server, mocker: MockerFixture): + """Ming audio voices defer extraction and cache ownership to Stage-0.""" speech_server.uploaded_speakers = { "audio_voice": { "name": "audio_voice", @@ -1691,26 +1699,12 @@ class TestTTSMethods: } ref_audio_source = "data:audio/wav;base64,ZmFrZWF1ZGlv" ref_audio_data = ([0.0, 0.1], 16000) - fake_embedding = [0.2] * SPEAKER_EMBEDDING_DIM - cache_key = speech_server._speaker_cache.make_cache_key( - "audio_voice", - model_type="ming_tts", - created_at=123, - ) - speech_server._speaker_cache.put( - cache_key, - {"speaker_embedding": torch.tensor(fake_embedding)}, - ) mocker.patch.object(speech_server, "_get_uploaded_audio_data", return_value=ref_audio_source) mocker.patch.object( speech_server, "_resolve_ref_audio", new=mocker.AsyncMock(return_value=ref_audio_data), ) - mock_extract = mocker.patch.object( - speech_server, - "_extract_ming_speaker_embeddings_from_ref_audio", - ) mock_prompt = mocker.patch.object( speech_server, "_build_ming_dense_prompt", @@ -1721,13 +1715,82 @@ class TestTTSMethods: req = OpenAICreateSpeechRequest(input="Hello", voice="audio_voice") prepared = asyncio.run(adapter.build(req, [], False)) - mock_extract.assert_not_called() - mock_prompt.assert_called_once_with(req, ref_audio_data=ref_audio_data) + mock_prompt.assert_called_once_with( + req, + ref_audio_data=ref_audio_data, + voice_name="audio_voice", + voice_created_at=123, + ) assert req.ref_text == "Reference transcript." - assert req.speaker_embedding == pytest.approx(fake_embedding) + assert req.speaker_embedding is None assert prepared.model_type == "ming_tts" - def test_ming_adapter_legacy_audio_only_voice_falls_back_and_caches( + def test_build_ming_prompt_forwards_audio_segments_and_worker_cache_metadata(self, speech_server): + class FakeTokenizer: + unk_token_id = -1 + + def encode(self, text): + return [777] if text == "<|vision_pad|>" else [1] + + def convert_tokens_to_ids(self, token): + return 888 if token == "" else self.unk_token_id + + speech_server._tts_tokenizer = FakeTokenizer() + request = OpenAICreateSpeechRequest( + input="Hello", + ref_text="Two reference speakers.", + ) + ref_audio_data = [ + ([0.1, 0.2, 0.3], 16000), + ([0.4, 0.5], 24000), + ] + + prompt = speech_server._build_ming_dense_prompt( + request, + ref_audio_data=ref_audio_data, + voice_name="uploaded_voice", + voice_created_at=123, + ) + + info = prompt["additional_information"] + assert KEY_SPEAKER_EMBEDDING not in info + torch.testing.assert_close( + info[KEY_SPEAKER_WAVEFORM], + torch.tensor([[0.1, 0.2, 0.3, 0.4, 0.5]], dtype=torch.float32), + ) + assert info[KEY_SPEAKER_WAVEFORM_LENGTHS].tolist() == [3, 2] + assert info[KEY_SPEAKER_SAMPLE_RATES].tolist() == [16000, 24000] + assert info["prompt_waveform"].shape[-1] > info[KEY_SPEAKER_WAVEFORM].shape[-1] + assert info["voice_name"] == "uploaded_voice" + assert info["voice_created_at"] == 123 + assert prompt["prompt_token_ids"].count(777) == 2 + + def test_build_ming_prompt_forwards_speaker_waveform_without_ref_text(self, speech_server): + class FakeTokenizer: + unk_token_id = -1 + + def encode(self, text): + return [1] + + def convert_tokens_to_ids(self, token): + return 888 if token == "" else self.unk_token_id + + speech_server._tts_tokenizer = FakeTokenizer() + request = OpenAICreateSpeechRequest(input="Hello") + + prompt = speech_server._build_ming_dense_prompt( + request, + ref_audio_data=([0.1, 0.2, 0.3], 16000), + ) + + info = prompt["additional_information"] + assert info[KEY_SPEAKER_WAVEFORM].shape == (1, 3) + assert info[KEY_SPEAKER_WAVEFORM_LENGTHS].tolist() == [3] + assert info[KEY_SPEAKER_SAMPLE_RATES].tolist() == [16000] + assert "prompt_waveform" not in info + assert KEY_SPEAKER_EMBEDDING not in info + + def test_ming_adapter_legacy_audio_only_voice_forwards_audio_to_worker( self, speech_server, mocker: MockerFixture, @@ -1753,20 +1816,13 @@ class TestTTSMethods: } ref_audio_source = "data:audio/wav;base64,ZmFrZWF1ZGlv" ref_audio_data = ([0.0, 0.1], 16000) - fake_embedding = [0.4] * SPEAKER_EMBEDDING_DIM mocker.patch.object(speech_server, "_get_uploaded_audio_data", return_value=ref_audio_source) mocker.patch.object( speech_server, "_resolve_ref_audio", new=mocker.AsyncMock(return_value=ref_audio_data), ) - mock_extract = mocker.patch.object( - speech_server, - "_extract_ming_speaker_embeddings_from_ref_audio", - return_value=[fake_embedding], - ) - mock_to_thread = mocker.spy(asyncio, "to_thread") - mocker.patch.object( + mock_prompt = mocker.patch.object( speech_server, "_build_ming_dense_prompt", return_value={"additional_information": {"speaker_count": 1}}, @@ -1778,22 +1834,25 @@ class TestTTSMethods: second_req = OpenAICreateSpeechRequest(input="Hello again", voice="legacy_audio_voice") second_prepared = asyncio.run(adapter.build(second_req, [], False)) - mock_extract.assert_called_once_with([ref_audio_data]) - mock_to_thread.assert_called_once_with(mock_extract, [ref_audio_data]) - assert first_req.speaker_embedding == pytest.approx(fake_embedding) - assert second_req.speaker_embedding == pytest.approx(fake_embedding) + assert first_req.speaker_embedding is None + assert second_req.speaker_embedding is None assert first_req.ref_text == "Reference transcript." assert second_req.ref_text == "Reference transcript." assert first_prepared.model_type == "ming_tts" assert second_prepared.model_type == "ming_tts" - cache_key = speech_server._speaker_cache.make_cache_key( - "legacy_audio_voice", - model_type="ming_tts", - created_at=456, + assert mock_prompt.call_count == 2 + mock_prompt.assert_any_call( + first_req, + ref_audio_data=ref_audio_data, + voice_name="legacy_audio_voice", + voice_created_at=456, + ) + mock_prompt.assert_any_call( + second_req, + ref_audio_data=ref_audio_data, + voice_name="legacy_audio_voice", + voice_created_at=456, ) - cached = speech_server._speaker_cache.get(cache_key) - assert cached is not None - torch.testing.assert_close(cached["speaker_embedding"], torch.tensor(fake_embedding)) # ── regression: full flow from issue #1603 ── diff --git a/tests/model_executor/models/ming_tts/test_ming_shared_modules.py b/tests/model_executor/models/ming_tts/test_ming_shared_modules.py index c5966dd60..7a0ad68de 100644 --- a/tests/model_executor/models/ming_tts/test_ming_shared_modules.py +++ b/tests/model_executor/models/ming_tts/test_ming_shared_modules.py @@ -12,17 +12,24 @@ from vllm_omni.model_executor.models.common.ming.fm import Solver from vllm_omni.model_executor.models.ming_tts.constants import ( AGGREGATOR_HIDDEN_SIZE, HISTORY_PATCH_SIZE, + KEY_SPEAKER_EMBEDDING, + KEY_SPEAKER_SAMPLE_RATES, + KEY_SPEAKER_WAVEFORM, + KEY_SPEAKER_WAVEFORM_LENGTHS, LATENT_DIM, LLM_HIDDEN_SIZE, LLM_VOCAB_SIZE, PATCH_SIZE, SAMPLE_RATE, + SPEAKER_EMBEDDING_DIM, VAE_PATCH_SIZE, ) from vllm_omni.model_executor.models.ming_tts.flowloss_head import FlowLoss from vllm_omni.model_executor.models.ming_tts.ming_tts_llm import MingLLMModel +from vllm_omni.model_executor.models.ming_tts.speaker_extractor import _resolve_speaker_embeddings from vllm_omni.model_executor.models.ming_tts.validation import validate_ming_tts_config from vllm_omni.model_executor.models.output_templates import OmniOutput +from vllm_omni.utils.speaker_cache import SpeakerEmbeddingCache pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.tts] @@ -99,6 +106,122 @@ def test_ming_instruction_parser_preserves_dense_and_flash_defaults(): assert flash_fields == {"风格": "calm"} +class _FakeMingSpeakerExtractor: + def __init__(self): + self.calls: list[tuple[torch.Tensor, int]] = [] + + def extract_from_waveform(self, waveform, sample_rate): + waveform = torch.as_tensor(waveform).detach().clone() + self.calls.append((waveform, int(sample_rate))) + return torch.full((SPEAKER_EMBEDDING_DIM,), float(len(self.calls)), dtype=torch.float32) + + +def _make_ming_speaker_wrapper(): + return SimpleNamespace( + _speaker_cache=SpeakerEmbeddingCache(max_bytes=1024 * 1024), + _speaker_extractor=_FakeMingSpeakerExtractor(), + ming_config=SimpleNamespace(sample_rate=SAMPLE_RATE), + ) + + +def test_ming_stage0_speaker_cache_extracts_once_and_honors_created_at(): + wrapper = _make_ming_speaker_wrapper() + waveform = torch.arange(12, dtype=torch.float32).reshape(1, -1) + info = { + KEY_SPEAKER_WAVEFORM: waveform, + KEY_SPEAKER_WAVEFORM_LENGTHS: torch.tensor([12]), + "voice_name": "UploadedVoice", + "voice_created_at": 123, + } + + first = _resolve_speaker_embeddings(wrapper, info) + cached = _resolve_speaker_embeddings( + wrapper, + { + "voice_name": "uploadedvoice", + "voice_created_at": 123, + }, + ) + refreshed = _resolve_speaker_embeddings( + wrapper, + { + **info, + "voice_created_at": 124, + }, + ) + + assert first is not None and cached is not None and refreshed is not None + torch.testing.assert_close(first[0], cached[0]) + assert torch.all(first[0] == 1) + assert torch.all(refreshed[0] == 2) + assert len(wrapper._speaker_extractor.calls) == 2 + + +def test_ming_stage0_speaker_extraction_splits_multiple_waveforms(): + wrapper = _make_ming_speaker_wrapper() + + embeddings = _resolve_speaker_embeddings( + wrapper, + { + KEY_SPEAKER_WAVEFORM: torch.arange(10, dtype=torch.float32).reshape(1, -1), + KEY_SPEAKER_WAVEFORM_LENGTHS: torch.tensor([4, 6]), + KEY_SPEAKER_SAMPLE_RATES: torch.tensor([16000, 24000]), + }, + ) + + assert embeddings is not None and len(embeddings) == 2 + assert [call[0].shape[-1] for call in wrapper._speaker_extractor.calls] == [4, 6] + assert [call[1] for call in wrapper._speaker_extractor.calls] == [16000, 24000] + + +def test_ming_stage0_speaker_extraction_rejects_mismatched_sample_rates(): + wrapper = _make_ming_speaker_wrapper() + + with pytest.raises(ValueError, match="Invalid Ming speaker sample rates"): + _resolve_speaker_embeddings( + wrapper, + { + KEY_SPEAKER_WAVEFORM: torch.arange(10, dtype=torch.float32).reshape(1, -1), + KEY_SPEAKER_WAVEFORM_LENGTHS: torch.tensor([4, 6]), + KEY_SPEAKER_SAMPLE_RATES: torch.tensor([16000]), + }, + ) + + +def test_ming_stage0_prompt_waveform_does_not_trigger_speaker_extraction(): + wrapper = _make_ming_speaker_wrapper() + + embeddings = _resolve_speaker_embeddings( + wrapper, + { + "prompt_waveform": torch.ones((1, 10)), + "prompt_text": "Reference transcript.", + }, + ) + + assert embeddings is None + assert not wrapper._speaker_extractor.calls + + +def test_ming_stage0_direct_speaker_embedding_bypasses_extractor(): + wrapper = _make_ming_speaker_wrapper() + direct = torch.full((SPEAKER_EMBEDDING_DIM,), 0.25) + + embeddings = _resolve_speaker_embeddings( + wrapper, + { + KEY_SPEAKER_EMBEDDING: direct, + KEY_SPEAKER_WAVEFORM: torch.ones((1, 10)), + "voice_name": "direct", + "voice_created_at": 1, + }, + ) + + assert embeddings is not None + torch.testing.assert_close(embeddings[0], direct) + assert not wrapper._speaker_extractor.calls + + def _make_ming_logits_model(vocab_size=8): model = object.__new__(MingLLMModel) model.ming_config = SimpleNamespace( diff --git a/vllm_omni/entrypoints/openai/serving_speech.py b/vllm_omni/entrypoints/openai/serving_speech.py index 2bb402caf..782b1c884 100644 --- a/vllm_omni/entrypoints/openai/serving_speech.py +++ b/vllm_omni/entrypoints/openai/serving_speech.py @@ -2124,37 +2124,18 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): waveform = resample_audio(waveform, int(sample_rate), SAMPLE_RATE) return waveform - def _build_ming_prompt_waveform( + def _build_ming_reference_waveforms( self, ref_audio_data: tuple[list[float], int] | list[tuple[list[float], int]] | None, - ): - if isinstance(ref_audio_data, list): - return torch.cat( - [self._coerce_ming_prompt_waveform(item[0], item[1]) for item in ref_audio_data], - dim=-1, - ) - if ref_audio_data is not None: - return self._coerce_ming_prompt_waveform(ref_audio_data[0], ref_audio_data[1]) - return None - - def _extract_ming_speaker_embeddings_from_ref_audio( - self, - ref_audio_data_list: list[tuple[list[float], int]], - ) -> list[list[float]]: - from vllm_omni.model_executor.models.ming_tts.speaker_extractor import MingSpeakerEmbeddingExtractor - - extractor = MingSpeakerEmbeddingExtractor(self.engine_client.model_config.model, target_sr=16000) - embeddings = [] - for wav_samples, sr in ref_audio_data_list: - waveform = torch.as_tensor(wav_samples, dtype=torch.float32).reshape(1, -1) - embedding = extractor.extract_from_waveform(waveform, int(sr)) - flat = embedding.detach().reshape(-1).to(torch.float32).cpu() - if int(flat.numel()) != SPEAKER_EMBEDDING_DIM: - raise ValueError( - f"Ming speaker extractor returned {int(flat.numel())} dims; expected {SPEAKER_EMBEDDING_DIM}" - ) - embeddings.append(flat.tolist()) - return embeddings + ) -> tuple[list[torch.Tensor], list[int]]: + if ref_audio_data is None: + return [], [] + items = ref_audio_data if isinstance(ref_audio_data, list) else [ref_audio_data] + waveforms = [torch.as_tensor(samples, dtype=torch.float32).reshape(1, -1) for samples, _ in items] + sample_rates = [int(sample_rate) for _, sample_rate in items] + if any(sample_rate <= 0 for sample_rate in sample_rates): + raise ValueError(f"Ming reference audio sample rates must be positive, got {sample_rates}") + return waveforms, sample_rates def _parse_ming_instruction_fields( self, @@ -2197,11 +2178,18 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): request: OpenAICreateSpeechRequest, *, ref_audio_data: tuple[list[float], int] | list[tuple[list[float], int]] | None = None, + voice_name: str | None = None, + voice_created_at: int = 0, ) -> dict[str, Any]: """Build a Ming dense prompt directly from the OpenAI speech request.""" from transformers import AutoTokenizer - from vllm_omni.model_executor.models.ming_tts.config_ming_tts import KEY_MAX_DECODE_STEPS + from vllm_omni.model_executor.models.ming_tts.config_ming_tts import ( + KEY_MAX_DECODE_STEPS, + KEY_SPEAKER_SAMPLE_RATES, + KEY_SPEAKER_WAVEFORM, + KEY_SPEAKER_WAVEFORM_LENGTHS, + ) from vllm_omni.model_executor.models.ming_tts.prompt_assembly import build_ming_dense_prompt if self._tts_tokenizer is None: @@ -2210,9 +2198,20 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): self._tts_tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=trust_remote_code) ref_text = request.ref_text - prompt_waveform = self._build_ming_prompt_waveform(ref_audio_data) if ref_text is not None else None + reference_waveforms, reference_sample_rates = self._build_ming_reference_waveforms(ref_audio_data) + reference_waveform = torch.cat(reference_waveforms, dim=-1) if reference_waveforms else None + prompt_waveforms = ( + [ + self._coerce_ming_prompt_waveform(waveform, sample_rate) + for waveform, sample_rate in zip(reference_waveforms, reference_sample_rates, strict=True) + ] + if ref_text is not None + else [] + ) + prompt_waveform = torch.cat(prompt_waveforms, dim=-1) if prompt_waveforms else None speaker_embedding = request.speaker_embedding - use_zero_spk_emb = prompt_waveform is None and speaker_embedding is None + pending_speaker_extraction = speaker_embedding is None and reference_waveform is not None + use_zero_spk_emb = not pending_speaker_extraction and prompt_waveform is None and speaker_embedding is None runtime_controls = {} if request.max_new_tokens is not None: @@ -2229,8 +2228,23 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): prompt_text=ref_text, prompt_waveform=prompt_waveform, speaker_embedding=speaker_embedding, + speaker_count=len(reference_waveforms) if pending_speaker_extraction else None, use_zero_spk_emb=use_zero_spk_emb, ) + additional_information = prompt_dict["additional_information"] + if pending_speaker_extraction: + additional_information[KEY_SPEAKER_WAVEFORM] = reference_waveform + additional_information[KEY_SPEAKER_WAVEFORM_LENGTHS] = torch.tensor( + [int(waveform.shape[-1]) for waveform in reference_waveforms], + dtype=torch.int32, + ) + additional_information[KEY_SPEAKER_SAMPLE_RATES] = torch.tensor( + reference_sample_rates, + dtype=torch.int32, + ) + if voice_name: + additional_information["voice_name"] = voice_name + additional_information["voice_created_at"] = int(voice_created_at) prompt = tokens_input(prompt_token_ids=prompt_dict["prompt_token_ids"]) prompt["prompt"] = prompt_dict["prompt"] prompt["text"] = prompt_dict["text"] diff --git a/vllm_omni/entrypoints/openai/tts_adapters/ming_tts.py b/vllm_omni/entrypoints/openai/tts_adapters/ming_tts.py index dff392c28..adce42867 100644 --- a/vllm_omni/entrypoints/openai/tts_adapters/ming_tts.py +++ b/vllm_omni/entrypoints/openai/tts_adapters/ming_tts.py @@ -1,10 +1,8 @@ # SPDX-License-Identifier: Apache-2.0 """Ming-TTS (dense) serving adapter.""" -import asyncio from typing import TYPE_CHECKING -import torch from vllm.logger import init_logger from vllm_omni.entrypoints.openai.tts_adapters import register_tts_adapter @@ -36,7 +34,8 @@ class MingTTSAdapter(ARTTSAdapter): ) -> PreparedRequest: server = self.ctx.server ref_audio_source = request.ref_audio - speaker_cache_key: tuple[str, str, int] | None = None + uploaded_audio_voice = None + uploaded_audio_created_at = 0 voice_lower = request.voice.lower() if isinstance(request.voice, str) else None if ref_audio_source is None and voice_lower in server.uploaded_speakers: speaker_info = server.uploaded_speakers[voice_lower] @@ -46,51 +45,25 @@ class MingTTSAdapter(ARTTSAdapter): if request.speaker_embedding is None: raise ValueError(f"Speaker embedding for uploaded voice '{request.voice}' is missing") else: - if request.speaker_embedding is None: - speaker_cache_key = server._speaker_cache.make_cache_key( - voice_lower, - model_type=self.name, - created_at=int(speaker_info.get("created_at") or 0), - ) - cached = server._speaker_cache.get(speaker_cache_key) - if cached is not None: - cached_embedding = cached.get("speaker_embedding") - if isinstance(cached_embedding, torch.Tensor): - flat = cached_embedding.detach().reshape(-1).to(torch.float32).cpu() - if int(flat.numel()) == SPEAKER_EMBEDDING_DIM: - request.speaker_embedding = flat.tolist() - speaker_cache_key = None - logger.debug("Speaker cache HIT for Ming-TTS speaker '%s'", voice_lower) ref_audio_source = server._get_uploaded_audio_data(request.voice) if not ref_audio_source: raise ValueError(f"Audio file for uploaded voice '{request.voice}' is missing") if request.ref_text is None: request.ref_text = speaker_info.get("ref_text") + uploaded_audio_voice = voice_lower + uploaded_audio_created_at = server._voice_created_at(voice_lower) ref_audio_data = None if isinstance(ref_audio_source, list): ref_audio_data = await server._resolve_ref_audio_many(ref_audio_source) - if request.speaker_embedding is None: - request.speaker_embedding = await asyncio.to_thread( - server._extract_ming_speaker_embeddings_from_ref_audio, - ref_audio_data, - ) elif ref_audio_source is not None and isinstance(ref_audio_source, str): wav_list, sr = await server._resolve_ref_audio(ref_audio_source) ref_audio_data = (wav_list, sr) - if request.speaker_embedding is None: - embeddings = await asyncio.to_thread( - server._extract_ming_speaker_embeddings_from_ref_audio, - [ref_audio_data], - ) - request.speaker_embedding = embeddings[0] - if speaker_cache_key is not None and request.speaker_embedding is not None: - embedding = torch.as_tensor(request.speaker_embedding, dtype=torch.float32).detach().reshape(-1).cpu() - server._speaker_cache.put( - speaker_cache_key, - {"speaker_embedding": embedding}, - ) - logger.debug("Speaker cache STORE for Ming-TTS speaker '%s'", voice_lower) - prompt = server._build_ming_dense_prompt(request, ref_audio_data=ref_audio_data) + prompt = server._build_ming_dense_prompt( + request, + ref_audio_data=ref_audio_data, + voice_name=uploaded_audio_voice, + voice_created_at=uploaded_audio_created_at, + ) tts_params = prompt.get("additional_information", {}) # Ming stop-token / max_tokens sampling stays in the orchestrator tail. return PreparedRequest(prompt=prompt, tts_params=tts_params, model_type="ming_tts") diff --git a/vllm_omni/model_executor/models/ming_tts/config_ming_tts.py b/vllm_omni/model_executor/models/ming_tts/config_ming_tts.py index fdac2a30e..a6dc270c3 100644 --- a/vllm_omni/model_executor/models/ming_tts/config_ming_tts.py +++ b/vllm_omni/model_executor/models/ming_tts/config_ming_tts.py @@ -35,6 +35,9 @@ from .constants import ( KEY_REQUEST_ID, KEY_SIGMA, KEY_SPEAKER_EMBEDDING, + KEY_SPEAKER_SAMPLE_RATES, + KEY_SPEAKER_WAVEFORM, + KEY_SPEAKER_WAVEFORM_LENGTHS, KEY_TEMPERATURE, KEY_TEXT_MODE, LATENT_CHUNK_SIZE, @@ -326,6 +329,9 @@ __all__ = [ "KEY_REQUEST_ID", "KEY_SIGMA", "KEY_SPEAKER_EMBEDDING", + "KEY_SPEAKER_SAMPLE_RATES", + "KEY_SPEAKER_WAVEFORM", + "KEY_SPEAKER_WAVEFORM_LENGTHS", "KEY_TEMPERATURE", "KEY_TEXT_MODE", "LATENT_CHUNK_SIZE", diff --git a/vllm_omni/model_executor/models/ming_tts/constants.py b/vllm_omni/model_executor/models/ming_tts/constants.py index 3db8ff3b8..397de3288 100644 --- a/vllm_omni/model_executor/models/ming_tts/constants.py +++ b/vllm_omni/model_executor/models/ming_tts/constants.py @@ -74,6 +74,9 @@ KEY_LAST_STOP_PROB = "ming_last_stop_prob" KEY_NEXT_EMBEDS = "ming_next_embeds" KEY_PROMPT_LATENTS = "ming_prompt_latents" KEY_SPEAKER_EMBEDDING = "ming_speaker_embedding" +KEY_SPEAKER_WAVEFORM = "ming_speaker_waveform" +KEY_SPEAKER_WAVEFORM_LENGTHS = "ming_speaker_waveform_lengths" +KEY_SPEAKER_SAMPLE_RATES = "ming_speaker_sample_rates" KEY_REQUEST_ID = "ming_request_id" KEY_CHUNK_ID = "ming_chunk_id" KEY_CFG = "ming_cfg" diff --git a/vllm_omni/model_executor/models/ming_tts/ming_tts.py b/vllm_omni/model_executor/models/ming_tts/ming_tts.py index 70a414186..d6e2ac92d 100644 --- a/vllm_omni/model_executor/models/ming_tts/ming_tts.py +++ b/vllm_omni/model_executor/models/ming_tts/ming_tts.py @@ -20,7 +20,6 @@ from .audio_prep import ( _find_speaker_placeholder_positions, _initial_history, _take_scalar, - coerce_speaker_embeddings, ) from .config_ming_tts import ( AUDIO_START_TOKEN_ID, @@ -33,13 +32,13 @@ from .config_ming_tts import ( KEY_NEXT_EMBEDS, KEY_REQUEST_ID, KEY_SIGMA, - KEY_SPEAKER_EMBEDDING, KEY_TEMPERATURE, KEY_TEXT_MODE, MingTTSConfig, ) from .patch_emission import MING_STOP_REASON_KEY from .prompt_encoder import _resolve_prompt_latents +from .speaker_extractor import _resolve_speaker_embeddings class _ModelSampleAdapter(nn.Module): @@ -64,8 +63,12 @@ class MingTTSForConditionalGeneration(nn.Module, SupportsPP, CustomProcessMixin) self.requires_raw_input_tokens = False self.model_stage = vllm_config.model_config.model_stage self._prompt_encoder = None + self._speaker_extractor = None if self.model_stage == "llm": + from vllm_omni.utils.speaker_cache import get_speaker_cache + + self._speaker_cache = get_speaker_cache() self.model = init_vllm_registered_model(vllm_config=vllm_config, architectures=["MingLLMModel"]) self.has_preprocess = True self.has_postprocess = True @@ -174,13 +177,7 @@ class MingTTSForConditionalGeneration(nn.Module, SupportsPP, CustomProcessMixin) ) update[KEY_LATENT_HISTORY] = history.detach().to("cpu").contiguous() - speaker_embedding = info_dict.get(KEY_SPEAKER_EMBEDDING, info_dict.get("speaker_embedding")) - speaker_embeddings = None - if speaker_embedding is not None: - speaker_embeddings = coerce_speaker_embeddings( - speaker_embedding, - use_zero_spk_emb=bool(info_dict.get("use_zero_spk_emb", False)), - ) + speaker_embeddings = _resolve_speaker_embeddings(self, info_dict) if speaker_embeddings is not None and len(speaker_embeddings) > 0: speaker_slots = _find_speaker_placeholder_positions(input_ids, self.ming_config) if len(speaker_slots) < len(speaker_embeddings): diff --git a/vllm_omni/model_executor/models/ming_tts/prompt_assembly.py b/vllm_omni/model_executor/models/ming_tts/prompt_assembly.py index 36817022a..5960af742 100644 --- a/vllm_omni/model_executor/models/ming_tts/prompt_assembly.py +++ b/vllm_omni/model_executor/models/ming_tts/prompt_assembly.py @@ -265,6 +265,7 @@ def build_ming_dense_prompt( prompt_waveform: Any = None, prompt_latents: Any = None, speaker_embedding: Any = None, + speaker_count: int | None = None, use_zero_spk_emb: bool = False, request_id: str | None = None, model_variant: str | None = None, @@ -273,6 +274,17 @@ def build_ming_dense_prompt( model_variant = detect_model_variant(tokenizer) instruction_text = create_instruction(instruction) speaker_embeddings = coerce_speaker_embeddings(speaker_embedding, use_zero_spk_emb=use_zero_spk_emb) + if speaker_count is None: + effective_speaker_count = 0 if speaker_embeddings is None else len(speaker_embeddings) + else: + effective_speaker_count = int(speaker_count) + if effective_speaker_count < 0: + raise ValueError(f"speaker_count must be >= 0, got {effective_speaker_count}") + if speaker_embeddings is not None and effective_speaker_count != len(speaker_embeddings): + raise ValueError( + f"speaker_count={effective_speaker_count} does not match " + f"{len(speaker_embeddings)} provided speaker embeddings" + ) effective_runtime_controls = resolve_effective_runtime_controls(text=text, runtime_controls=runtime_controls) prompt_waveform_tensor = None @@ -305,7 +317,7 @@ def build_ming_dense_prompt( text=text, instruction=instruction_text, prompt_text=prompt_text if prompt_patch_count > 0 else None, - speaker_count=0 if speaker_embeddings is None else len(speaker_embeddings), + speaker_count=effective_speaker_count, prompt_patch_count=prompt_patch_count, ) diff --git a/vllm_omni/model_executor/models/ming_tts/speaker_extractor.py b/vllm_omni/model_executor/models/ming_tts/speaker_extractor.py index 00523c26f..8e9f69068 100644 --- a/vllm_omni/model_executor/models/ming_tts/speaker_extractor.py +++ b/vllm_omni/model_executor/models/ming_tts/speaker_extractor.py @@ -1,14 +1,33 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project # Adopted from https://github.com/inclusionAI/Ming-omni-tts/blob/main/spkemb_extractor.py +from __future__ import annotations + import os +import threading +from typing import TYPE_CHECKING, Any import torch import torchaudio +from vllm.logger import init_logger from vllm.multimodal.media.audio import load_audio from vllm_omni.model_executor.models.ming_flash_omni.spk_embedding import SpkembExtractor +from .audio_prep import coerce_prompt_waveform, coerce_speaker_embeddings +from .constants import ( + KEY_SPEAKER_EMBEDDING, + KEY_SPEAKER_SAMPLE_RATES, + KEY_SPEAKER_WAVEFORM, + KEY_SPEAKER_WAVEFORM_LENGTHS, +) + +if TYPE_CHECKING: + from .ming_tts import MingTTSForConditionalGeneration + +logger = init_logger(__name__) +_EXTRACTOR_LOAD_LOCK = threading.Lock() + def resolve_model_to_local_path(model): if os.path.isdir(model): @@ -49,3 +68,114 @@ class MingSpeakerEmbeddingExtractor: def extract_many(self, audio_paths): return [self.extract_from_file(path) for path in audio_paths] + + +def _load_speaker_extractor(wrapper: MingTTSForConditionalGeneration) -> MingSpeakerEmbeddingExtractor: + extractor = getattr(wrapper, "_speaker_extractor", None) + if extractor is not None: + return extractor + + with _EXTRACTOR_LOAD_LOCK: + extractor = getattr(wrapper, "_speaker_extractor", None) + if extractor is None: + model_path = wrapper.vllm_config.model_config.model + extractor = MingSpeakerEmbeddingExtractor(model_path, target_sr=16000) + wrapper._speaker_extractor = extractor + return extractor + + +def _speaker_waveforms_from_info( + info_dict: dict[str, Any], + default_sample_rate: int, +) -> list[tuple[torch.Tensor, int]]: + raw_waveform = info_dict.get(KEY_SPEAKER_WAVEFORM) + if raw_waveform is None: + return [] + + waveform = coerce_prompt_waveform(raw_waveform) + raw_lengths = info_dict.get(KEY_SPEAKER_WAVEFORM_LENGTHS) + if raw_lengths is None: + parts = [waveform] + else: + lengths = torch.as_tensor(raw_lengths).detach().reshape(-1).to(torch.int64).cpu().tolist() + if not lengths or any(int(length) <= 0 for length in lengths): + raise ValueError(f"Invalid Ming speaker waveform lengths: {lengths}") + if sum(int(length) for length in lengths) > int(waveform.shape[-1]): + raise ValueError( + "Ming speaker waveform lengths exceed the waveform size: " + f"lengths={lengths}, waveform_samples={int(waveform.shape[-1])}" + ) + + parts = [] + offset = 0 + for length in lengths: + end = offset + int(length) + parts.append(waveform[:, offset:end]) + offset = end + + raw_sample_rates = info_dict.get(KEY_SPEAKER_SAMPLE_RATES) + if raw_sample_rates is None: + sample_rates = [int(default_sample_rate)] * len(parts) + else: + sample_rates = torch.as_tensor(raw_sample_rates).detach().reshape(-1).to(torch.int64).cpu().tolist() + if len(sample_rates) != len(parts) or any(int(sample_rate) <= 0 for sample_rate in sample_rates): + raise ValueError( + f"Invalid Ming speaker sample rates: sample_rates={sample_rates}, waveform_count={len(parts)}" + ) + return [(waveform_part, int(sample_rate)) for waveform_part, sample_rate in zip(parts, sample_rates, strict=True)] + + +def _resolve_speaker_embeddings( + wrapper: MingTTSForConditionalGeneration, + info_dict: dict[str, Any], +) -> list[torch.Tensor] | None: + direct_embedding = info_dict.get(KEY_SPEAKER_EMBEDDING, info_dict.get("speaker_embedding")) + use_zero_spk_emb = bool(info_dict.get("use_zero_spk_emb", False)) + if direct_embedding is not None or use_zero_spk_emb: + return coerce_speaker_embeddings(direct_embedding, use_zero_spk_emb=use_zero_spk_emb) + + voice_name = info_dict.get("voice_name") + if isinstance(voice_name, (list, tuple)): + voice_name = voice_name[0] if voice_name else None + created_at = info_dict.get("voice_created_at", 0) + if isinstance(created_at, (list, tuple)): + created_at = created_at[0] if created_at else 0 + if isinstance(created_at, torch.Tensor): + created_at = created_at.detach().reshape(-1)[0].item() if created_at.numel() else 0 + + cache = wrapper._speaker_cache + cache_key = None + if isinstance(voice_name, str) and voice_name.strip(): + cache_key = cache.make_cache_key(voice_name, model_type="ming_tts", created_at=int(created_at or 0)) + cached = cache.get(cache_key) + if cached is not None: + cached_embeddings = coerce_speaker_embeddings(cached.get("speaker_embedding")) + if cached_embeddings is not None: + logger.debug("Speaker cache HIT for Ming-TTS speaker '%s'", voice_name) + return cached_embeddings + + waveform_inputs = _speaker_waveforms_from_info(info_dict, int(wrapper.ming_config.sample_rate)) + if not waveform_inputs: + if cache_key is not None: + raise ValueError(f"Ming-TTS speaker '{voice_name}' was requested without reference audio on cache miss") + return None + + extractor = _load_speaker_extractor(wrapper) + embeddings = [ + extractor.extract_from_waveform(waveform, sample_rate).detach().reshape(-1).to(torch.float32).cpu() + for waveform, sample_rate in waveform_inputs + ] + resolved = coerce_speaker_embeddings(embeddings) + if cache_key is not None and resolved is not None: + cached_embedding = resolved[0] if len(resolved) == 1 else torch.stack(resolved, dim=0) + cache.put(cache_key, {"speaker_embedding": cached_embedding}) + logger.debug("Speaker cache STORE for Ming-TTS speaker '%s'", voice_name) + return resolved + + +__all__ = [ + "MingSpeakerEmbeddingExtractor", + "_load_speaker_extractor", + "_resolve_speaker_embeddings", + "resolve_model_to_local_path", +] From cbab1d4d8f814e48ee691b69ac4cee9d0419f218 Mon Sep 17 00:00:00 2001 From: Zhou Taichang Date: Mon, 27 Jul 2026 23:35:49 +0800 Subject: [PATCH 118/240] [Rebase] Rebase to vllm 0.26.0 (#5443) Signed-off-by: tzhouam Signed-off-by: tzhouam Co-authored-by: Claude Fable 5 Co-authored-by: tzhouam --- .buildkite/cuda/test-merge.yml | 8 ++- docker/Dockerfile.ci | 4 +- docker/Dockerfile.cuda | 2 +- docker/Dockerfile.rocm | 4 +- docker/Dockerfile.xpu | 2 +- docs/getting_started/installation/gpu.md | 2 +- .../installation/gpu/cuda.inc.md | 16 +++--- .../installation/gpu/rocm.inc.md | 14 ++--- docs/getting_started/quickstart.md | 6 +-- tests/attention/test_fish_kvcache_attn.py | 53 ++++++++++--------- .../sched/test_omni_ar_scheduler_streaming.py | 20 +++++++ ...st_omni_scheduler_finish_requests_purge.py | 17 +++--- ...mni_scheduler_input_coordinator_cleanup.py | 10 ++-- .../test_boogu_image_transformer.py | 45 +++++++++++----- .../test_chunk_transfer_adapter.py | 12 +++-- .../omni_connectors/test_kv_flow.py | 25 +++++++++ .../test_qwen3_omni_realtime_websocket.py | 12 ++++- tests/entrypoints/test_pd_disaggregation.py | 27 ++-------- .../qwen2_5_omni/test_qwen2_5_omni_embed.py | 47 +++++++++++----- .../qwen3_omni/test_qwen3_omni_moe_lora.py | 1 + tests/worker/test_gpu_ar_model_runner.py | 3 ++ .../test_gpu_generation_model_runner.py | 2 +- tests/worker/test_omni_connector_mixin.py | 2 +- vllm_omni/__init__.py | 10 +++- vllm_omni/attention/fish_kvcache_backend.py | 10 ++-- vllm_omni/core/sched/omni_ar_scheduler.py | 35 +++++++----- .../core/sched/omni_generation_scheduler.py | 29 ++++++---- .../models/cosmos3/pipeline_cosmos3.py | 3 +- .../hunyuan_image3_tokenizer.py | 2 +- .../omni_connectors/kv_transfer_manager.py | 2 +- .../omni_connectors/utils/kv_utils.py | 26 ++++++++- .../ar_diffusion/kv_cache/manager.py | 5 ++ .../minicpmo_4_5/minicpmo_4_5_omni_llm.py | 14 +---- .../models/moss_tts/moss_codec_cudagraph.py | 10 ++-- .../qwen2_5_omni/qwen2_5_omni_thinker.py | 2 - .../qwen3_omni/qwen3_omni_moe_thinker.py | 10 +++- vllm_omni/platforms/cuda/platform.py | 10 ++++ vllm_omni/worker/gpu_ar_model_runner.py | 5 +- .../worker/gpu_generation_model_runner.py | 5 +- vllm_omni/worker/gpu_model_runner.py | 3 +- .../omni_connector_model_runner_mixin.py | 2 +- 41 files changed, 334 insertions(+), 183 deletions(-) diff --git a/.buildkite/cuda/test-merge.yml b/.buildkite/cuda/test-merge.yml index 22da9617b..96ea162b4 100644 --- a/.buildkite/cuda/test-merge.yml +++ b/.buildkite/cuda/test-merge.yml @@ -49,13 +49,17 @@ steps: depends_on: upload-merge-pipeline steps: - label: "Entrypoint Test with H100" - timeout_in_minutes: 60 + # TEMPORARY 60 -> 90: engine startup/sleep-wake is ~13-15 min slower per + # run on the vLLM 0.26 image (model loading ~2x, init ~2x; see builds + # 2915 vs 2916), which pushed this suite past 60 min. Revert once the + # 0.26 startup regression is understood and fixed. + timeout_in_minutes: 90 commands: - pytest -s -v tests/entrypoints/ -m "advanced_model and cuda and H100" --run-level "advanced_model" mirror_hardwares: h100_2 - label: "Entrypoint Test with L4" - timeout_in_minutes: 30 + timeout_in_minutes: 60 commands: - pytest -s -v tests/entrypoints/ -m "advanced_model and cuda and L4" --run-level "advanced_model" mirror_hardwares: l4_1 diff --git a/docker/Dockerfile.ci b/docker/Dockerfile.ci index caae57abf..02a7be452 100644 --- a/docker/Dockerfile.ci +++ b/docker/Dockerfile.ci @@ -1,7 +1,7 @@ -# Target vllm version: v0.25.0 (releases/v0.25.0 branch). +# Target vllm version: v0.26.0 (releases/v0.26.0 branch). # Keep VLLM_BASE_TAG aligned with the target vLLM release on every rebase. ARG VLLM_BASE_IMAGE=vllm/vllm-openai -ARG VLLM_BASE_TAG=v0.25.0 +ARG VLLM_BASE_TAG=v0.26.0 # We use registry-based layer caching for the dependencies, # where the tag is determined based on the hash of the contents diff --git a/docker/Dockerfile.cuda b/docker/Dockerfile.cuda index 4e9075f2a..f2e80eaf3 100644 --- a/docker/Dockerfile.cuda +++ b/docker/Dockerfile.cuda @@ -1,4 +1,4 @@ -ARG BASE_IMAGE=vllm/vllm-openai:v0.25.0 +ARG BASE_IMAGE=vllm/vllm-openai:v0.26.0 FROM ${BASE_IMAGE} ARG COMMON_WORKDIR=/app diff --git a/docker/Dockerfile.rocm b/docker/Dockerfile.rocm index 9be3e5fb5..6b5a12309 100644 --- a/docker/Dockerfile.rocm +++ b/docker/Dockerfile.rocm @@ -1,4 +1,4 @@ -ARG BASE_IMAGE=vllm/vllm-openai-rocm:v0.25.0 +ARG BASE_IMAGE=vllm/vllm-openai-rocm:v0.26.0 FROM ${BASE_IMAGE} AS base # Declare a variable to know if we want to use the nightly build or the stable build. @@ -10,7 +10,7 @@ FROM ${BASE_IMAGE} AS base # we should swap over to use stable release ASAP. # We should avoid relying on custom commits. ARG USE_NIGHTLY_BUILD=0 -ARG VLLM_VERSION_OR_COMMIT_HASH=89138b21cc246ae944c741d5c399c148e2b770ab +ARG VLLM_VERSION_OR_COMMIT_HASH=v0.26.0 ARG ARG_PYTORCH_ROCM_ARCH ENV PYTORCH_ROCM_ARCH=${ARG_PYTORCH_ROCM_ARCH:-${PYTORCH_ROCM_ARCH}} diff --git a/docker/Dockerfile.xpu b/docker/Dockerfile.xpu index b70dd274e..5d298040c 100644 --- a/docker/Dockerfile.xpu +++ b/docker/Dockerfile.xpu @@ -88,7 +88,7 @@ ENV UV_EXTRA_INDEX_URL=${PIP_EXTRA_INDEX_URL} ENV UV_INDEX_STRATEGY="unsafe-best-match" ENV UV_LINK_MODE="copy" -ARG VLLM_VERSION=v0.25.0 +ARG VLLM_VERSION=v0.26.0 RUN git clone -b ${VLLM_VERSION} https://github.com/vllm-project/vllm WORKDIR /workspace/vllm diff --git a/docs/getting_started/installation/gpu.md b/docs/getting_started/installation/gpu.md index e21a11cee..59d2aa482 100644 --- a/docs/getting_started/installation/gpu.md +++ b/docs/getting_started/installation/gpu.md @@ -34,7 +34,7 @@ vLLM-Omni is a Python library that supports the following GPU variants. The libr ### Pre-built wheels -Note: Pre-built wheels are currently available for vLLM-Omni 0.11.0rc1, 0.12.0rc1, 0.14.0rc1, 0.14.0, 0.16.0, 0.18.0, 0.20.0, 0.21.0, 0.22.0, 0.23.0, 0.24.0, and 0.25.0. If you need a newer unreleased revision, please [build from source](https://docs.vllm.ai/projects/vllm-omni/en/latest/getting_started/installation/gpu/#build-wheel-from-source). +Note: Pre-built wheels are currently available for vLLM-Omni 0.11.0rc1, 0.12.0rc1, 0.14.0rc1, 0.14.0, 0.16.0, 0.18.0, 0.20.0, 0.21.0, 0.22.0, 0.23.0, 0.24.0, 0.25.0, and 0.26.0. If you need a newer unreleased revision, please [build from source](https://docs.vllm.ai/projects/vllm-omni/en/latest/getting_started/installation/gpu/#build-wheel-from-source). === "NVIDIA CUDA" diff --git a/docs/getting_started/installation/gpu/cuda.inc.md b/docs/getting_started/installation/gpu/cuda.inc.md index d4808139e..9d568bde3 100644 --- a/docs/getting_started/installation/gpu/cuda.inc.md +++ b/docs/getting_started/installation/gpu/cuda.inc.md @@ -20,7 +20,7 @@ Therefore, it is recommended to install vLLM and vLLM-Omni with a **fresh new** vLLM-Omni is built based on vLLM. Please install it with command below. ```bash -uv pip install vllm==0.25.0 --torch-backend=auto +uv pip install vllm==0.26.0 --torch-backend=auto ``` #### Installation of vLLM-Omni @@ -39,13 +39,13 @@ uv pip install 'vllm-omni[demo]' # --8<-- [start:build-wheel-from-source] #### Installation of vLLM -If you do not need to modify source code of vLLM, you can directly install the stable 0.25.0 release version of the library +If you do not need to modify source code of vLLM, you can directly install the stable 0.26.0 release version of the library ```bash -uv pip install vllm==0.25.0 --torch-backend=auto +uv pip install vllm==0.26.0 --torch-backend=auto ``` -The 0.25.0 release of vLLM ships CUDA 13.0-compatible binaries by default. If you need a different CUDA variant or want to reuse an existing PyTorch installation, build vLLM from source instead. +The 0.26.0 release of vLLM ships CUDA 13.0-compatible binaries by default. If you need a different CUDA variant or want to reuse an existing PyTorch installation, build vLLM from source instead. #### Installation of vLLM-Omni Since vllm-omni is rapidly evolving, it's recommended to install it from source @@ -66,12 +66,12 @@ If you want to check, modify or debug with source code of vLLM, install the libr ```bash git clone https://github.com/vllm-project/vllm.git cd vllm -git checkout v0.25.0 +git checkout v0.26.0 ``` Set up environment variables to get pre-built wheels. If there are internet problems, just download the whl file manually. And set `VLLM_PRECOMPILED_WHEEL_LOCATION` as your local absolute path of whl file. ```bash -#For CUDA 13.0 (the default for v0.25.0; the wheel filename has no `+cu130` suffix) -export VLLM_PRECOMPILED_WHEEL_LOCATION=https://github.com/vllm-project/vllm/releases/download/v0.25.0/vllm-0.25.0-cp38-abi3-manylinux_2_28_x86_64.whl +#For CUDA 13.0 (the default for v0.26.0; the wheel filename has no `+cu130` suffix) +export VLLM_PRECOMPILED_WHEEL_LOCATION=https://github.com/vllm-project/vllm/releases/download/v0.26.0/vllm-0.26.0-cp38-abi3-manylinux_2_28_x86_64.whl ``` Install vllm with command below (If you have no existing PyTorch). ```bash @@ -102,7 +102,7 @@ docker run --runtime nvidia --gpus 2 \ --env "HF_TOKEN=$HF_TOKEN" \ -p 8091:8091 \ --ipc=host \ - vllm/vllm-omni:v0.25.0 \ + vllm/vllm-omni:v0.26.0 \ vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 ``` diff --git a/docs/getting_started/installation/gpu/rocm.inc.md b/docs/getting_started/installation/gpu/rocm.inc.md index 4a64e8723..c44407299 100644 --- a/docs/getting_started/installation/gpu/rocm.inc.md +++ b/docs/getting_started/installation/gpu/rocm.inc.md @@ -13,7 +13,7 @@ vLLM-Omni current recommends the steps in under setup through Docker Images. vLLM-Omni is built based on vLLM. Please install it with command below. ```bash -uv pip install vllm==0.25.0+rocm723 --extra-index-url https://wheels.vllm.ai/rocm/0.25.0/rocm723 +uv pip install vllm==0.26.0+rocm723 --extra-index-url https://wheels.vllm.ai/rocm/0.26.0/rocm723 ``` #### Installation of vLLM-Omni @@ -34,13 +34,13 @@ uv pip install onnxruntime-rocm # --8<-- [start:build-wheel-from-source] #### Installation of vLLM -If you do not need to modify source code of vLLM, you can directly install the stable 0.25.0 release version of the library +If you do not need to modify source code of vLLM, you can directly install the stable 0.26.0 release version of the library ```bash -uv pip install vllm==0.25.0+rocm723 --extra-index-url https://wheels.vllm.ai/rocm/0.25.0/rocm723 +uv pip install vllm==0.26.0+rocm723 --extra-index-url https://wheels.vllm.ai/rocm/0.26.0/rocm723 ``` -The pre-built 0.25.0 vLLM wheel targets ROCm 7.2.3. If you need a different ROCm stack or want to reuse an existing PyTorch installation, build vLLM from source instead. +The pre-built 0.26.0 vLLM wheel targets ROCm 7.2.3. If you need a different ROCm stack or want to reuse an existing PyTorch installation, build vLLM from source instead. #### Installation of vLLM-Omni Since vllm-omni is rapidly evolving, it's recommended to install it from source @@ -58,7 +58,7 @@ If you want to check, modify or debug with source code of vLLM, install the libr ```bash git clone https://github.com/vllm-project/vllm.git cd vllm -git checkout v0.25.0 +git checkout v0.26.0 python3 -m pip install -r requirements/rocm.txt python3 setup.py develop ``` @@ -130,7 +130,7 @@ docker run --rm \ -v ~/.cache/huggingface:/root/.cache/huggingface \ --env "HF_TOKEN=$HF_TOKEN" \ -p 8091:8091 \ - vllm/vllm-omni-rocm:v0.25.0 \ + vllm/vllm-omni-rocm:v0.26.0 \ --model Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 ``` @@ -149,7 +149,7 @@ docker run --rm -it \ -v ~/.cache/huggingface:/root/.cache/huggingface \ --env "HF_TOKEN=$HF_TOKEN" \ --entrypoint bash \ - vllm/vllm-omni-rocm:v0.25.0 + vllm/vllm-omni-rocm:v0.26.0 ``` # --8<-- [end:pre-built-images] diff --git a/docs/getting_started/quickstart.md b/docs/getting_started/quickstart.md index 914f1f7b4..15ceb395e 100644 --- a/docs/getting_started/quickstart.md +++ b/docs/getting_started/quickstart.md @@ -19,10 +19,10 @@ uv venv --python 3.12 --seed source .venv/bin/activate # On CUDA -uv pip install vllm==0.25.0 --torch-backend=auto +uv pip install vllm==0.26.0 --torch-backend=auto # On ROCm -uv pip install vllm==0.25.0+rocm723 --extra-index-url https://wheels.vllm.ai/rocm/0.25.0/rocm723 +uv pip install vllm==0.26.0+rocm723 --extra-index-url https://wheels.vllm.ai/rocm/0.26.0/rocm723 git clone https://github.com/vllm-project/vllm-omni.git cd vllm-omni @@ -34,7 +34,7 @@ For additional installation methods — please see the [installation guide](inst !!! note It is important to install the same major & minor version of vLLM and vLLM Omni, otherwise things may not work as expected. If the versions are misaligned, you will see a warning when you import vLLM Omni. - If you are seeing strange behavior with the `vllm` command not handling the `--omni` flag correctly, you most likely have a version mismatch with vLLM < `0.25.0` and vLLM Omni `0.25.0`, as vLLM Omni no longer hijacks the vLLM entrypoint. Updating vLLM should resolve this issue. + If you are seeing strange behavior with the `vllm` command not handling the `--omni` flag correctly, you most likely have a version mismatch with vLLM < `0.26.0` and vLLM Omni `0.26.0`, as vLLM Omni no longer hijacks the vLLM entrypoint. Updating vLLM should resolve this issue. ## Offline Inference diff --git a/tests/attention/test_fish_kvcache_attn.py b/tests/attention/test_fish_kvcache_attn.py index c82fd1858..d38380e13 100644 --- a/tests/attention/test_fish_kvcache_attn.py +++ b/tests/attention/test_fish_kvcache_attn.py @@ -100,8 +100,9 @@ def _decode_kv_cache( head_size: int = 128, dtype: torch.dtype = torch.float16, ) -> torch.Tensor: - # vLLM >=0.23.0 KV cache layout: key/value live on dim 1. - return torch.zeros((num_blocks, 2, block_size, num_kv_heads, head_size), dtype=dtype) + # vLLM >=0.23.0 KV cache layout: (num_blocks, num_kv_heads, block_size, + # 2*head_size) with K and V packed in the last dimension. + return torch.zeros((num_blocks, num_kv_heads, block_size, 2 * head_size), dtype=dtype) def test_fish_kvcache_enabled_by_default(monkeypatch): @@ -221,10 +222,11 @@ def _vllm_kv_cache_shape(num_blocks, block_size, num_kv_heads, head_size): def test_fish_kvcache_backend_unbinds_kv_on_vllm_cache_layout(monkeypatch): - # vLLM >=0.23.0 lays the KV cache out as (num_blocks, 2, block_size, - # num_kv_heads, head_size). The backend must unbind on dim 1 so each of - # key/value comes back as the 4-D (num_blocks, block_size, num_kv_heads, - # head_size) tensor the decode kernel expects. + # vLLM >=0.23.0 lays out the KV cache as (num_blocks, num_kv_heads, + # block_size, 2*head_size) with K and V packed in the last dimension. + # The backend must split on the content dim so each of key/value comes + # back as the 4-D (num_blocks, block_size, num_kv_heads, head_size) + # tensor the decode kernel expects. monkeypatch.setenv("VLLM_OMNI_FISH_KVCACHE_ATTN", "1") monkeypatch.setattr(fish_kvcache_backend, "is_available", lambda: True) monkeypatch.setattr(fish_kvcache_backend, "load_error", lambda: None) @@ -232,7 +234,8 @@ def test_fish_kvcache_backend_unbinds_kv_on_vllm_cache_layout(monkeypatch): num_blocks, block_size, num_kv_heads, head_size = 8, 16, 8, 128 cache_shape = _vllm_kv_cache_shape(num_blocks, block_size, num_kv_heads, head_size) - assert cache_shape == (num_blocks, 2, block_size, num_kv_heads, head_size) + # Upstream now returns a 4-D packed shape: (B, H, N, 2*D). + assert cache_shape == (num_blocks, num_kv_heads, block_size, 2 * head_size) captured = {} @@ -262,20 +265,21 @@ def test_fish_kvcache_backend_unbinds_kv_on_vllm_cache_layout(monkeypatch): def test_fish_kvcache_vllm_cache_unbinds_then_falls_back_until_contiguous(monkeypatch): # End-to-end with the real vLLM cache layout and the real (unmocked) guard. # - # The point of the unbind(1) fix: unbind(0) raised "too many values to - # unpack" on the 5-D vLLM 0.23.0 layout, so the request crashed outright. - # unbind(1) makes the request *work* again -- it unpacks cleanly and the - # backend gracefully falls back to the original attention with correct - # output. This test guards that fix and must NOT depend on the fast path - # firing. + # The point of the transpose+split fix: unbind(1) on the old 5-D layout + # raised "too many values to unpack" on the vLLM 0.23.0 layout, and the + # upstream now packs K/V into the last dimension (4-D shape). The + # transpose(1, 2) + split correctly unpacks into separate key/value tensors + # and the backend gracefully falls back to the original attention with + # correct output. This test guards that fix and must NOT depend on the fast + # path firing. # - # It also documents a known limitation (tracked separately): unbind(1) on - # the interleaved (num_blocks, 2, ...) layout yields *non-contiguous* - # key/value views, which the guard rejects on the is_contiguous() check. - # So the dimensions are compatible (4-D, block_size=16, head_size=128) -- - # the only thing keeping the fast path from engaging is contiguity, not - # shape. Making the fast path actually fire would require a .contiguous() - # copy and is out of scope here. + # It also documents a known limitation (tracked separately): + # transpose(1, 2) on the interleaved (B, H, N, 2*D) layout yields + # *non-contiguous* key/value views, which the guard rejects on the + # is_contiguous() check. So the dimensions are compatible (4-D, + # block_size=16, head_size=128) -- the only thing keeping the fast path + # from engaging is contiguity, not shape. Making the fast path actually + # fire would require a .contiguous() copy and is out of scope here. fish_kvcache_backend.reset_fish_kvcache_attn_stats() monkeypatch.setenv("VLLM_OMNI_FISH_KVCACHE_ATTN", "1") monkeypatch.setattr(fish_kvcache_backend, "is_available", lambda: True) @@ -300,13 +304,14 @@ def test_fish_kvcache_vllm_cache_unbinds_then_falls_back_until_contiguous(monkey output = torch.zeros_like(query) kv_cache = torch.zeros(cache_shape, dtype=torch.float16) - # Root cause record: unbind(1) gives the right 4-D shape but a non-contiguous view. - key_cache, value_cache = kv_cache.unbind(1) + # Root cause record: transpose(1,2) + split gives the right 4-D shape but + # a non-contiguous view. + key_cache, value_cache = kv_cache.transpose(1, 2).split(head_size, dim=-1) assert tuple(key_cache.shape) == (num_blocks, block_size, num_kv_heads, head_size) assert not key_cache.is_contiguous() - # unbind(1) no longer crashes (unbind(0) would have raised here) and the - # request completes via the correct fallback path. + # transpose+split no longer crashes (unbind(1) would have raised here) and + # the request completes via the correct fallback path. result = model.layers[0].self_attn.attn.impl.forward(None, query, None, None, kv_cache, _metadata(), output) assert result is output diff --git a/tests/core/sched/test_omni_ar_scheduler_streaming.py b/tests/core/sched/test_omni_ar_scheduler_streaming.py index 06037d4cf..4ae4f7684 100644 --- a/tests/core/sched/test_omni_ar_scheduler_streaming.py +++ b/tests/core/sched/test_omni_ar_scheduler_streaming.py @@ -73,6 +73,9 @@ def _run_resumable_segment_stop( sched._update_request_with_output.side_effect = stop_request sched._handle_stopped_request.return_value = session_finished + # vLLM 0.26 returns (kv_xfer_params, ec_xfer_params); an unconfigured + # MagicMock iterates empty and fails to unpack at the call site. + sched._free_request.return_value = (None, None) sched.chunk_transfer_adapter = None sched.running = [session] sched.waiting_for_transfer_free = set() @@ -137,6 +140,23 @@ def test_resumable_session_terminal_is_not_marked_as_segment_boundary() -> None: assert output.is_segment_finished is False +def test_update_from_output_settles_in_flight_tokens() -> None: + """vLLM 0.26: schedule() increments num_in_flight_tokens per scheduled + token; update_from_output must decrement it symmetrically. If the + decrement is dropped the counter grows monotonically and both readers + (allocate_slots, _connector_finished) clamp + max(0, num_computed_tokens - num_in_flight_tokens) to zero forever, + silently freezing sliding-window block freeing. + """ + session = _make_request() + session.status = RequestStatus.RUNNING + session.num_in_flight_tokens = 1 # as left by schedule() for this step + + _run_resumable_segment_stop(session) + + assert session.num_in_flight_tokens == 0 + + def test_running_decode_step_without_inter_stage_payload_does_not_raise() -> None: """A decode step that neither stops nor carries an inter-stage payload. diff --git a/tests/core/sched/test_omni_scheduler_finish_requests_purge.py b/tests/core/sched/test_omni_scheduler_finish_requests_purge.py index 9b564d7d5..fbfeb384f 100644 --- a/tests/core/sched/test_omni_scheduler_finish_requests_purge.py +++ b/tests/core/sched/test_omni_scheduler_finish_requests_purge.py @@ -76,8 +76,9 @@ def test_finish_requests_purges_finished_request_from_running( ) def fake_finish_requests(self, request_ids, finished_status): - # super() returning the abort tuple but leaving self.running untouched - return [(finished.request_id, 0)] + # vLLM 0.26 super() returns the finished Request objects but here + # leaves self.running untouched to simulate the missed removal. + return [finished] monkeypatch.setattr(scheduler_mod.VLLMScheduler, "finish_requests", fake_finish_requests) @@ -88,7 +89,7 @@ def test_finish_requests_purges_finished_request_from_running( ) assert scheduler.running == [] - assert result == [(finished.request_id, 0)] + assert result == [finished] @pytest.mark.parametrize(("scheduler_cls", "scheduler_mod"), _SCHEDULER_PARAMS) @@ -111,7 +112,7 @@ def test_finish_requests_purges_untracked_request_from_running( ) def fake_finish_requests(self, request_ids, finished_status): - return [(untracked.request_id, 0)] + return [untracked] monkeypatch.setattr(scheduler_mod.VLLMScheduler, "finish_requests", fake_finish_requests) @@ -122,7 +123,7 @@ def test_finish_requests_purges_untracked_request_from_running( ) assert scheduler.running == [] - assert result == [(untracked.request_id, 0)] + assert result == [untracked] @pytest.mark.parametrize(("scheduler_cls", "scheduler_mod"), _SCHEDULER_PARAMS) @@ -136,7 +137,7 @@ def test_finish_requests_leaves_healthy_running_intact( is conservative and won't aggressively drop healthy entries. """ alive = _StubRequest("req-alive", RequestStatus.RUNNING) - leaving_id = "req-leaving" + leaving = _StubRequest("req-leaving", RequestStatus.FINISHED_STOPPED) scheduler = _make_scheduler( scheduler_cls, requests={alive.request_id: alive}, @@ -145,13 +146,13 @@ def test_finish_requests_leaves_healthy_running_intact( ) def fake_finish_requests(self, request_ids, finished_status): - return [(leaving_id, 0)] + return [leaving] monkeypatch.setattr(scheduler_mod.VLLMScheduler, "finish_requests", fake_finish_requests) scheduler_cls.finish_requests( scheduler, - [leaving_id], + [leaving.request_id], RequestStatus.FINISHED_STOPPED, ) diff --git a/tests/core/sched/test_omni_scheduler_input_coordinator_cleanup.py b/tests/core/sched/test_omni_scheduler_input_coordinator_cleanup.py index fcac8172b..cb0f50a5d 100644 --- a/tests/core/sched/test_omni_scheduler_input_coordinator_cleanup.py +++ b/tests/core/sched/test_omni_scheduler_input_coordinator_cleanup.py @@ -53,7 +53,11 @@ def test_finish_requests_cleans_input_coordinator_for_finished_ids( def fake_finish_requests(self, request_ids, finished_status): assert request_ids == ["req-a", "req-b"] assert finished_status == RequestStatus.FINISHED_STOPPED - return [("req-a", 0), ("req-b", 1)] + # vLLM 0.26 returns the finished Request objects, not (req_id, client_index) tuples. + return [ + SimpleNamespace(request_id="req-a", client_index=0), + SimpleNamespace(request_id="req-b", client_index=1), + ] monkeypatch.setattr(scheduler_mod.VLLMScheduler, "finish_requests", fake_finish_requests) @@ -63,7 +67,7 @@ def test_finish_requests_cleans_input_coordinator_for_finished_ids( RequestStatus.FINISHED_STOPPED, ) - assert result == [("req-a", 0), ("req-b", 1)] + assert [(r.request_id, r.client_index) for r in result] == [("req-a", 0), ("req-b", 1)] assert coordinator.freed == ["req-a", "req-b"] @@ -82,7 +86,7 @@ def test_ar_free_request_cleans_input_coordinator_on_normal_free() -> None: result = OmniARScheduler._free_request(scheduler, DummyRequest()) - assert result is None + assert result == (None, None) assert coordinator.freed == ["req-free"] assert scheduler._omits_kv_transfer_cache == {} assert scheduler._new_prompt_len_snapshot == {} diff --git a/tests/diffusion/models/boogu_image/test_boogu_image_transformer.py b/tests/diffusion/models/boogu_image/test_boogu_image_transformer.py index 5551ba191..08ba4af09 100644 --- a/tests/diffusion/models/boogu_image/test_boogu_image_transformer.py +++ b/tests/diffusion/models/boogu_image/test_boogu_image_transformer.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import logging import os from types import SimpleNamespace @@ -338,27 +339,43 @@ def test_transformer_load_weights_round_trip(): assert torch.allclose(reloaded[native_name], expected) -def test_transformer_load_weights_warns_for_unexpected_and_unloaded(caplog): +def test_transformer_load_weights_warns_for_unexpected_and_unloaded(): from vllm_omni.diffusion.models.boogu_image.boogu_image_transformer import ( BooguImageTransformer2DModel, ) - model = BooguImageTransformer2DModel(od_config=_tiny_od_config()) - native_params = dict(model.named_parameters()) - loaded_name = next(iter(native_params)) - checkpoint_name = _native_to_checkpoint_name(loaded_name) + # vllm_omni's logger hierarchy hangs off vLLM's `vllm` logger, which sets + # propagate=False, so records never reach the root logger that pytest's + # caplog listens on. Capture at the emitting logger instead. + messages: list[str] = [] - loaded = model.load_weights( - [ - (checkpoint_name, torch.randn_like(native_params[loaded_name])), - ("unexpected.weight", torch.ones(1)), - ] - ) + class _Capture(logging.Handler): + def emit(self, record: logging.LogRecord) -> None: + messages.append(record.getMessage()) + log = logging.getLogger("vllm_omni.diffusion.models.boogu_image.boogu_image_transformer") + handler = _Capture(level=logging.WARNING) + log.addHandler(handler) + try: + model = BooguImageTransformer2DModel(od_config=_tiny_od_config()) + native_params = dict(model.named_parameters()) + loaded_name = next(iter(native_params)) + checkpoint_name = _native_to_checkpoint_name(loaded_name) + + loaded = model.load_weights( + [ + (checkpoint_name, torch.randn_like(native_params[loaded_name])), + ("unexpected.weight", torch.ones(1)), + ] + ) + finally: + log.removeHandler(handler) + + text = "\n".join(messages) assert loaded == {loaded_name} - assert "Skipping unexpected checkpoint weight unexpected.weight" in caplog.text - assert "Model parameters not loaded from checkpoint" in caplog.text - assert next(name for name in native_params if name != loaded_name) in caplog.text + assert "Skipping unexpected checkpoint weight unexpected.weight" in text + assert "Model parameters not loaded from checkpoint" in text + assert next(name for name in native_params if name != loaded_name) in text def test_transformer_forward_t2i_shape(): diff --git a/tests/distributed/omni_connectors/test_chunk_transfer_adapter.py b/tests/distributed/omni_connectors/test_chunk_transfer_adapter.py index 1485ace60..1a045f306 100644 --- a/tests/distributed/omni_connectors/test_chunk_transfer_adapter.py +++ b/tests/distributed/omni_connectors/test_chunk_transfer_adapter.py @@ -780,6 +780,10 @@ def test_restore_queues_skips_requests_missing_from_scheduler_requests(build_ada class _HashableRequest(SimpleNamespace): """SimpleNamespace that can be added to a set (needed by scheduler internals).""" + # vLLM 0.26: update_from_output settles this counter for every scheduled + # request; real Requests initialise it to 0 (vllm/v1/request.py). + num_in_flight_tokens = 0 + def __hash__(self): return hash(self.request_id) @@ -833,7 +837,7 @@ def test_generation_scheduler_calls_cleanup_on_finished(monkeypatch, mocker: Moc scheduler.requests = {"req-s1": request} scheduler._handle_stopped_request = mocker.MagicMock(return_value=True) - scheduler._free_request = mocker.MagicMock(return_value=None) + scheduler._free_request = mocker.MagicMock(return_value=(None, None)) scheduler._get_routed_experts = mocker.MagicMock(return_value=None) scheduler.running = [request] scheduler.waiting = mocker.MagicMock() @@ -918,7 +922,7 @@ def test_ar_scheduler_defers_cleanup_and_queues_save_on_finished(mocker: MockerF scheduler._update_request_with_output = mocker.MagicMock(return_value=([], True)) scheduler._process_kv_transfer_trigger = mocker.MagicMock(return_value=False) scheduler._handle_stopped_request = mocker.MagicMock(return_value=True) - scheduler._free_request = mocker.MagicMock(return_value=None) + scheduler._free_request = mocker.MagicMock(return_value=(None, None)) scheduler._get_routed_experts = mocker.MagicMock(return_value=None) scheduler.running = [request] scheduler.waiting = mocker.MagicMock() @@ -1047,7 +1051,7 @@ def _build_deferred_finish_scheduler(mocker, *, running, pending_finish_reqs): scheduler._pending_finish_reqs = list(pending_finish_reqs) scheduler._handle_stopped_request = mocker.MagicMock(return_value=True) - scheduler._free_request = mocker.MagicMock(return_value=None) + scheduler._free_request = mocker.MagicMock(return_value=(None, None)) scheduler._get_routed_experts = mocker.MagicMock(return_value=None) scheduler.running = list(running) scheduler.waiting = mocker.MagicMock() @@ -1103,7 +1107,7 @@ def test_deferred_finish_emits_finished_output(mocker: MockerFixture): running=[request], pending_finish_reqs=[request], ) - scheduler._free_request.return_value = {"mock": "kv_params"} + scheduler._free_request.return_value = ({"mock": "kv_params"}, None) result = OmniGenerationScheduler.update_from_output(scheduler, sched_out, model_out) diff --git a/tests/distributed/omni_connectors/test_kv_flow.py b/tests/distributed/omni_connectors/test_kv_flow.py index 83206fefb..b738c095e 100644 --- a/tests/distributed/omni_connectors/test_kv_flow.py +++ b/tests/distributed/omni_connectors/test_kv_flow.py @@ -330,6 +330,31 @@ def test_normalize_layer_kv_rejects_invalid_inputs(kv_config, common_constants, assert normalized is None +def test_normalize_layer_kv_unpacks_packed_layout(common_constants): + """The vLLM >= 0.23 packed layout (num_blocks, n_heads, block_size, + 2*head_dim) is unpacked into (num_blocks, block_size, n_heads, head_dim) + key/value blocks when block_size is provided.""" + block_size = common_constants["block_size"] + num_heads = common_constants["num_heads"] + head_dim = common_constants["head_dim"] + req_id = common_constants["req_id"] + num_blocks = 10 + + layer_kv = torch.randn(num_blocks, num_heads, block_size, 2 * head_dim) + + # Without block_size the 4-D packed layout is not recognized. + assert normalize_layer_kv(layer_kv, req_id=req_id, layer_idx=0) is None + + normalized = normalize_layer_kv(layer_kv, req_id=req_id, layer_idx=0, block_size=block_size) + assert normalized is not None + key_blocks, value_blocks = normalized + assert key_blocks.shape == (num_blocks, block_size, num_heads, head_dim) + assert value_blocks.shape == (num_blocks, block_size, num_heads, head_dim) + reference = layer_kv.transpose(1, 2) + assert torch.equal(key_blocks, reference[..., :head_dim]) + assert torch.equal(value_blocks, reference[..., head_dim:]) + + def test_manager_reception(kv_config, mock_connector, common_constants): """Test reception and injection logic in OmniKVTransferManager.""" num_layers = common_constants["num_layers"] diff --git a/tests/entrypoints/openai_api/test_qwen3_omni_realtime_websocket.py b/tests/entrypoints/openai_api/test_qwen3_omni_realtime_websocket.py index 50f8eef91..325daabdd 100644 --- a/tests/entrypoints/openai_api/test_qwen3_omni_realtime_websocket.py +++ b/tests/entrypoints/openai_api/test_qwen3_omni_realtime_websocket.py @@ -201,6 +201,7 @@ def _assert_realtime_smoke(result: dict) -> None: def _assert_realtime_accuracy( result: dict, whisper_model_size: str = "large-v3", + threshold: float = 0.8, ) -> None: """Assert that whisper transcription of audio output matches model text. @@ -214,6 +215,12 @@ def _assert_realtime_accuracy( variability even though audio generation was correct. large-v3 transcribes these clips reliably, so a failure here now points at the model, not the ASR grader. + threshold: Minimum cosine similarity (with length penalty) required to + pass. Default 0.8. Do not lower per-callsite without data: + at 0.35 the assertion no longer detects real audio + regressions. If a variant genuinely needs a different gate + (e.g. whisper partial transcripts under async_chunk), propose + it in its own PR with measurements. """ final_text = (result["transcription_text"] or "").strip() assert final_text, "Expected non-empty transcription (model text stream)" @@ -223,8 +230,9 @@ def _assert_realtime_accuracy( assert whisper_text, "Whisper returned empty string for synthesized output audio" sim = cosine_similarity_text(whisper_text.lower(), final_text.lower()) - assert sim > 0.8, ( - f"Output audio transcript should match model text (sim={sim:.3f}): " + assert sim > threshold, ( + f"Output audio transcript should match model text (sim={sim:.3f}, " + f"threshold={threshold}): " f"whisper={whisper_text!r}, model_text={final_text!r}" ) diff --git a/tests/entrypoints/test_pd_disaggregation.py b/tests/entrypoints/test_pd_disaggregation.py index 57e484af1..774e6d1a8 100644 --- a/tests/entrypoints/test_pd_disaggregation.py +++ b/tests/entrypoints/test_pd_disaggregation.py @@ -305,21 +305,11 @@ def mock_get_config(monkeypatch): """Auto-mock get_config and related model loading functions.""" import sys - fake_tokenizer = _ns() - fake_tokenizer.encode = lambda *args, **kwargs: [1, 2, 3] - fake_tokenizer.decode = lambda *args, **kwargs: "test" - - def _mock_init_tokenizer_from_configs(model_config=None, **kwargs): - return fake_tokenizer - - monkeypatch.setattr( - "vllm.transformers_utils.tokenizer.init_tokenizer_from_configs", - _mock_init_tokenizer_from_configs, - raising=False, - ) - tokenizer_module_path = "vllm.transformers_utils.tokenizer" - if tokenizer_module_path in sys.modules: - setattr(sys.modules[tokenizer_module_path], "init_tokenizer_from_configs", _mock_init_tokenizer_from_configs) + # TODO(pd-unskip): the tokenizer mock that lived here targeted + # vllm.transformers_utils.tokenizer.init_tokenizer_from_configs, which was + # deleted in vLLM 0.26 (the raising=False setattr made it a silent no-op). + # When the module-level skip is lifted, mock whatever tokenizer seam the + # revived PD entrypoint code actually uses. def _mock_length_from_prompt_token_ids_or_embeds(prompt_token_ids=None, prompt_embeds=None): if prompt_token_ids is not None: @@ -344,13 +334,6 @@ def mock_get_config(monkeypatch): _mock_length_from_prompt_token_ids_or_embeds, ) - monkeypatch.setattr( - "vllm_omni.entrypoints.async_omni.init_tokenizer_from_configs", _mock_init_tokenizer_from_configs, raising=False - ) - async_omni_path = "vllm_omni.entrypoints.async_omni" - if async_omni_path in sys.modules: - setattr(sys.modules[async_omni_path], "init_tokenizer_from_configs", _mock_init_tokenizer_from_configs) - fake_hf_config = _ns() fake_hf_config.model_type = "qwen2_5_omni" diff --git a/tests/model_executor/models/qwen2_5_omni/test_qwen2_5_omni_embed.py b/tests/model_executor/models/qwen2_5_omni/test_qwen2_5_omni_embed.py index d4778b7e2..13899d47d 100644 --- a/tests/model_executor/models/qwen2_5_omni/test_qwen2_5_omni_embed.py +++ b/tests/model_executor/models/qwen2_5_omni/test_qwen2_5_omni_embed.py @@ -15,6 +15,7 @@ import functools import pytest import torch from pytest_mock import MockerFixture +from vllm.multimodal.utils import set_mm_embedding_modality pytestmark = [pytest.mark.core_model, pytest.mark.cpu] @@ -117,13 +118,30 @@ class TestCheckInterleavedAudioVideo: # --------------------------------------------------------------------------- -def make_mock_model(mocker: MockerFixture, hidden: int = 8): +@functools.lru_cache(maxsize=1) +def _omni_qwen2_5_thinker_cls(): + """vllm_omni's thinker override, loaded lazily like the upstream one.""" + from vllm_omni.model_executor.models.qwen2_5_omni.qwen2_5_omni_thinker import ( + Qwen2_5OmniThinkerForConditionalGeneration, + ) + + return Qwen2_5OmniThinkerForConditionalGeneration + + +def make_mock_model(mocker: MockerFixture, hidden: int = 8, thinker_cls=None): """ Return a minimal mock of Qwen2_5OmniThinkerForConditionalGeneration that has enough structure to run embed_input_ids. + + ``thinker_cls`` selects whose ``embed_input_ids`` the mock dispatches to. + It defaults to upstream's. Only the interleaved case passes omni's override: + omni's non-interleaved path falls through to a zero-arg ``super()`` call, + which raises TypeError when ``self`` is a Mock rather than an instance. """ m = _qwen2_5_omni_thinker_mod() Qwen2_5OmniThinkerForConditionalGeneration = m.Qwen2_5OmniThinkerForConditionalGeneration + if thinker_cls is None: + thinker_cls = Qwen2_5OmniThinkerForConditionalGeneration model = mocker.Mock(spec=Qwen2_5OmniThinkerForConditionalGeneration) @@ -155,9 +173,7 @@ def make_mock_model(mocker: MockerFixture, hidden: int = 8): is_multimodal=is_multimodal, ) - model.embed_input_ids = lambda *a, **kw: m.Qwen2_5OmniThinkerForConditionalGeneration.embed_input_ids( # noqa: E501 - model, *a, **kw - ) + model.embed_input_ids = lambda *a, **kw: thinker_cls.embed_input_ids(model, *a, **kw) model._super_embed_input_ids = fake_super_embed @@ -171,11 +187,11 @@ def build_mm_embeds(audio_n, image_n, video_n, hidden, audio_val=10.0, image_val """ embs = [] if audio_n: - embs.append(torch.full((audio_n, hidden), audio_val)) + embs.append(set_mm_embedding_modality(torch.full((audio_n, hidden), audio_val), "audio")) if image_n: - embs.append(torch.full((image_n, hidden), image_val)) + embs.append(set_mm_embedding_modality(torch.full((image_n, hidden), image_val), "image")) if video_n: - embs.append(torch.full((video_n, hidden), video_val)) + embs.append(set_mm_embedding_modality(torch.full((video_n, hidden), video_val), "video")) return embs @@ -259,6 +275,11 @@ class TestEmbedInputIds: """ Interleaved (use_audio_in_video): video chunks interleaved with audio. Video embeddings must go to video positions, audio to audio positions. + + Runs against **omni's** thinker override, not upstream's: this is the + only unit-level coverage of omni's call into + ``merge_interleaved_embeddings``, whose upstream signature dropped + ``num_video``/``num_audio`` in vLLM #46213. """ hidden = 8 audio_val, video_val = 10.0, 30.0 @@ -270,11 +291,11 @@ class TestEmbedInputIds: audio_n = sum(audio_chunks) # 6 mm_embeds = [ - torch.full((video_n, hidden), video_val), - torch.full((audio_n, hidden), audio_val), + set_mm_embedding_modality(torch.full((video_n, hidden), video_val), "video"), + set_mm_embedding_modality(torch.full((audio_n, hidden), audio_val), "audio"), ] - model, _ = make_mock_model(mocker, hidden) + model, _ = make_mock_model(mocker, hidden, thinker_cls=_omni_qwen2_5_thinker_cls()) result = model.embed_input_ids(input_ids, mm_embeds, is_multimodal=is_multimodal) video_pos = (input_ids == VIDEO_TOKEN_ID).nonzero(as_tuple=True)[0] @@ -308,8 +329,8 @@ class TestMergeInterleavedEmbeddings: inputs_embeds = torch.zeros(len(input_ids), hidden) mm_embeds = [ - torch.full((num_video, hidden), 30.0), - torch.full((num_audio, hidden), 10.0), + set_mm_embedding_modality(torch.full((num_video, hidden), 30.0), "video"), + set_mm_embedding_modality(torch.full((num_audio, hidden), 10.0), "audio"), ] result = m.merge_interleaved_embeddings( @@ -318,8 +339,6 @@ class TestMergeInterleavedEmbeddings: is_video, is_audio, is_multimodal, - num_video, - num_audio, ) video_pos = is_video.nonzero(as_tuple=True)[0] diff --git a/tests/model_executor/models/qwen3_omni/test_qwen3_omni_moe_lora.py b/tests/model_executor/models/qwen3_omni/test_qwen3_omni_moe_lora.py index e5fa38a6b..75d81eed5 100644 --- a/tests/model_executor/models/qwen3_omni/test_qwen3_omni_moe_lora.py +++ b/tests/model_executor/models/qwen3_omni/test_qwen3_omni_moe_lora.py @@ -84,6 +84,7 @@ class _CpuFusedMoE3DWithLoRA(FusedMoE3DWithLoRA): self.tp_size = 1 self.tp_rank = 0 self._w13_slices = 1 + self.enable_moe_shared_loras = False def _save_peft_adapter(tmp_path: Path) -> Path: diff --git a/tests/worker/test_gpu_ar_model_runner.py b/tests/worker/test_gpu_ar_model_runner.py index a1f885c15..050063978 100644 --- a/tests/worker/test_gpu_ar_model_runner.py +++ b/tests/worker/test_gpu_ar_model_runner.py @@ -69,6 +69,9 @@ def test_speech_extra_params_reach_model_sampler_as_sampling_metadata(monkeypatc vocab_size=1024, block_sizes=[1], kernel_block_sizes=[1], + # vLLM 0.26: one entry per KV cache group; max_model_len=8 with + # block_sizes=[1] means one group of 8 blocks. + max_num_blocks_per_req=[8], ) input_batch.add_request( CachedRequestState( diff --git a/tests/worker/test_gpu_generation_model_runner.py b/tests/worker/test_gpu_generation_model_runner.py index e64eecb57..a6df4732f 100644 --- a/tests/worker/test_gpu_generation_model_runner.py +++ b/tests/worker/test_gpu_generation_model_runner.py @@ -4,7 +4,7 @@ from types import SimpleNamespace import pytest import torch from vllm.config import CacheConfig -from vllm.v1.worker.gpu_model_runner import EMPTY_MODEL_RUNNER_OUTPUT +from vllm.v1.outputs import EMPTY_MODEL_RUNNER_OUTPUT import vllm_omni.worker.gpu_generation_model_runner as gen_runner_module from vllm_omni.worker.gpu_generation_model_runner import ( diff --git a/tests/worker/test_omni_connector_mixin.py b/tests/worker/test_omni_connector_mixin.py index 87cf4ba73..cf4e09cb9 100644 --- a/tests/worker/test_omni_connector_mixin.py +++ b/tests/worker/test_omni_connector_mixin.py @@ -1245,7 +1245,7 @@ class TestRankAwareKVRouting(unittest.TestCase): class TestAttachOmniConnectorOutput(unittest.TestCase): def test_wraps_empty_model_runner_output_when_signals_exist(self): - from vllm.v1.worker.gpu_model_runner import EMPTY_MODEL_RUNNER_OUTPUT + from vllm.v1.outputs import EMPTY_MODEL_RUNNER_OUTPUT host = MixinHost() host.get_omni_connector_output = lambda: OmniConnectorOutput(chunk_ready_req_ids={"req-1"}) diff --git a/vllm_omni/__init__.py b/vllm_omni/__init__.py index f3c9d9afd..c2cbe9d5c 100644 --- a/vllm_omni/__init__.py +++ b/vllm_omni/__init__.py @@ -27,8 +27,14 @@ except ModuleNotFoundError as exc: # pragma: no cover - optional dependency patch = None # type: ignore # Register custom configs (AutoConfig, AutoTokenizer) as early as possible. -from vllm_omni.transformers_utils import configs as _configs # noqa: F401, E402 -from vllm_omni.transformers_utils import parsers as _parsers # noqa: F401, E402 +try: + from vllm_omni.transformers_utils import configs as _configs # noqa: F401, E402 + from vllm_omni.transformers_utils import parsers as _parsers # noqa: F401, E402 +except ModuleNotFoundError as exc: # pragma: no cover - optional dependency + if exc.name != "vllm": + raise + # Allow importing vllm_omni without vllm (e.g., documentation builds) + pass from .config import OmniModelConfig diff --git a/vllm_omni/attention/fish_kvcache_backend.py b/vllm_omni/attention/fish_kvcache_backend.py index 0178f722e..4ca6d467b 100644 --- a/vllm_omni/attention/fish_kvcache_backend.py +++ b/vllm_omni/attention/fish_kvcache_backend.py @@ -234,11 +234,11 @@ def _forward_with_fish_kvcache( if attn_metadata is not None and not attn_metadata.use_cascade: num_actual_tokens = attn_metadata.num_actual_tokens # vLLM >=0.23.0 lays out the KV cache as - # (num_blocks, 2, block_size, num_kv_heads, head_size), so key/value - # live on dim 1 (matches FlashAttentionImpl's kv_cache.unbind(1)). - # The pre-0.23 layout had them on dim 0; unbind(0) here would split - # along num_blocks and raise "too many values to unpack". - key_cache, value_cache = kv_cache.unbind(1) + # (num_blocks, num_kv_heads, block_size, 2*head_size) with K and V + # packed in the last dimension. Match FlashAttentionImpl's own + # unpacking: transpose(1, 2) then split on the content dim. + head_size = kv_cache.shape[-1] // 2 + key_cache, value_cache = kv_cache.transpose(1, 2).split(head_size, dim=-1) q = query[:num_actual_tokens] out = output[:num_actual_tokens] active_batch = int(q.shape[0]) diff --git a/vllm_omni/core/sched/omni_ar_scheduler.py b/vllm_omni/core/sched/omni_ar_scheduler.py index 3892bd389..22917197d 100644 --- a/vllm_omni/core/sched/omni_ar_scheduler.py +++ b/vllm_omni/core/sched/omni_ar_scheduler.py @@ -1,6 +1,7 @@ from __future__ import annotations from collections import defaultdict +from collections.abc import Iterable from time import time from typing import Any @@ -341,10 +342,18 @@ class OmniARScheduler(OmniSchedulerMixin, VLLMScheduler): stopped_preempted_reqs: set[Request] = set() for req_id, num_tokens_scheduled in num_scheduled_tokens.items(): assert num_tokens_scheduled > 0 + request = self.requests.get(req_id) + if request is not None: + # vLLM 0.26: settle the in-flight tokens counted in schedule(). + # Must happen before the skips below — failed-KV-load and + # already-finished requests were incremented too, and the two + # readers (allocate_slots, _connector_finished) clamp with + # max(0, computed - in_flight), so a leaked counter silently + # freezes sliding-window block freeing. + request.num_in_flight_tokens -= num_tokens_scheduled if failed_kv_load_req_ids and req_id in failed_kv_load_req_ids: # Skip requests that were recovered from KV load failure continue - request = self.requests.get(req_id) if request is None or request.is_finished(): # The request is already finished. This can happen if the # request is aborted while the model is executing it (e.g., @@ -451,7 +460,7 @@ class OmniARScheduler(OmniSchedulerMixin, VLLMScheduler): request.spec_token_ids = [] request._output_token_ids.clear() if finished: - kv_transfer_params = self._free_request(request) + kv_transfer_params, _ = self._free_request(request) if status_before_stop == RequestStatus.RUNNING: stopped_running_reqs.add(request) elif status_before_stop == RequestStatus.WAITING_FOR_CHUNK: @@ -642,7 +651,7 @@ class OmniARScheduler(OmniSchedulerMixin, VLLMScheduler): return engine_core_outputs - def finish_requests(self, request_ids: Any, finished_status: RequestStatus) -> list[tuple[str, int]]: + def finish_requests(self, request_ids: str | Iterable[str] | None, finished_status: RequestStatus) -> list[Request]: """Handles the finish signal from outside the scheduler. For example, the API server can abort a request when the client @@ -651,8 +660,8 @@ class OmniARScheduler(OmniSchedulerMixin, VLLMScheduler): If request_ids is None, all requests will be finished. Returns: - Tuple of (req_id, client_index) for requests that were aborted. Will not - include any that were already finished. + The Request objects that were aborted. Will not include any that + were already finished. """ # TODO(yrr): chunk transfer adapter & input_coordinator unified to one if self.chunk_transfer_adapter: @@ -688,8 +697,8 @@ class OmniARScheduler(OmniSchedulerMixin, VLLMScheduler): input_coordinator = getattr(self, "input_coordinator", None) if input_coordinator is not None: - for request_id, _ in finished: - self._free_input_coordinator_request(request_id) + for request in finished: + self._free_input_coordinator_request(request.request_id) return finished def _update_request_as_session(self, session: Request, update: StreamingUpdate) -> None: @@ -740,7 +749,9 @@ class OmniARScheduler(OmniSchedulerMixin, VLLMScheduler): if hasattr(update, "model_intermediate_buffer"): session.model_intermediate_buffer = update.model_intermediate_buffer - def _free_request(self, request: Request, delay_free_blocks: bool = False) -> dict[str, Any] | None: + def _free_request( + self, request: Request, delay_free_blocks: bool = False + ) -> tuple[dict[str, Any] | None, dict[str, Any] | None]: # TODO(wzliu)! for offline mode, we should not end process until all data is transferred """Mark a request as finished and free its resources.""" assert request.is_finished() @@ -779,10 +790,10 @@ class OmniARScheduler(OmniSchedulerMixin, VLLMScheduler): logger.debug(f"[Omni] Request {request_id} finished but transfer is still ACTIVE. Waiting.") self.waiting_for_transfer_free.add(request_id) kv_xfer_params = None - return kv_xfer_params + return kv_xfer_params, None elif request_id in self.waiting_for_transfer_free: # Blocks held until KV extraction completes in a future step. - return None + return None, None else: logger.debug( f"[Omni] Request {request_id} finished and transfer no longer ACTIVE (extracted/acked). " @@ -819,14 +830,14 @@ class OmniARScheduler(OmniSchedulerMixin, VLLMScheduler): if isinstance(add_info, dict): add_info.update(kv_xfer_params) - return kv_xfer_params + return kv_xfer_params, None # 3. Standard Freeing delay_free_blocks |= connector_delay_free_blocks if not delay_free_blocks: self._free_blocks(request) - return kv_xfer_params + return kv_xfer_params, None finally: self._free_input_coordinator_request(request_id) diff --git a/vllm_omni/core/sched/omni_generation_scheduler.py b/vllm_omni/core/sched/omni_generation_scheduler.py index 534019513..06082b0dc 100644 --- a/vllm_omni/core/sched/omni_generation_scheduler.py +++ b/vllm_omni/core/sched/omni_generation_scheduler.py @@ -2,6 +2,7 @@ from __future__ import annotations import time from collections import defaultdict +from collections.abc import Iterable from typing import Any from vllm.compilation.cuda_graph import CUDAGraphStat @@ -348,7 +349,7 @@ class OmniGenerationScheduler(OmniSchedulerMixin, VLLMScheduler): return self._wrap_omni_scheduler_output(scheduler_output) - def finish_requests(self, request_ids, finished_status: RequestStatus) -> list[tuple[str, int]]: + def finish_requests(self, request_ids: str | Iterable[str] | None, finished_status: RequestStatus) -> list[Request]: """Handles the finish signal from outside the scheduler. For example, the API server can abort a request when the client @@ -357,8 +358,8 @@ class OmniGenerationScheduler(OmniSchedulerMixin, VLLMScheduler): If request_ids is None, all requests will be finished. Returns: - Tuple of (req_id, client_index) for requests that were aborted. Will not - include any that were already finished. + The Request objects that were aborted. Will not include any that + were already finished. """ if self.chunk_transfer_adapter: @@ -382,11 +383,13 @@ class OmniGenerationScheduler(OmniSchedulerMixin, VLLMScheduler): self._purge_finished_from_running() if self.input_coordinator is not None: - for request_id, _ in finished: - self._free_input_coordinator_request(request_id) + for request in finished: + self._free_input_coordinator_request(request.request_id) return finished - def _free_request(self, request: Request, delay_free_blocks: bool = False) -> dict[str, Any] | None: + def _free_request( + self, request: Request, delay_free_blocks: bool = False + ) -> tuple[dict[str, Any] | None, dict[str, Any] | None]: if self.input_coordinator is None: return super()._free_request(request, delay_free_blocks) @@ -432,10 +435,18 @@ class OmniGenerationScheduler(OmniSchedulerMixin, VLLMScheduler): stopped_preempted_reqs: set[Request] = set() for req_id, num_tokens_scheduled in num_scheduled_tokens.items(): assert num_tokens_scheduled > 0 + request = self.requests.get(req_id) + if request is not None: + # vLLM 0.26: settle the in-flight tokens counted in schedule(). + # Must happen before the skips below — failed-KV-load and + # already-finished requests were incremented too, and the two + # readers (allocate_slots, _connector_finished) clamp with + # max(0, computed - in_flight), so a leaked counter silently + # freezes sliding-window block freeing. + request.num_in_flight_tokens -= num_tokens_scheduled if failed_kv_load_req_ids and req_id in failed_kv_load_req_ids: # Skip requests that were recovered from KV load failure continue - request = self.requests.get(req_id) if request is None or request.is_finished(): # Request may already be finished (e.g., aborted during # execution / pipeline parallelism / async scheduling). @@ -510,7 +521,7 @@ class OmniGenerationScheduler(OmniSchedulerMixin, VLLMScheduler): if self.chunk_transfer_adapter: self.chunk_transfer_adapter.segment_finished_requests.discard(req_id) if finished: - kv_transfer_params = self._free_request(request) + kv_transfer_params, _ = self._free_request(request) if self.chunk_transfer_adapter is not None: self.chunk_transfer_adapter.cleanup( request.request_id, @@ -576,7 +587,7 @@ class OmniGenerationScheduler(OmniSchedulerMixin, VLLMScheduler): is_segment_finished = not finished kv_transfer_params = None if finished: - kv_transfer_params = self._free_request(request) + kv_transfer_params, _ = self._free_request(request) if self.chunk_transfer_adapter is not None: self.chunk_transfer_adapter.cleanup( request.request_id, diff --git a/vllm_omni/diffusion/models/cosmos3/pipeline_cosmos3.py b/vllm_omni/diffusion/models/cosmos3/pipeline_cosmos3.py index 41a3e80fc..fcc2a8b6c 100644 --- a/vllm_omni/diffusion/models/cosmos3/pipeline_cosmos3.py +++ b/vllm_omni/diffusion/models/cosmos3/pipeline_cosmos3.py @@ -956,11 +956,10 @@ class Cosmos3OmniDiffusersPipeline( self.weights_sources = [ DiffusersPipelineLoader.ComponentSource( model_or_path=model_path, - subfolder=None, + subfolder="transformer", revision=None, prefix="transformer.", fall_back_to_pt=True, - allow_patterns_overrides=["transformer/*.safetensors"], ), ] diff --git a/vllm_omni/diffusion/models/hunyuan_image3/hunyuan_image3_tokenizer.py b/vllm_omni/diffusion/models/hunyuan_image3/hunyuan_image3_tokenizer.py index 751bfb21a..e7bb75fbe 100644 --- a/vllm_omni/diffusion/models/hunyuan_image3/hunyuan_image3_tokenizer.py +++ b/vllm_omni/diffusion/models/hunyuan_image3/hunyuan_image3_tokenizer.py @@ -384,7 +384,7 @@ class TokenizerWrapper: drop_last_break = True break if source.get("front_boi", use_front_boi_token): - token_seq.append(self.boi_token_id) # Use patched boi for Janus, otherwise useing default + token_seq.append(self.boi_token_id) # Use patched boi for Janus, otherwise using default extra_token_pos["boi"].append(token_count) token_count += 1 token_count = self._add_image_meta_info_token( diff --git a/vllm_omni/distributed/omni_connectors/kv_transfer_manager.py b/vllm_omni/distributed/omni_connectors/kv_transfer_manager.py index b75a7089a..efed75241 100644 --- a/vllm_omni/distributed/omni_connectors/kv_transfer_manager.py +++ b/vllm_omni/distributed/omni_connectors/kv_transfer_manager.py @@ -1023,7 +1023,7 @@ class OmniKVTransferManager: value_cache: list[torch.Tensor | None] = [None] * num_layers for layer_idx, layer_kv in enumerate(kv_caches): - kv_pair = normalize_layer_kv(layer_kv, req_id=req_id, layer_idx=layer_idx) + kv_pair = normalize_layer_kv(layer_kv, req_id=req_id, layer_idx=layer_idx, block_size=block_size) if kv_pair is None: continue key_blocks, value_blocks = kv_pair diff --git a/vllm_omni/distributed/omni_connectors/utils/kv_utils.py b/vllm_omni/distributed/omni_connectors/utils/kv_utils.py index fe0df21f2..b7f51460c 100644 --- a/vllm_omni/distributed/omni_connectors/utils/kv_utils.py +++ b/vllm_omni/distributed/omni_connectors/utils/kv_utils.py @@ -402,6 +402,7 @@ def normalize_layer_kv( *, req_id: str = "", layer_idx: int = -1, + block_size: int | None = None, ) -> tuple[torch.Tensor, torch.Tensor] | None: """Normalize one layer KV cache to a ``(key_blocks, value_blocks)`` tuple. @@ -424,6 +425,10 @@ def normalize_layer_kv( dim-0 selects key / value. * **Stacked tensor** ``[num_blocks, 2, block_size, n_heads, head_dim]`` – dim-1 selects key / value. + * **Packed tensor** ``[num_blocks, n_heads, block_size, 2 * head_dim]`` – + vLLM >= 0.23 FlashAttention packs K and V into the last (content) + dimension. Only recognized when *block_size* is provided and matches + dim 2, to avoid misreading other 4-D layouts. * **Tuple** ``(key_tensor, value_tensor)`` – returned as-is after validation. @@ -431,13 +436,29 @@ def normalize_layer_kv( layer_kv: The raw KV cache (tensor or tuple) for the layer. req_id: Request ID used only for diagnostic log messages. layer_idx: Layer index used only for diagnostic log messages. + block_size: The paged-attention block size, required to recognize + the packed 4-D layout. Returns: ``(key_blocks, value_blocks)`` if *layer_kv* is valid, ``None`` otherwise. """ if isinstance(layer_kv, torch.Tensor): - if layer_kv.ndim >= 3 and layer_kv.shape[0] == 2: + if ( + layer_kv.ndim == 4 + and block_size is not None + and layer_kv.shape[2] == block_size + and layer_kv.shape[-1] % 2 == 0 + ): + # vLLM >= 0.23 FlashAttention layout: (num_blocks, n_heads, + # block_size, 2 * head_dim). Unpack the same way + # FlashAttentionImpl does: move block_size before heads, then + # split the content dim into the K and V halves. Checked before + # the stacked layouts (which are 5-D in practice) so that a + # 2-KV-head packed tensor is not misread as a dim-1 stack. + head_dim = layer_kv.shape[-1] // 2 + key_blocks, value_blocks = layer_kv.transpose(1, 2).split(head_dim, dim=-1) + elif layer_kv.ndim >= 3 and layer_kv.shape[0] == 2: key_blocks = layer_kv[0] value_blocks = layer_kv[1] elif layer_kv.ndim >= 3 and layer_kv.shape[1] == 2: @@ -446,7 +467,8 @@ def normalize_layer_kv( else: logger.warning( f"Layer {layer_idx} for request {req_id} has invalid stacked KV shape: " - f"expected [2, ...] or [..., 2, ...] at dim 0/1, got {tuple(layer_kv.shape)}" + f"expected [2, ...], [..., 2, ...] at dim 0/1, or packed " + f"[num_blocks, n_heads, block_size, 2*head_dim], got {tuple(layer_kv.shape)}" ) return None elif isinstance(layer_kv, tuple): diff --git a/vllm_omni/experimental/ar_diffusion/kv_cache/manager.py b/vllm_omni/experimental/ar_diffusion/kv_cache/manager.py index 8c7d4b2ce..9ef3cca0c 100644 --- a/vllm_omni/experimental/ar_diffusion/kv_cache/manager.py +++ b/vllm_omni/experimental/ar_diffusion/kv_cache/manager.py @@ -71,6 +71,11 @@ class ARDiffusionRequestAdapter: self.num_preemptions = 0 # vLLM watermark gate reads this; map the request lifecycle onto it. self.status = RequestStatus.WAITING + # vLLM 0.26 KVCacheManager.allocate_slots reads this when computing + # remove_skipped_blocks. Always 0 here: an AR-Diffusion request only + # advances num_computed_tokens on on_chunk_committed(), so there are + # never optimistically-counted in-flight tokens to subtract. + self.num_in_flight_tokens = 0 @property def num_computed_tokens(self) -> int: diff --git a/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py b/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py index 6f6841db3..577ce5aa7 100644 --- a/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py +++ b/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py @@ -105,11 +105,6 @@ from vllm.multimodal.inputs import ( MultiModalKwargsItems, NestedTensors, ) - -try: - from vllm.multimodal.inputs import ModalityData, MultiModalDataDict -except ImportError: - from vllm.multimodal.parse import ModalityData, MultiModalDataDict from vllm.multimodal.parse import ( AudioItem, AudioProcessorItems, @@ -132,15 +127,10 @@ from vllm.multimodal.processing import ( ) from vllm.sequence import IntermediateTensors -try: - from vllm.transformers_utils.tokenizer import encode_tokens as _vllm_encode_tokens -except ImportError: - _vllm_encode_tokens = None - +# vllm.transformers_utils.tokenizer no longer exists in upstream vLLM; +# _encode_tokens uses tokenizer.encode() as the only code path. def _encode_tokens(tokenizer: Any, prompt: str) -> list[int]: - if _vllm_encode_tokens is not None: - return _vllm_encode_tokens(tokenizer, prompt) return tokenizer.encode(prompt, add_special_tokens=False) diff --git a/vllm_omni/model_executor/models/moss_tts/moss_codec_cudagraph.py b/vllm_omni/model_executor/models/moss_tts/moss_codec_cudagraph.py index 87f255072..ed03c018e 100644 --- a/vllm_omni/model_executor/models/moss_tts/moss_codec_cudagraph.py +++ b/vllm_omni/model_executor/models/moss_tts/moss_codec_cudagraph.py @@ -1,6 +1,6 @@ """CUDA Graph acceleration for the MOSS Audio Tokenizer codec decoder. -Captures MossAudioTokenizerModel._decode_frame for a set of fixed frame-count +Captures MossAudioTokenizerModel._decode for a set of fixed frame-count bucket sizes, then replays the captured graph at inference time to eliminate kernel-launch overhead. Inputs that exceed all captured sizes fall back to eager execution transparently. @@ -24,7 +24,7 @@ logger = init_logger(__name__) class MossTTSCUDAGraphCodecWrapper: - """CUDA Graph wrapper for MossAudioTokenizerModel._decode_frame. + """CUDA Graph wrapper for MossAudioTokenizerModel._decode. Graphs are keyed by padded_T (int). On each call the actual T is bucket-matched to the smallest pre-captured size >= T. The static code @@ -99,7 +99,7 @@ class MossTTSCUDAGraphCodecWrapper: dummy_codes = torch.zeros(nq, 1, size, dtype=torch.long, device=device) dummy_lengths = torch.tensor([size], dtype=torch.long, device=device) with torch.no_grad(): - _ = self.model._decode_frame(dummy_codes, dummy_lengths) + _ = self.model._decode(dummy_codes, dummy_lengths) torch.accelerator.synchronize(device) @@ -132,13 +132,13 @@ class MossTTSCUDAGraphCodecWrapper: # Extra eager warmup inside capture to ensure all kernels are compiled. with torch.no_grad(): - _ = self.model._decode_frame(static_codes, static_lengths) + _ = self.model._decode(static_codes, static_lengths) torch.accelerator.synchronize(device) graph = CUDAGraph() with torch.no_grad(): with torch.cuda.graph(graph, pool=current_platform.get_global_graph_pool()): - static_out = self.model._decode_frame(static_codes, static_lengths) + static_out = self.model._decode(static_codes, static_lengths) self.graphs[size] = graph self.static_codes[size] = static_codes diff --git a/vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py b/vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py index 827f355a6..2648cacbd 100644 --- a/vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py +++ b/vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py @@ -1342,8 +1342,6 @@ class Qwen2_5OmniThinkerForConditionalGeneration( is_video, is_audio, is_multimodal, - num_video, - num_audio, ) # Default: standard merge (no interleaving), same as parent class diff --git a/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py b/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py index 65ec102a1..f7d0553d4 100644 --- a/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py +++ b/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py @@ -114,6 +114,7 @@ from vllm.multimodal.processing.processor import ( PromptUpdate, PromptUpdateDetails, ) +from vllm.multimodal.utils import set_mm_embedding_modality from vllm.sequence import IntermediateTensors from vllm.transformers_utils.processor import cached_processor_from_config @@ -1420,6 +1421,13 @@ class Qwen3OmniMoeThinkerForConditionalGeneration( visual_dim = embeddings.shape[-1] // (multiscale_len + 1) multi_dim = visual_dim * multiscale_len embeddings_main, embeddings_multiscale = torch.split(embeddings, [visual_dim, multi_dim], dim=-1) + # torch.split returns fresh views, so the `modality` tag the + # encoder gather attached (gpu_model_runner._gather_mm_embeddings) + # does not survive. merge_interleaved_embeddings groups by that + # tag since vLLM #46213, so carry it across the split. + modality = getattr(embeddings, "modality", None) + if modality is not None: + set_mm_embedding_modality(embeddings_main, modality) multimodal_embeddings[index] = embeddings_main multimodal_embeddings_multiscale.append(embeddings_multiscale) if not is_interleaved: @@ -1457,8 +1465,6 @@ class Qwen3OmniMoeThinkerForConditionalGeneration( is_video, is_audio, is_mm_device, - num_video, - num_audio, ) # Default: standard merge (no interleaving), same as parent class. diff --git a/vllm_omni/platforms/cuda/platform.py b/vllm_omni/platforms/cuda/platform.py index 4f203cfc2..5eb7406a5 100644 --- a/vllm_omni/platforms/cuda/platform.py +++ b/vllm_omni/platforms/cuda/platform.py @@ -287,6 +287,16 @@ class CudaOmniPlatform(OmniPlatform, CudaPlatformBase): def get_device_name(cls, device_id: int = 0) -> str: return torch.cuda.get_device_name(device_id) + @classmethod + def get_device_total_memory(cls, device_id: int = 0) -> int: + device_props = torch.cuda.get_device_properties(device_id) + return device_props.total_memory + + @classmethod + def is_fully_connected(cls, device_ids: list[int]) -> bool: + logger.debug("NVLink detection not available on CudaOmniPlatform; assuming no NVLink.") + return False + @classmethod def get_default_ir_op_priority(cls, vllm_config: VllmConfig) -> IrOpPriorityConfig: """Prefer ``vllm_c`` CUDA kernels over ``native`` for diffusion IR ops.""" diff --git a/vllm_omni/worker/gpu_ar_model_runner.py b/vllm_omni/worker/gpu_ar_model_runner.py index ca7c2a21a..7d9580c91 100644 --- a/vllm_omni/worker/gpu_ar_model_runner.py +++ b/vllm_omni/worker/gpu_ar_model_runner.py @@ -22,9 +22,10 @@ from vllm.distributed.kv_transfer import get_kv_transfer_group, has_kv_transfer_ from vllm.distributed.parallel_state import get_pp_group, get_tp_group from vllm.forward_context import set_forward_context from vllm.logger import init_logger +from vllm.sequence import IntermediateTensors from vllm.utils.platform_utils import is_pin_memory_available from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput -from vllm.v1.outputs import AsyncModelRunnerOutput, make_empty_encoder_model_runner_output +from vllm.v1.outputs import EMPTY_MODEL_RUNNER_OUTPUT, AsyncModelRunnerOutput, make_empty_encoder_model_runner_output from vllm.v1.spec_decode.dflash import DFlashProposer from vllm.v1.spec_decode.draft_model import DraftModelProposer from vllm.v1.spec_decode.eagle import EagleProposer @@ -33,9 +34,7 @@ from vllm.v1.spec_decode.gemma4 import Gemma4Proposer from vllm.v1.structured_output.utils import apply_grammar_bitmask from vllm.v1.utils import record_function_or_nullcontext from vllm.v1.worker.gpu_model_runner import ( - EMPTY_MODEL_RUNNER_OUTPUT, AsyncGPUModelRunnerOutput, - IntermediateTensors, ) from vllm.v1.worker.ubatch_utils import maybe_create_ubatch_slices from vllm.v1.worker.utils import is_residual_scattered_for_sp diff --git a/vllm_omni/worker/gpu_generation_model_runner.py b/vllm_omni/worker/gpu_generation_model_runner.py index 2c81643dd..3c00fad38 100644 --- a/vllm_omni/worker/gpu_generation_model_runner.py +++ b/vllm_omni/worker/gpu_generation_model_runner.py @@ -18,9 +18,10 @@ from vllm.distributed.ec_transfer import get_ec_transfer, has_ec_transfer from vllm.distributed.kv_transfer import get_kv_transfer_group, has_kv_transfer_group from vllm.distributed.parallel_state import get_pp_group from vllm.forward_context import set_forward_context +from vllm.sequence import IntermediateTensors from vllm.utils.math_utils import cdiv from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput -from vllm.v1.outputs import AsyncModelRunnerOutput, make_empty_encoder_model_runner_output +from vllm.v1.outputs import EMPTY_MODEL_RUNNER_OUTPUT, AsyncModelRunnerOutput, make_empty_encoder_model_runner_output from vllm.v1.spec_decode.dflash import DFlashProposer from vllm.v1.spec_decode.draft_model import DraftModelProposer from vllm.v1.spec_decode.eagle import EagleProposer @@ -28,9 +29,7 @@ from vllm.v1.spec_decode.extract_hidden_states import ExtractHiddenStatesPropose from vllm.v1.spec_decode.gemma4 import Gemma4Proposer from vllm.v1.utils import record_function_or_nullcontext from vllm.v1.worker.gpu_model_runner import ( - EMPTY_MODEL_RUNNER_OUTPUT, AsyncGPUModelRunnerOutput, - IntermediateTensors, PerLayerAttnMetadata, ) from vllm.v1.worker.ubatch_utils import maybe_create_ubatch_slices diff --git a/vllm_omni/worker/gpu_model_runner.py b/vllm_omni/worker/gpu_model_runner.py index cb52a2b67..9fb781dc4 100644 --- a/vllm_omni/worker/gpu_model_runner.py +++ b/vllm_omni/worker/gpu_model_runner.py @@ -13,6 +13,7 @@ from vllm.logger import init_logger from vllm.model_executor.models.interfaces import supports_mrope from vllm.model_executor.models.interfaces_base import VllmModelForPooling from vllm.sampling_params import SamplingType +from vllm.sequence import IntermediateTensors from vllm.tracing import instrument from vllm.utils.import_utils import LazyLoader from vllm.utils.math_utils import cdiv @@ -26,7 +27,7 @@ from vllm.v1.spec_decode.ngram_proposer_gpu import ( update_scheduler_for_invalid_drafts, ) from vllm.v1.worker.gpu_input_batch import CachedRequestState -from vllm.v1.worker.gpu_model_runner import GPUModelRunner, IntermediateTensors, PerLayerAttnMetadata +from vllm.v1.worker.gpu_model_runner import GPUModelRunner, PerLayerAttnMetadata from vllm.v1.worker.ubatch_utils import maybe_create_ubatch_slices from vllm_omni.core.prefix_cache import OmniTensorPrefixCache diff --git a/vllm_omni/worker/omni_connector_model_runner_mixin.py b/vllm_omni/worker/omni_connector_model_runner_mixin.py index b31b04688..363b850d4 100644 --- a/vllm_omni/worker/omni_connector_model_runner_mixin.py +++ b/vllm_omni/worker/omni_connector_model_runner_mixin.py @@ -1575,7 +1575,7 @@ class OmniConnectorModelRunnerMixin: from copy import copy - from vllm.v1.worker.gpu_model_runner import EMPTY_MODEL_RUNNER_OUTPUT + from vllm.v1.outputs import EMPTY_MODEL_RUNNER_OUTPUT wrapped = copy(result if result is not None else EMPTY_MODEL_RUNNER_OUTPUT) wrapped.omni_connector_output = omni_output From 8f115164c236e140755b82f26edc7d8567de7757 Mon Sep 17 00:00:00 2001 From: Yueqian Lin <70319226+linyueqian@users.noreply.github.com> Date: Mon, 27 Jul 2026 09:04:08 -0700 Subject: [PATCH 119/240] [Bugfix][Connectors] Flush processor tail on terminal async-chunk sends (#5414) Signed-off-by: Yueqian Lin <70319226+linyueqian@users.noreply.github.com> --- .../test_chunk_transfer_adapter.py | 31 +++++++ .../minicpmo_4_5/test_code2wav_model_dir.py | 88 +++++++++++++++++++ .../chunk_transfer_adapter.py | 8 +- .../minicpmo_4_5/minicpmo_4_5_code2wav.py | 32 +++++-- 4 files changed, 152 insertions(+), 7 deletions(-) create mode 100644 tests/model_executor/models/minicpmo_4_5/test_code2wav_model_dir.py diff --git a/tests/distributed/omni_connectors/test_chunk_transfer_adapter.py b/tests/distributed/omni_connectors/test_chunk_transfer_adapter.py index 1a045f306..3ab2528fb 100644 --- a/tests/distributed/omni_connectors/test_chunk_transfer_adapter.py +++ b/tests/distributed/omni_connectors/test_chunk_transfer_adapter.py @@ -252,6 +252,37 @@ def test_save_async_uses_confirmed_tokens_for_async_scheduler_watermark(build_ad assert len(adapter._pending_save_reqs) == 1 +def test_send_single_request_terminal_chunk_still_flushes_processor(build_adapter, monkeypatch): + """A terminal stop is not a segment boundary (#5383), but the producer-side + processor must still receive the flush signal on the terminal chunk. + Passing only ``is_segment_finished`` starved processors of their final + accumulated payload once terminal stops stopped setting it (#5413: the + downstream stage got ``meta.finished`` with the tail data missing). + """ + adapter, connector = build_adapter(stage_id=0) + request = _req("req-terminal", RequestStatus.FINISHED_STOPPED, external_req_id="ext-terminal") + + seen_flush_flags = [] + + def recording_processor(**kwargs): + seen_flush_flags.append(kwargs["is_finished"]) + return OmniPayloadStruct( + codes=CodesStruct(audio=torch.tensor([1, 2, 3], dtype=torch.long)), + ) + + adapter.custom_process_next_stage_input_func = recording_processor + monkeypatch.setattr(adapter, "cleanup", lambda *a, **kw: None) + + adapter._send_single_request( + {"multimodal_output": None, "request": request, "is_finished": True, "is_segment_finished": False} + ) + + assert seen_flush_flags == [True] + sent_payload = connector.put.call_args.kwargs["data"] + assert bool(sent_payload.meta.finished.item()) is True + assert bool(sent_payload.meta.is_segment_finished.item()) is False + + def test_send_single_request_struct_without_meta_does_not_crash(build_adapter, monkeypatch): """Producer may return a struct with ``meta=None`` (e.g. payload that carries only ``embed`` or ``codes``). The sender's ``meta is not None`` diff --git a/tests/model_executor/models/minicpmo_4_5/test_code2wav_model_dir.py b/tests/model_executor/models/minicpmo_4_5/test_code2wav_model_dir.py new file mode 100644 index 000000000..5aa78262a --- /dev/null +++ b/tests/model_executor/models/minicpmo_4_5/test_code2wav_model_dir.py @@ -0,0 +1,88 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Unit tests for MiniCPM-o 4.5 code2wav model-dir resolution (#5442). + +In hub/CI deployments ``model_config.model`` is a repo id rather than a local +directory, so asset lookups must resolve to the downloaded snapshot. The +resolution must stay lazy: constructing the model with a fake path (as the +CPU unit tests do) must not touch the hub. +""" + +from __future__ import annotations + +from types import SimpleNamespace + +import huggingface_hub +import pytest + +from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_code2wav import ( + MiniCPMO45Code2Wav, + _resolve_model_dir, +) + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +def _no_hub(monkeypatch): + def _fail(*args, **kwargs): # pragma: no cover - must not be reached + raise AssertionError("snapshot_download must not be called here") + + monkeypatch.setattr(huggingface_hub, "snapshot_download", _fail) + + +def test_local_directory_is_returned_unchanged(tmp_path, monkeypatch): + _no_hub(monkeypatch) + assert _resolve_model_dir(str(tmp_path)) == str(tmp_path) + + +def test_repo_id_resolves_via_snapshot_download(tmp_path, monkeypatch): + calls = {} + + def _fake_snapshot_download(model_ref, revision=None, allow_patterns=None): + calls["model_ref"] = model_ref + calls["revision"] = revision + calls["allow_patterns"] = allow_patterns + return str(tmp_path / "snapshot") + + monkeypatch.setattr(huggingface_hub, "snapshot_download", _fake_snapshot_download) + resolved = _resolve_model_dir("openbmb/MiniCPM-o-4_5", revision="abc123") + assert resolved == str(tmp_path / "snapshot") + assert calls["model_ref"] == "openbmb/MiniCPM-o-4_5" + assert calls["revision"] == "abc123" + assert calls["allow_patterns"] == ["assets/*"] + + +def test_snapshot_download_failure_propagates(monkeypatch): + def _raise(*args, **kwargs): + raise FileNotFoundError("offline and not cached") + + monkeypatch.setattr(huggingface_hub, "snapshot_download", _raise) + with pytest.raises(FileNotFoundError): + _resolve_model_dir("openbmb/MiniCPM-o-4_5") + + +def test_init_with_fake_path_does_not_resolve(monkeypatch): + """Mirrors the CPU-test construction: fake model path, no ``revision``.""" + _no_hub(monkeypatch) + config = SimpleNamespace( + model_config=SimpleNamespace( + model="/fake/model", + stage_connector_config=None, + ) + ) + model = MiniCPMO45Code2Wav(vllm_config=config) + assert model.model_path == "/fake/model" + assert model._default_prompt_wav == "/fake/model/assets/HT_ref_audio.wav" + + +def test_default_prompt_wav_follows_resolved_model_path(monkeypatch): + _no_hub(monkeypatch) + config = SimpleNamespace( + model_config=SimpleNamespace( + model="openbmb/MiniCPM-o-4_5", + stage_connector_config=None, + ) + ) + model = MiniCPMO45Code2Wav(vllm_config=config) + model.model_path = "/resolved/snapshot" + assert model._default_prompt_wav == "/resolved/snapshot/assets/HT_ref_audio.wav" diff --git a/vllm_omni/distributed/omni_connectors/transfer_adapter/chunk_transfer_adapter.py b/vllm_omni/distributed/omni_connectors/transfer_adapter/chunk_transfer_adapter.py index 1a41372a8..83def2dc0 100644 --- a/vllm_omni/distributed/omni_connectors/transfer_adapter/chunk_transfer_adapter.py +++ b/vllm_omni/distributed/omni_connectors/transfer_adapter/chunk_transfer_adapter.py @@ -317,7 +317,13 @@ class OmniChunkTransferAdapter(OmniTransferAdapterBase): multimodal_output=multimodal_output, request=request, # Existing processors use is_finished as a flush signal. - is_finished=is_segment_finished, + # Terminal stops no longer count as segment boundaries + # (is_segment_finished is False when the request finishes, + # see #5383), but the processor must still flush its + # accumulated tail on the terminal chunk — otherwise the + # downstream stage receives the finished marker without + # the final payload (#5413). + is_finished=is_segment_finished or is_finished, ) except Exception as e: diff --git a/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_code2wav.py b/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_code2wav.py index a9ffa6e0a..3eac2e49d 100644 --- a/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_code2wav.py +++ b/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_code2wav.py @@ -30,6 +30,20 @@ from .batched_token2wav import ( logger = init_logger(__name__) +def _resolve_model_dir(model_ref: str, revision: str | None = None) -> str: + """Resolve ``model_ref`` to a local directory containing the repo assets. + + ``model_config.model`` is a filesystem path in local deployments but a + Hugging Face repo id in hub/CI deployments; the prompt-audio and + token2wav asset lookups need a real directory either way. + """ + if Path(model_ref).is_dir(): + return model_ref + from huggingface_hub import snapshot_download + + return snapshot_download(model_ref, revision=revision, allow_patterns=["assets/*"]) + + def _batch_error(reason: str, **details: Any) -> RuntimeError: payload = {"reason": reason, **details} return RuntimeError(f"MiniCPMO45Code2WavBatchError {json.dumps(payload, sort_keys=True)}") @@ -118,6 +132,7 @@ class MiniCPMO45Code2Wav(nn.Module): del prefix self.vllm_config = vllm_config self.model_path = str(vllm_config.model_config.model) + self._model_revision = getattr(vllm_config.model_config, "revision", None) self.backend: BatchedToken2Wav | None = None self._states: dict[str, _RequestState] = {} self._owned_prompt_wavs: dict[str, tuple[str, str]] = {} @@ -126,12 +141,13 @@ class MiniCPMO45Code2Wav(nn.Module): if self._min_batch_size < 1: raise ValueError("MiniCPM-o Code2Wav code2wav_min_batch_size must be >= 1") self._default_prompt_id = str(extra.get("prompt_cache_id", "HT_ref_audio")) - self._default_prompt_wav = str( - extra.get( - "prompt_wav", - Path(self.model_path) / "assets" / "HT_ref_audio.wav", - ) - ) + self._prompt_wav_override = extra.get("prompt_wav") + + @property + def _default_prompt_wav(self) -> str: + if self._prompt_wav_override is not None: + return str(self._prompt_wav_override) + return str(Path(self.model_path) / "assets" / "HT_ref_audio.wav") def _extra_config(self) -> dict[str, Any]: model_config = getattr(self.vllm_config, "model_config", None) @@ -600,6 +616,10 @@ class MiniCPMO45Code2Wav(nn.Module): from stepaudio2.token2wav import Token2wav extra = self._extra_config() + # Hub repo ids only need to become local directories once the vocoder + # assets are actually read; unit tests construct this model with fake + # paths and must not trigger a hub download (#5442). + self.model_path = _resolve_model_dir(self.model_path, self._model_revision) prompt_path = Path(self._default_prompt_wav) if not prompt_path.is_file(): raise FileNotFoundError(f"MiniCPM-o Code2Wav prompt audio not found: {prompt_path}") From d4a5eaac9081cb8345aa535aba4eb4218d8730dc Mon Sep 17 00:00:00 2001 From: Yueqian Lin <70319226+linyueqian@users.noreply.github.com> Date: Mon, 27 Jul 2026 10:04:17 -0700 Subject: [PATCH 120/240] [Bugfix][Realtime] Stop disconnected sessions from cycling through stages (#5388) Signed-off-by: Yueqian Lin <70319226+linyueqian@users.noreply.github.com> Co-authored-by: Sy03 <1370724210@qq.com> --- tests/engine/test_orchestrator.py | 50 +++++++++++++++++++ vllm_omni/engine/orchestrator.py | 22 +++----- vllm_omni/entrypoints/async_omni.py | 12 ++++- .../entrypoints/openai/realtime_connection.py | 19 ++++++- 4 files changed, 86 insertions(+), 17 deletions(-) diff --git a/tests/engine/test_orchestrator.py b/tests/engine/test_orchestrator.py index 4217e2156..5c7066e0c 100644 --- a/tests/engine/test_orchestrator.py +++ b/tests/engine/test_orchestrator.py @@ -1813,6 +1813,56 @@ async def test_stage_pool_submit_update_refreshes_output_processor_state() -> No @pytest.mark.asyncio +async def test_handle_streaming_update_unknown_request_is_dropped() -> None: + """A streaming update for an untracked request must not resurrect it. + + Updates always follow the first-chunk add_request through the same + ordered submission queue, so an unknown id can only mean the request + already finished or was aborted (e.g. the client disconnected). + Falling back to add_request created headless sessions that kept + cycling through the stages (issue #4271). + """ + + class RecordingPool: + def __init__(self) -> None: + self.calls: list[tuple[str, Any]] = [] + + async def submit_update(self, request_id, req_state, request, *, prompt_text=None) -> int: + self.calls.append((request_id, prompt_text)) + return 0 + + pool = RecordingPool() + orchestrator = object.__new__(Orchestrator) + orchestrator.async_chunk = False + orchestrator.request_states = {} + orchestrator.stage_pools = [pool] + add_request_calls: list[str] = [] + + async def record_add_request(msg) -> None: + add_request_calls.append(msg.request_id) + + orchestrator._handle_add_request = record_add_request + + await orchestrator._handle_streaming_update( + StageSubmissionMessage( + type="streaming_update", + request_id="req-unknown", + prompt=SimpleNamespace(request_id="req-unknown", prompt_token_ids=[1], resumable=True), + original_prompt={"prompt": "segment-2"}, + output_prompt_text="segment-2", + sampling_params_list=[_sampling_params()], + final_stage_id=0, + preprocess_ms=0.0, + request_timestamp=time.time(), + enqueue_ts=time.perf_counter(), + ) + ) + + assert pool.calls == [] + assert add_request_calls == [] + assert "req-unknown" not in orchestrator.request_states + + async def test_handle_streaming_update_passes_prompt_text_to_stage_pool() -> None: class RecordingPool: def __init__(self) -> None: diff --git a/vllm_omni/engine/orchestrator.py b/vllm_omni/engine/orchestrator.py index affd0e452..fa10653be 100644 --- a/vllm_omni/engine/orchestrator.py +++ b/vllm_omni/engine/orchestrator.py @@ -681,24 +681,16 @@ class Orchestrator: final_stage_id = msg.final_stage_id req_state = self.request_states.get(request_id) if req_state is None: + # Streaming updates always follow the first-chunk add_request + # through the same ordered submission queue, so an unknown id + # here can only mean the request already finished or was + # aborted (e.g. the client disconnected). Re-adding it would + # resurrect a headless session that keeps cycling through the + # stages with nobody consuming its outputs (issue #4271). logger.warning( - "[Orchestrator] streaming_update for unknown req=%s, falling back to add_request", + "[Orchestrator] streaming_update for unknown req=%s; dropping (request finished or aborted)", request_id, ) - fallback_msg = StageSubmissionMessage( - type="add_request", - request_id=msg.request_id, - prompt=msg.prompt, - original_prompt=msg.original_prompt, - output_prompt_text=msg.output_prompt_text, - sampling_params_list=msg.sampling_params_list, - final_stage_id=msg.final_stage_id, - final_output_stage_ids=msg.final_output_stage_ids, - preprocess_ms=msg.preprocess_ms, - request_timestamp=msg.request_timestamp, - enqueue_ts=msg.enqueue_ts, - ) - await self._handle_add_request(fallback_msg) return if msg.sampling_params_list: diff --git a/vllm_omni/entrypoints/async_omni.py b/vllm_omni/entrypoints/async_omni.py index 332e5b921..1437cc680 100644 --- a/vllm_omni/entrypoints/async_omni.py +++ b/vllm_omni/entrypoints/async_omni.py @@ -628,6 +628,13 @@ class AsyncOmni(EngineClient, OmniBase): logger.debug(f"[AsyncOmni] Request {request_id} completed") + # The input pump can outlive a normally-completed generation + # (e.g. a realtime client that keeps streaming after the final + # output); a live pump keeps issuing streaming updates for a + # request the orchestrator no longer tracks. + if input_stream_task is not None and not input_stream_task.done(): + input_stream_task.cancel() + self._log_summary_and_cleanup(request_id) except (asyncio.CancelledError, GeneratorExit): @@ -1059,7 +1066,10 @@ class AsyncOmni(EngineClient, OmniBase): """Submit request IDs to be aborted to the engine.""" await self.engine.abort_async(request_ids) for rid in request_ids: - self.request_states.pop(rid, None) + state = self.request_states.pop(rid, None) + input_stream_task = getattr(state, "input_stream_task", None) + if input_stream_task is not None and not input_stream_task.done(): + input_stream_task.cancel() if self.log_stats: logger.info("[AsyncOmni] Aborted request(s) %s", ",".join(request_ids)) diff --git a/vllm_omni/entrypoints/openai/realtime_connection.py b/vllm_omni/entrypoints/openai/realtime_connection.py index 0f0efbef8..aad7ea658 100644 --- a/vllm_omni/entrypoints/openai/realtime_connection.py +++ b/vllm_omni/entrypoints/openai/realtime_connection.py @@ -138,6 +138,7 @@ class RealtimeConnection(VllmRealtimeConnection): is_streaming=True, ) + result_gen = None try: result_gen = self.engine.generate( prompt=streaming_input_gen, @@ -196,6 +197,16 @@ class RealtimeConnection(VllmRealtimeConnection): logger.exception("Error in generation: %s", e) await self.send_error(str(e), "processing_error") finally: + # Close the generator explicitly so AsyncOmni.generate's cleanup + # (input-pump cancellation and engine-side abort) runs now rather + # than whenever the event loop garbage-collects the async + # generator; the delay window is where a disconnected session + # keeps cycling through the stages (issue #4271). + if result_gen is not None: + try: + await result_gen.aclose() + except Exception: + logger.exception("Failed to close realtime result generator") # Always send terminal event so clients don't hang forever. if self._is_connected and not audio_done_sent: try: @@ -206,4 +217,10 @@ class RealtimeConnection(VllmRealtimeConnection): self.audio_queue.get_nowait() async def send_json(self, payload: dict): - await self.websocket.send_text(json.dumps(payload)) + try: + await self.websocket.send_text(json.dumps(payload)) + except Exception: + # A failed send means the client is gone; flag it so the + # generation loop stops instead of retrying into a dead socket. + self._is_connected = False + raise From 807db6efd70ff2e9b55a63d6e1b0530e2b74f8f2 Mon Sep 17 00:00:00 2001 From: Joshna-Medisetty Date: Mon, 27 Jul 2026 16:35:35 -0700 Subject: [PATCH 121/240] [Refactor] Migrate Voxtral TTS XPU stage config to --deploy-config (#5467) Signed-off-by: Joshna Medisetty --- .../test_voxtral_tts_expansion.py | 2 +- .../test_voxtral_tts_expansion.py | 4 +- vllm_omni/deploy/voxtral_tts.yaml | 11 ++ .../xpu/stage_configs/voxtral_tts.yaml | 106 ------------------ 4 files changed, 14 insertions(+), 109 deletions(-) delete mode 100644 vllm_omni/platforms/xpu/stage_configs/voxtral_tts.yaml diff --git a/tests/e2e/offline_inference/test_voxtral_tts_expansion.py b/tests/e2e/offline_inference/test_voxtral_tts_expansion.py index 56529dbbe..2441c46dc 100644 --- a/tests/e2e/offline_inference/test_voxtral_tts_expansion.py +++ b/tests/e2e/offline_inference/test_voxtral_tts_expansion.py @@ -58,7 +58,7 @@ def _compose_request(model_name: str, text: str, voice: str) -> dict: @pytest.mark.parametrize("omni_runner_function", [_OMNI_RUNNER_PARAM], indirect=True) -@hardware_test(res={"cuda": "L4"}, num_cards=1) +@hardware_test(res={"cuda": "L4", "xpu": "B60"}, num_cards=1) def test_voxtral_tts_offline_basic(omni_runner_function: OmniRunner) -> None: """Offline sync path; function-scoped runner so AsyncOmni test does not overlap a live Omni.""" omni = omni_runner_function.omni diff --git a/tests/e2e/online_serving/test_voxtral_tts_expansion.py b/tests/e2e/online_serving/test_voxtral_tts_expansion.py index 1448567d3..264778ab9 100644 --- a/tests/e2e/online_serving/test_voxtral_tts_expansion.py +++ b/tests/e2e/online_serving/test_voxtral_tts_expansion.py @@ -30,7 +30,7 @@ pytestmark = [ ] -@hardware_test(res={"cuda": "L4"}, num_cards=1) +@hardware_test(res={"cuda": "L4", "xpu": "B60"}, num_cards=1) def test_speech_english_basic(omni_server, openai_client) -> None: """Test basic English TTS generation.""" openai_client.send_audio_speech_request( @@ -46,7 +46,7 @@ def test_speech_english_basic(omni_server, openai_client) -> None: ) -@hardware_test(res={"cuda": "L4"}, num_cards=1) +@hardware_test(res={"cuda": "L4", "xpu": "B60"}, num_cards=1) def test_speech_english_streaming(omni_server, openai_client) -> None: """Test basic streaming English TTS generation (PCM via streaming API).""" openai_client.send_audio_speech_request( diff --git a/vllm_omni/deploy/voxtral_tts.yaml b/vllm_omni/deploy/voxtral_tts.yaml index d024c3900..5524c5cc8 100644 --- a/vllm_omni/deploy/voxtral_tts.yaml +++ b/vllm_omni/deploy/voxtral_tts.yaml @@ -69,3 +69,14 @@ stages: config_format: mistral load_format: mistral skip_mm_profiling: true + +platforms: + xpu: + stages: + - stage_id: 0 + enforce_eager: true + gpu_memory_utilization: 0.7 + distributed_executor_backend: "mp" + - stage_id: 1 + gpu_memory_utilization: 0.28 + distributed_executor_backend: "mp" diff --git a/vllm_omni/platforms/xpu/stage_configs/voxtral_tts.yaml b/vllm_omni/platforms/xpu/stage_configs/voxtral_tts.yaml deleted file mode 100644 index cf7937e52..000000000 --- a/vllm_omni/platforms/xpu/stage_configs/voxtral_tts.yaml +++ /dev/null @@ -1,106 +0,0 @@ -# Voxtral TTS — Intel XPU (AR → audio tokenizer). Matches CUDA stage config knobs where noted. - -async_chunk: true -stage_args: - - stage_id: 0 - stage_type: llm - runtime: - process: true - devices: "0" - max_batch_size: 1 - engine_args: - max_num_seqs: 32 - model_stage: audio_generation - model_arch: VoxtralTTSForConditionalGeneration - worker_type: ar - worker_cls: vllm_omni.platforms.xpu.worker.xpu_ar_worker.XPUARWorker - scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler - gpu_memory_utilization: 0.7 - enforce_eager: true - trust_remote_code: true - async_scheduling: true - engine_output_type: latent - enable_prefix_caching: false - tokenizer_mode: mistral - config_format: mistral - load_format: mistral - skip_mm_profiling: true - enable_chunked_prefill: false - max_model_len: 4096 - distributed_executor_backend: "mp" - custom_process_next_stage_input_func: vllm_omni.model_executor.stage_input_processors.voxtral_tts.generator2tokenizer_async_chunk - output_connectors: - to_stage_1: connector_of_shared_memory - is_comprehension: true - final_output: false - final_output_type: text - default_sampling_params: - temperature: 0.0 - top_p: 1.0 - top_k: -1 - max_tokens: 2048 - seed: 42 - detokenize: True - repetition_penalty: 1.1 - - - stage_id: 1 - stage_type: llm - runtime: - process: true - devices: "0" - max_batch_size: 1 - engine_args: - model_stage: audio_tokenizer - model_arch: VoxtralTTSForConditionalGeneration - worker_type: generation - worker_cls: vllm_omni.platforms.xpu.worker.xpu_generation_worker.XPUGenerationWorker - scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler - async_scheduling: false - max_num_seqs: 1 - gpu_memory_utilization: 0.28 - enforce_eager: true - trust_remote_code: true - enable_prefix_caching: false - skip_mm_profiling: true - engine_output_type: audio - tokenizer_mode: mistral - config_format: mistral - load_format: mistral - max_num_batched_tokens: 16384 - max_model_len: 16384 - distributed_executor_backend: "mp" - engine_input_source: [0] - is_comprehension: false - final_output: true - final_output_type: audio - input_connectors: - from_stage_0: connector_of_shared_memory - tts_args: - max_instructions_length: 500 - default_sampling_params: - temperature: 0.9 - top_p: 0.8 - top_k: 40 - max_tokens: 2048 - seed: 42 - detokenize: True - repetition_penalty: 1.05 - -runtime: - enabled: true - - connectors: - connector_of_shared_memory: - name: SharedMemoryConnector - extra: - codec_streaming: true - connector_get_sleep_s: 0.01 - connector_get_max_wait_first_chunk: 3000 - connector_get_max_wait: 300 - codec_chunk_frames: 25 - codec_chunk_frames_at_begin: 5 - codec_left_context_frames: 25 - - edges: - - from: 0 - to: 1 From d3e0171cd77eb530791a5b11f4ed71afbc1c6132 Mon Sep 17 00:00:00 2001 From: Wei <48665416+hbhflw2000@users.noreply.github.com> Date: Tue, 28 Jul 2026 10:34:19 +0800 Subject: [PATCH 122/240] [bugfix] validate AR sampled-token logprob alignment (#5273) Signed-off-by: hbhflw2000 <417911774@qq.com> Signed-off-by: Samit <285365963@qq.com> Co-authored-by: Samit <285365963@qq.com> --- .../sched/test_omni_ar_scheduler_logprobs.py | 181 ++++++++++++++++++ vllm_omni/core/sched/omni_ar_scheduler.py | 75 +++++++- 2 files changed, 249 insertions(+), 7 deletions(-) create mode 100644 tests/core/sched/test_omni_ar_scheduler_logprobs.py diff --git a/tests/core/sched/test_omni_ar_scheduler_logprobs.py b/tests/core/sched/test_omni_ar_scheduler_logprobs.py new file mode 100644 index 000000000..155bd894f --- /dev/null +++ b/tests/core/sched/test_omni_ar_scheduler_logprobs.py @@ -0,0 +1,181 @@ +"""Tests for the AR sampled-token logprob contract.""" + +from __future__ import annotations + +from collections import defaultdict +from types import SimpleNamespace + +import numpy as np +import pytest +from vllm.v1.engine import FinishReason +from vllm.v1.outputs import LogprobsLists +from vllm.v1.request import RequestStatus + +from vllm_omni.core.sched.omni_ar_scheduler import OmniARScheduler, _slice_sampled_logprobs + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +def _logprobs( + token_ids: list[list[int]], + values: list[list[float]], + *, + cumulative_rows: list[int] | None = None, +) -> LogprobsLists: + return LogprobsLists( + logprob_token_ids=np.asarray(token_ids, dtype=np.int32), + logprobs=np.asarray(values, dtype=np.float32), + sampled_token_ranks=np.zeros(len(token_ids), dtype=np.int32), + cu_num_generated_tokens=cumulative_rows, + ) + + +def test_slices_requested_rows_and_preserves_sampled_token_values() -> None: + all_logprobs = _logprobs( + [[3, 8], [4, 9], [11, 2], [12, 1]], + [[-0.3, -1.0], [-0.4, -1.1], [-0.11, -2.0], [-0.12, -2.1]], + cumulative_rows=[0, 2], + ) + + sliced = _slice_sampled_logprobs(all_logprobs, req_index=1, sampled_token_ids=[11, 12]) + + np.testing.assert_array_equal(sliced.logprob_token_ids[:, 0], [11, 12]) + np.testing.assert_allclose(sliced.logprobs[:, 0], [-0.11, -0.12]) + + +def test_rejects_missing_logprobs() -> None: + with pytest.raises(RuntimeError, match="model runner returned none"): + _slice_sampled_logprobs(None, req_index=0, sampled_token_ids=[7]) + + +def test_rejects_row_count_mismatch() -> None: + logprobs = _logprobs([[7, 1]], [[-0.7, -1.0]]) + + with pytest.raises(RuntimeError, match="row count does not match"): + _slice_sampled_logprobs(logprobs, req_index=0, sampled_token_ids=[7, 8]) + + +def test_rejects_sampled_token_mismatch() -> None: + logprobs = _logprobs([[9, 7]], [[-0.9, -1.0]]) + + with pytest.raises(RuntimeError, match="generated_token=7 logprob_token=9"): + _slice_sampled_logprobs(logprobs, req_index=0, sampled_token_ids=[7]) + + +def test_rejects_non_finite_sampled_logprob() -> None: + logprobs = _logprobs([[7, 1]], [[float("nan"), -1.0]]) + + with pytest.raises(RuntimeError, match="non-finite values at rows \\[0\\]"): + _slice_sampled_logprobs(logprobs, req_index=0, sampled_token_ids=[7]) + + +class _Request: + def __init__(self, request_id: str) -> None: + self.request_id = request_id + self.client_index = 0 + self.status = RequestStatus.RUNNING + self.sampling_params = SimpleNamespace(num_logprobs=1) + self.num_computed_tokens = 0 + self.num_output_placeholders = 0 + self.has_encoder_inputs = False + self.pooling_params = None + self.resumable = True + self.stop_reason = None + self.trace_headers = None + self.num_nans_in_logits = 0 + + def is_finished(self) -> bool: + return RequestStatus.is_finished(self.status) + + def get_finished_reason(self): + return RequestStatus.get_finished_reason(self.status) + + def take_events(self): + return None + + def take_prefill_stats(self): + return None + + +class _RequestQueue(list): + def remove_requests(self, requests) -> None: + for request in requests: + if request in self: + self.remove(request) + + +def test_invalid_logprobs_finish_only_the_affected_scheduler_request() -> None: + """A bad runner response must not bring down unrelated batch requests.""" + bad = _Request("bad") + good = _Request("good") + scheduler = SimpleNamespace( + perf_metrics=None, + connector=None, + chunk_transfer_adapter=None, + requests={bad.request_id: bad, good.request_id: good}, + running=[bad, good], + waiting=_RequestQueue(), + skipped_waiting=_RequestQueue(), + structured_output_manager=SimpleNamespace(should_advance=lambda _request: False), + transfer_triggered_requests=set(), + active_kv_transfers=set(), + pending_stop_after_extraction=set(), + waiting_for_transfer_free=set(), + _new_prompt_len_snapshot={}, + finished_req_ids=set(), + finished_req_ids_dict=defaultdict(set), + kv_cache_manager=SimpleNamespace(take_events=lambda: None), + kv_event_publisher=SimpleNamespace(publish=lambda _events: None), + ) + update_calls: list[str] = [] + + def update_request(request, token_ids): + update_calls.append(request.request_id) + return token_ids, False + + def free_request(request): + scheduler.requests.pop(request.request_id, None) + scheduler.finished_req_ids.add(request.request_id) + scheduler.finished_req_ids_dict[request.client_index].add(request.request_id) + return None + + scheduler._update_request_with_output = update_request + scheduler._process_kv_transfer_trigger = lambda _request, _tokens: False + scheduler._handle_stopped_request = lambda _request: True + scheduler._free_request = free_request + scheduler.make_spec_decoding_stats = lambda *args, **kwargs: None + scheduler.make_stats = lambda *args, **kwargs: None + scheduler._capture_omni_connector_output = lambda _output: None + + scheduler_output = SimpleNamespace( + num_scheduled_tokens={"bad": 1, "good": 1}, + scheduled_spec_decode_tokens={}, + num_invalid_spec_tokens=0, + ) + model_runner_output = SimpleNamespace( + sampled_token_ids=[[7], [8]], + logprobs=_logprobs( + [[9, 1], [8, 2]], + [[-0.9, -1.0], [-0.8, -1.0]], + cumulative_rows=[0, 1], + ), + prompt_logprobs_dict={}, + pooler_output=None, + num_nans_in_logits=None, + kv_connector_output=None, + cudagraph_stats=None, + req_id_to_index={"bad": 0, "good": 1}, + routed_experts=None, + ) + + outputs = OmniARScheduler.update_from_output(scheduler, scheduler_output, model_runner_output) + output_by_id = {output.request_id: output for output in outputs[0].outputs} + + assert update_calls == ["good"] + assert bad.status is RequestStatus.FINISHED_ERROR + assert bad.stop_reason is not None and "misaligned" in bad.stop_reason + assert "bad" not in scheduler.requests + assert output_by_id["bad"].finish_reason is FinishReason.ERROR + assert output_by_id["bad"].new_token_ids == [] + assert output_by_id["good"].new_token_ids == [8] + np.testing.assert_array_equal(output_by_id["good"].new_logprobs.logprob_token_ids[:, 0], [8]) diff --git a/vllm_omni/core/sched/omni_ar_scheduler.py b/vllm_omni/core/sched/omni_ar_scheduler.py index 22917197d..d87fc0c8a 100644 --- a/vllm_omni/core/sched/omni_ar_scheduler.py +++ b/vllm_omni/core/sched/omni_ar_scheduler.py @@ -5,6 +5,7 @@ from collections.abc import Iterable from time import time from typing import Any +import numpy as np from vllm.compilation.cuda_graph import CUDAGraphStat from vllm.distributed.kv_events import KVEventBatch from vllm.distributed.kv_transfer.kv_connector.v1.metrics import KVConnectorStats @@ -36,6 +37,51 @@ from vllm_omni.outputs import OmniConnectorOutput logger = init_logger(__name__) +class SampledLogprobContractError(RuntimeError): + """The model runner returned unusable sampled-token logprobs.""" + + +def _slice_sampled_logprobs(logprobs: Any, req_index: int, sampled_token_ids: list[int]) -> Any: + """Slice and validate the sampled-token logprobs for one AR request.""" + if logprobs is None: + raise SampledLogprobContractError("AR logprobs were requested, but the model runner returned none") + + sliced = logprobs.slice_request(req_index, len(sampled_token_ids)) + token_rows = np.asarray(sliced.logprob_token_ids) + value_rows = np.asarray(sliced.logprobs) + expected_rows = len(sampled_token_ids) + + if token_rows.ndim != 2 or value_rows.ndim != 2: + raise SampledLogprobContractError( + "AR sampled-token logprobs must be rank-2 arrays, " + f"got token_ids={token_rows.shape} logprobs={value_rows.shape}" + ) + if token_rows.shape[0] != expected_rows or value_rows.shape[0] != expected_rows: + raise SampledLogprobContractError( + "AR sampled-token logprob row count does not match generated tokens: " + f"tokens={expected_rows} token_id_rows={token_rows.shape[0]} " + f"logprob_rows={value_rows.shape[0]}" + ) + if expected_rows == 0: + return sliced + if token_rows.shape[1] == 0 or value_rows.shape[1] == 0: + raise SampledLogprobContractError("AR sampled-token logprob rows are empty") + + sampled = np.asarray(sampled_token_ids) + if not np.array_equal(token_rows[:, 0], sampled): + mismatch = np.flatnonzero(token_rows[:, 0] != sampled) + first = int(mismatch[0]) + raise SampledLogprobContractError( + "AR sampled-token logprobs are misaligned: " + f"row={first} generated_token={int(sampled[first])} " + f"logprob_token={int(token_rows[first, 0])}" + ) + if not np.isfinite(value_rows[:, 0]).all(): + bad_rows = np.flatnonzero(~np.isfinite(value_rows[:, 0])).tolist() + raise SampledLogprobContractError(f"AR sampled-token logprobs contain non-finite values at rows {bad_rows}") + return sliced + + class OmniARScheduler(OmniSchedulerMixin, VLLMScheduler): """Synchronous AutoRegressive scheduler for vLLM-Omni. This class is also used as a base class for the OmniARAsyncScheduler and holds most of the @@ -362,6 +408,27 @@ class OmniARScheduler(OmniSchedulerMixin, VLLMScheduler): req_index = model_runner_output.req_id_to_index[req_id] generated_token_ids = sampled_token_ids[req_index] if sampled_token_ids else [] + status_before_stop = request.status + new_logprobs = None + logprob_validation_failed = False + + # Validate before mutating request token state. A bad runner output + # is request-local: terminate only this request and keep processing + # the rest of the batch. + if ( + generated_token_ids + and request.sampling_params is not None + and request.sampling_params.num_logprobs is not None + ): + try: + new_logprobs = _slice_sampled_logprobs(logprobs, req_index, generated_token_ids) + except SampledLogprobContractError as exc: + logger.error("Invalid AR sampled-token logprobs for request %s: %s", req_id, exc) + request.status = RequestStatus.FINISHED_ERROR + request.stop_reason = str(exc) + request.resumable = False + generated_token_ids = [] + logprob_validation_failed = True scheduled_spec_token_ids = scheduler_output.scheduled_spec_decode_tokens.get(req_id) if scheduled_spec_token_ids and generated_token_ids: @@ -391,16 +458,14 @@ class OmniARScheduler(OmniSchedulerMixin, VLLMScheduler): if request.has_encoder_inputs: self._free_encoder_inputs(request) - stopped = False + stopped = logprob_validation_failed is_segment_finished = False finished = False - new_logprobs = None new_token_ids = generated_token_ids pooler_output = pooler_outputs[req_index] if pooler_outputs else None mm_output = mm_outputs[req_index] if mm_outputs else None inter_stage_output = inter_stage_outputs[req_index] if inter_stage_outputs else None kv_transfer_params = None - status_before_stop = request.status finish_reason = None routed_experts = None @@ -471,10 +536,6 @@ class OmniARScheduler(OmniSchedulerMixin, VLLMScheduler): else: stopped_preempted_reqs.add(request) - # Extract sample logprobs if needed. - if request.sampling_params is not None and request.sampling_params.num_logprobs is not None and logprobs: - new_logprobs = logprobs.slice_request(req_index, len(new_token_ids)) - if num_nans_in_logits is not None and req_id in num_nans_in_logits: request.num_nans_in_logits = num_nans_in_logits[req_id] From f2c5152df07c0c71f9b08bf9c67dc46309935b5d Mon Sep 17 00:00:00 2001 From: Weiming Liao Date: Tue, 28 Jul 2026 11:44:30 +0800 Subject: [PATCH 123/240] [CI][NPU]: Add nightly performance test for NPU - qwen3-omni/wan2.2/qwen-image-edit-2511 (#5339) Signed-off-by: Weiming Liao Co-authored-by: wangyu <53896905+yenuo26@users.noreply.github.com> --- .buildkite/npu/test-npu-nightly.yml | 107 ++++++++++++++---- docker/Dockerfile.npu.ci | 4 + docker/Dockerfile.npu.ci.a3 | 4 + .../perf/scripts/run_diffusion_benchmark.py | 13 ++- .../tests/test_qwen3_omni_async_chunk.json | 3 +- .../tests/test_qwen3_omni_no_async_chunk.json | 3 +- .../test_qwen_image_edit_2511_vllm_omni.json | 3 +- .../perf/tests/test_wan22_i2v_vllm_omni.json | 3 +- 8 files changed, 114 insertions(+), 26 deletions(-) diff --git a/.buildkite/npu/test-npu-nightly.yml b/.buildkite/npu/test-npu-nightly.yml index e05178c52..5aa003241 100644 --- a/.buildkite/npu/test-npu-nightly.yml +++ b/.buildkite/npu/test-npu-nightly.yml @@ -1,25 +1,59 @@ steps: - - group: ":card_index_dividers: Diffusion X2V Model Test" - key: nightly-diffusion-x2v-group + - group: ":card_index_dividers: Omni Model Test" + key: nightly-omni-test-group depends_on: upload-nightly-pipeline if: >- build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" || - build.pull_request.labels includes "diffusion-x2v-test" + build.pull_request.labels includes "omni-test" steps: - - label: ":full_moon: Diffusion X2V · Function Test" - mirror_hardwares: a2b3_npu_4 + - label: ":full_moon: Omni · Perf Test · No Async Chunk" + key: nightly-omni-performance-no-async-chunk + timeout_in_minutes: 180 + mirror_hardwares: a3_npu_4 env: VLLM_WORKER_MULTIPROC_METHOD: spawn + VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1" HF_TOKEN: "${HF_TOKEN}" commands: - - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k npu_i2v --run-level "full_model" + - export BENCHMARK_DIR=tests/dfx/perf/results + - | + set +e + pytest -s -v tests/dfx/perf/scripts/run_benchmark.py -m npu --test-config-file tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json + EXIT=$$? + buildkite-agent artifact upload "tests/dfx/perf/results/*.json" + exit $$EXIT + - label: ":full_moon: Omni · Perf Test · Async Chunk" + key: nightly-omni-performance-async-chunk + timeout_in_minutes: 180 + mirror_hardwares: a3_npu_4 + env: + VLLM_WORKER_MULTIPROC_METHOD: spawn + VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1" + HF_TOKEN: "${HF_TOKEN}" + commands: + - export BENCHMARK_DIR=tests/dfx/perf/results + - | + set +e + pytest -s -v tests/dfx/perf/scripts/run_benchmark.py -m npu --test-config-file tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json + EXIT=$$? + buildkite-agent artifact upload "tests/dfx/perf/results/*.json" + exit $$EXIT + - label: ":robot: Omni · MiniCPM-o 4.5 Test" + mirror_hardwares: a3_npu_2 + env: + VLLM_WORKER_MULTIPROC_METHOD: spawn + VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1" + HF_TOKEN: "${HF_TOKEN}" + commands: + - pytest -s -v tests/e2e/offline_inference/test_minicpmo_4_5.py tests/e2e/online_serving/test_minicpmo_4_5.py tests/e2e/online_serving/test_minicpmo_4_5_expansion.py -m 'full_model' --run-level 'full_model' - group: ":card_index_dividers: TTS Model Test" key: nightly-tts-test-group depends_on: upload-nightly-pipeline if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" || build.pull_request.labels includes "tts-test" steps: - label: ":full_moon: TTS · Perf Test" + timeout_in_minutes: 180 mirror_hardwares: a3_npu_2 env: VLLM_WORKER_MULTIPROC_METHOD: spawn @@ -34,25 +68,56 @@ steps: EXIT=$$? buildkite-agent artifact upload "tests/dfx/perf/results/*.json" exit $$EXIT - - - group: ":card_index_dividers: Omni Model Test" - key: nightly-omni-group + - group: ":card_index_dividers: Diffusion X2I(&A&T) Model Test" + key: nightly-diffusion-x2iat-group depends_on: upload-nightly-pipeline - if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" || build.pull_request.labels includes "omni-test" + if: >- + build.env("NIGHTLY") == "1" || + build.pull_request.labels includes "nightly-test" || + build.pull_request.labels includes "diffusion-x2iat-test" steps: - - label: ":robot: Omni · MiniCPM-o 4.5 Test" - agents: - queue: "ascend-a3" - resource_class: "npu-2" - image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}-a3" - plugins: - - kubernetes: - podSpecPatch: - imagePullSecrets: - - name: swr-secret + - label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image-Edit-2511" + key: nightly-diffusion-x2iat-performance-qwen-image-edit-2511 + timeout_in_minutes: 180 + mirror_hardwares: a3_npu_8 env: VLLM_WORKER_MULTIPROC_METHOD: spawn VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1" HF_TOKEN: "${HF_TOKEN}" + DIFFUSION_ATTENTION_BACKEND: TORCH_SDPA commands: - - pytest -s -v tests/e2e/offline_inference/test_minicpmo_4_5.py tests/e2e/online_serving/test_minicpmo_4_5.py tests/e2e/online_serving/test_minicpmo_4_5_expansion.py -m 'full_model' --run-level 'full_model' + - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results + - export CACHE_DIT_VERSION=1.3.0 + - | + set +e + pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py -m npu --test-config-file tests/dfx/perf/tests/test_qwen_image_edit_2511_vllm_omni.json + EXIT=$$? + buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json" + buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log" + exit $$EXIT + - group: ":card_index_dividers: Diffusion X2V Model Test" + key: nightly-diffusion-x2v-group + depends_on: upload-nightly-pipeline + if: >- + build.env("NIGHTLY") == "1" || + build.pull_request.labels includes "nightly-test" || + build.pull_request.labels includes "diffusion-x2v-test" + steps: + - label: ":full_moon: Diffusion X2V · Perf Test" + key: nightly-diffusion-x2v-performance + timeout_in_minutes: 180 + mirror_hardwares: a3_npu_4 + env: + VLLM_WORKER_MULTIPROC_METHOD: spawn + VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1" + HF_TOKEN: "${HF_TOKEN}" + DIFFUSION_ATTENTION_BACKEND: TORCH_SDPA + commands: + - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results + - | + set +e + pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py -m npu --test-config-file tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json + EXIT=$$? + buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json" + buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log" + exit $$EXIT diff --git a/docker/Dockerfile.npu.ci b/docker/Dockerfile.npu.ci index c0b4af9b6..10a62fcac 100644 --- a/docker/Dockerfile.npu.ci +++ b/docker/Dockerfile.npu.ci @@ -37,6 +37,10 @@ RUN source /usr/local/Ascend/ascend-toolkit/set_env.sh && \ "pyttsx3>=2.99" \ "opencv-python>=4.12.0.88" \ "imageio[ffmpeg]>=0.6.0" \ + "jiwer>=3.0.0" \ + "zhon>=2.0.0" \ + "zhconv>=1.4.2" \ + "funasr>=1.0.0" \ "hyperpyyaml>=1.2.0" \ "onnx>=1.16.0" \ "librosa>=0.10.0" \ diff --git a/docker/Dockerfile.npu.ci.a3 b/docker/Dockerfile.npu.ci.a3 index b86d7c0f8..5ce526c96 100644 --- a/docker/Dockerfile.npu.ci.a3 +++ b/docker/Dockerfile.npu.ci.a3 @@ -37,6 +37,10 @@ RUN source /usr/local/Ascend/ascend-toolkit/set_env.sh && \ "pyttsx3>=2.99" \ "opencv-python>=4.12.0.88" \ "imageio[ffmpeg]>=0.6.0" \ + "jiwer>=3.0.0" \ + "zhon>=2.0.0" \ + "zhconv>=1.4.2" \ + "funasr>=1.0.0" \ "hyperpyyaml>=1.2.0" \ "onnx>=1.16.0" \ "librosa>=0.10.0" \ diff --git a/tests/dfx/perf/scripts/run_diffusion_benchmark.py b/tests/dfx/perf/scripts/run_diffusion_benchmark.py index d19c0900e..375d2a440 100644 --- a/tests/dfx/perf/scripts/run_diffusion_benchmark.py +++ b/tests/dfx/perf/scripts/run_diffusion_benchmark.py @@ -355,6 +355,17 @@ def _kill_process_tree(pid: int) -> None: # --------------------------------------------------------------------------- +def _resolve_offline_model(model: str) -> str: + """Under HF_HUB_OFFLINE, resolve a HF repo id to its local snapshot dir.""" + import huggingface_hub + + if not model or os.path.isdir(model) or not huggingface_hub.constants.HF_HUB_OFFLINE: + return model + from huggingface_hub import snapshot_download + + return snapshot_download(model, local_files_only=True) + + class DiffusionServer: """Start a vLLM-Omni diffusion model server as a subprocess. @@ -372,7 +383,7 @@ class DiffusionServer: port: int | None = None, ) -> None: self.server_cfg: dict[str, Any] = server_cfg - self.model = server_cfg["model"] + self.model = _resolve_offline_model(server_cfg["model"]) self.serve_args = server_cfg["serve_args"] self.host = "127.0.0.1" self.port = port if port is not None else get_open_port(self.host) diff --git a/tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json b/tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json index 013436d80..be108cbd6 100644 --- a/tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json +++ b/tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json @@ -5,7 +5,8 @@ { "hardware_marks": { "res": { - "cuda": "H100" + "cuda": "H100", + "npu": "A3" }, "num_cards": 2 } diff --git a/tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json b/tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json index 71c90f0e7..3fbeafb9d 100644 --- a/tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json +++ b/tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json @@ -5,7 +5,8 @@ { "hardware_marks": { "res": { - "cuda": "H100" + "cuda": "H100", + "npu": "A3" }, "num_cards": 2 } diff --git a/tests/dfx/perf/tests/test_qwen_image_edit_2511_vllm_omni.json b/tests/dfx/perf/tests/test_qwen_image_edit_2511_vllm_omni.json index 350cfb6bb..75a3d7744 100644 --- a/tests/dfx/perf/tests/test_qwen_image_edit_2511_vllm_omni.json +++ b/tests/dfx/perf/tests/test_qwen_image_edit_2511_vllm_omni.json @@ -66,7 +66,8 @@ { "hardware_marks": { "res": { - "cuda": "H100" + "cuda": "H100", + "npu": "A3" }, "num_cards": 4 } diff --git a/tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json b/tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json index 53c7da9b7..52441dae4 100644 --- a/tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json +++ b/tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json @@ -55,7 +55,8 @@ { "hardware_marks": { "res": { - "cuda": "H100" + "cuda": "H100", + "npu": "A3" }, "num_cards": 2 } From 5014fea4a0652c61feaf72672b587fcf10d259ab Mon Sep 17 00:00:00 2001 From: Yuekai Zhang Date: Tue, 28 Jul 2026 11:58:22 +0800 Subject: [PATCH 124/240] [New Model] support nemotron Audex (#4976) Signed-off-by: Yuekai Zhang Signed-off-by: Yueqian Lin Co-authored-by: Claude Fable 5 Co-authored-by: Yueqian Lin Co-authored-by: Yueqian Lin <70319226+linyueqian@users.noreply.github.com> --- examples/offline_inference/audex/README.md | 114 ++++ examples/offline_inference/audex/audio_qa.py | 93 +++ examples/offline_inference/audex/common.py | 149 +++++ .../audex/speech_to_speech.py | 277 +++++++++ .../offline_inference/audex/text_to_audio.py | 157 +++++ .../offline_inference/audex/text_to_speech.py | 162 +++++ examples/online_serving/audex/README.md | 89 +++ examples/online_serving/audex/client.py | 319 ++++++++++ examples/online_serving/audex/run_server.sh | 57 ++ recipes/NVIDIA/Nemotron-Labs-Audex.md | 142 +++++ recipes/README.md | 1 + tests/config/test_omni_config.py | 5 + .../sched/test_omni_scheduling_coordinator.py | 4 + .../test_invalid_audio_speech.py | 46 +- tests/e2e/offline_inference/test_audex_s2s.py | 262 ++++++++ tests/e2e/offline_inference/test_audex_s2t.py | 106 ++++ tests/e2e/offline_inference/test_audex_tta.py | 145 +++++ tests/e2e/offline_inference/test_audex_tts.py | 161 +++++ tests/e2e/online_serving/test_audex_tta.py | 84 +++ tests/e2e/online_serving/test_audex_tts.py | 230 +++++++ .../openai_api/test_audex_serving_guards.py | 376 ++++++++++++ tests/helpers/media.py | 21 + vllm_omni/config/config_factory.py | 45 +- vllm_omni/config/model.py | 5 +- vllm_omni/config/omni_config.py | 9 + vllm_omni/config/pipeline_registry.py | 14 + vllm_omni/config/stage_config.py | 2 +- .../core/sched/omni_scheduling_coordinator.py | 6 + vllm_omni/deploy/audex_s2s.yaml | 65 ++ vllm_omni/deploy/audex_s2s_30b.yaml | 60 ++ vllm_omni/deploy/audex_thinker_only.yaml | 25 + vllm_omni/deploy/audex_thinker_only_30b.yaml | 22 + vllm_omni/deploy/audex_tta.yaml | 69 +++ vllm_omni/deploy/audex_tta_30b.yaml | 57 ++ vllm_omni/deploy/audex_tts.yaml | 83 +++ vllm_omni/deploy/audex_tts_30b.yaml | 76 +++ vllm_omni/engine/async_omni_engine.py | 17 +- vllm_omni/engine/stage_engine_core_proc.py | 9 +- vllm_omni/engine/stage_init_utils.py | 65 ++ vllm_omni/entrypoints/cli/serve.py | 4 +- .../entrypoints/openai/serving_speech.py | 184 ++++++ .../openai/tts_adapters/__init__.py | 2 + .../entrypoints/openai/tts_adapters/audex.py | 67 +++ .../openai/tts_adapters/audex_tta.py | 69 +++ vllm_omni/entrypoints/utils.py | 26 +- .../model_executor/models/audex/__init__.py | 2 + .../models/audex/audex_code2wav.py | 227 +++++++ .../model_executor/models/audex/audex_omni.py | 322 ++++++++++ .../models/audex/audex_omni_processing.py | 260 ++++++++ .../models/audex/audex_thinker.py | 414 +++++++++++++ .../models/audex/audex_xcodec.py | 178 ++++++ vllm_omni/model_executor/models/audex/cfg.py | 567 ++++++++++++++++++ .../model_executor/models/audex/checkpoint.py | 186 ++++++ .../model_executor/models/audex/pipeline.py | 195 ++++++ .../model_executor/models/audex/prompt.py | 83 +++ .../models/audex/speech_decoder/__init__.py | 22 + ...nfiguration_audex_causal_speech_decoder.py | 50 ++ .../modeling_audex_causal_speech_decoder.py | 478 +++++++++++++++ .../audex/speech_decoder/streaming_utils.py | 140 +++++ vllm_omni/model_executor/models/audex/tta.py | 293 +++++++++ vllm_omni/model_executor/models/registry.py | 25 + .../stage_input_processors/audex.py | 356 +++++++++++ vllm_omni/worker/gpu_ar_model_runner.py | 18 +- .../worker/gpu_generation_model_runner.py | 2 + 64 files changed, 7760 insertions(+), 39 deletions(-) create mode 100644 examples/offline_inference/audex/README.md create mode 100644 examples/offline_inference/audex/audio_qa.py create mode 100644 examples/offline_inference/audex/common.py create mode 100644 examples/offline_inference/audex/speech_to_speech.py create mode 100644 examples/offline_inference/audex/text_to_audio.py create mode 100644 examples/offline_inference/audex/text_to_speech.py create mode 100644 examples/online_serving/audex/README.md create mode 100644 examples/online_serving/audex/client.py create mode 100755 examples/online_serving/audex/run_server.sh create mode 100644 recipes/NVIDIA/Nemotron-Labs-Audex.md create mode 100644 tests/e2e/offline_inference/test_audex_s2s.py create mode 100644 tests/e2e/offline_inference/test_audex_s2t.py create mode 100644 tests/e2e/offline_inference/test_audex_tta.py create mode 100644 tests/e2e/offline_inference/test_audex_tts.py create mode 100644 tests/e2e/online_serving/test_audex_tta.py create mode 100644 tests/e2e/online_serving/test_audex_tts.py create mode 100644 tests/entrypoints/openai_api/test_audex_serving_guards.py create mode 100644 vllm_omni/deploy/audex_s2s.yaml create mode 100644 vllm_omni/deploy/audex_s2s_30b.yaml create mode 100644 vllm_omni/deploy/audex_thinker_only.yaml create mode 100644 vllm_omni/deploy/audex_thinker_only_30b.yaml create mode 100644 vllm_omni/deploy/audex_tta.yaml create mode 100644 vllm_omni/deploy/audex_tta_30b.yaml create mode 100644 vllm_omni/deploy/audex_tts.yaml create mode 100644 vllm_omni/deploy/audex_tts_30b.yaml create mode 100644 vllm_omni/entrypoints/openai/tts_adapters/audex.py create mode 100644 vllm_omni/entrypoints/openai/tts_adapters/audex_tta.py create mode 100644 vllm_omni/model_executor/models/audex/__init__.py create mode 100644 vllm_omni/model_executor/models/audex/audex_code2wav.py create mode 100644 vllm_omni/model_executor/models/audex/audex_omni.py create mode 100644 vllm_omni/model_executor/models/audex/audex_omni_processing.py create mode 100644 vllm_omni/model_executor/models/audex/audex_thinker.py create mode 100644 vllm_omni/model_executor/models/audex/audex_xcodec.py create mode 100644 vllm_omni/model_executor/models/audex/cfg.py create mode 100644 vllm_omni/model_executor/models/audex/checkpoint.py create mode 100644 vllm_omni/model_executor/models/audex/pipeline.py create mode 100644 vllm_omni/model_executor/models/audex/prompt.py create mode 100644 vllm_omni/model_executor/models/audex/speech_decoder/__init__.py create mode 100644 vllm_omni/model_executor/models/audex/speech_decoder/configuration_audex_causal_speech_decoder.py create mode 100644 vllm_omni/model_executor/models/audex/speech_decoder/modeling_audex_causal_speech_decoder.py create mode 100644 vllm_omni/model_executor/models/audex/speech_decoder/streaming_utils.py create mode 100644 vllm_omni/model_executor/models/audex/tta.py create mode 100644 vllm_omni/model_executor/stage_input_processors/audex.py diff --git a/examples/offline_inference/audex/README.md b/examples/offline_inference/audex/README.md new file mode 100644 index 000000000..793b69eb9 --- /dev/null +++ b/examples/offline_inference/audex/README.md @@ -0,0 +1,114 @@ +# Audex (Nemotron-Labs-Audex-2B) offline inference + +One checkpoint, four tasks — each script builds its own `Omni` engine with +the right deploy yaml, so they run as plain `python